diff --git a/include/fluent-bit/tls/flb_tls.h b/include/fluent-bit/tls/flb_tls.h index 018231218e2..f3c1f485eee 100644 --- a/include/fluent-bit/tls/flb_tls.h +++ b/include/fluent-bit/tls/flb_tls.h @@ -35,6 +35,13 @@ #define FLB_TLS_WANT_READ -0x7e4 #define FLB_TLS_WANT_WRITE -0x7e6 +/* + * Maximum consecutive WANT_READ/WANT_WRITE retries before treating the + * connection as stale. Prevents a dead TLS connection from spinning the + * engine thread in a tight yield/resume loop (see #9927). + */ +#define FLB_TLS_WANT_READ_MAX_RETRIES 800 + /* Cert Flags */ #define FLB_TLS_CA_ROOT 1 #define FLB_TLS_CERT 2 diff --git a/src/tls/flb_tls.c b/src/tls/flb_tls.c index 40961b6a12a..0239b6475cf 100644 --- a/src/tls/flb_tls.c +++ b/src/tls/flb_tls.c @@ -17,6 +17,8 @@ * limitations under the License. */ +#include + #include #include #include @@ -393,17 +395,50 @@ int flb_tls_net_read_async(struct flb_coro *co, struct mk_event event_backup; struct flb_tls *tls; int ret; + int want_read_retries; tls = session->tls; event_restore_needed = FLB_FALSE; + /* + * Track consecutive SSL_ERROR_WANT_READ iterations with no progress. + * When a TLS connection's underlying socket continuously reports + * EPOLLIN but SSL_read() returns WANT_READ (no actual TLS data), + * the coroutine bounces between yield and resume in a tight loop, + * monopolizing the engine thread and starving all other processing + * (output flushes, timers, new connections). + * + * This happens when a dead TCP connection stays ESTABLISHED but + * has no TLS records to deliver — the kernel signals "readable" + * but OpenSSL needs more data. + * + * See: https://github.com/fluent/fluent-bit/issues/9927 + */ + want_read_retries = 0; + io_tls_backup_event(session->connection, &event_backup); retry_read: ret = tls->api->net_read(session, buf, len); if (ret == FLB_TLS_WANT_READ) { + want_read_retries++; + + if (want_read_retries > FLB_TLS_WANT_READ_MAX_RETRIES) { + flb_warn("[tls] connection #%i exceeded maximum read retries (%i), " + "closing stale connection", + session->connection->fd, + FLB_TLS_WANT_READ_MAX_RETRIES); + + session->connection->coroutine = NULL; + session->connection->net_error = ETIMEDOUT; + + io_tls_restore_event(session->connection, &event_backup); + + return -1; + } + event_restore_needed = FLB_TRUE; session->connection->coroutine = co; @@ -414,6 +449,8 @@ int flb_tls_net_read_async(struct flb_coro *co, goto retry_read; } else if (ret == FLB_TLS_WANT_WRITE) { + /* Progress on a different axis, reset the read retry counter */ + want_read_retries = 0; event_restore_needed = FLB_TRUE; session->connection->coroutine = co; @@ -500,11 +537,13 @@ int flb_tls_net_write_async(struct flb_coro *co, size_t total; int ret; struct flb_tls *tls; + int want_retries; total = 0; tls = session->tls; event_restore_needed = FLB_FALSE; + want_retries = 0; io_tls_backup_event(session->connection, &event_backup); @@ -516,6 +555,23 @@ int flb_tls_net_write_async(struct flb_coro *co, len - total); if (ret == FLB_TLS_WANT_WRITE) { + want_retries++; + + if (want_retries > FLB_TLS_WANT_READ_MAX_RETRIES) { + flb_warn("[tls] connection #%i exceeded maximum write retries (%i), " + "closing stale connection", + session->connection->fd, + FLB_TLS_WANT_READ_MAX_RETRIES); + + session->connection->coroutine = NULL; + session->connection->net_error = ETIMEDOUT; + *out_len = total; + + io_tls_restore_event(session->connection, &event_backup); + + return -1; + } + event_restore_needed = FLB_TRUE; io_tls_event_switch(session, MK_EVENT_WRITE); @@ -525,6 +581,23 @@ int flb_tls_net_write_async(struct flb_coro *co, goto retry_write; } else if (ret == FLB_TLS_WANT_READ) { + want_retries++; + + if (want_retries > FLB_TLS_WANT_READ_MAX_RETRIES) { + flb_warn("[tls] connection #%i exceeded maximum write retries (%i), " + "closing stale connection", + session->connection->fd, + FLB_TLS_WANT_READ_MAX_RETRIES); + + session->connection->coroutine = NULL; + session->connection->net_error = ETIMEDOUT; + *out_len = total; + + io_tls_restore_event(session->connection, &event_backup); + + return -1; + } + event_restore_needed = FLB_TRUE; io_tls_event_switch(session, MK_EVENT_READ);