From f03d8c277c4fe9731abdd76e02871f46ef6cb97c Mon Sep 17 00:00:00 2001 From: Akash Kumar <116457960+akashchamp@users.noreply.github.com> Date: Thu, 24 Sep 2026 03:32:02 +0530 Subject: [PATCH] fix(opentelemetry): guard span:finish() against shutdown-time export errors The vendored opentelemetry-lua's batch_span_processor falls back to a synchronous flush_all() when ngx.timer.at() fails to schedule its background flush timer, which happens near the end of every graceful worker shutdown. That synchronous flush opens a cosocket, which trips APISIX's own phase guard (apisix/patch.lua) when span:finish() runs inside a restricted phase such as log_by_lua* or body_filter_by_lua*, crashing the last span(s) on effectively every rolling deploy that has the opentelemetry plugin's tracing enabled. Wrap span:finish() in pcall as defense in depth, since this class of "third-party tracer code doing something disallowed in a restricted phase" can't be fully prevented from the plugin side alone. The underlying fix belongs upstream (yangxikun/opentelemetry-lua#106/#107); this only stops it from crashing the request phase in the meantime. Fixes #13980 --- apisix/plugins/opentelemetry.lua | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua index f1ebeea5bf17..2e325eb78f23 100644 --- a/apisix/plugins/opentelemetry.lua +++ b/apisix/plugins/opentelemetry.lua @@ -51,6 +51,7 @@ local string_format = string.format local string_lower = string.lower local update_time = ngx.update_time local tostring = tostring +local pcall = pcall local lrucache = core.lrucache.new({ type = 'plugin', count = 128, ttl = 24 * 60 * 60, @@ -451,6 +452,20 @@ function _M.rewrite(conf, api_ctx) end +-- the vendored opentelemetry-lua's batch_span_processor falls back to a +-- synchronous flush (opening a cosocket) when it fails to schedule the +-- background timer, which happens near the end of every worker shutdown; +-- that synchronous call trips APISIX's own phase guard (patch.lua) when +-- span:finish() runs inside a restricted phase like log_by_lua*, so guard +-- it here as defense in depth (see opentelemetry-lua#106/#107 upstream) +local function finish_span(span, end_time) + local ok, err = pcall(span.finish, span, end_time) + if not ok then + core.log.warn("failed to finish opentelemetry span, ignore error: ", err) + end +end + + local function create_child_span(tracer, parent_span_ctx, spans, span) if not span or span.finished then return @@ -469,7 +484,7 @@ local function create_child_span(tracer, parent_span_ctx, spans, span) if span.status then new_span:set_status(span.status.code, span.status.message) end - new_span:finish(span.end_time) + finish_span(new_span, span.end_time) end @@ -556,7 +571,7 @@ function _M.log(conf, api_ctx) end update_time() - span:finish() + finish_span(span) end end