From f0bec26be49b1ddd4591a144722af51b13fd1000 Mon Sep 17 00:00:00 2001 From: Kayla Reopelle Date: Wed, 8 Apr 2026 14:50:49 +0200 Subject: [PATCH 01/10] fix: Limit length of response body read to 4mb Limiting the read size may help prevent memory exhaustion exploits when the configured collector endpoint is attacker-controlled. --- .../exporter/otlp/http/trace_exporter.rb | 44 ++++++- .../exporter/otlp/http/trace_exporter_test.rb | 77 +++++++++++ .../exporter/otlp/logs/logs_exporter.rb | 44 ++++++- .../exporter/otlp/logs_exporter_test.rb | 77 +++++++++++ .../exporter/otlp/metrics/metrics_exporter.rb | 9 +- .../exporter/otlp/metrics/util.rb | 35 ++++- .../otlp/metrics/metrics_exporter_test.rb | 77 +++++++++++ .../opentelemetry/exporter/otlp/exporter.rb | 44 ++++++- .../exporter/otlp/exporter_test.rb | 121 ++++++++++++++++++ 9 files changed, 501 insertions(+), 27 deletions(-) diff --git a/exporter/otlp-http/lib/opentelemetry/exporter/otlp/http/trace_exporter.rb b/exporter/otlp-http/lib/opentelemetry/exporter/otlp/http/trace_exporter.rb index 74c92a4f4c..55e862402c 100644 --- a/exporter/otlp-http/lib/opentelemetry/exporter/otlp/http/trace_exporter.rb +++ b/exporter/otlp-http/lib/opentelemetry/exporter/otlp/http/trace_exporter.rb @@ -25,7 +25,8 @@ class TraceExporter # rubocop:disable Metrics/ClassLength # Default timeouts in seconds. KEEP_ALIVE_TIMEOUT = 30 RETRY_COUNT = 5 - private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT) + RESPONSE_BODY_LIMIT = 4_194_304 # 4 MB + private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT, :RESPONSE_BODY_LIMIT) ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash' private_constant(:ERROR_MESSAGE_INVALID_HEADERS) @@ -158,18 +159,19 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/MethodLength case response when Net::HTTPSuccess - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory SUCCESS when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code) FAILURE when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory redo if backoff?(retry_count: retry_count += 1, reason: response.code) FAILURE when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError - log_status(response.body) + body = read_response_body(response) + log_status(body) @metrics_reporter.add_to_counter('otel.otlp_exporter.failure', labels: { 'reason' => response.code }) FAILURE when Net::HTTPRedirection @@ -216,14 +218,42 @@ def handle_redirect(location) end def log_status(body) + truncation_note = @body_truncated ? ' (body truncated due to size limit)' : '' status = Google::Rpc::Status.decode(body) details = status.details.map do |detail| klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(detail.type_name).msgclass detail.unpack(klass_or_nil) if klass_or_nil end.compact - OpenTelemetry.handle_error(message: "OTLP exporter received rpc.Status{message=#{status.message}, details=#{details}}") + OpenTelemetry.handle_error(message: "OTLP exporter received rpc.Status{message=#{status.message}, details=#{details}}#{truncation_note}") rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: 'unexpected error decoding rpc.Status in OTLP::Exporter#log_status') + OpenTelemetry.handle_error(exception: e, message: "unexpected error decoding rpc.Status in OTLP::Exporter#log_status#{truncation_note}") + ensure + @body_truncated = false + end + + def read_response_body(response) + return '' if response.nil? + + body = +'' + truncated = false + + response.read_body do |chunk| + if body.bytesize + chunk.bytesize <= RESPONSE_BODY_LIMIT + body << chunk + else + remaining = RESPONSE_BODY_LIMIT - body.bytesize + body << chunk.byteslice(0, remaining) if remaining > 0 + truncated = true + break + end + end + + body.force_encoding('UTF-8') + @body_truncated = truncated + body + rescue StandardError => e + OpenTelemetry.handle_error(exception: e, message: 'error reading response body') + '' end def measure_request_duration diff --git a/exporter/otlp-http/test/opentelemetry/exporter/otlp/http/trace_exporter_test.rb b/exporter/otlp-http/test/opentelemetry/exporter/otlp/http/trace_exporter_test.rb index e5d3ef2160..c7dfe4826e 100644 --- a/exporter/otlp-http/test/opentelemetry/exporter/otlp/http/trace_exporter_test.rb +++ b/exporter/otlp-http/test/opentelemetry/exporter/otlp/http/trace_exporter_test.rb @@ -742,4 +742,81 @@ end end end + + describe 'response body reading' do + let(:exporter) { OpenTelemetry::Exporter::OTLP::HTTP::TraceExporter.new } + let(:span_data) { OpenTelemetry::TestHelpers.create_span_data } + + it 'discards body for successful responses without reading into memory' do + stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 200, body: 'success body') + + result = exporter.export([span_data]) + + _(result).must_equal(success) + end + + it 'discards body for retryable responses without reading into memory' do + stub_request(:post, 'http://localhost:4318/v1/traces') + .to_return(status: 503, body: 'service unavailable', headers: { 'Retry-After' => '0' }) + .then.to_return(status: 200) + + result = exporter.export([span_data]) + + _(result).must_equal(success) + end + + it 'reads and parses error response body smaller than limit' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: 'error details'))] + status = ::Google::Rpc::Status.encode(::Google::Rpc::Status.new(code: 3, message: 'invalid argument', details: details)) + stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 400, body: status) + + result = exporter.export([span_data]) + + _(result).must_equal(export_failure) + _(log_stream.string).must_match(/invalid argument/) + _(log_stream.string).wont_match(/truncated/) + ensure + OpenTelemetry.logger = logger + end + + it 'truncates error response body larger than 4 MB limit' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + # Create a body larger than 4 MB + large_message = 'x' * 5_000_000 # 5 MB + details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: large_message))] + large_status = ::Google::Rpc::Status.new(code: 3, message: 'large error', details: details) + large_body = ::Google::Rpc::Status.encode(large_status) + + stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 400, body: large_body) + + result = exporter.export([span_data]) + + _(result).must_equal(export_failure) + _(log_stream.string).must_match(/body truncated due to size limit/) + ensure + OpenTelemetry.logger = logger + end + + it 'handles malformed error response body gracefully' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 400, body: 'not valid protobuf') + + result = exporter.export([span_data]) + + _(result).must_equal(export_failure) + _(log_stream.string).must_match(/unexpected error decoding rpc.Status/) + ensure + OpenTelemetry.logger = logger + end + end end diff --git a/exporter/otlp-logs/lib/opentelemetry/exporter/otlp/logs/logs_exporter.rb b/exporter/otlp-logs/lib/opentelemetry/exporter/otlp/logs/logs_exporter.rb index 91f5d7e4db..1d32540516 100644 --- a/exporter/otlp-logs/lib/opentelemetry/exporter/otlp/logs/logs_exporter.rb +++ b/exporter/otlp-logs/lib/opentelemetry/exporter/otlp/logs/logs_exporter.rb @@ -31,7 +31,8 @@ class LogsExporter # rubocop:disable Metrics/ClassLength # Default timeouts in seconds. KEEP_ALIVE_TIMEOUT = 30 RETRY_COUNT = 5 - private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT) + RESPONSE_BODY_LIMIT = 4_194_304 # 4 MB + private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT, :RESPONSE_BODY_LIMIT) ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash' private_constant(:ERROR_MESSAGE_INVALID_HEADERS) @@ -167,15 +168,15 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity, case response when Net::HTTPSuccess - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory SUCCESS when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory handle_http_error(response) redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1) FAILURE when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory handle_http_error(response) redo if backoff?(retry_count: retry_count += 1) FAILURE @@ -183,7 +184,8 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity, handle_http_error(response) FAILURE when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError - log_status(response.body) + body = read_response_body(response) + log_status(body) FAILURE when Net::HTTPRedirection @http.finish @@ -234,14 +236,42 @@ def handle_redirect(location) end def log_status(body) + truncation_note = @body_truncated ? ' (body truncated due to size limit)' : '' status = Google::Rpc::Status.decode(body) details = status.details.map do |detail| klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(detail.type_name).msgclass detail.unpack(klass_or_nil) if klass_or_nil end.compact - OpenTelemetry.handle_error(message: "OTLP logs exporter received rpc.Status{message=#{status.message}, details=#{details}}") + OpenTelemetry.handle_error(message: "OTLP logs exporter received rpc.Status{message=#{status.message}, details=#{details}}#{truncation_note}") rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: 'unexpected error decoding rpc.Status in OTLP::Exporter#log_status') + OpenTelemetry.handle_error(exception: e, message: "unexpected error decoding rpc.Status in OTLP::Exporter#log_status#{truncation_note}") + ensure + @body_truncated = false + end + + def read_response_body(response) + return '' if response.nil? + + body = +'' + truncated = false + + response.read_body do |chunk| + if body.bytesize + chunk.bytesize <= RESPONSE_BODY_LIMIT + body << chunk + else + remaining = RESPONSE_BODY_LIMIT - body.bytesize + body << chunk.byteslice(0, remaining) if remaining > 0 + truncated = true + break + end + end + + body.force_encoding('UTF-8') + @body_truncated = truncated + body + rescue StandardError => e + OpenTelemetry.handle_error(exception: e, message: 'error reading response body') + '' end def backoff?(retry_count:, retry_after: nil) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity diff --git a/exporter/otlp-logs/test/opentelemetry/exporter/otlp/logs_exporter_test.rb b/exporter/otlp-logs/test/opentelemetry/exporter/otlp/logs_exporter_test.rb index d4dccd34ea..eaf012bba9 100644 --- a/exporter/otlp-logs/test/opentelemetry/exporter/otlp/logs_exporter_test.rb +++ b/exporter/otlp-logs/test/opentelemetry/exporter/otlp/logs_exporter_test.rb @@ -955,4 +955,81 @@ end end end + + describe 'response body reading' do + let(:exporter) { OpenTelemetry::Exporter::OTLP::Logs::LogsExporter.new } + let(:log_record_data) { OpenTelemetry::TestHelpers.create_log_record_data } + + it 'discards body for successful responses without reading into memory' do + stub_request(:post, 'http://localhost:4318/v1/logs').to_return(status: 200, body: 'success body') + + result = exporter.export([log_record_data]) + + _(result).must_equal(SUCCESS) + end + + it 'discards body for retryable responses without reading into memory' do + stub_request(:post, 'http://localhost:4318/v1/logs') + .to_return(status: 503, body: 'service unavailable', headers: { 'Retry-After' => '0' }) + .then.to_return(status: 200) + + result = exporter.export([log_record_data]) + + _(result).must_equal(SUCCESS) + end + + it 'reads and parses error response body smaller than limit' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: 'error details'))] + status = ::Google::Rpc::Status.encode(::Google::Rpc::Status.new(code: 3, message: 'invalid argument', details: details)) + stub_request(:post, 'http://localhost:4318/v1/logs').to_return(status: 400, body: status) + + result = exporter.export([log_record_data]) + + _(result).must_equal(FAILURE) + _(log_stream.string).must_match(/invalid argument/) + _(log_stream.string).wont_match(/truncated/) + ensure + OpenTelemetry.logger = logger + end + + it 'truncates error response body larger than 4 MB limit' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + # Create a body larger than 4 MB + large_message = 'x' * 5_000_000 # 5 MB + details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: large_message))] + large_status = ::Google::Rpc::Status.new(code: 3, message: 'large error', details: details) + large_body = ::Google::Rpc::Status.encode(large_status) + + stub_request(:post, 'http://localhost:4318/v1/logs').to_return(status: 400, body: large_body) + + result = exporter.export([log_record_data]) + + _(result).must_equal(FAILURE) + _(log_stream.string).must_match(/body truncated due to size limit/) + ensure + OpenTelemetry.logger = logger + end + + it 'handles malformed error response body gracefully' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + stub_request(:post, 'http://localhost:4318/v1/logs').to_return(status: 400, body: 'not valid protobuf') + + result = exporter.export([log_record_data]) + + _(result).must_equal(FAILURE) + _(log_stream.string).must_match(/unexpected error decoding rpc.Status/) + ensure + OpenTelemetry.logger = logger + end + end end diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb index 9157e1fdfe..d032762d82 100644 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb @@ -120,15 +120,15 @@ def send_bytes(bytes, timeout:) response = @http.request(request) case response when Net::HTTPSuccess - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory SUCCESS when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code) OpenTelemetry.logger.warn('Net::HTTPServiceUnavailable/Net::HTTPTooManyRequests in MetricsExporter#send_bytes') FAILURE when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory redo if backoff?(retry_count: retry_count += 1, reason: response.code) OpenTelemetry.logger.warn('Net::HTTPRequestTimeOut/Net::HTTPGatewayTimeOut/Net::HTTPBadGateway in MetricsExporter#send_bytes') FAILURE @@ -136,7 +136,8 @@ def send_bytes(bytes, timeout:) OpenTelemetry.handle_error(message: "OTLP metrics_exporter received http.code=404 for uri: '#{@path}'") FAILURE when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError - log_status(response.body) + body = read_response_body(response) + log_status(body) OpenTelemetry.logger.warn('Net::HTTPBadRequest/Net::HTTPClientError/Net::HTTPServerError in MetricsExporter#send_bytes') FAILURE when Net::HTTPRedirection diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb index efcbc6ff87..f4a7a51a62 100644 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb @@ -12,8 +12,10 @@ module Metrics module Util # rubocop:disable Metrics/ModuleLength KEEP_ALIVE_TIMEOUT = 30 RETRY_COUNT = 5 + RESPONSE_BODY_LIMIT = 4_194_304 # 4 MB ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash' DEFAULT_USER_AGENT = "OTel-OTLP-MetricsExporter-Ruby/#{OpenTelemetry::Exporter::OTLP::Metrics::VERSION} Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM}; #{RUBY_ENGINE}/#{RUBY_ENGINE_VERSION})".freeze + private_constant(:RESPONSE_BODY_LIMIT) def http_connection(uri, ssl_verify_mode, certificate_file, client_certificate_file, client_key_file) http = Net::HTTP.new(uri.hostname, uri.port) @@ -115,15 +117,44 @@ def backoff?(retry_count:, reason:, retry_after: nil) end def log_status(body) + truncation_note = @body_truncated ? ' (body truncated due to size limit)' : '' status = Google::Rpc::Status.decode(body) details = status.details.map do |detail| type_name = detail.type_url.to_s.split('/').last.to_s klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(type_name)&.msgclass detail.unpack(klass_or_nil) if klass_or_nil end.compact - OpenTelemetry.handle_error(message: "OTLP metrics_exporter received rpc.Status{message=#{status.message}, details=#{details}}") + OpenTelemetry.handle_error(message: "OTLP metrics_exporter received rpc.Status{message=#{status.message}, details=#{details}}#{truncation_note}") rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: 'unexpected error decoding rpc.Status in OTLP::MetricsExporter#log_status') + OpenTelemetry.handle_error(exception: e, message: "unexpected error decoding rpc.Status in OTLP::MetricsExporter#log_status#{truncation_note}") + ensure + @body_truncated = false + end + + def read_response_body(response) + return '' if response.nil? + + # Stream read with 4 MB limit + body = +'' + truncated = false + + response.read_body do |chunk| + if body.bytesize + chunk.bytesize <= RESPONSE_BODY_LIMIT + body << chunk + else + remaining = RESPONSE_BODY_LIMIT - body.bytesize + body << chunk.byteslice(0, remaining) if remaining > 0 + truncated = true + break + end + end + + body.force_encoding('UTF-8') + @body_truncated = truncated + body + rescue StandardError => e + OpenTelemetry.handle_error(exception: e, message: 'error reading response body') + '' end def handle_redirect(location); end diff --git a/exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics/metrics_exporter_test.rb b/exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics/metrics_exporter_test.rb index 7debfd74c3..9eecbc12ce 100644 --- a/exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics/metrics_exporter_test.rb +++ b/exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics/metrics_exporter_test.rb @@ -913,4 +913,81 @@ end end end + + describe 'response body reading' do + let(:exporter) { OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new } + let(:metric_data) { create_metrics_data } + + it 'discards body for successful responses without reading into memory' do + stub_request(:post, 'http://localhost:4318/v1/metrics').to_return(status: 200, body: 'success body') + + result = exporter.export([metric_data]) + + _(result).must_equal(METRICS_SUCCESS) + end + + it 'discards body for retryable responses without reading into memory' do + stub_request(:post, 'http://localhost:4318/v1/metrics') + .to_return(status: 503, body: 'service unavailable', headers: { 'Retry-After' => '0' }) + .then.to_return(status: 200) + + result = exporter.export([metric_data]) + + _(result).must_equal(METRICS_SUCCESS) + end + + it 'reads and parses error response body smaller than limit' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: 'error details'))] + status = ::Google::Rpc::Status.encode(::Google::Rpc::Status.new(code: 3, message: 'invalid argument', details: details)) + stub_request(:post, 'http://localhost:4318/v1/metrics').to_return(status: 400, body: status) + + result = exporter.export([metric_data]) + + _(result).must_equal(METRICS_FAILURE) + _(log_stream.string).must_match(/invalid argument/) + _(log_stream.string).wont_match(/truncated/) + ensure + OpenTelemetry.logger = logger + end + + it 'truncates error response body larger than 4 MB limit' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + # Create a body larger than 4 MB + large_message = 'x' * 5_000_000 # 5 MB + details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: large_message))] + large_status = ::Google::Rpc::Status.new(code: 3, message: 'large error', details: details) + large_body = ::Google::Rpc::Status.encode(large_status) + + stub_request(:post, 'http://localhost:4318/v1/metrics').to_return(status: 400, body: large_body) + + result = exporter.export([metric_data]) + + _(result).must_equal(METRICS_FAILURE) + _(log_stream.string).must_match(/body truncated due to size limit/) + ensure + OpenTelemetry.logger = logger + end + + it 'handles malformed error response body gracefully' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + stub_request(:post, 'http://localhost:4318/v1/metrics').to_return(status: 400, body: 'not valid protobuf') + + result = exporter.export([metric_data]) + + _(result).must_equal(METRICS_FAILURE) + _(log_stream.string).must_match(/unexpected error decoding rpc.Status/) + ensure + OpenTelemetry.logger = logger + end + end end diff --git a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb index 9647177f61..5f61e7f9a7 100644 --- a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb +++ b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb @@ -28,7 +28,8 @@ class Exporter # rubocop:disable Metrics/ClassLength # Default timeouts in seconds. KEEP_ALIVE_TIMEOUT = 30 RETRY_COUNT = 5 - private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT) + RESPONSE_BODY_LIMIT = 4_194_304 # 4 MB + private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT, :RESPONSE_BODY_LIMIT) ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash' private_constant(:ERROR_MESSAGE_INVALID_HEADERS) @@ -178,21 +179,22 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity, case response when Net::HTTPSuccess - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory SUCCESS when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code) FAILURE when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway - response.body # Read and discard body + response.read_body(nil) # Discard without reading into memory redo if backoff?(retry_count: retry_count += 1, reason: response.code) FAILURE when Net::HTTPNotFound log_request_failure(response.code) FAILURE when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError - log_status(response.body) + body = read_response_body(response) + log_status(body) @metrics_reporter.add_to_counter('otel.otlp_exporter.failure', labels: { 'reason' => response.code }) FAILURE when Net::HTTPRedirection @@ -240,14 +242,42 @@ def handle_redirect(location) end def log_status(body) + truncation_note = @body_truncated ? ' (body truncated due to size limit)' : '' status = Google::Rpc::Status.decode(body) details = status.details.map do |detail| klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(detail.type_name).msgclass detail.unpack(klass_or_nil) if klass_or_nil end.compact - OpenTelemetry.handle_error(message: "OTLP exporter received rpc.Status{message=#{status.message}, details=#{details}} for uri=#{@uri}") + OpenTelemetry.handle_error(message: "OTLP exporter received rpc.Status{message=#{status.message}, details=#{details}}#{truncation_note} for uri=#{@uri}") rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: 'unexpected error decoding rpc.Status in OTLP::Exporter#log_status') + OpenTelemetry.handle_error(exception: e, message: "unexpected error decoding rpc.Status in OTLP::Exporter#log_status#{truncation_note}") + ensure + @body_truncated = false + end + + def read_response_body(response) + return '' if response.nil? + + body = +'' + truncated = false + + response.read_body do |chunk| + if body.bytesize + chunk.bytesize <= RESPONSE_BODY_LIMIT + body << chunk + else + remaining = RESPONSE_BODY_LIMIT - body.bytesize + body << chunk.byteslice(0, remaining) if remaining > 0 + truncated = true + break + end + end + + body.force_encoding('UTF-8') + @body_truncated = truncated + body + rescue StandardError => e + OpenTelemetry.handle_error(exception: e, message: 'error reading response body') + '' end def log_request_failure(response_code) diff --git a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb index e5a1dc5e0a..4ba7bd8224 100644 --- a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb +++ b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb @@ -1041,4 +1041,125 @@ def create_link(span_context) OpenTelemetry::Trace::Link.new(span_context, { 'link-attribute' => 'link-value' }) end end + + describe 'response body reading' do + let(:exporter) { OpenTelemetry::Exporter::OTLP::Exporter.new } + let(:span_data) { OpenTelemetry::TestHelpers.create_span_data } + + it 'discards body for successful responses without reading into memory' do + stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 200, body: 'success body') + + result = exporter.export([span_data]) + + _(result).must_equal(SUCCESS) + end + + it 'discards body for retryable responses without reading into memory' do + stub_request(:post, 'http://localhost:4318/v1/traces') + .to_return(status: 503, body: 'service unavailable', headers: { 'Retry-After' => '0' }) + .then.to_return(status: 200) + + result = exporter.export([span_data]) + + _(result).must_equal(SUCCESS) + end + + it 'reads and parses error response body smaller than limit' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: 'error details'))] + status = ::Google::Rpc::Status.encode(::Google::Rpc::Status.new(code: 3, message: 'invalid argument', details: details)) + stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 400, body: status) + + result = exporter.export([span_data]) + + _(result).must_equal(FAILURE) + _(log_stream.string).must_match(/invalid argument/) + _(log_stream.string).wont_match(/truncated/) + ensure + OpenTelemetry.logger = logger + end + + it 'truncates error response body larger than 4 MB limit' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + # Create a body larger than 4 MB + large_message = 'x' * 5_000_000 # 5 MB + details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: large_message))] + large_status = ::Google::Rpc::Status.new(code: 3, message: 'large error', details: details) + large_body = ::Google::Rpc::Status.encode(large_status) + + stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 400, body: large_body) + + result = exporter.export([span_data]) + + _(result).must_equal(FAILURE) + _(log_stream.string).must_match(/body truncated due to size limit/) + ensure + OpenTelemetry.logger = logger + end + + it 'handles error response body at exactly 4 MB limit' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + # Create a body at exactly 4 MB + exact_size_message = 'y' * (4_194_304 - 100) # Account for protobuf overhead + details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: exact_size_message))] + exact_status = ::Google::Rpc::Status.new(code: 3, message: 'exact size', details: details) + exact_body = ::Google::Rpc::Status.encode(exact_status) + + # Skip if encoded body is still larger than 4 MB due to protobuf overhead + skip 'Protobuf overhead makes this test impractical' if exact_body.bytesize > 4_194_304 + + stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 400, body: exact_body) + + result = exporter.export([span_data]) + + _(result).must_equal(FAILURE) + ensure + OpenTelemetry.logger = logger + end + + it 'handles malformed error response body gracefully' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 400, body: 'not valid protobuf') + + result = exporter.export([span_data]) + + _(result).must_equal(FAILURE) + _(log_stream.string).must_match(/unexpected error decoding rpc.Status/) + ensure + OpenTelemetry.logger = logger + end + + it 'handles truncated protobuf in error response' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + # Create a large protobuf that will be truncated, making it invalid + large_message = 'z' * 5_000_000 + details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: large_message))] + large_status = ::Google::Rpc::Status.new(code: 3, message: 'truncation test', details: details) + large_body = ::Google::Rpc::Status.encode(large_status) + + stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 400, body: large_body) + + result = exporter.export([span_data]) + + _(result).must_equal(FAILURE) + _(log_stream.string).must_match(/body truncated due to size limit/) + ensure + OpenTelemetry.logger = logger + end + end end From a2081842da1f2f2d854e4153f935c15b76642001 Mon Sep 17 00:00:00 2001 From: Robb Kidd Date: Wed, 15 Apr 2026 13:48:47 -0400 Subject: [PATCH 02/10] test: add real-socket test for response body limit Adds a failing test that exercises the OTLP exporter's response body limit against a real TCPServer instead of WebMock stubs. WebMock patches Net::HTTP's read_body internals, so the existing stub-based tests pass even though the chunked reader doesn't work against real HTTP. With WebMock's adapter fully disabled, Net::HTTP#request eagerly reads the full body before read_response_body runs, causing IOError: "read_body called twice". The full response body is already in memory by that point, defeating the 4 MB cap. --- .../exporter/otlp/exporter_test.rb | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb index 4ba7bd8224..de198e6db3 100644 --- a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb +++ b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb @@ -1042,6 +1042,70 @@ def create_link(span_context) end end + describe 'response body reading with real sockets' do + # These tests use a real TCPServer instead of WebMock to verify + # behavior against actual Net::HTTP socket I/O. WebMock's patching + # of Net::HTTP changes how read_body works — even with + # allow_net_connect!, responses go through WebMock's adapter which + # doesn't exercise the same code paths as real Net::HTTP. + + def with_fake_server(response_body_size:, status: 400) + server = TCPServer.new('127.0.0.1', 0) + port = server.addr[1] + body = 'X' * response_body_size + + server_thread = Thread.new do + client = server.accept + content_length = 0 + while (line = client.gets) && line != "\r\n" + content_length = line.split(': ', 2).last.to_i if line.start_with?('Content-Length') + end + client.read(content_length) if content_length > 0 + + client.print "HTTP/1.1 #{status} Bad Request\r\n" + client.print "Content-Type: application/octet-stream\r\n" + client.print "Content-Length: #{body.bytesize}\r\n" + client.print "Connection: close\r\n" + client.print "\r\n" + client.write body + client.close + rescue => e + # client may disconnect early + end + + # Fully disable WebMock's Net::HTTP adapter so we get real + # socket behavior, not WebMock's patched read_body. + WebMock::HttpLibAdapters::NetHttpAdapter.disable! + yield port + ensure + WebMock::HttpLibAdapters::NetHttpAdapter.enable! + server&.close + server_thread&.join(2) + end + + it 'limits error response body read to 4 MB against a real HTTP server' do + log_stream = StringIO.new + logger = OpenTelemetry.logger + OpenTelemetry.logger = ::Logger.new(log_stream) + + with_fake_server(response_body_size: 5_000_000) do |port| + exporter = OpenTelemetry::Exporter::OTLP::Exporter.new( + endpoint: "http://127.0.0.1:#{port}/v1/traces" + ) + span_data = OpenTelemetry::TestHelpers.create_span_data + result = exporter.export([span_data]) + + _(result).must_equal(FAILURE) + # The body cap should work without hitting "read_body called twice". + # If we see this error, it means read_response_body was called after + # Net::HTTP already read the full body during @http.request(). + _(log_stream.string).wont_match(/read_body called twice/) + end + ensure + OpenTelemetry.logger = logger + end + end + describe 'response body reading' do let(:exporter) { OpenTelemetry::Exporter::OTLP::Exporter.new } let(:span_data) { OpenTelemetry::TestHelpers.create_span_data } From 7af99b40abd0c3430e9269965b735081177c015a Mon Sep 17 00:00:00 2001 From: Robb Kidd Date: Wed, 15 Apr 2026 13:54:21 -0400 Subject: [PATCH 03/10] fix: use block form of request for streaming body access Net::HTTP#request without a block eagerly reads the entire response body into memory via reading_body's self.body call. Refactor send_bytes to use @http.request(request) { |response| } so the case statement runs before the body is read. The error path uses streaming access for the chunked reader. Non-error paths drain-and-discard with read_body { |_| } to preserve keep-alive connections (read_body(nil) leaves the socket undrained). redo doesn't work from within a nested block, so add a should_redo flag to trigger it. --- .../opentelemetry/exporter/otlp/exporter.rb | 72 +++++++++++-------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb index 5f61e7f9a7..85fd1cbc93 100644 --- a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb +++ b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb @@ -175,37 +175,49 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity, @http.read_timeout = remaining_timeout @http.write_timeout = remaining_timeout @http.start unless @http.started? - response = measure_request_duration { @http.request(request) } - - case response - when Net::HTTPSuccess - response.read_body(nil) # Discard without reading into memory - SUCCESS - when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests - response.read_body(nil) # Discard without reading into memory - redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code) - FAILURE - when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway - response.read_body(nil) # Discard without reading into memory - redo if backoff?(retry_count: retry_count += 1, reason: response.code) - FAILURE - when Net::HTTPNotFound - log_request_failure(response.code) - FAILURE - when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError - body = read_response_body(response) - log_status(body) - @metrics_reporter.add_to_counter('otel.otlp_exporter.failure', labels: { 'reason' => response.code }) - FAILURE - when Net::HTTPRedirection - @http.finish - handle_redirect(response['location']) - redo if backoff?(retry_after: 0, retry_count: retry_count += 1, reason: response.code) - else - @http.finish - log_request_failure(response.code) - FAILURE + result = nil + should_redo = false + + measure_request_duration do + @http.request(request) do |response| + case response + when Net::HTTPSuccess + response.read_body { |_| } # Drain and discard, preserves keep-alive + result = SUCCESS + when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests + response.read_body { |_| } + should_redo = backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code) + result = FAILURE + when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway + response.read_body { |_| } + should_redo = backoff?(retry_count: retry_count += 1, reason: response.code) + result = FAILURE + when Net::HTTPNotFound + response.read_body { |_| } + log_request_failure(response.code) + result = FAILURE + when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError + body = read_response_body(response) + log_status(body) + @metrics_reporter.add_to_counter('otel.otlp_exporter.failure', labels: { 'reason' => response.code }) + result = FAILURE + when Net::HTTPRedirection + response.read_body { |_| } + @http.finish + handle_redirect(response['location']) + should_redo = backoff?(retry_after: 0, retry_count: retry_count += 1, reason: response.code) + else + response.read_body { |_| } + @http.finish + log_request_failure(response.code) + result = FAILURE + end + end end + + redo if should_redo + + result rescue Net::OpenTimeout, Net::ReadTimeout retry if backoff?(retry_count: retry_count += 1, reason: 'timeout') return FAILURE From e1a8c001a0f122cdd6da186896337fbb6545793c Mon Sep 17 00:00:00 2001 From: Robb Kidd Date: Wed, 15 Apr 2026 14:00:46 -0400 Subject: [PATCH 04/10] test: assert body truncation in real-socket test Got past the IOError from trying to read the body twice, verify the chunked reader hit the 4 MB cap against a real HTTP response. --- .../otlp/test/opentelemetry/exporter/otlp/exporter_test.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb index de198e6db3..414881ef2b 100644 --- a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb +++ b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb @@ -1096,9 +1096,10 @@ def with_fake_server(response_body_size:, status: 400) result = exporter.export([span_data]) _(result).must_equal(FAILURE) - # The body cap should work without hitting "read_body called twice". - # If we see this error, it means read_response_body was called after - # Net::HTTP already read the full body during @http.request(). + # The chunked reader should cap at 4 MB and note the truncation. + _(log_stream.string).must_match(/body truncated due to size limit/) + # And it should work without hitting "read_body called twice", + # which would mean the body was already fully read into memory. _(log_stream.string).wont_match(/read_body called twice/) end ensure From 54022d8fe0105b957830a08f82643a28038fe013 Mon Sep 17 00:00:00 2001 From: Robb Kidd Date: Wed, 15 Apr 2026 14:16:26 -0400 Subject: [PATCH 05/10] test: verify Net::HTTP doesn't buffer beyond the cap After read_response_body breaks out of the chunked read at 4 MB, Net::HTTP's reading_body still calls self.body which reads more data from the socket into memory. Our String is capped but response.body held 5.8 MB against a 10 MB response. Spies on the exporter to observe response.body size after read_response_body returns. --- .../exporter/otlp/exporter_test.rb | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb index 414881ef2b..822a0ee427 100644 --- a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb +++ b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb @@ -1105,6 +1105,35 @@ def with_fake_server(response_body_size:, status: 400) ensure OpenTelemetry.logger = logger end + + it 'does not buffer beyond the limit internally' do + response_body_internal_size = nil + + spy = Module.new do + define_method(:read_response_body) do |response| + super(response).tap do + internal = response.body + response_body_internal_size = internal.bytesize if internal.is_a?(String) + end + end + end + + limit = OpenTelemetry::Exporter::OTLP::Exporter.const_get(:RESPONSE_BODY_LIMIT) + + with_fake_server(response_body_size: 10_000_000) do |port| + exporter = OpenTelemetry::Exporter::OTLP::Exporter.new( + endpoint: "http://127.0.0.1:#{port}/v1/traces" + ) + exporter.singleton_class.prepend(spy) + + OpenTelemetry.logger = ::Logger.new(File::NULL) + exporter.export([OpenTelemetry::TestHelpers.create_span_data]) + + _(response_body_internal_size).wont_be_nil + _(response_body_internal_size).must_be :<=, limit, + "Net::HTTP buffered #{response_body_internal_size} bytes internally, exceeding the #{limit} byte limit" + end + end end describe 'response body reading' do From 558562c09e4fe14dd71f0bd54a3e4020242521c2 Mon Sep 17 00:00:00 2001 From: Robb Kidd Date: Wed, 15 Apr 2026 15:24:49 -0400 Subject: [PATCH 06/10] fix: close socket on oversized response to prevent full buffering When read_response_body truncates at the 4MB limit, Net::HTTP's reading_body calls self.body after our block returns. This re-reads the entire remaining response from the socket into memory. Our break doesn't prevent this because reading_body has its "ensure to read body" safety net that fires before the socket closes. Calling @http.finish before break closes the socket. reading_body's self.body call hits a closed stream instead of re-reading. The resulting IOError is rescued only when truncation caused it. Unexpected IOErrors still propagate. --- .../opentelemetry/exporter/otlp/exporter.rb | 7 ++++++ .../exporter/otlp/exporter_test.rb | 23 +++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb index 85fd1cbc93..d7d49cb99e 100644 --- a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb +++ b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb @@ -280,6 +280,7 @@ def read_response_body(response) remaining = RESPONSE_BODY_LIMIT - body.bytesize body << chunk.byteslice(0, remaining) if remaining > 0 truncated = true + @http.finish # closes socket, nil's the body or else net/http will attempt to read the rest of the response break end end @@ -287,6 +288,12 @@ def read_response_body(response) body.force_encoding('UTF-8') @body_truncated = truncated body + rescue IOError => e + raise unless truncated # we'll handle this when we know net/http is upset trying to read after http.finish + + body&.force_encoding('UTF-8') + @body_truncated = truncated + body || '' rescue StandardError => e OpenTelemetry.handle_error(exception: e, message: 'error reading response body') '' diff --git a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb index 822a0ee427..c36c188896 100644 --- a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb +++ b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb @@ -1107,13 +1107,14 @@ def with_fake_server(response_body_size:, status: 400) end it 'does not buffer beyond the limit internally' do - response_body_internal_size = nil + captured_body_size = nil + internal_body = :not_checked spy = Module.new do define_method(:read_response_body) do |response| - super(response).tap do - internal = response.body - response_body_internal_size = internal.bytesize if internal.is_a?(String) + super(response).tap do |result| + captured_body_size = result.bytesize + internal_body = response.body end end end @@ -1129,9 +1130,17 @@ def with_fake_server(response_body_size:, status: 400) OpenTelemetry.logger = ::Logger.new(File::NULL) exporter.export([OpenTelemetry::TestHelpers.create_span_data]) - _(response_body_internal_size).wont_be_nil - _(response_body_internal_size).must_be :<=, limit, - "Net::HTTP buffered #{response_body_internal_size} bytes internally, exceeding the #{limit} byte limit" + # read_response_body must cap the returned body at the limit + _(captured_body_size).wont_be_nil + _(captured_body_size).must_be :<=, limit + + # Net::HTTP must NOT have the full 10 MB response buffered internally. + # After @http.finish closes the socket mid-read, response.body is nil + # (block-form read_body doesn't accumulate into @body). + if internal_body.is_a?(String) + _(internal_body.bytesize).must_be :<=, limit, + "Net::HTTP buffered #{internal_body.bytesize} bytes internally, exceeding the #{limit} byte limit" + end end end end From 7be8257b83e5d9c73bd0b52670b56487cefa2f1d Mon Sep 17 00:00:00 2001 From: Robb Kidd Date: Wed, 15 Apr 2026 15:38:00 -0400 Subject: [PATCH 07/10] refactor: return truncation flag instead of ivar send_bytes()'s sad path needs to know if the chucking reader truncated the response body so it can log that appropriately. Instead of managing a shared-state instance variable, return that state from the reader and pass it into the logger. --- .../opentelemetry/exporter/otlp/exporter.rb | 20 ++++++++----------- .../exporter/otlp/exporter_test.rb | 4 ++-- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb index d7d49cb99e..ebb5b563e3 100644 --- a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb +++ b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb @@ -197,8 +197,8 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity, log_request_failure(response.code) result = FAILURE when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError - body = read_response_body(response) - log_status(body) + body, truncated = read_response_body(response) + log_status(body, truncated: truncated) @metrics_reporter.add_to_counter('otel.otlp_exporter.failure', labels: { 'reason' => response.code }) result = FAILURE when Net::HTTPRedirection @@ -253,8 +253,8 @@ def handle_redirect(location) # TODO: figure out destination and reinitialize @http and @path end - def log_status(body) - truncation_note = @body_truncated ? ' (body truncated due to size limit)' : '' + def log_status(body, truncated: false) + truncation_note = truncated ? ' (body truncated due to size limit)' : '' status = Google::Rpc::Status.decode(body) details = status.details.map do |detail| klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(detail.type_name).msgclass @@ -263,12 +263,10 @@ def log_status(body) OpenTelemetry.handle_error(message: "OTLP exporter received rpc.Status{message=#{status.message}, details=#{details}}#{truncation_note} for uri=#{@uri}") rescue StandardError => e OpenTelemetry.handle_error(exception: e, message: "unexpected error decoding rpc.Status in OTLP::Exporter#log_status#{truncation_note}") - ensure - @body_truncated = false end def read_response_body(response) - return '' if response.nil? + return ['', false] if response.nil? body = +'' truncated = false @@ -286,17 +284,15 @@ def read_response_body(response) end body.force_encoding('UTF-8') - @body_truncated = truncated - body + [body, truncated] rescue IOError => e raise unless truncated # we'll handle this when we know net/http is upset trying to read after http.finish body&.force_encoding('UTF-8') - @body_truncated = truncated - body || '' + [body || '', truncated] rescue StandardError => e OpenTelemetry.handle_error(exception: e, message: 'error reading response body') - '' + ['', false] end def log_request_failure(response_code) diff --git a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb index c36c188896..60799fc8b5 100644 --- a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb +++ b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb @@ -1112,8 +1112,8 @@ def with_fake_server(response_body_size:, status: 400) spy = Module.new do define_method(:read_response_body) do |response| - super(response).tap do |result| - captured_body_size = result.bytesize + super(response).tap do |result_body, _truncated| + captured_body_size = result_body.bytesize internal_body = response.body end end From eefa743368564f8cf5f94b5e893bcf04a15ddf2d Mon Sep 17 00:00:00 2001 From: Robb Kidd Date: Wed, 22 Apr 2026 14:01:52 -0400 Subject: [PATCH 08/10] test: assert body truncated logged, not a decode error Decode will raise an error on a truncated body and then the exporter will log a confusing "unexpected error decoding rpc.Status". --- .../test/opentelemetry/exporter/otlp/exporter_test.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb index 60799fc8b5..b4e6ec943f 100644 --- a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb +++ b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb @@ -1096,8 +1096,9 @@ def with_fake_server(response_body_size:, status: 400) result = exporter.export([span_data]) _(result).must_equal(FAILURE) - # The chunked reader should cap at 4 MB and note the truncation. - _(log_stream.string).must_match(/body truncated due to size limit/) + # The chunked reader should cap at 4 MB and log a clear oversized message. + _(log_stream.string).must_match(/oversized error response body/) + _(log_stream.string).wont_match(/unexpected error decoding/) # And it should work without hitting "read_body called twice", # which would mean the body was already fully read into memory. _(log_stream.string).wont_match(/read_body called twice/) @@ -1201,7 +1202,8 @@ def with_fake_server(response_body_size:, status: 400) result = exporter.export([span_data]) _(result).must_equal(FAILURE) - _(log_stream.string).must_match(/body truncated due to size limit/) + _(log_stream.string).must_match(/oversized error response body/) + _(log_stream.string).wont_match(/unexpected error decoding/) ensure OpenTelemetry.logger = logger end @@ -1260,7 +1262,7 @@ def with_fake_server(response_body_size:, status: 400) result = exporter.export([span_data]) _(result).must_equal(FAILURE) - _(log_stream.string).must_match(/body truncated due to size limit/) + _(log_stream.string).must_match(/oversized error response body/) ensure OpenTelemetry.logger = logger end From 83a8d4ee86c6e2f75fd38d0727f760b4a73ed980 Mon Sep 17 00:00:00 2001 From: Robb Kidd Date: Wed, 22 Apr 2026 15:14:00 -0400 Subject: [PATCH 09/10] fix: return early when response body was truncated A truncated protobuf is invalid binary. Passing it to Google::Rpc::Status.decode raises "unexpected error decoding rpc.Status" which is rescued by our own error handling. The resulting log output about decode error mentions nothing about a known-truncated body. When we know the body was truncated, log a direct oversized-body message and return early. --- .../otlp/lib/opentelemetry/exporter/otlp/exporter.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb index ebb5b563e3..0287fd6187 100644 --- a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb +++ b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb @@ -254,15 +254,19 @@ def handle_redirect(location) end def log_status(body, truncated: false) - truncation_note = truncated ? ' (body truncated due to size limit)' : '' + if truncated + OpenTelemetry.handle_error(message: "OTLP exporter received an oversized error response body (truncated at #{RESPONSE_BODY_LIMIT} bytes) for uri=#{@uri}") + return + end + status = Google::Rpc::Status.decode(body) details = status.details.map do |detail| klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(detail.type_name).msgclass detail.unpack(klass_or_nil) if klass_or_nil end.compact - OpenTelemetry.handle_error(message: "OTLP exporter received rpc.Status{message=#{status.message}, details=#{details}}#{truncation_note} for uri=#{@uri}") + OpenTelemetry.handle_error(message: "OTLP exporter received rpc.Status{message=#{status.message}, details=#{details}} for uri=#{@uri}") rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: "unexpected error decoding rpc.Status in OTLP::Exporter#log_status#{truncation_note}") + OpenTelemetry.handle_error(exception: e, message: 'unexpected error decoding rpc.Status in OTLP::Exporter#log_status') end def read_response_body(response) From 97d3196d6ef30e6b8250b129b2ee8952f5542b69 Mon Sep 17 00:00:00 2001 From: Kayla Reopelle Date: Fri, 24 Apr 2026 20:34:39 +0200 Subject: [PATCH 10/10] Rubocop * Metrics/CyclomaticComplexity * Metrics/MethodLength * Metrics/PerceivedComplexity --- .../opentelemetry/exporter/otlp/exporter.rb | 25 ++++++----- .../exporter/otlp/exporter_test.rb | 45 +++++++++++-------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb index 0287fd6187..737cdbc656 100644 --- a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb +++ b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb @@ -272,31 +272,34 @@ def log_status(body, truncated: false) def read_response_body(response) return ['', false] if response.nil? + body, truncated = collect_response_chunks(response) + body.force_encoding('UTF-8') + [body, truncated] + rescue StandardError => e + OpenTelemetry.handle_error(exception: e, message: 'error reading response body') + ['', false] + end + + def collect_response_chunks(response) body = +'' truncated = false response.read_body do |chunk| - if body.bytesize + chunk.bytesize <= RESPONSE_BODY_LIMIT - body << chunk - else - remaining = RESPONSE_BODY_LIMIT - body.bytesize - body << chunk.byteslice(0, remaining) if remaining > 0 + remaining = RESPONSE_BODY_LIMIT - body.bytesize + body << chunk.byteslice(0, remaining) + + if chunk.bytesize > remaining truncated = true @http.finish # closes socket, nil's the body or else net/http will attempt to read the rest of the response break end end - body.force_encoding('UTF-8') [body, truncated] - rescue IOError => e + rescue IOError raise unless truncated # we'll handle this when we know net/http is upset trying to read after http.finish - body&.force_encoding('UTF-8') [body || '', truncated] - rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: 'error reading response body') - ['', false] end def log_request_failure(response_code) diff --git a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb index b4e6ec943f..7be054c30e 100644 --- a/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb +++ b/exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb @@ -1054,24 +1054,7 @@ def with_fake_server(response_body_size:, status: 400) port = server.addr[1] body = 'X' * response_body_size - server_thread = Thread.new do - client = server.accept - content_length = 0 - while (line = client.gets) && line != "\r\n" - content_length = line.split(': ', 2).last.to_i if line.start_with?('Content-Length') - end - client.read(content_length) if content_length > 0 - - client.print "HTTP/1.1 #{status} Bad Request\r\n" - client.print "Content-Type: application/octet-stream\r\n" - client.print "Content-Length: #{body.bytesize}\r\n" - client.print "Connection: close\r\n" - client.print "\r\n" - client.write body - client.close - rescue => e - # client may disconnect early - end + server_thread = Thread.new { handle_fake_request(server, body, status) } # Fully disable WebMock's Net::HTTP adapter so we get real # socket behavior, not WebMock's patched read_body. @@ -1083,6 +1066,30 @@ def with_fake_server(response_body_size:, status: 400) server_thread&.join(2) end + def handle_fake_request(server, body, status) + client = server.accept + content_length = read_content_length(client) + client.read(content_length) if content_length > 0 + + client.print "HTTP/1.1 #{status} Bad Request\r\n" + client.print "Content-Type: application/octet-stream\r\n" + client.print "Content-Length: #{body.bytesize}\r\n" + client.print "Connection: close\r\n" + client.print "\r\n" + client.write body + client.close + rescue StandardError + # client may disconnect early + end + + def read_content_length(client) + content_length = 0 + while (line = client.gets) && line != "\r\n" + content_length = line.split(': ', 2).last.to_i if line.start_with?('Content-Length') + end + content_length + end + it 'limits error response body read to 4 MB against a real HTTP server' do log_stream = StringIO.new logger = OpenTelemetry.logger @@ -1140,7 +1147,7 @@ def with_fake_server(response_body_size:, status: 400) # (block-form read_body doesn't accumulate into @body). if internal_body.is_a?(String) _(internal_body.bytesize).must_be :<=, limit, - "Net::HTTP buffered #{internal_body.bytesize} bytes internally, exceeding the #{limit} byte limit" + "Net::HTTP buffered #{internal_body.bytesize} bytes internally, exceeding the #{limit} byte limit" end end end