From 560347ada6da0b5e7399f57f93e6415777c042d6 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Fri, 7 Nov 2025 16:40:34 -0800 Subject: [PATCH 01/19] Add a support for standard SSL configurations, proxy and basic auth configs. --- CHANGELOG.md | 14 ++ docs/index.asciidoc | 225 ++++++++++++++++++ lib/logstash/codecs/avro.rb | 204 +++++++++++++++- logstash-codec-avro.gemspec | 3 +- spec/{codecs => unit}/avro_spec.rb | 146 +++++++++++- .../resources/do_not_remove_path/.gitignore | 2 + 6 files changed, 587 insertions(+), 7 deletions(-) rename spec/{codecs => unit}/avro_spec.rb (62%) create mode 100644 spec/unit/resources/do_not_remove_path/.gitignore diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a0dd55..e770f20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## 3.5.0 + - Add SSL/TLS support for HTTPS schema registry connections + - Add `ssl_enabled` option to enable/disable SSL + - Add `ssl_certificate` and `ssl_key` options for PEM-based client authentication (unencrypted keys only) + - Add `ssl_certificate_authorities` option for PEM-based server certificate validation + - Add `ssl_verification_mode` option to control SSL verification (full, none) + - Add `ssl_cipher_suites` option to configure cipher suites + - Add `ssl_supported_protocols` option to configure TLS protocol versions (TLSv1.1, TLSv1.2, TLSv1.3) + - Add `ssl_truststore_path` and `ssl_truststore_password` options for server certificate validation (JKS/PKCS12) + - Add `ssl_keystore_path` and `ssl_keystore_password` options for mutual TLS authentication (JKS/PKCS12) + - Add `ssl_truststore_type` and `ssl_keystore_type` options (JKS or PKCS12) + - Add HTTP proxy support with `proxy` option + - Add HTTP basic authentication support with `username` and `password` options + ## 3.4.1 - Fixes `(Errno::ENOENT) No such file or directory` error [#43](https://github.com/logstash-plugins/logstash-codec-avro/pull/43) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 7695ac2..becb067 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -83,9 +83,25 @@ output { |Setting |Input type|Required | <> | <>|No | <> | <>, one of `["binary", "base64"]`|No +| <> |<>|No +| <> |<>|No | <> |<>|Yes +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No |=======================================================================   @@ -112,6 +128,23 @@ Use `base64` (default) to indicate that this codec sends or expects to receive b Set this option to `binary` to indicate that this codec sends or expects to receive binary Avro data. +[id="plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is <> +* There is no default value for this setting. + +Password for HTTP basic authentication when fetching the schema from a remote server. +Must be used in combination with the `username` setting. + +[id="plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is <> +* There is no default value for this setting. + +Proxy server URL for schema registry connections. + [id="plugins-{type}s-{plugin}-schema_uri"] ===== `schema_uri` @@ -134,6 +167,175 @@ example: tag events with `_avroparsefailure` when decode fails +[id="plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is <> +* There is no default value for this setting. + +Path to PEM encoded certificate file for client authentication (mutual TLS). +This is an alternative to using `ssl_keystore_path`. + +*Example* +[source,ruby] +---------------------------------- +ssl_certificate => "/path/to/client.crt" +---------------------------------- + +[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is <> +* There is no default value for this setting. + +Path to PEM encoded CA certificate file(s) for server verification. +This is an alternative to using `ssl_truststore_path`. + +*Example* +[source,ruby] +---------------------------------- +ssl_certificate_authorities => "/path/to/ca.crt" +---------------------------------- + +[id="plugins-{type}s-{plugin}-ssl_cipher_suites"] +===== `ssl_cipher_suites` + +* Value type is <> +* There is no default value for this setting. + +The list of cipher suites to use, listed by priorities. +Supported cipher suites vary depending on which version of Java is used. + +[id="plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is <> +* There is no default value for this setting. + +Path to PEM encoded private key file for client authentication. +Must be used together with `ssl_certificate`. +The private key must be unencrypted (passphrase-protected keys are not supported). + +*Example* +[source,ruby] +---------------------------------- +ssl_key => "/path/to/client.key" +---------------------------------- + +[id="plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is <> +* There is no default value for this setting. + +Enable SSL/TLS secured communication to remote schema registry. +When using HTTPS schema URIs, SSL is automatically enabled. + +[id="plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is <> +* There is no default value for this setting. + +The path to the JKS or PKCS12 keystore file for client certificate authentication. +Use this when the schema registry requires mutual TLS (mTLS) authentication. + +This setting is only used when `schema_uri` uses the `https://` scheme. + +[id="plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is <> +* There is no default value for this setting. + +The password for the keystore file specified in `ssl_keystore_path`. + + +[id="plugins-{type}s-{plugin}-ssl_keystore_type"] +===== `ssl_keystore_type` + +* Value type is <> +* Default value is `"JKS"` + +The type of the keystore file. Can be either `JKS` or `PKCS12`. + +[id="plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + +* Value type is <> +* Default value is `[]` (uses Java defaults) +* Valid values are: `TLSv1.1`, `TLSv1.2`, `TLSv1.3` + +List of allowed SSL/TLS protocol versions. +When not specified, the JVM defaults are used. + +[id="plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is <> +* There is no default value for this setting. + +The path to the JKS or PKCS12 truststore file containing certificates to verify +the schema registry server's certificate. + +This setting is only used when `schema_uri` uses the `https://` scheme. + +*Example* +[source,ruby] +---------------------------------- +input { + kafka { + codec => avro { + schema_uri => "https://schema-registry.example.com:8081/schemas/ids/1" + ssl_truststore_path => "/path/to/truststore.jks" + ssl_truststore_password => "${TRUSTSTORE_PASSWORD}" + } + } +} +---------------------------------- + +[id="plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is <> +* There is no default value for this setting. + +The password for the truststore file specified in `ssl_truststore_path`. + +[id="plugins-{type}s-{plugin}-ssl_truststore_type"] +===== `ssl_truststore_type` + +* Value type is <> +* Default value is `"JKS"` + +The type of the truststore file. Can be either `JKS` or `PKCS12`. + +[id="plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is <> +* Default value is `"full"` +* Valid options are: `full`, `none` + +Options to verify the server's certificate: + +* `full`: Validates that the provided certificate has an issue date that's within the not_before and not_after dates; chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate. (recommended) +* `none`: Performs no certificate validation. **Warning:** Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf) + +*Example* +[source,ruby] +---------------------------------- +input { + kafka { + codec => avro { + schema_uri => "https://schema-registry.example.com:8081/schemas/ids/1" + ssl_certificate_authorities => "/path/to/ca.crt" + ssl_verification_mode => "full" + } + } +} +---------------------------------- + [id="plugins-{type}s-{plugin}-target"] ===== `target` @@ -156,3 +358,26 @@ input { } } ---------------------------------- + +[id="plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is <> +* There is no default value for this setting. + +Username for HTTP basic authentication when fetching the schema from a remote server. +Must be used in combination with the `password` setting. + +*Example* +[source,ruby] +---------------------------------- +input { + kafka { + codec => avro { + schema_uri => "https://schema-registry.example.com:8081/schemas/ids/1" + username => "registry_user" + password => "${REGISTRY_PASSWORD}" + } + } +} +---------------------------------- diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index 31c543c..3ef8c13 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -1,5 +1,6 @@ # encoding: utf-8 require "open-uri" +require "manticore" require "avro" require "base64" require "logstash/codecs/base" @@ -84,9 +85,59 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base # NOTE: the target is only relevant while decoding data into a new event. config :target, :validate => :field_reference - def open_and_read(uri_string) - URI.open(uri_string, &:read) - end + # Proxy server URL for schema registry connections + config :proxy, :validate => :uri + + # Username for HTTP basic authentication + config :username, :validate => :string + + # Password for HTTP basic authentication + config :password, :validate => :password + + # Enable SSL/TLS secured communication to remote schema registry + config :ssl_enabled, :validate => :boolean + + # PEM-based SSL configuration (alternative to keystore/truststore) + # Path to PEM encoded certificate file for client authentication + config :ssl_certificate, :validate => :path + + # Path to PEM encoded private key file for client authentication + config :ssl_key, :validate => :path + + # Path to PEM encoded CA certificate file(s) for server verification + # Can be a single file or directory containing multiple CA certificates + config :ssl_certificate_authorities, :validate => :path + + # Options to verify the server's certificate. + # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates; + # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate. + # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf) + config :ssl_verification_mode, :validate => %w[full none], :default => 'full' + + # The keystore path + config :ssl_keystore_path, :validate => :path + + # The keystore password + config :ssl_keystore_password, :validate => :password + + # Keystore type (JKS or PKCS12). Defaults to JKS. + config :ssl_keystore_type, :validate => %w[JKS PKCS12], :default => "JKS" + + # The truststore path + config :ssl_truststore_path, :validate => :path + + # The truststore password + config :ssl_truststore_password, :validate => :password + + # Truststore type (JKS or PKCS12). Defaults to JKS. + config :ssl_truststore_type, :validate => %w[JKS PKCS12], :default => "JKS" + + # The list of cipher suites to use, listed by priorities. + # Supported cipher suites vary depending on which version of Java is used. + config :ssl_cipher_suites, :validate => :string, :list => true + + # SSL supported protocols + config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true public def initialize(*params) @@ -95,7 +146,7 @@ def initialize(*params) end def register - @schema = Avro::Schema.parse(open_and_read(schema_uri)) + @schema = Avro::Schema.parse(fetch_schema(schema_uri)) end public @@ -131,4 +182,149 @@ def encode(event) @on_event.call(event, buffer.string) end end + + private + def fetch_schema(uri_string) + http_connection = uri_string.start_with?('http://') + https_connection = uri_string.start_with?('https://') + + if http_connection + ssl_config_provided = original_params.keys.select {|k| k.start_with?("ssl_") && k != "ssl_enabled" } + if ssl_config_provided.any? + raise_config_error! "When SSL is disabled, the following provided parameters are not allowed: #{ssl_config_provided}" + end + + credentials_configured = @username && @password + @logger.warn("Credentials are being sent over unencrypted HTTP. This may bring security risk.") if credentials_configured && http_connection + fetch_remote_schema(uri_string) + elsif https_connection + validate_ssl_settings! + fetch_remote_schema(uri_string) + else + # local schema + URI.open(uri_string, &:read) + end + end + + def fetch_remote_schema(uri_string) + client_options = {} + + unless @proxy&.empty? + client_options[:proxy] = @proxy.to_s + end + + basic_auth_options = build_basic_auth + client_options[:auth] = basic_auth_options unless basic_auth_options.empty? + + if @ssl_enabled + ssl_options = build_ssl_options + client_options[:ssl] = ssl_options unless ssl_options.empty? + end + + client = Manticore::Client.new(client_options) + response = client.get(uri_string).call + + unless response.code == 200 + raise "HTTP request failed: #{response.code} #{response.message}" + end + + response.body + ensure + client.close if client + end + + def build_basic_auth + if !@username && !@password + return {} + end + + raise LogStash::ConfigurationError, "`username` requires `password`" if @username && !@password + raise LogStash::ConfigurationError, "`password` is not allowed unless `username` is specified" if !@username && @password + + if @username && @password + raise LogStash::ConfigurationError, "Empty `username` or `password` is not allowed" if @username.empty? || @password.value.empty? + end + + {:user => @username, :password => @password.value} + end + + def validate_ssl_settings! + @ssl_enabled = true if @ssl_enabled.nil? + @ssl_verification_mode = "full".freeze if @ssl_verification_mode.nil? + + # optional: presenting our identity + raise_config_error! "`ssl_certificate` and `ssl_keystore_path` cannot be used together." if @ssl_certificate && @ssl_keystore_path + raise_config_error! "`ssl_certificate` requires `ssl_key`" if @ssl_certificate && !@ssl_key + ensure_readable_and_non_writable! "ssl_certificate", @ssl_certificate if @ssl_certificate + + raise_config_error! "`ssl_key` is not allowed unless `ssl_certificate` is specified" if @ssl_key && !@ssl_certificate + ensure_readable_and_non_writable! "ssl_key", @ssl_key if @ssl_key + + raise_config_error! "`ssl_keystore_path` requires `ssl_keystore_password`" if @ssl_keystore_path && !@ssl_keystore_password + raise_config_error! "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is specified" if @ssl_keystore_password && !@ssl_keystore_path + raise_config_error! "`ssl_keystore_password` cannot be empty" if @ssl_keystore_password && @ssl_keystore_password.value.empty? + raise_config_error! "`ssl_keystore_type` is not allowed unless `ssl_keystore_path` is specified" if @ssl_keystore_type && !@ssl_keystore_path + + ensure_readable_and_non_writable! "ssl_keystore_path", @ssl_keystore_path if @ssl_keystore_path + + # establishing trust of the server we connect to + # system-provided trust requires verification mode enabled + if @ssl_verification_mode == "none" + raise_config_error! "`ssl_truststore_path` requires `ssl_verification_mode` to be either `full`" if @ssl_truststore_path + raise_config_error! "`ssl_truststore_password` requires `ssl_truststore_path` and `ssl_verification_mode => 'full'`" if @ssl_truststore_password + raise_config_error! "`ssl_certificate_authorities` requires `ssl_verification_mode` to be `full`" if @ssl_certificate_authorities + end + + raise_config_error! "`ssl_truststore_path` and `ssl_certificate_authorities` cannot be used together." if @ssl_truststore_path && @ssl_certificate_authorities + raise_config_error! "`ssl_truststore_path` requires `ssl_truststore_password`" if @ssl_truststore_path && !@ssl_truststore_password + ensure_readable_and_non_writable! "ssl_truststore_path", @ssl_truststore_path if @ssl_truststore_path + + raise_config_error! "`ssl_truststore_password` is not allowed unless `ssl_truststore_path` is specified" if !@ssl_truststore_path && @ssl_truststore_password + raise_config_error! "`ssl_truststore_password` cannot be empty" if @ssl_truststore_password && @ssl_truststore_password.value.empty? + + if !@ssl_truststore_path && @ssl_certificate_authorities&.empty? + raise_config_error! "`ssl_certificate_authorities` cannot be empty" + end + + raise_config_error! "Multiple values on `ssl_certificate_authorities` are not supported by this plugin" if @ssl_certificate_authorities.size > 1 + ensure_readable_and_non_writable! "ssl_certificate_authorities", @ssl_certificate_authorities&.first + end + + def build_ssl_options + ssl_options = {} + + ssl_options[:client_cert] = @ssl_certificate if @ssl_certificate + ssl_options[:client_key] = @ssl_key if @ssl_key + + ssl_options[:ca_file] = @ssl_certificate_authorities&.first if @ssl_certificate_authorities + + ssl_options[:cipher_suites] = @ssl_cipher_suites if @ssl_cipher_suites + + ssl_options[:verify] = :default if @ssl_verification_mode == 'full' + ssl_options[:verify] = :disable if @ssl_verification_mode == 'none' + + ssl_options[:keystore] = @ssl_keystore_path if @ssl_keystore_path + ssl_options[:keystore_password] = @ssl_keystore_password&.value if @ssl_keystore_path && @ssl_keystore_password + ssl_options[:keystore_type] = @ssl_keystore_type.downcase if @ssl_keystore_path && @ssl_keystore_type + + ssl_options[:truststore] = @ssl_truststore_path if @ssl_truststore_path + ssl_options[:truststore_password] = @ssl_truststore_password&.value if @ssl_truststore_path && @ssl_truststore_password + ssl_options[:truststore_type] = @ssl_truststore_type.downcase if @ssl_truststore_path && @ssl_truststore_type + + ssl_options[:protocols] = @ssl_supported_protocols if @ssl_supported_protocols && @ssl_supported_protocols&.any? + + ssl_options + end + + ## + # @param message [String] + # @raise [LogStash::ConfigurationError] + def raise_config_error!(message) + raise LogStash::ConfigurationError, message + end + + def ensure_readable_and_non_writable!(name, path) + raise_config_error! "Specified #{name} #{path} path must be readable." unless File.readable?(path) + raise_config_error! "Specified #{name} #{path} path must not be writable." if File.writable?(path) + end end diff --git a/logstash-codec-avro.gemspec b/logstash-codec-avro.gemspec index de5eccf..7fda54c 100644 --- a/logstash-codec-avro.gemspec +++ b/logstash-codec-avro.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-codec-avro' - s.version = '3.4.1' + s.version = '3.5.0' s.platform = 'java' s.licenses = ['Apache-2.0'] s.summary = "Reads serialized Avro records as Logstash events" @@ -23,6 +23,7 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" s.add_runtime_dependency "avro", "~> 1.10.2" #(Apache 2.0 license) + s.add_runtime_dependency "manticore", '>= 0.8.0', '< 1.0.0' s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3' s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0' s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0' diff --git a/spec/codecs/avro_spec.rb b/spec/unit/avro_spec.rb similarity index 62% rename from spec/codecs/avro_spec.rb rename to spec/unit/avro_spec.rb index a584f80..af2268c 100644 --- a/spec/codecs/avro_spec.rb +++ b/spec/unit/avro_spec.rb @@ -8,6 +8,13 @@ require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper' describe LogStash::Codecs::Avro, :ecs_compatibility_support, :aggregate_failures do + let(:paths) do + { + # path has to be created, otherwise config :path validation fails + # and since we cannot control the chmod operations on paths, we should stub file readable? and writable? operations + :test_path => "spec/unit/resources/do_not_remove_path" + } + end ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select| before(:each) do @@ -24,7 +31,7 @@ subject do allow_any_instance_of(LogStash::Codecs::Avro).to \ - receive(:open_and_read).and_return(avro_config['schema_uri']) + receive(:fetch_schema).and_return(avro_config['schema_uri']) next LogStash::Codecs::Avro.new(avro_config) end @@ -177,7 +184,7 @@ subject do allow_any_instance_of(LogStash::Codecs::Avro).to \ - receive(:open_and_read).and_return(avro_config['schema_uri']) + receive(:fetch_schema).and_return(avro_config['schema_uri']) next LogStash::Codecs::Avro.new(avro_config) end @@ -199,5 +206,140 @@ end end + + context "remote schema registry" do + + context "basic authentication" do + let(:test_schema) do + '{"type": "record", "name": "Test", + "fields": [{"name": "foo", "type": ["null", "string"]}, + {"name": "bar", "type": "int"}]}' + end + + before do + allow_any_instance_of(described_class).to receive(:fetch_remote_schema).and_return(test_schema) + end + + subject do + LogStash::Codecs::Avro.new(avro_config) + end + + context "with both username and password" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'username' => 'test_user', + 'password' => 'test_&^%$!password' + } + end + + it "uses user and password" do + auth = subject.send(:build_basic_auth) + expect(auth).to eq({:user => 'test_user', :password => 'test_&^%$!password'}) + end + + it "includes valid credentials in auth hash" do + auth = subject.send(:build_basic_auth) + expect(auth[:user]).not_to be_empty + expect(auth[:password]).not_to be_empty + end + end + + context "with only username" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'username' => 'test_user' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /`username` requires `password`/) + end + end + + context "with only password" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'password' => 'test_&^%$!password' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /`password` is not allowed unless `username` is specified/) + end + end + + context "with empty username" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'username' => '', + 'password' => 'test_&^%$!password' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /Empty `username` or `password` is not allowed/) + end + end + + context "with empty password" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'username' => 'test_user', + 'password' => '' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /Empty `username` or `password` is not allowed/) + end + end + + context "with neither username nor password" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc' + } + end + + it "returns empty hash" do + auth = subject.send(:build_basic_auth) + expect(auth).to be_empty + end + end + + context "with unsecure connection and credentials" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'username' => 'test_user', + 'password' => 'test_&^%$!password' + } + end + + it "warns about credentials over unencrypted HTTP" do + expect(subject.logger).to receive(:warn).with(/Credentials are being sent over unencrypted HTTP/) + subject.register + end + + it "still returns valid auth hash" do + allow(subject.logger).to receive(:warn) + auth = subject.send(:build_basic_auth) + expect(auth).to eq({:user => 'test_user', :password => 'test_&^%$!password'}) + end + end + end + + context "secured connection against schema registry" do + + # TODO: Add unit tests for secured connection against schema registry + # - specified and inferred ssl_enabled + # - use "ssl_keystore_path" => paths[:test_path] to overcome File.readable?/writable? operations + end + end end end diff --git a/spec/unit/resources/do_not_remove_path/.gitignore b/spec/unit/resources/do_not_remove_path/.gitignore new file mode 100644 index 0000000..79bf7a1 --- /dev/null +++ b/spec/unit/resources/do_not_remove_path/.gitignore @@ -0,0 +1,2 @@ +# Empty dir for test cases to run, imitates readable/non-readable/writable folder +# When configs are :path validated, existed path is required \ No newline at end of file From 8295eb04931a424bbbf3fd484d5656126469de9c Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 17 Nov 2025 15:32:45 -0800 Subject: [PATCH 02/19] Increase test coverage with unit tests. --- docs/index.asciidoc | 24 +- lib/logstash/codecs/avro.rb | 14 +- spec/unit/avro_spec.rb | 560 +++++++++++++++++++++++++++++++++++- 3 files changed, 577 insertions(+), 21 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index becb067..98c06a5 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -87,7 +87,7 @@ output { | <> |<>|No | <> |<>|Yes | <> |<>|No -| <> |<>|No +| <> |list of <>|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -174,7 +174,8 @@ tag events with `_avroparsefailure` when decode fails * There is no default value for this setting. Path to PEM encoded certificate file for client authentication (mutual TLS). -This is an alternative to using `ssl_keystore_path`. +This is an alternative to using <>. +You cannot use this setting and <> at the same time. *Example* [source,ruby] @@ -185,16 +186,17 @@ ssl_certificate => "/path/to/client.crt" [id="plugins-{type}s-{plugin}-ssl_certificate_authorities"] ===== `ssl_certificate_authorities` -* Value type is <> +* Value type is a list of <> * There is no default value for this setting. Path to PEM encoded CA certificate file(s) for server verification. -This is an alternative to using `ssl_truststore_path`. +This is an alternative to using <>. +You cannot use this setting and <> at the same time. *Example* [source,ruby] ---------------------------------- -ssl_certificate_authorities => "/path/to/ca.crt" +ssl_certificate_authorities => ["/path/to/ca.crt"] ---------------------------------- [id="plugins-{type}s-{plugin}-ssl_cipher_suites"] @@ -213,7 +215,7 @@ Supported cipher suites vary depending on which version of Java is used. * There is no default value for this setting. Path to PEM encoded private key file for client authentication. -Must be used together with `ssl_certificate`. +Must be used together with <>. The private key must be unencrypted (passphrase-protected keys are not supported). *Example* @@ -248,14 +250,14 @@ This setting is only used when `schema_uri` uses the `https://` scheme. * Value type is <> * There is no default value for this setting. -The password for the keystore file specified in `ssl_keystore_path`. +The password for the keystore file specified in <>. [id="plugins-{type}s-{plugin}-ssl_keystore_type"] ===== `ssl_keystore_type` * Value type is <> -* Default value is `"JKS"` +* There is no default value for this setting. The type of the keystore file. Can be either `JKS` or `PKCS12`. @@ -300,13 +302,13 @@ input { * Value type is <> * There is no default value for this setting. -The password for the truststore file specified in `ssl_truststore_path`. +The password for the truststore file specified in <>. [id="plugins-{type}s-{plugin}-ssl_truststore_type"] ===== `ssl_truststore_type` * Value type is <> -* Default value is `"JKS"` +* There is no default value for this setting. The type of the truststore file. Can be either `JKS` or `PKCS12`. @@ -329,7 +331,7 @@ input { kafka { codec => avro { schema_uri => "https://schema-registry.example.com:8081/schemas/ids/1" - ssl_certificate_authorities => "/path/to/ca.crt" + ssl_certificate_authorities => ["/path/to/ca.crt"] ssl_verification_mode => "full" } } diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index 3ef8c13..4d8c9a9 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -106,7 +106,7 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base # Path to PEM encoded CA certificate file(s) for server verification # Can be a single file or directory containing multiple CA certificates - config :ssl_certificate_authorities, :validate => :path + config :ssl_certificate_authorities, :validate => :path, :list => true # Options to verify the server's certificate. # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates; @@ -120,8 +120,8 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base # The keystore password config :ssl_keystore_password, :validate => :password - # Keystore type (JKS or PKCS12). Defaults to JKS. - config :ssl_keystore_type, :validate => %w[JKS PKCS12], :default => "JKS" + # Keystore type (JKS or PKCS12) + config :ssl_keystore_type, :validate => %w[JKS PKCS12] # The truststore path config :ssl_truststore_path, :validate => :path @@ -129,8 +129,8 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base # The truststore password config :ssl_truststore_password, :validate => :password - # Truststore type (JKS or PKCS12). Defaults to JKS. - config :ssl_truststore_type, :validate => %w[JKS PKCS12], :default => "JKS" + # Truststore type (JKS or PKCS12) + config :ssl_truststore_type, :validate => %w[JKS PKCS12] # The list of cipher suites to use, listed by priorities. # Supported cipher suites vary depending on which version of Java is used. @@ -270,7 +270,7 @@ def validate_ssl_settings! # establishing trust of the server we connect to # system-provided trust requires verification mode enabled if @ssl_verification_mode == "none" - raise_config_error! "`ssl_truststore_path` requires `ssl_verification_mode` to be either `full`" if @ssl_truststore_path + raise_config_error! "`ssl_truststore_path` requires `ssl_verification_mode` to be `full`" if @ssl_truststore_path raise_config_error! "`ssl_truststore_password` requires `ssl_truststore_path` and `ssl_verification_mode => 'full'`" if @ssl_truststore_password raise_config_error! "`ssl_certificate_authorities` requires `ssl_verification_mode` to be `full`" if @ssl_certificate_authorities end @@ -286,7 +286,7 @@ def validate_ssl_settings! raise_config_error! "`ssl_certificate_authorities` cannot be empty" end - raise_config_error! "Multiple values on `ssl_certificate_authorities` are not supported by this plugin" if @ssl_certificate_authorities.size > 1 + raise_config_error! "Multiple values on `ssl_certificate_authorities` are not supported by this plugin" if @ssl_certificate_authorities && @ssl_certificate_authorities&.size > 1 ensure_readable_and_non_writable! "ssl_certificate_authorities", @ssl_certificate_authorities&.first end diff --git a/spec/unit/avro_spec.rb b/spec/unit/avro_spec.rb index af2268c..ea602f5 100644 --- a/spec/unit/avro_spec.rb +++ b/spec/unit/avro_spec.rb @@ -335,10 +335,564 @@ end context "secured connection against schema registry" do + let(:test_schema) do + '{"type": "record", "name": "Test", + "fields": [{"name": "foo", "type": ["null", "string"]}, + {"name": "bar", "type": "int"}]}' + end + + before do + allow_any_instance_of(described_class).to receive(:fetch_remote_schema).and_return(test_schema) + allow(File).to receive(:readable?).and_return(true) + allow(File).to receive(:writable?).and_return(false) + end + + subject do + LogStash::Codecs::Avro.new(avro_config) + end + + context "explicit and inferred SSL" do + context "with explicit ssl_enabled => true" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'ssl_enabled' => true + } + end + + it "enables SSL" do + expect(subject.instance_variable_get(:@ssl_enabled)).to be true + end + end + + context "with HTTPS URI (inferred ssl_enabled)" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' + } + end + + it "automatically enables SSL for HTTPS URIs" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to be true + end + end + + context "with explicit ssl_enabled => false" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'ssl_enabled' => false + } + end + + it "disables SSL" do + expect(subject.instance_variable_get(:@ssl_enabled)).to be false + end + end + end + + context "SSL verification" do + context "with ssl_verification_mode => 'full'" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_verification_mode' => 'full' + } + end + + it "sets verification mode to full" do + expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('full') + end + + it "builds SSL options with default verify mode" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:verify]).to eq(:default) + end + end + + context "with ssl_verification_mode => 'none'" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_verification_mode' => 'none' + } + end + + it "sets verification mode to none" do + expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('none') + end - # TODO: Add unit tests for secured connection against schema registry - # - specified and inferred ssl_enabled - # - use "ssl_keystore_path" => paths[:test_path] to overcome File.readable?/writable? operations + it "builds SSL options with disable verify mode" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:verify]).to eq(:disable) + end + end + + context "with default ssl_verification_mode" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' + } + end + + it "defaults to 'full'" do + expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('full') + end + end + end + + context "keystore configuration" do + context "with ssl_keystore_path" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_keystore_path' => paths[:test_path], + 'ssl_keystore_password' => 'keystore_pass', + 'ssl_keystore_type' => 'JKS' + } + end + + it "configures keystore options" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:keystore]).to eq(paths[:test_path]) + expect(ssl_options[:keystore_password]).to eq('keystore_pass') + expect(ssl_options[:keystore_type]).to eq('jks') + end + end + + context "with ssl_keystore_path and PKCS12 type" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_keystore_path' => paths[:test_path], + 'ssl_keystore_password' => 'keystore_pass', + 'ssl_keystore_type' => 'PKCS12' + } + end + + it "configures PKCS12 keystore" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:keystore_type]).to eq('pkcs12') + end + end + + context "with ssl_keystore_path but no password" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_keystore_path' => paths[:test_path], + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_keystore_path` requires `ssl_keystore_password`/ + ) + end + end + end + + context "truststore configuration" do + context "with ssl_truststore_path" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_path' => paths[:test_path], + 'ssl_truststore_password' => 'truststore_pass', + 'ssl_truststore_type' => 'JKS' + } + end + + it "configures truststore options" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:truststore]).to eq(paths[:test_path]) + expect(ssl_options[:truststore_password]).to eq('truststore_pass') + expect(ssl_options[:truststore_type]).to eq('jks') + end + end + + context "with ssl_truststore_path and PKCS12 type" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_path' => paths[:test_path], + 'ssl_truststore_password' => 'truststore_pass', + 'ssl_truststore_type' => 'PKCS12' + } + end + + it "configures PKCS12 truststore" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:truststore_type]).to eq('pkcs12') + end + end + + context "with ssl_truststore_path but no password" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_path' => paths[:test_path] + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_path` requires `ssl_truststore_password`/ + ) + end + end + end + + context "CA configuration" do + + context "with single CA file" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_certificate_authorities' => [paths[:test_path]] + } + end + + it "configures CA certificate" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:ca_file]).to eq(paths[:test_path]) + end + end + + context "with multiple CA files" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_certificate_authorities' => [paths[:test_path], paths[:test_path]] + } + end + + it "raises ConfigurationError for multiple CAs" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /Multiple values on `ssl_certificate_authorities` are not supported/ + ) + end + end + + context "with empty ssl_certificate_authorities" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_certificate_authorities' => [] + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_certificate_authorities` cannot be empty/ + ) + end + end + end + + context "cipher suites" do + context "with specified cipher suites" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_cipher_suites' => %w[TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256] + } + end + + it "configures cipher suites" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:cipher_suites]).to eq(%w[TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]) + end + end + + context "without cipher suites" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' + } + end + + it "does not include cipher_suites in SSL options" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options).not_to have_key(:cipher_suites) + end + end + end + + context "supported protocols" do + context "with TLSv1.2 and TLSv1.3" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_supported_protocols' => %w[TLSv1.2 TLSv1.3] + } + end + + it "configures supported protocols" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:protocols]).to eq(%w[TLSv1.2 TLSv1.3]) + end + end + + context "with only TLSv1.3" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_supported_protocols' => ['TLSv1.3'] + } + end + + it "configures only TLSv1.3" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:protocols]).to eq(['TLSv1.3']) + end + end + + context "with empty ssl_supported_protocols" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_supported_protocols' => [] + } + end + + it "does not include protocols in SSL options" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options).not_to have_key(:protocols) + end + end + + context "without ssl_supported_protocols" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' + } + end + + it "uses default (no protocols key)" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options).not_to have_key(:protocols) + end + end + end + + context "SSL validations" do + context "PEM certificate validation" do + context "when both ssl_certificate and ssl_keystore_path are set" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_certificate' => paths[:test_path], + 'ssl_key' => paths[:test_path], + 'ssl_keystore_path' => paths[:test_path], + 'ssl_keystore_password' => 'password' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_certificate` and `ssl_keystore_path` cannot be used together/ + ) + end + end + + context "when ssl_certificate is set without ssl_key" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_certificate' => paths[:test_path] + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_certificate` requires `ssl_key`/ + ) + end + end + + context "when ssl_key is set without ssl_certificate" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_key' => paths[:test_path] + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_key` is not allowed unless `ssl_certificate` is specified/ + ) + end + end + end + + context "keystore validation" do + context "when ssl_keystore_password is set without ssl_keystore_path" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_keystore_password' => 'password' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is specified/ + ) + end + end + + context "when ssl_keystore_password is empty" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_keystore_path' => paths[:test_path], + 'ssl_keystore_password' => '' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_keystore_password` cannot be empty/ + ) + end + end + + context "when ssl_keystore_type is set without ssl_keystore_path" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_keystore_type' => 'JKS' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_keystore_type` is not allowed unless `ssl_keystore_path` is specified/ + ) + end + end + end + + context "truststore validation" do + context "when ssl_truststore_password is set without ssl_truststore_path" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_password' => 'password' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_password` is not allowed unless `ssl_truststore_path` is specified/ + ) + end + end + + context "when ssl_truststore_password is empty" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_path' => paths[:test_path], + 'ssl_truststore_password' => '' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_password` cannot be empty/ + ) + end + end + + context "when both ssl_truststore_path and ssl_certificate_authorities are set" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_path' => paths[:test_path], + 'ssl_truststore_password' => 'password', + 'ssl_certificate_authorities' => [paths[:test_path]] + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_path` and `ssl_certificate_authorities` cannot be used together/ + ) + end + end + end + + context "verification mode validation" do + context "when ssl_truststore_path is set with verification mode none" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_path' => paths[:test_path], + 'ssl_truststore_password' => 'password', + 'ssl_verification_mode' => 'none' + } + end + + it "requires `ssl_verification_mode` => 'full'" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_path` requires `ssl_verification_mode` to be `full`/ + ) + end + end + + context "when ssl_truststore_password is set with verification mode none" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_password' => 'password', + 'ssl_verification_mode' => 'none' + } + end + + it "requires `ssl_verification_mode => 'full'" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_password` requires `ssl_truststore_path` and `ssl_verification_mode => 'full'`/ + ) + end + end + + context "when ssl_certificate_authorities is set with verification mode none" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_certificate_authorities' => [paths[:test_path]], + 'ssl_verification_mode' => 'none' + } + end + + it "requires `ssl_verification_mode => 'full'" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_certificate_authorities` requires `ssl_verification_mode` to be `full`/ + ) + end + end + end + end end end end From 78283c771f940bddf05957629a1bebeceb3b7718 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 17 Nov 2025 20:03:46 -0800 Subject: [PATCH 03/19] Separate remote schema registry tests from encode and decode. --- spec/unit/avro_spec.rb | 918 ++++++++++++++++++++--------------------- 1 file changed, 453 insertions(+), 465 deletions(-) diff --git a/spec/unit/avro_spec.rb b/spec/unit/avro_spec.rb index ea602f5..ebc43f5 100644 --- a/spec/unit/avro_spec.rb +++ b/spec/unit/avro_spec.rb @@ -206,690 +206,678 @@ end end + end - context "remote schema registry" do - - context "basic authentication" do - let(:test_schema) do - '{"type": "record", "name": "Test", + context "remote schema registry" do + let(:test_schema) do + '{"type": "record", "name": "Test", "fields": [{"name": "foo", "type": ["null", "string"]}, {"name": "bar", "type": "int"}]}' + end + + subject do + allow_any_instance_of(LogStash::Codecs::Avro).to \ + receive(:fetch_remote_schema).and_return(test_schema) + next LogStash::Codecs::Avro.new(avro_config) + end + + context "basic authentication" do + + context "with both username and password" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'username' => 'test_user', + 'password' => 'test_&^%$!password' + } + end + + it "uses user and password" do + auth = subject.send(:build_basic_auth) + expect(auth).to eq({:user => 'test_user', :password => 'test_&^%$!password'}) + end + + it "includes valid credentials in auth hash" do + auth = subject.send(:build_basic_auth) + expect(auth[:user]).not_to be_empty + expect(auth[:password]).not_to be_empty + end + end + + context "with only username" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'username' => 'test_user' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /`username` requires `password`/) + end + end + + context "with only password" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'password' => 'test_&^%$!password' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /`password` is not allowed unless `username` is specified/) + end + end + + context "with empty username" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'username' => '', + 'password' => 'test_&^%$!password' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /Empty `username` or `password` is not allowed/) + end + end + + context "with empty password" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'username' => 'test_user', + 'password' => '' + } + end + + it "raises ConfigurationError" do + expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /Empty `username` or `password` is not allowed/) + end + end + + context "with neither username nor password" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc' + } end - before do - allow_any_instance_of(described_class).to receive(:fetch_remote_schema).and_return(test_schema) + it "returns empty hash" do + auth = subject.send(:build_basic_auth) + expect(auth).to be_empty end + end - subject do - LogStash::Codecs::Avro.new(avro_config) + context "with unsecure connection and credentials" do + let(:avro_config) do + { + 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', + 'username' => 'test_user', + 'password' => 'test_&^%$!password' + } end - context "with both username and password" do + it "warns about credentials over unencrypted HTTP" do + expect(subject.logger).to receive(:warn).with(/Credentials are being sent over unencrypted HTTP/) + subject.register + end + + it "still returns valid auth hash" do + allow(subject.logger).to receive(:warn) + auth = subject.send(:build_basic_auth) + expect(auth).to eq({:user => 'test_user', :password => 'test_&^%$!password'}) + end + end + end + + context "secured connection against schema registry" do + + before do + allow(File).to receive(:readable?).and_return(true) + allow(File).to receive(:writable?).and_return(false) + end + + context "explicit and inferred SSL" do + context "with explicit ssl_enabled => true" do let(:avro_config) do { 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', - 'username' => 'test_user', - 'password' => 'test_&^%$!password' + 'ssl_enabled' => true } end - it "uses user and password" do - auth = subject.send(:build_basic_auth) - expect(auth).to eq({:user => 'test_user', :password => 'test_&^%$!password'}) + it "enables SSL" do + expect(subject.instance_variable_get(:@ssl_enabled)).to be true end + end - it "includes valid credentials in auth hash" do - auth = subject.send(:build_basic_auth) - expect(auth[:user]).not_to be_empty - expect(auth[:password]).not_to be_empty + context "with HTTPS URI (inferred ssl_enabled)" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' + } + end + + it "automatically enables SSL for HTTPS URIs" do + subject.register + expect(subject.instance_variable_get(:@ssl_enabled)).to be true end end - context "with only username" do + context "with explicit ssl_enabled => false" do let(:avro_config) do { 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', - 'username' => 'test_user' + 'ssl_enabled' => false } end - it "raises ConfigurationError" do - expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /`username` requires `password`/) + it "disables SSL" do + expect(subject.instance_variable_get(:@ssl_enabled)).to be false end end + end - context "with only password" do + context "SSL verification" do + context "with ssl_verification_mode => 'full'" do let(:avro_config) do { - 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', - 'password' => 'test_&^%$!password' + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_verification_mode' => 'full' } end - it "raises ConfigurationError" do - expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /`password` is not allowed unless `username` is specified/) + it "sets verification mode to full" do + expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('full') + end + + it "builds SSL options with default verify mode" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:verify]).to eq(:default) end end - context "with empty username" do + context "with ssl_verification_mode => 'none'" do let(:avro_config) do { - 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', - 'username' => '', - 'password' => 'test_&^%$!password' + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_verification_mode' => 'none' } end - it "raises ConfigurationError" do - expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /Empty `username` or `password` is not allowed/) + it "sets verification mode to none" do + expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('none') + end + + it "builds SSL options with disable verify mode" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:verify]).to eq(:disable) end end - context "with empty password" do + context "with default ssl_verification_mode" do let(:avro_config) do { - 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', - 'username' => 'test_user', - 'password' => '' + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' } end - it "raises ConfigurationError" do - expect { subject.send(:build_basic_auth) }.to raise_error(LogStash::ConfigurationError, /Empty `username` or `password` is not allowed/) + it "defaults to 'full'" do + expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('full') end end + end - context "with neither username nor password" do + context "keystore configuration" do + context "with ssl_keystore_path" do let(:avro_config) do { - 'schema_uri' => 'http://schema-registry.example.com/schema.avsc' + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_keystore_path' => paths[:test_path], + 'ssl_keystore_password' => 'keystore_pass', + 'ssl_keystore_type' => 'JKS' } end - it "returns empty hash" do - auth = subject.send(:build_basic_auth) - expect(auth).to be_empty + it "configures keystore options" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:keystore]).to eq(paths[:test_path]) + expect(ssl_options[:keystore_password]).to eq('keystore_pass') + expect(ssl_options[:keystore_type]).to eq('jks') end end - context "with unsecure connection and credentials" do + context "with ssl_keystore_path and PKCS12 type" do let(:avro_config) do { - 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', - 'username' => 'test_user', - 'password' => 'test_&^%$!password' + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_keystore_path' => paths[:test_path], + 'ssl_keystore_password' => 'keystore_pass', + 'ssl_keystore_type' => 'PKCS12' } end - it "warns about credentials over unencrypted HTTP" do - expect(subject.logger).to receive(:warn).with(/Credentials are being sent over unencrypted HTTP/) - subject.register + it "configures PKCS12 keystore" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:keystore_type]).to eq('pkcs12') end + end - it "still returns valid auth hash" do - allow(subject.logger).to receive(:warn) - auth = subject.send(:build_basic_auth) - expect(auth).to eq({:user => 'test_user', :password => 'test_&^%$!password'}) + context "with ssl_keystore_path but no password" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_keystore_path' => paths[:test_path], + } + end + + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_keystore_path` requires `ssl_keystore_password`/ + ) end end end - context "secured connection against schema registry" do - let(:test_schema) do - '{"type": "record", "name": "Test", - "fields": [{"name": "foo", "type": ["null", "string"]}, - {"name": "bar", "type": "int"}]}' - end + context "truststore configuration" do + context "with ssl_truststore_path" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_path' => paths[:test_path], + 'ssl_truststore_password' => 'truststore_pass', + 'ssl_truststore_type' => 'JKS' + } + end - before do - allow_any_instance_of(described_class).to receive(:fetch_remote_schema).and_return(test_schema) - allow(File).to receive(:readable?).and_return(true) - allow(File).to receive(:writable?).and_return(false) + it "configures truststore options" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:truststore]).to eq(paths[:test_path]) + expect(ssl_options[:truststore_password]).to eq('truststore_pass') + expect(ssl_options[:truststore_type]).to eq('jks') + end end - subject do - LogStash::Codecs::Avro.new(avro_config) + context "with ssl_truststore_path and PKCS12 type" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_path' => paths[:test_path], + 'ssl_truststore_password' => 'truststore_pass', + 'ssl_truststore_type' => 'PKCS12' + } + end + + it "configures PKCS12 truststore" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:truststore_type]).to eq('pkcs12') + end end - context "explicit and inferred SSL" do - context "with explicit ssl_enabled => true" do - let(:avro_config) do - { - 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', - 'ssl_enabled' => true - } - end + context "with ssl_truststore_path but no password" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_path' => paths[:test_path] + } + end - it "enables SSL" do - expect(subject.instance_variable_get(:@ssl_enabled)).to be true - end + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_path` requires `ssl_truststore_password`/ + ) end + end + end - context "with HTTPS URI (inferred ssl_enabled)" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' - } - end + context "CA configuration" do - it "automatically enables SSL for HTTPS URIs" do - subject.register - expect(subject.instance_variable_get(:@ssl_enabled)).to be true - end + context "with single CA file" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_certificate_authorities' => [paths[:test_path]] + } end - context "with explicit ssl_enabled => false" do - let(:avro_config) do - { - 'schema_uri' => 'http://schema-registry.example.com/schema.avsc', - 'ssl_enabled' => false - } - end + it "configures CA certificate" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:ca_file]).to eq(paths[:test_path]) + end + end - it "disables SSL" do - expect(subject.instance_variable_get(:@ssl_enabled)).to be false - end + context "with multiple CA files" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_certificate_authorities' => [paths[:test_path], paths[:test_path]] + } + end + + it "raises ConfigurationError for multiple CAs" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /Multiple values on `ssl_certificate_authorities` are not supported/ + ) end end - context "SSL verification" do - context "with ssl_verification_mode => 'full'" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_verification_mode' => 'full' - } - end + context "with empty ssl_certificate_authorities" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_certificate_authorities' => [] + } + end - it "sets verification mode to full" do - expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('full') - end + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_certificate_authorities` cannot be empty/ + ) + end + end + end - it "builds SSL options with default verify mode" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options[:verify]).to eq(:default) - end + context "cipher suites" do + context "with specified cipher suites" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_cipher_suites' => %w[TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256] + } end - context "with ssl_verification_mode => 'none'" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_verification_mode' => 'none' - } - end + it "configures cipher suites" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:cipher_suites]).to eq(%w[TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]) + end + end - it "sets verification mode to none" do - expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('none') - end + context "without cipher suites" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' + } + end - it "builds SSL options with disable verify mode" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options[:verify]).to eq(:disable) - end + it "does not include cipher_suites in SSL options" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options).not_to have_key(:cipher_suites) end + end + end - context "with default ssl_verification_mode" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' - } - end + context "supported protocols" do + context "with TLSv1.2 and TLSv1.3" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_supported_protocols' => %w[TLSv1.2 TLSv1.3] + } + end - it "defaults to 'full'" do - expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('full') - end + it "configures supported protocols" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:protocols]).to eq(%w[TLSv1.2 TLSv1.3]) end end - context "keystore configuration" do - context "with ssl_keystore_path" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_keystore_path' => paths[:test_path], - 'ssl_keystore_password' => 'keystore_pass', - 'ssl_keystore_type' => 'JKS' - } - end + context "with only TLSv1.3" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_supported_protocols' => ['TLSv1.3'] + } + end - it "configures keystore options" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options[:keystore]).to eq(paths[:test_path]) - expect(ssl_options[:keystore_password]).to eq('keystore_pass') - expect(ssl_options[:keystore_type]).to eq('jks') - end + it "configures only TLSv1.3" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options[:protocols]).to eq(['TLSv1.3']) end + end - context "with ssl_keystore_path and PKCS12 type" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_keystore_path' => paths[:test_path], - 'ssl_keystore_password' => 'keystore_pass', - 'ssl_keystore_type' => 'PKCS12' - } - end + context "with empty ssl_supported_protocols" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_supported_protocols' => [] + } + end - it "configures PKCS12 keystore" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options[:keystore_type]).to eq('pkcs12') - end + it "does not include protocols in SSL options" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options).not_to have_key(:protocols) end + end - context "with ssl_keystore_path but no password" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_keystore_path' => paths[:test_path], - } - end + context "without ssl_supported_protocols" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' + } + end - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_keystore_path` requires `ssl_keystore_password`/ - ) - end + it "uses default (no protocols key)" do + ssl_options = subject.send(:build_ssl_options) + expect(ssl_options).not_to have_key(:protocols) end end + end - context "truststore configuration" do - context "with ssl_truststore_path" do + context "SSL validations" do + context "PEM certificate validation" do + context "when both ssl_certificate and ssl_keystore_path are set" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_truststore_path' => paths[:test_path], - 'ssl_truststore_password' => 'truststore_pass', - 'ssl_truststore_type' => 'JKS' + 'ssl_certificate' => paths[:test_path], + 'ssl_key' => paths[:test_path], + 'ssl_keystore_path' => paths[:test_path], + 'ssl_keystore_password' => 'password' } end - it "configures truststore options" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options[:truststore]).to eq(paths[:test_path]) - expect(ssl_options[:truststore_password]).to eq('truststore_pass') - expect(ssl_options[:truststore_type]).to eq('jks') + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_certificate` and `ssl_keystore_path` cannot be used together/ + ) end end - context "with ssl_truststore_path and PKCS12 type" do + context "when ssl_certificate is set without ssl_key" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_truststore_path' => paths[:test_path], - 'ssl_truststore_password' => 'truststore_pass', - 'ssl_truststore_type' => 'PKCS12' + 'ssl_certificate' => paths[:test_path] } end - it "configures PKCS12 truststore" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options[:truststore_type]).to eq('pkcs12') + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_certificate` requires `ssl_key`/ + ) end end - context "with ssl_truststore_path but no password" do + context "when ssl_key is set without ssl_certificate" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_truststore_path' => paths[:test_path] + 'ssl_key' => paths[:test_path] } end it "raises ConfigurationError" do expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_truststore_path` requires `ssl_truststore_password`/ - ) + LogStash::ConfigurationError, + /`ssl_key` is not allowed unless `ssl_certificate` is specified/ + ) end end end - context "CA configuration" do - - context "with single CA file" do + context "keystore validation" do + context "when ssl_keystore_password is set without ssl_keystore_path" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_certificate_authorities' => [paths[:test_path]] + 'ssl_keystore_password' => 'password' } end - it "configures CA certificate" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options[:ca_file]).to eq(paths[:test_path]) + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is specified/ + ) end end - context "with multiple CA files" do + context "when ssl_keystore_password is empty" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_certificate_authorities' => [paths[:test_path], paths[:test_path]] + 'ssl_keystore_path' => paths[:test_path], + 'ssl_keystore_password' => '' } end - it "raises ConfigurationError for multiple CAs" do + it "raises ConfigurationError" do expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /Multiple values on `ssl_certificate_authorities` are not supported/ - ) + LogStash::ConfigurationError, + /`ssl_keystore_password` cannot be empty/ + ) end end - context "with empty ssl_certificate_authorities" do + context "when ssl_keystore_type is set without ssl_keystore_path" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_certificate_authorities' => [] + 'ssl_keystore_type' => 'JKS' } end it "raises ConfigurationError" do expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_certificate_authorities` cannot be empty/ - ) + LogStash::ConfigurationError, + /`ssl_keystore_type` is not allowed unless `ssl_keystore_path` is specified/ + ) end end end - context "cipher suites" do - context "with specified cipher suites" do + context "truststore validation" do + context "when ssl_truststore_password is set without ssl_truststore_path" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_cipher_suites' => %w[TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256] + 'ssl_truststore_password' => 'password' } end - it "configures cipher suites" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options[:cipher_suites]).to eq(%w[TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]) + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_password` is not allowed unless `ssl_truststore_path` is specified/ + ) end end - context "without cipher suites" do + context "when ssl_truststore_password is empty" do let(:avro_config) do { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_truststore_path' => paths[:test_path], + 'ssl_truststore_password' => '' } end - it "does not include cipher_suites in SSL options" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options).not_to have_key(:cipher_suites) + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_password` cannot be empty/ + ) end end - end - context "supported protocols" do - context "with TLSv1.2 and TLSv1.3" do + context "when both ssl_truststore_path and ssl_certificate_authorities are set" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_supported_protocols' => %w[TLSv1.2 TLSv1.3] + 'ssl_truststore_path' => paths[:test_path], + 'ssl_truststore_password' => 'password', + 'ssl_certificate_authorities' => [paths[:test_path]] } end - it "configures supported protocols" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options[:protocols]).to eq(%w[TLSv1.2 TLSv1.3]) + it "raises ConfigurationError" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_path` and `ssl_certificate_authorities` cannot be used together/ + ) end end + end - context "with only TLSv1.3" do + context "verification mode validation" do + context "when ssl_truststore_path is set with verification mode none" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_supported_protocols' => ['TLSv1.3'] + 'ssl_truststore_path' => paths[:test_path], + 'ssl_truststore_password' => 'password', + 'ssl_verification_mode' => 'none' } end - it "configures only TLSv1.3" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options[:protocols]).to eq(['TLSv1.3']) + it "requires `ssl_verification_mode` => 'full'" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_path` requires `ssl_verification_mode` to be `full`/ + ) end end - context "with empty ssl_supported_protocols" do + context "when ssl_truststore_password is set with verification mode none" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_supported_protocols' => [] + 'ssl_truststore_password' => 'password', + 'ssl_verification_mode' => 'none' } end - it "does not include protocols in SSL options" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options).not_to have_key(:protocols) + it "requires `ssl_verification_mode => 'full'" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_truststore_password` requires `ssl_truststore_path` and `ssl_verification_mode => 'full'`/ + ) end end - context "without ssl_supported_protocols" do + context "when ssl_certificate_authorities is set with verification mode none" do let(:avro_config) do { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc' + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_certificate_authorities' => [paths[:test_path]], + 'ssl_verification_mode' => 'none' } end - it "uses default (no protocols key)" do - ssl_options = subject.send(:build_ssl_options) - expect(ssl_options).not_to have_key(:protocols) - end - end - end - - context "SSL validations" do - context "PEM certificate validation" do - context "when both ssl_certificate and ssl_keystore_path are set" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_certificate' => paths[:test_path], - 'ssl_key' => paths[:test_path], - 'ssl_keystore_path' => paths[:test_path], - 'ssl_keystore_password' => 'password' - } - end - - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_certificate` and `ssl_keystore_path` cannot be used together/ - ) - end - end - - context "when ssl_certificate is set without ssl_key" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_certificate' => paths[:test_path] - } - end - - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_certificate` requires `ssl_key`/ - ) - end - end - - context "when ssl_key is set without ssl_certificate" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_key' => paths[:test_path] - } - end - - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_key` is not allowed unless `ssl_certificate` is specified/ - ) - end - end - end - - context "keystore validation" do - context "when ssl_keystore_password is set without ssl_keystore_path" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_keystore_password' => 'password' - } - end - - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is specified/ - ) - end - end - - context "when ssl_keystore_password is empty" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_keystore_path' => paths[:test_path], - 'ssl_keystore_password' => '' - } - end - - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_keystore_password` cannot be empty/ - ) - end - end - - context "when ssl_keystore_type is set without ssl_keystore_path" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_keystore_type' => 'JKS' - } - end - - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_keystore_type` is not allowed unless `ssl_keystore_path` is specified/ - ) - end - end - end - - context "truststore validation" do - context "when ssl_truststore_password is set without ssl_truststore_path" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_truststore_password' => 'password' - } - end - - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_truststore_password` is not allowed unless `ssl_truststore_path` is specified/ - ) - end - end - - context "when ssl_truststore_password is empty" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_truststore_path' => paths[:test_path], - 'ssl_truststore_password' => '' - } - end - - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_truststore_password` cannot be empty/ - ) - end - end - - context "when both ssl_truststore_path and ssl_certificate_authorities are set" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_truststore_path' => paths[:test_path], - 'ssl_truststore_password' => 'password', - 'ssl_certificate_authorities' => [paths[:test_path]] - } - end - - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_truststore_path` and `ssl_certificate_authorities` cannot be used together/ - ) - end - end - end - - context "verification mode validation" do - context "when ssl_truststore_path is set with verification mode none" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_truststore_path' => paths[:test_path], - 'ssl_truststore_password' => 'password', - 'ssl_verification_mode' => 'none' - } - end - - it "requires `ssl_verification_mode` => 'full'" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_truststore_path` requires `ssl_verification_mode` to be `full`/ - ) - end - end - - context "when ssl_truststore_password is set with verification mode none" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_truststore_password' => 'password', - 'ssl_verification_mode' => 'none' - } - end - - it "requires `ssl_verification_mode => 'full'" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_truststore_password` requires `ssl_truststore_path` and `ssl_verification_mode => 'full'`/ - ) - end - end - - context "when ssl_certificate_authorities is set with verification mode none" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_certificate_authorities' => [paths[:test_path]], - 'ssl_verification_mode' => 'none' - } - end - - it "requires `ssl_verification_mode => 'full'" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_certificate_authorities` requires `ssl_verification_mode` to be `full`/ - ) - end + it "requires `ssl_verification_mode => 'full'" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /`ssl_certificate_authorities` requires `ssl_verification_mode` to be `full`/ + ) end end end From d0478a7c0156c3e5df01081f01f6fb846af0c3ac Mon Sep 17 00:00:00 2001 From: Mashhur Date: Tue, 18 Nov 2025 13:19:35 -0800 Subject: [PATCH 04/19] Fix CI failures. --- spec/unit/avro_spec.rb | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/spec/unit/avro_spec.rb b/spec/unit/avro_spec.rb index ebc43f5..53f79fd 100644 --- a/spec/unit/avro_spec.rb +++ b/spec/unit/avro_spec.rb @@ -216,8 +216,7 @@ end subject do - allow_any_instance_of(LogStash::Codecs::Avro).to \ - receive(:fetch_remote_schema).and_return(test_schema) + allow_any_instance_of(LogStash::Codecs::Avro).to receive(:fetch_schema).and_return(test_schema) next LogStash::Codecs::Avro.new(avro_config) end @@ -320,11 +319,6 @@ } end - it "warns about credentials over unencrypted HTTP" do - expect(subject.logger).to receive(:warn).with(/Credentials are being sent over unencrypted HTTP/) - subject.register - end - it "still returns valid auth hash" do allow(subject.logger).to receive(:warn) auth = subject.send(:build_basic_auth) @@ -362,7 +356,7 @@ end it "automatically enables SSL for HTTPS URIs" do - subject.register + subject.send(:validate_ssl_settings!) expect(subject.instance_variable_get(:@ssl_enabled)).to be true end end From 8643636b37d185d000d8d51fa2286a7b080d7920 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Wed, 19 Nov 2025 10:50:08 -0800 Subject: [PATCH 05/19] Prepare kafka and confluent schema registry environment to use in the integration tests. --- spec/integration/fixtures/jaas.config | 5 ++ spec/integration/fixtures/pwd | 2 + .../integration/fixtures/trust-store_stub.jks | 0 spec/integration/kafka_test_setup.sh | 80 +++++++++++++++++++ spec/integration/kafka_test_teardown.sh | 17 ++++ .../setup_keystore_and_truststore.sh | 12 +++ .../integration/start_auth_schema_registry.sh | 9 +++ spec/integration/start_schema_registry.sh | 6 ++ spec/integration/stop_schema_registry.sh | 7 ++ 9 files changed, 138 insertions(+) create mode 100644 spec/integration/fixtures/jaas.config create mode 100644 spec/integration/fixtures/pwd create mode 100644 spec/integration/fixtures/trust-store_stub.jks create mode 100755 spec/integration/kafka_test_setup.sh create mode 100755 spec/integration/kafka_test_teardown.sh create mode 100755 spec/integration/setup_keystore_and_truststore.sh create mode 100755 spec/integration/start_auth_schema_registry.sh create mode 100755 spec/integration/start_schema_registry.sh create mode 100755 spec/integration/stop_schema_registry.sh diff --git a/spec/integration/fixtures/jaas.config b/spec/integration/fixtures/jaas.config new file mode 100644 index 0000000..f1c29ac --- /dev/null +++ b/spec/integration/fixtures/jaas.config @@ -0,0 +1,5 @@ +SchemaRegistry-Props { + org.eclipse.jetty.security.jaas.spi.PropertyFileLoginModule required + file="build/confluent_platform/etc/schema-registry/pwd" + debug="true"; +}; diff --git a/spec/integration/fixtures/pwd b/spec/integration/fixtures/pwd new file mode 100644 index 0000000..7d2a92a --- /dev/null +++ b/spec/integration/fixtures/pwd @@ -0,0 +1,2 @@ +barney: changeme,user,developer +admin:admin,admin \ No newline at end of file diff --git a/spec/integration/fixtures/trust-store_stub.jks b/spec/integration/fixtures/trust-store_stub.jks new file mode 100644 index 0000000..e69de29 diff --git a/spec/integration/kafka_test_setup.sh b/spec/integration/kafka_test_setup.sh new file mode 100755 index 0000000..490fc6a --- /dev/null +++ b/spec/integration/kafka_test_setup.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Setup Kafka and create test topics + +set -ex +# check if KAFKA_VERSION env var is set +if [ -n "${KAFKA_VERSION+1}" ]; then + echo "KAFKA_VERSION is $KAFKA_VERSION" +else + KAFKA_VERSION=4.1.0 +fi + +KAFKA_MAJOR_VERSION="${KAFKA_VERSION%%.*}" + +export _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true" + +rm -rf build +mkdir build + +echo "Setup Kafka version $KAFKA_VERSION" +if [ ! -e "kafka_2.13-$KAFKA_VERSION.tgz" ]; then + echo "Kafka not present locally, downloading" + curl -s -o "kafka_2.13-$KAFKA_VERSION.tgz" "https://downloads.apache.org/kafka/$KAFKA_VERSION/kafka_2.13-$KAFKA_VERSION.tgz" +fi +cp "kafka_2.13-$KAFKA_VERSION.tgz" "build/kafka.tgz" +mkdir "build/kafka" && tar xzf "build/kafka.tgz" -C "build/kafka" --strip-components 1 + +echo "Use KRaft for Kafka version $KAFKA_VERSION" +echo "log.dirs=${PWD}/build/kafka-logs" >> build/kafka/config/server.properties + +build/kafka/bin/kafka-storage.sh format \ + --cluster-id $(build/kafka/bin/kafka-storage.sh random-uuid) \ + --config build/kafka/config/server.properties \ + --ignore-formatted \ + --standalone + +echo "Starting Kafka broker" +build/kafka/bin/kafka-server-start.sh -daemon "build/kafka/config/server.properties" --override advertised.host.name=127.0.0.1 --override log.dirs="${PWD}/build/kafka-logs" +sleep 10 + +echo "Setup Confluent Platform" +# check if CONFLUENT_VERSION env var is set +if [ -n "${CONFLUENT_VERSION+1}" ]; then + echo "CONFLUENT_VERSION is $CONFLUENT_VERSION" +else + CONFLUENT_VERSION=8.0.0 +fi +if [ ! -e "confluent-community-$CONFLUENT_VERSION.tar.gz" ]; then + echo "Confluent Platform not present locally, downloading" + CONFLUENT_MINOR=$(echo "$CONFLUENT_VERSION" | sed -n 's/^\([[:digit:]]*\.[[:digit:]]*\)\.[[:digit:]]*$/\1/p') + echo "CONFLUENT_MINOR is $CONFLUENT_MINOR" + curl -s -o "confluent-community-$CONFLUENT_VERSION.tar.gz" "http://packages.confluent.io/archive/$CONFLUENT_MINOR/confluent-community-$CONFLUENT_VERSION.tar.gz" +fi +echo "Extracting confluent-community-$CONFLUENT_VERSION.tar.gz to build" +mkdir "build/confluent_platform" && tar xzf "confluent-community-$CONFLUENT_VERSION.tar.gz" -C "build/confluent_platform" --strip-components 1 + +echo "Configuring TLS on Schema registry" +rm -Rf tls_repository +mkdir tls_repository +./setup_keystore_and_truststore.sh +# configure schema-registry to handle https on 8083 port +if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' 's/http:\/\/0.0.0.0:8081/http:\/\/0.0.0.0:8081, https:\/\/0.0.0.0:8083/g' "build/confluent_platform/etc/schema-registry/schema-registry.properties" +else + sed -i 's/http:\/\/0.0.0.0:8081/http:\/\/0.0.0.0:8081, https:\/\/0.0.0.0:8083/g' "build/confluent_platform/etc/schema-registry/schema-registry.properties" +fi +echo "ssl.keystore.location=`pwd`/tls_repository/schema_reg.jks" >> "build/confluent_platform/etc/schema-registry/schema-registry.properties" +echo "ssl.keystore.password=changeit" >> "build/confluent_platform/etc/schema-registry/schema-registry.properties" +echo "ssl.key.password=changeit" >> "build/confluent_platform/etc/schema-registry/schema-registry.properties" + +cp "build/confluent_platform/etc/schema-registry/schema-registry.properties" "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" +echo "authentication.method=BASIC" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" +echo "authentication.roles=admin,developer,user,sr-user" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" +echo "authentication.realm=SchemaRegistry-Props" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" +cp fixtures/jaas.config "build/confluent_platform/etc/schema-registry" + +echo "Setting up a test topic" +build/kafka/bin/kafka-topics.sh --create --partitions 3 --replication-factor 1 --topic logstash_integration_topic_plain --bootstrap-server localhost:9092 + +cp fixtures/pwd "build/confluent_platform/etc/schema-registry" +echo "Setup complete, running specs" diff --git a/spec/integration/kafka_test_teardown.sh b/spec/integration/kafka_test_teardown.sh new file mode 100755 index 0000000..42419e9 --- /dev/null +++ b/spec/integration/kafka_test_teardown.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Setup Kafka and create test topics +set -ex + +echo "Unregistering test topics" +#build/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --delete --topic 'topic_avro.*' 2>&1 + +echo "Stopping Kafka broker" +build/kafka/bin/kafka-server-stop.sh + +if [ -f "build/kafka/bin/zookeeper-server-stop.sh" ]; then + echo "Stopping ZooKeeper" + build/kafka/bin/zookeeper-server-stop.sh +fi + +echo "Clean TLS folder" +rm -Rf tls_repository diff --git a/spec/integration/setup_keystore_and_truststore.sh b/spec/integration/setup_keystore_and_truststore.sh new file mode 100755 index 0000000..a43d41e --- /dev/null +++ b/spec/integration/setup_keystore_and_truststore.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Setup Schema Registry keystore and Kafka's schema registry client's truststore +set -ex + +echo "Generating schema registry key store" +keytool -genkey -alias schema_reg -keyalg RSA -keystore tls_repository/schema_reg.jks -keypass changeit -storepass changeit -validity 365 -keysize 2048 -dname "CN=localhost, OU=John Doe, O=Acme Inc, L=Unknown, ST=Unknown, C=IT" + +echo "Exporting schema registry certificate" +keytool -exportcert -rfc -keystore tls_repository/schema_reg.jks -storepass changeit -alias schema_reg -file tls_repository/schema_reg_certificate.pem + +echo "Creating client's truststore and importing schema registry's certificate" +keytool -import -trustcacerts -file tls_repository/schema_reg_certificate.pem -keypass changeit -storepass changeit -keystore tls_repository/clienttruststore.jks -noprompt \ No newline at end of file diff --git a/spec/integration/start_auth_schema_registry.sh b/spec/integration/start_auth_schema_registry.sh new file mode 100755 index 0000000..a1d6497 --- /dev/null +++ b/spec/integration/start_auth_schema_registry.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Setup Kafka and create test topics +set -ex + +echo "Starting authed SchemaRegistry" +SCHEMA_REGISTRY_OPTS="-Djava.security.auth.login.config=build/confluent_platform/etc/schema-registry/jaas.config" \ + build/confluent_platform/bin/schema-registry-start \ + build/confluent_platform/etc/schema-registry/authed-schema-registry.properties \ + > /dev/null 2>&1 & \ No newline at end of file diff --git a/spec/integration/start_schema_registry.sh b/spec/integration/start_schema_registry.sh new file mode 100755 index 0000000..b1d343c --- /dev/null +++ b/spec/integration/start_schema_registry.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Setup Kafka and create test topics +set -ex + +echo "Starting SchemaRegistry" +build/confluent_platform/bin/schema-registry-start build/confluent_platform/etc/schema-registry/schema-registry.properties > /dev/null 2>&1 & diff --git a/spec/integration/stop_schema_registry.sh b/spec/integration/stop_schema_registry.sh new file mode 100755 index 0000000..93e7adb --- /dev/null +++ b/spec/integration/stop_schema_registry.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Setup Kafka and create test topics +set -ex + +echo "Stopping SchemaRegistry" +build/confluent_platform/bin/schema-registry-stop +sleep 5 \ No newline at end of file From f20a085744ad85b54783d971774bcab5e1e2a5c4 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Wed, 19 Nov 2025 16:10:07 -0800 Subject: [PATCH 06/19] Basic integration test with schema registry. --- .ci/run.sh | 34 ++++ lib/logstash/codecs/avro.rb | 19 ++- spec/integration/avro_integration_spec.rb | 186 ++++++++++++++++++++++ 3 files changed, 237 insertions(+), 2 deletions(-) create mode 100755 .ci/run.sh create mode 100644 spec/integration/avro_integration_spec.rb diff --git a/.ci/run.sh b/.ci/run.sh new file mode 100755 index 0000000..7b9ef93 --- /dev/null +++ b/.ci/run.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# This is intended to be run inside the docker container as the command of the docker-compose. + +env + +set -ex + +if [[ "$INTEGRATION" != "true" ]]; then + bundle exec rake test +else + # Define the Kafka:Confluent version pairs + VERSIONS=( + # "3.9.1:7.4.0" + "4.1.0:8.0.0" + ) + + for pair in "${VERSIONS[@]}"; do + KAFKA_VERSION="${pair%%:*}" + CONFLUENT_VERSION="${pair##*:}" + + echo "==================================================" + echo " Testing with Kafka $KAFKA_VERSION / Confluent $CONFLUENT_VERSION" + echo "==================================================" + + export KAFKA_VERSION + export CONFLUENT_VERSION + + cd spec/integration && ./kafka_test_setup.sh && cd ../.. + bundle exec rspec -fd --tag integration + cd spec/integration && ./kafka_test_teardown.sh && cd ../.. + done +fi + + diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index 4d8c9a9..0f76070 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -3,6 +3,7 @@ require "manticore" require "avro" require "base64" +require "json" require "logstash/codecs/base" require "logstash/event" require "logstash/timestamp" @@ -209,7 +210,7 @@ def fetch_schema(uri_string) def fetch_remote_schema(uri_string) client_options = {} - unless @proxy&.empty? + if @proxy && !@proxy.empty? client_options[:proxy] = @proxy.to_s end @@ -228,7 +229,21 @@ def fetch_remote_schema(uri_string) raise "HTTP request failed: #{response.code} #{response.message}" end - response.body + body = response.body + # Response may contain schema metadata, schema field is what we need + # Example response: {"subject":"test-no-auth-1763597024","version":1,"id":1,"guid":"5c6c5f26-e876-e5ab-02b0-8d9bebbc90d7","schemaType":"AVRO","schema":"{"type":"record","name":"TestRecord","namespace":"com.example","fields":[{"name":"message","type":"string"},{"name":"timestamp","type":"long"}]}","ts":1763597024561,"deleted":false} + begin + parsed = JSON.parse(body) + if parsed.is_a?(Hash) && parsed.has_key?('schema') + parsed['schema'] + else + # fallback to use the response as it is + body + end + rescue JSON::ParserError + # Not JSON, return as-is (probably a direct schema) + body + end ensure client.close if client end diff --git a/spec/integration/avro_integration_spec.rb b/spec/integration/avro_integration_spec.rb new file mode 100644 index 0000000..b2fd186 --- /dev/null +++ b/spec/integration/avro_integration_spec.rb @@ -0,0 +1,186 @@ +# encoding: utf-8 +require 'logstash/devutils/rspec/spec_helper' +require 'logstash/codecs/avro' +require 'logstash/event' +require 'avro' +require 'base64' +require 'manticore' + +describe "Avro Codec Integration Tests", :integration => true do + INTEGRATION_DIR = File.expand_path('../', __FILE__) + + let(:test_schema) do + { + "type" => "record", + "name" => "TestRecord", + "namespace" => "com.example", + "fields" => [ + { "name" => "message", "type" => "string" }, + { "name" => "timestamp", "type" => "long" } + ] + } + end + + let(:test_schema_json) { test_schema.to_json } + let(:test_event_data) do + { + "message" => "test message", + "timestamp" => Time.now.to_i + } + end + + let(:config) {{ }} + let(:codec) { LogStash::Codecs::Avro.new(config).tap { |c| c.register } } + + def run_integration_script(script_name) + Dir.chdir(INTEGRATION_DIR) do + result = system("./#{script_name}") + puts "Script #{script_name} #{result ? 'succeeded' : 'failed'}" + result + end + end + + def register_schema(schema_registry_url, schema_json, username: nil, password: nil, ssl_options: {}) + client_options = {} + + if username && password + client_options[:auth] = { user: username, password: password } + end + + client_options[:ssl] = ssl_options unless ssl_options.empty? + + client = Manticore::Client.new(client_options) + + response = client.post("#{schema_registry_url}/subjects/test-value/versions", + headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, + body: { schema: schema_json }.to_json + ).call + + raise "Failed to register schema: #{response.code} #{response.body}" unless response.code == 200 + + JSON.parse(response.body)["id"] + ensure + client&.close + end + + def encode_avro_data(schema_json, data) + schema = Avro::Schema.parse(schema_json) + dw = Avro::IO::DatumWriter.new(schema) + buffer = StringIO.new + encoder = Avro::IO::BinaryEncoder.new(buffer) + dw.write(data, encoder) + Base64.strict_encode64(buffer.string) + end + + def create_test_schema_file(filename = "test_schema.avsc") + schema_path = File.join(Dir.tmpdir, filename) + File.write(schema_path, test_schema_json) + schema_path + end + + def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl_options: {}) + start_time = Time.now + client_options = {} + + if username && password + client_options[:auth] = { user: username, password: password } + end + + client_options[:ssl] = ssl_options unless ssl_options.empty? + + puts "Waiting for Schema Registry at #{url}..." + attempt = 0 + + loop do + attempt += 1 + begin + client = Manticore::Client.new(client_options) + response = client.get(url).call + if response.code == 200 + puts "✓ Schema Registry is ready after #{attempt} attempts" + return true + end + rescue => e + # Continue waiting + puts " Attempt #{attempt}: #{e.class.name} - #{e.message[0..80]}" if attempt % 5 == 0 + ensure + client&.close if client + end + + if Time.now - start_time > timeout + raise "Schema Registry at #{url} did not become available within #{timeout} seconds after #{attempt} attempts" + end + + sleep 2 + end + end + + context "Schema Registry without authentication" do + let(:schema_registry_url) { "http://localhost:8081" } + + before(:all) do + run_integration_script("start_schema_registry.sh") + wait_for_schema_registry("http://localhost:8081") + end + + after(:all) do + run_integration_script("stop_schema_registry.sh") + sleep 2 + end + + context "fetching schema via HTTP" do + let(:schema_subject) { "test-no-auth-#{Time.now.to_i}" } + let(:full_schema_url) do + url = "#{schema_registry_url}/subjects/#{schema_subject}/versions/latest" + puts "Constructed schema URL: #{url}" + raise "Schema URL is empty!" if url.nil? || url.empty? || !url.start_with?('http') + url + end + + let(:config) { super().merge({'schema_uri' => full_schema_url}) } + + before do + client = Manticore::Client.new + response = client.post("#{schema_registry_url}/subjects/#{schema_subject}/versions", + headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, + body: { schema: test_schema_json }.to_json + ).call + puts "Schema registration response: #{response.code}" + expect(response.code).to eq(200) + client.close + end + + it "fetches and decodes schema from Schema Registry" do + encoded_data = encode_avro_data(test_schema_json, test_event_data) + events = [] + codec.decode(encoded_data) do |event| + events << event + end + + expect(events.size).to eq(1) + expect(events.first.get("message")).to eq(test_event_data["message"]) + expect(events.first.get("timestamp")).to eq(test_event_data["timestamp"]) + end + + it "encodes data using schema from schema registry" do + event = LogStash::Event.new(test_event_data) + encoded_data = nil + + codec.on_event do |e, data| + encoded_data = data + end + + codec.encode(event) + + expect(encoded_data).not_to be_nil + + events = [] + codec.decode(encoded_data) do |decoded_event| + events << decoded_event + end + + expect(events.first.get("message")).to eq(test_event_data["message"]) + end + end + end +end From 7c731621a5245765be0a1b855a8b88a03504f354 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Wed, 19 Nov 2025 20:10:06 -0800 Subject: [PATCH 07/19] Add integration tests for SSL and auth based schema registry. Add integration steps in the Travis. --- .travis.yml | 12 +- lib/logstash/codecs/avro.rb | 6 +- spec/integration/avro_integration_spec.rb | 282 +++++++++++++++++- .../setup_keystore_and_truststore.sh | 7 +- 4 files changed, 299 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index a50fc73..e248321 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,2 +1,12 @@ import: -- logstash-plugins/.ci:travis/travis.yml@1.x \ No newline at end of file +- logstash-plugins/.ci:travis/travis.yml@1.x + +jobs: + include: + - stage: "Integration Tests" + env: INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current + - env: INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current + - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=main \ No newline at end of file diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index 0f76070..5c366d3 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -301,8 +301,10 @@ def validate_ssl_settings! raise_config_error! "`ssl_certificate_authorities` cannot be empty" end - raise_config_error! "Multiple values on `ssl_certificate_authorities` are not supported by this plugin" if @ssl_certificate_authorities && @ssl_certificate_authorities&.size > 1 - ensure_readable_and_non_writable! "ssl_certificate_authorities", @ssl_certificate_authorities&.first + if @ssl_certificate_authorities && !@ssl_certificate_authorities.empty? + raise_config_error! "Multiple values on `ssl_certificate_authorities` are not supported by this plugin" if @ssl_certificate_authorities.size > 1 + ensure_readable_and_non_writable! "ssl_certificate_authorities", @ssl_certificate_authorities.first + end end def build_ssl_options diff --git a/spec/integration/avro_integration_spec.rb b/spec/integration/avro_integration_spec.rb index b2fd186..1c4c703 100644 --- a/spec/integration/avro_integration_spec.rb +++ b/spec/integration/avro_integration_spec.rb @@ -20,7 +20,6 @@ ] } end - let(:test_schema_json) { test_schema.to_json } let(:test_event_data) do { @@ -28,8 +27,7 @@ "timestamp" => Time.now.to_i } end - - let(:config) {{ }} + let(:config) { {} } let(:codec) { LogStash::Codecs::Avro.new(config).tap { |c| c.register } } def run_integration_script(script_name) @@ -137,7 +135,7 @@ def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl url end - let(:config) { super().merge({'schema_uri' => full_schema_url}) } + let(:config) { super().merge({ 'schema_uri' => full_schema_url }) } before do client = Manticore::Client.new @@ -183,4 +181,280 @@ def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl end end end + + context "Schema Registry with authentication" do + let(:schema_registry_url) { "http://localhost:8081" } + let(:username) { "barney" } + let(:password) { "changeme" } + + before(:all) do + run_integration_script("stop_schema_registry.sh") + sleep 2 + run_integration_script("start_auth_schema_registry.sh") + sleep 5 + wait_for_schema_registry("http://localhost:8081", username: "barney", password: "changeme") + end + + after(:all) do + run_integration_script("stop_schema_registry.sh") + sleep 2 + end + + context "with valid credentials" do + let(:schema_subject) { "test-auth-#{Time.now.to_i}" } + let(:full_schema_url) { "#{schema_registry_url}/subjects/#{schema_subject}/versions/latest" } + let(:config) { super().merge({ 'schema_uri' => full_schema_url, 'username' => username, 'password' => password }) } + + before do + client_options = { auth: { user: username, password: password } } + client = Manticore::Client.new(client_options) + response = client.post("#{schema_registry_url}/subjects/#{schema_subject}/versions", + headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, + body: { schema: test_schema_json }.to_json + ).call + expect(response.code).to eq(200) + client.close + end + + it "fetches schema with valid credentials" do + encoded_data = encode_avro_data(test_schema_json, test_event_data) + events = [] + codec.decode(encoded_data) do |event| + events << event + end + + expect(events.size).to eq(1) + expect(events.first.get("message")).to eq(test_event_data["message"]) + end + + it "encodes data with authentication" do + event = LogStash::Event.new(test_event_data) + encoded_data = nil + + codec.on_event do |e, data| + encoded_data = data + end + + codec.encode(event) + expect(encoded_data).not_to be_nil + end + end + + context "with invalid credentials" do + let(:schema_subject) { "test-invalid-auth-#{Time.now.to_i}" } + let(:full_schema_url) { "#{schema_registry_url}/subjects/#{schema_subject}/versions/latest" } + + before do + client_options = { auth: { user: username, password: password } } + client = Manticore::Client.new(client_options) + response = client.post("#{schema_registry_url}/subjects/#{schema_subject}/versions", + headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, + body: { schema: test_schema_json }.to_json + ).call + expect(response.code).to eq(200) + client.close + end + + it "fails with invalid credentials" do + expect { + invalid_config = { 'schema_uri' => full_schema_url, 'username' => 'invalid', 'password' => 'wrong' } + LogStash::Codecs::Avro.new(invalid_config).tap { |c| c.register } + }.to raise_error(/401|403|HTTP request failed/) + end + end + end + + context "Schema Registry with SSL/TLS" do + let(:schema_registry_https_url) { "https://localhost:8083" } + let(:truststore_path) { File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks") } + let(:truststore_password) { "changeit" } + let(:ca_cert_path) { File.join(INTEGRATION_DIR, "tls_repository", "schema_reg_certificate.pem") } + + before(:all) do + # Ensure non-auth registry is running (it includes HTTPS on 8083) + run_integration_script("stop_schema_registry.sh") + sleep 2 + run_integration_script("start_schema_registry.sh") + sleep 5 + + ssl_options = { + truststore: File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks"), + truststore_password: "changeit", + truststore_type: "jks", + verify: :default + } + wait_for_schema_registry("https://localhost:8083", ssl_options: ssl_options) + end + + after(:all) do + run_integration_script("stop_schema_registry.sh") + sleep 2 + end + + context "with truststore configuration" do + let(:schema_subject) { "test-ssl-truststore-#{Time.now.to_i}" } + let(:full_schema_url) { "#{schema_registry_https_url}/subjects/#{schema_subject}/versions/latest" } + let(:config) do + super().merge({ + 'schema_uri' => full_schema_url, + 'ssl_enabled' => true, + 'ssl_truststore_path' => truststore_path, + 'ssl_truststore_password' => truststore_password, + 'ssl_truststore_type' => 'JKS', + 'ssl_verification_mode' => 'full' + }) + end + + before do + ssl_options = { + truststore: truststore_path, + truststore_password: truststore_password, + truststore_type: "jks", + verify: :default + } + client = Manticore::Client.new(ssl: ssl_options) + response = client.post("#{schema_registry_https_url}/subjects/#{schema_subject}/versions", + headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, + body: { schema: test_schema_json }.to_json + ).call + expect(response.code).to eq(200) + client.close + end + + it "fetches schema using truststore" do + encoded_data = encode_avro_data(test_schema_json, test_event_data) + events = [] + codec.decode(encoded_data) do |event| + events << event + end + + expect(events.size).to eq(1) + expect(events.first.get("message")).to eq(test_event_data["message"]) + end + end + + context "with CA certificate configuration" do + let(:schema_subject) { "test-ssl-ca-#{Time.now.to_i}" } + let(:full_schema_url) { "#{schema_registry_https_url}/subjects/#{schema_subject}/versions/latest" } + let(:config) do + super().merge({ + 'schema_uri' => full_schema_url, + 'ssl_enabled' => true, + 'ssl_certificate_authorities' => [ca_cert_path], + 'ssl_verification_mode' => 'full' + }) + end + + before do + ssl_options = { ca_file: ca_cert_path, verify: :default } + client = Manticore::Client.new(ssl: ssl_options) + response = client.post("#{schema_registry_https_url}/subjects/#{schema_subject}/versions", + headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, + body: { schema: test_schema_json }.to_json + ).call + expect(response.code).to eq(200) + client.close + end + + it "fetches schema using CA certificate" do + encoded_data = encode_avro_data(test_schema_json, test_event_data) + events = [] + codec.decode(encoded_data) do |event| + events << event + end + + expect(events.size).to eq(1) + expect(events.first.get("message")).to eq(test_event_data["message"]) + end + end + end + + context "Schema Registry with authentication and SSL" do + let(:schema_registry_https_url) { "https://localhost:8083" } + let(:username) { "barney" } + let(:password) { "changeme" } + let(:truststore_path) { File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks") } + let(:truststore_password) { "changeit" } + + before(:all) do + # Start authenticated registry (includes HTTPS) + run_integration_script("stop_schema_registry.sh") + sleep 3 + run_integration_script("start_auth_schema_registry.sh") + sleep 10 + + ssl_options = { + truststore: File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks"), + truststore_password: "changeit", + truststore_type: "jks", + verify: :default + } + wait_for_schema_registry("https://localhost:8083", username: "barney", password: "changeme", ssl_options: ssl_options) + end + + after(:all) do + run_integration_script("stop_schema_registry.sh") + sleep 2 + end + + context "with valid credentials and truststore" do + let(:schema_subject) { "test-auth-ssl-#{Time.now.to_i}" } + let(:full_schema_url) { "#{schema_registry_https_url}/subjects/#{schema_subject}/versions/latest" } + let(:config) do + super().merge({ + 'schema_uri' => full_schema_url, + 'username' => username, + 'password' => password, + 'ssl_enabled' => true, + 'ssl_truststore_path' => truststore_path, + 'ssl_truststore_password' => truststore_password, + 'ssl_truststore_type' => 'JKS', + 'ssl_verification_mode' => 'full' + }) + end + + before do + ssl_options = { + truststore: truststore_path, + truststore_password: truststore_password, + truststore_type: "jks", + verify: :default + } + client_options = { + auth: { user: username, password: password }, + ssl: ssl_options + } + client = Manticore::Client.new(client_options) + response = client.post("#{schema_registry_https_url}/subjects/#{schema_subject}/versions", + headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, + body: { schema: test_schema_json }.to_json + ).call + expect(response.code).to eq(200) + client.close + end + + it "fetches schema with both authentication and SSL" do + encoded_data = encode_avro_data(test_schema_json, test_event_data) + events = [] + codec.decode(encoded_data) do |event| + events << event + end + + expect(events.size).to eq(1) + expect(events.first.get("message")).to eq(test_event_data["message"]) + end + + it "encodes data with authentication and SSL" do + event = LogStash::Event.new(test_event_data) + encoded_data = nil + + codec.on_event do |e, data| + encoded_data = data + end + + codec.encode(event) + expect(encoded_data).not_to be_nil + end + end + end end diff --git a/spec/integration/setup_keystore_and_truststore.sh b/spec/integration/setup_keystore_and_truststore.sh index a43d41e..64f8b49 100755 --- a/spec/integration/setup_keystore_and_truststore.sh +++ b/spec/integration/setup_keystore_and_truststore.sh @@ -9,4 +9,9 @@ echo "Exporting schema registry certificate" keytool -exportcert -rfc -keystore tls_repository/schema_reg.jks -storepass changeit -alias schema_reg -file tls_repository/schema_reg_certificate.pem echo "Creating client's truststore and importing schema registry's certificate" -keytool -import -trustcacerts -file tls_repository/schema_reg_certificate.pem -keypass changeit -storepass changeit -keystore tls_repository/clienttruststore.jks -noprompt \ No newline at end of file +keytool -import -trustcacerts -file tls_repository/schema_reg_certificate.pem -keypass changeit -storepass changeit -keystore tls_repository/clienttruststore.jks -noprompt + +# make files read only +chmod 444 tls_repository/schema_reg.jks +chmod 444 tls_repository/schema_reg_certificate.pem +chmod 444 tls_repository/clienttruststore.jks \ No newline at end of file From d2f288c2b689182dfdb1342f091de1ad429ffa05 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Thu, 20 Nov 2025 08:20:04 -0800 Subject: [PATCH 08/19] Exclude 7.current from travis integration jobs. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e248321..890a1c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ jobs: - stage: "Integration Tests" env: INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current - env: INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current - - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=main \ No newline at end of file From cc137ab52bdd6a8e840a44beded03f6047ac39c6 Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Mon, 24 Nov 2025 15:08:46 -0800 Subject: [PATCH 09/19] Apply suggestions from code review Apply code review comments which I agree with. Co-authored-by: kaisecheng <69120390+kaisecheng@users.noreply.github.com> --- docs/index.asciidoc | 19 +++++++------------ lib/logstash/codecs/avro.rb | 4 +--- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 98c06a5..4d186e2 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -134,8 +134,8 @@ Set this option to `binary` to indicate that this codec sends or expects to rece * Value type is <> * There is no default value for this setting. -Password for HTTP basic authentication when fetching the schema from a remote server. -Must be used in combination with the `username` setting. +Password for HTTP basic authentication when fetching remote schemas. +Used together with `username`. [id="plugins-{type}s-{plugin}-proxy"] ===== `proxy` @@ -143,7 +143,7 @@ Must be used in combination with the `username` setting. * Value type is <> * There is no default value for this setting. -Proxy server URL for schema registry connections. +The address of a forward HTTP proxy to use when contacting a remote schema registry. [id="plugins-{type}s-{plugin}-schema_uri"] ===== `schema_uri` @@ -174,8 +174,7 @@ tag events with `_avroparsefailure` when decode fails * There is no default value for this setting. Path to PEM encoded certificate file for client authentication (mutual TLS). -This is an alternative to using <>. -You cannot use this setting and <> at the same time. +You may use this setting or <>, but not both simultaneously. *Example* [source,ruby] @@ -242,8 +241,6 @@ When using HTTPS schema URIs, SSL is automatically enabled. The path to the JKS or PKCS12 keystore file for client certificate authentication. Use this when the schema registry requires mutual TLS (mTLS) authentication. -This setting is only used when `schema_uri` uses the `https://` scheme. - [id="plugins-{type}s-{plugin}-ssl_keystore_password"] ===== `ssl_keystore_password` @@ -280,8 +277,6 @@ When not specified, the JVM defaults are used. The path to the JKS or PKCS12 truststore file containing certificates to verify the schema registry server's certificate. -This setting is only used when `schema_uri` uses the `https://` scheme. - *Example* [source,ruby] ---------------------------------- @@ -310,7 +305,7 @@ The password for the truststore file specified in <> * There is no default value for this setting. -The type of the truststore file. Can be either `JKS` or `PKCS12`. +The format of the truststore file. It must be either `JKS` or `PKCS12`. [id="plugins-{type}s-{plugin}-ssl_verification_mode"] ===== `ssl_verification_mode` @@ -367,8 +362,8 @@ input { * Value type is <> * There is no default value for this setting. -Username for HTTP basic authentication when fetching the schema from a remote server. -Must be used in combination with the `password` setting. +Username for HTTP basic authentication when fetching remote schemas. +Used together with `password`. *Example* [source,ruby] diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index 5c366d3..b572ddd 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -256,9 +256,7 @@ def build_basic_auth raise LogStash::ConfigurationError, "`username` requires `password`" if @username && !@password raise LogStash::ConfigurationError, "`password` is not allowed unless `username` is specified" if !@username && @password - if @username && @password - raise LogStash::ConfigurationError, "Empty `username` or `password` is not allowed" if @username.empty? || @password.value.empty? - end + raise LogStash::ConfigurationError, "Empty `username` or `password` is not allowed" if @username.empty? || @password.value.empty? {:user => @username, :password => @password.value} end From c0e2846182127c49c46d59bd4ba8801306ba7a24 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 24 Nov 2025 16:12:27 -0800 Subject: [PATCH 10/19] Simplifications. --- lib/logstash/codecs/avro.rb | 28 ++++----- spec/integration/avro_integration_spec.rb | 71 ++--------------------- spec/integration/kafka_test_setup.sh | 2 +- 3 files changed, 21 insertions(+), 80 deletions(-) diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index b572ddd..93d519d 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -190,13 +190,13 @@ def fetch_schema(uri_string) https_connection = uri_string.start_with?('https://') if http_connection - ssl_config_provided = original_params.keys.select {|k| k.start_with?("ssl_") && k != "ssl_enabled" } + ssl_config_provided = original_params.keys.select {|k| k.start_with?("ssl_") } if ssl_config_provided.any? raise_config_error! "When SSL is disabled, the following provided parameters are not allowed: #{ssl_config_provided}" end credentials_configured = @username && @password - @logger.warn("Credentials are being sent over unencrypted HTTP. This may bring security risk.") if credentials_configured && http_connection + @logger.warn("Credentials are being sent over unencrypted HTTP. This may bring security risk.") if credentials_configured fetch_remote_schema(uri_string) elsif https_connection validate_ssl_settings! @@ -208,6 +208,7 @@ def fetch_schema(uri_string) end def fetch_remote_schema(uri_string) + client = nil client_options = {} if @proxy && !@proxy.empty? @@ -232,18 +233,16 @@ def fetch_remote_schema(uri_string) body = response.body # Response may contain schema metadata, schema field is what we need # Example response: {"subject":"test-no-auth-1763597024","version":1,"id":1,"guid":"5c6c5f26-e876-e5ab-02b0-8d9bebbc90d7","schemaType":"AVRO","schema":"{"type":"record","name":"TestRecord","namespace":"com.example","fields":[{"name":"message","type":"string"},{"name":"timestamp","type":"long"}]}","ts":1763597024561,"deleted":false} - begin - parsed = JSON.parse(body) - if parsed.is_a?(Hash) && parsed.has_key?('schema') - parsed['schema'] - else - # fallback to use the response as it is - body - end - rescue JSON::ParserError - # Not JSON, return as-is (probably a direct schema) + parsed = JSON.parse(body) + if parsed.is_a?(Hash) && parsed.has_key?('schema') + parsed['schema'] + else + # fallback to use the response as it is body end + rescue JSON::ParserError + # Not JSON, return as-is (probably a direct schema) + body ensure client.close if client end @@ -263,6 +262,7 @@ def build_basic_auth def validate_ssl_settings! @ssl_enabled = true if @ssl_enabled.nil? + raise_config_error! "Secured #{@schema_uri} connection requires `ssl_enabled => true`. " unless @ssl_enabled @ssl_verification_mode = "full".freeze if @ssl_verification_mode.nil? # optional: presenting our identity @@ -270,8 +270,8 @@ def validate_ssl_settings! raise_config_error! "`ssl_certificate` requires `ssl_key`" if @ssl_certificate && !@ssl_key ensure_readable_and_non_writable! "ssl_certificate", @ssl_certificate if @ssl_certificate - raise_config_error! "`ssl_key` is not allowed unless `ssl_certificate` is specified" if @ssl_key && !@ssl_certificate - ensure_readable_and_non_writable! "ssl_key", @ssl_key if @ssl_key + raise_config_error! "`ssl_key` is not allowed unless `ssl_certificate` is specified" if @ssl_key && !@ssl_certificate + ensure_readable_and_non_writable! "ssl_key", @ssl_key if @ssl_key raise_config_error! "`ssl_keystore_path` requires `ssl_keystore_password`" if @ssl_keystore_path && !@ssl_keystore_password raise_config_error! "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is specified" if @ssl_keystore_password && !@ssl_keystore_path diff --git a/spec/integration/avro_integration_spec.rb b/spec/integration/avro_integration_spec.rb index 1c4c703..b5cd47c 100644 --- a/spec/integration/avro_integration_spec.rb +++ b/spec/integration/avro_integration_spec.rb @@ -38,29 +38,6 @@ def run_integration_script(script_name) end end - def register_schema(schema_registry_url, schema_json, username: nil, password: nil, ssl_options: {}) - client_options = {} - - if username && password - client_options[:auth] = { user: username, password: password } - end - - client_options[:ssl] = ssl_options unless ssl_options.empty? - - client = Manticore::Client.new(client_options) - - response = client.post("#{schema_registry_url}/subjects/test-value/versions", - headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, - body: { schema: schema_json }.to_json - ).call - - raise "Failed to register schema: #{response.code} #{response.body}" unless response.code == 200 - - JSON.parse(response.body)["id"] - ensure - client&.close - end - def encode_avro_data(schema_json, data) schema = Avro::Schema.parse(schema_json) dw = Avro::IO::DatumWriter.new(schema) @@ -70,14 +47,7 @@ def encode_avro_data(schema_json, data) Base64.strict_encode64(buffer.string) end - def create_test_schema_file(filename = "test_schema.avsc") - schema_path = File.join(Dir.tmpdir, filename) - File.write(schema_path, test_schema_json) - schema_path - end - - def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl_options: {}) - start_time = Time.now + def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) client_options = {} if username && password @@ -87,29 +57,11 @@ def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl client_options[:ssl] = ssl_options unless ssl_options.empty? puts "Waiting for Schema Registry at #{url}..." - attempt = 0 - - loop do - attempt += 1 - begin - client = Manticore::Client.new(client_options) - response = client.get(url).call - if response.code == 200 - puts "✓ Schema Registry is ready after #{attempt} attempts" - return true - end - rescue => e - # Continue waiting - puts " Attempt #{attempt}: #{e.class.name} - #{e.message[0..80]}" if attempt % 5 == 0 - ensure - client&.close if client - end - - if Time.now - start_time > timeout - raise "Schema Registry at #{url} did not become available within #{timeout} seconds after #{attempt} attempts" - end + client = Manticore::Client.new(client_options) - sleep 2 + Stud.try(20.times, [Manticore::SocketException, StandardError, RSpec::Expectations::ExpectationNotMetError]) do + response = client.get(url).call + expect(response.code).to eq(200) end end @@ -123,7 +75,6 @@ def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl after(:all) do run_integration_script("stop_schema_registry.sh") - sleep 2 end context "fetching schema via HTTP" do @@ -131,7 +82,6 @@ def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl let(:full_schema_url) do url = "#{schema_registry_url}/subjects/#{schema_subject}/versions/latest" puts "Constructed schema URL: #{url}" - raise "Schema URL is empty!" if url.nil? || url.empty? || !url.start_with?('http') url end @@ -189,15 +139,12 @@ def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl before(:all) do run_integration_script("stop_schema_registry.sh") - sleep 2 run_integration_script("start_auth_schema_registry.sh") - sleep 5 wait_for_schema_registry("http://localhost:8081", username: "barney", password: "changeme") end after(:all) do run_integration_script("stop_schema_registry.sh") - sleep 2 end context "with valid credentials" do @@ -259,7 +206,7 @@ def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl expect { invalid_config = { 'schema_uri' => full_schema_url, 'username' => 'invalid', 'password' => 'wrong' } LogStash::Codecs::Avro.new(invalid_config).tap { |c| c.register } - }.to raise_error(/401|403|HTTP request failed/) + }.to raise_error(/401 Unauthorized/) end end end @@ -273,9 +220,7 @@ def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl before(:all) do # Ensure non-auth registry is running (it includes HTTPS on 8083) run_integration_script("stop_schema_registry.sh") - sleep 2 run_integration_script("start_schema_registry.sh") - sleep 5 ssl_options = { truststore: File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks"), @@ -288,7 +233,6 @@ def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl after(:all) do run_integration_script("stop_schema_registry.sh") - sleep 2 end context "with truststore configuration" do @@ -379,9 +323,7 @@ def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl before(:all) do # Start authenticated registry (includes HTTPS) run_integration_script("stop_schema_registry.sh") - sleep 3 run_integration_script("start_auth_schema_registry.sh") - sleep 10 ssl_options = { truststore: File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks"), @@ -394,7 +336,6 @@ def wait_for_schema_registry(url, timeout: 60, username: nil, password: nil, ssl after(:all) do run_integration_script("stop_schema_registry.sh") - sleep 2 end context "with valid credentials and truststore" do diff --git a/spec/integration/kafka_test_setup.sh b/spec/integration/kafka_test_setup.sh index 490fc6a..b8e82e4 100755 --- a/spec/integration/kafka_test_setup.sh +++ b/spec/integration/kafka_test_setup.sh @@ -69,7 +69,7 @@ echo "ssl.key.password=changeit" >> "build/confluent_platform/etc/schema-registr cp "build/confluent_platform/etc/schema-registry/schema-registry.properties" "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" echo "authentication.method=BASIC" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" -echo "authentication.roles=admin,developer,user,sr-user" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" +echo "authentication.roles=admin,developer,user" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" echo "authentication.realm=SchemaRegistry-Props" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" cp fixtures/jaas.config "build/confluent_platform/etc/schema-registry" From cfd567612f075afffccfb9a1ab00ffa9ffb5372e Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 24 Nov 2025 16:36:45 -0800 Subject: [PATCH 11/19] Implement a retry mechanism when hitting the schema registry. --- lib/logstash/codecs/avro.rb | 16 ++++-- spec/integration/avro_integration_spec.rb | 68 +++++++++-------------- spec/integration/stop_schema_registry.sh | 2 +- spec/unit/avro_spec.rb | 16 ++++++ 4 files changed, 55 insertions(+), 47 deletions(-) diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index 93d519d..3f2346d 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -4,6 +4,7 @@ require "avro" require "base64" require "json" +require "stud/try" require "logstash/codecs/base" require "logstash/event" require "logstash/timestamp" @@ -224,13 +225,20 @@ def fetch_remote_schema(uri_string) end client = Manticore::Client.new(client_options) - response = client.get(uri_string).call - unless response.code == 200 - raise "HTTP request failed: #{response.code} #{response.message}" + body = Stud.try(3.times, [Manticore::SocketException, Manticore::Timeout, StandardError]) do + @logger.debug("Fetching schema from #{uri_string}") if @logger + response = client.get(uri_string).call + + unless response.code == 200 + error_msg = "HTTP request failed: #{response.code} #{response.message}" + @logger.warn(error_msg) if @logger + raise error_msg + end + + response.body end - body = response.body # Response may contain schema metadata, schema field is what we need # Example response: {"subject":"test-no-auth-1763597024","version":1,"id":1,"guid":"5c6c5f26-e876-e5ab-02b0-8d9bebbc90d7","schemaType":"AVRO","schema":"{"type":"record","name":"TestRecord","namespace":"com.example","fields":[{"name":"message","type":"string"},{"name":"timestamp","type":"long"}]}","ts":1763597024561,"deleted":false} parsed = JSON.parse(body) diff --git a/spec/integration/avro_integration_spec.rb b/spec/integration/avro_integration_spec.rb index b5cd47c..7b2b1a2 100644 --- a/spec/integration/avro_integration_spec.rb +++ b/spec/integration/avro_integration_spec.rb @@ -47,6 +47,20 @@ def encode_avro_data(schema_json, data) Base64.strict_encode64(buffer.string) end + def decode_with_codec(codec, encoded_data) + events = [] + codec.decode(encoded_data) do |event| + events << event + end + events + end + + def expect_decoded_event_matches(events, expected_data) + expect(events.size).to eq(1) + expect(events.first.get("message")).to eq(expected_data["message"]) + expect(events.first.get("timestamp")).to eq(expected_data["timestamp"]) + end + def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) client_options = {} @@ -100,14 +114,8 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) it "fetches and decodes schema from Schema Registry" do encoded_data = encode_avro_data(test_schema_json, test_event_data) - events = [] - codec.decode(encoded_data) do |event| - events << event - end - - expect(events.size).to eq(1) - expect(events.first.get("message")).to eq(test_event_data["message"]) - expect(events.first.get("timestamp")).to eq(test_event_data["timestamp"]) + events = decode_with_codec(codec, encoded_data) + expect_decoded_event_matches(events, test_event_data) end it "encodes data using schema from schema registry" do @@ -122,12 +130,8 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) expect(encoded_data).not_to be_nil - events = [] - codec.decode(encoded_data) do |decoded_event| - events << decoded_event - end - - expect(events.first.get("message")).to eq(test_event_data["message"]) + events = decode_with_codec(codec, encoded_data) + expect_decoded_event_matches(events, test_event_data) end end end @@ -165,13 +169,8 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) it "fetches schema with valid credentials" do encoded_data = encode_avro_data(test_schema_json, test_event_data) - events = [] - codec.decode(encoded_data) do |event| - events << event - end - - expect(events.size).to eq(1) - expect(events.first.get("message")).to eq(test_event_data["message"]) + events = decode_with_codec(codec, encoded_data) + expect_decoded_event_matches(events, test_event_data) end it "encodes data with authentication" do @@ -267,13 +266,8 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) it "fetches schema using truststore" do encoded_data = encode_avro_data(test_schema_json, test_event_data) - events = [] - codec.decode(encoded_data) do |event| - events << event - end - - expect(events.size).to eq(1) - expect(events.first.get("message")).to eq(test_event_data["message"]) + events = decode_with_codec(codec, encoded_data) + expect_decoded_event_matches(events, test_event_data) end end @@ -302,13 +296,8 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) it "fetches schema using CA certificate" do encoded_data = encode_avro_data(test_schema_json, test_event_data) - events = [] - codec.decode(encoded_data) do |event| - events << event - end - - expect(events.size).to eq(1) - expect(events.first.get("message")).to eq(test_event_data["message"]) + events = decode_with_codec(codec, encoded_data) + expect_decoded_event_matches(events, test_event_data) end end end @@ -376,13 +365,8 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) it "fetches schema with both authentication and SSL" do encoded_data = encode_avro_data(test_schema_json, test_event_data) - events = [] - codec.decode(encoded_data) do |event| - events << event - end - - expect(events.size).to eq(1) - expect(events.first.get("message")).to eq(test_event_data["message"]) + events = decode_with_codec(codec, encoded_data) + expect_decoded_event_matches(events, test_event_data) end it "encodes data with authentication and SSL" do diff --git a/spec/integration/stop_schema_registry.sh b/spec/integration/stop_schema_registry.sh index 93e7adb..4f255b0 100755 --- a/spec/integration/stop_schema_registry.sh +++ b/spec/integration/stop_schema_registry.sh @@ -4,4 +4,4 @@ set -ex echo "Stopping SchemaRegistry" build/confluent_platform/bin/schema-registry-stop -sleep 5 \ No newline at end of file +sleep 2 \ No newline at end of file diff --git a/spec/unit/avro_spec.rb b/spec/unit/avro_spec.rb index 53f79fd..b875d33 100644 --- a/spec/unit/avro_spec.rb +++ b/spec/unit/avro_spec.rb @@ -373,6 +373,22 @@ expect(subject.instance_variable_get(:@ssl_enabled)).to be false end end + + context "with explicit ssl_enabled => true and HTTPS URI" do + let(:avro_config) do + { + 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', + 'ssl_enabled' => false + } + end + + it "requires ssl_enabled => true" do + expect { subject.send(:validate_ssl_settings!) }.to raise_error( + LogStash::ConfigurationError, + /Secured https:\/\/schema-registry.example.com\/schema.avsc connection requires `ssl_enabled => true`. / + ) + end + end end context "SSL verification" do From 71ee0175ac28470c25fc03e2e9627bc937b86742 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 24 Nov 2025 16:41:49 -0800 Subject: [PATCH 12/19] Put back accidental removed piece while testing. --- lib/logstash/codecs/avro.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index 3f2346d..646750a 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -191,7 +191,7 @@ def fetch_schema(uri_string) https_connection = uri_string.start_with?('https://') if http_connection - ssl_config_provided = original_params.keys.select {|k| k.start_with?("ssl_") } + ssl_config_provided = original_params.keys.select {|k| k.start_with?("ssl_") && k != "ssl_enabled" } if ssl_config_provided.any? raise_config_error! "When SSL is disabled, the following provided parameters are not allowed: #{ssl_config_provided}" end From fe2b99ee4780ab0c65b5d994d9debef88a99222f Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 24 Nov 2025 20:36:04 -0800 Subject: [PATCH 13/19] Setup schema registry for mutual TLS and add integration test cases for it. --- spec/integration/avro_integration_spec.rb | 151 +++++++++++------- spec/integration/kafka_test_setup.sh | 5 + spec/integration/kafka_test_teardown.sh | 1 - .../integration/start_auth_schema_registry.sh | 1 - spec/integration/start_schema_registry.sh | 1 - .../start_schema_registry_mutual.sh | 5 + spec/integration/stop_schema_registry.sh | 1 - 7 files changed, 107 insertions(+), 58 deletions(-) create mode 100755 spec/integration/start_schema_registry_mutual.sh diff --git a/spec/integration/avro_integration_spec.rb b/spec/integration/avro_integration_spec.rb index 7b2b1a2..efab6aa 100644 --- a/spec/integration/avro_integration_spec.rb +++ b/spec/integration/avro_integration_spec.rb @@ -61,22 +61,35 @@ def expect_decoded_event_matches(events, expected_data) expect(events.first.get("timestamp")).to eq(expected_data["timestamp"]) end - def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) + def create_manticore_client(username: nil, password: nil, ssl_options: {}) client_options = {} - if username && password client_options[:auth] = { user: username, password: password } end - client_options[:ssl] = ssl_options unless ssl_options.empty? + Manticore::Client.new(client_options) + end + + def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) puts "Waiting for Schema Registry at #{url}..." - client = Manticore::Client.new(client_options) + client = create_manticore_client(username: username, password: password, ssl_options: ssl_options) Stud.try(20.times, [Manticore::SocketException, StandardError, RSpec::Expectations::ExpectationNotMetError]) do response = client.get(url).call expect(response.code).to eq(200) end + client.close + end + + def register_schema(base_url, subject, schema_json, username: nil, password: nil, ssl_options: {}) + client = create_manticore_client(username: username, password: password, ssl_options: ssl_options) + response = client.post("#{base_url}/subjects/#{subject}/versions", + headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, + body: { schema: schema_json }.to_json + ).call + expect(response.code).to eq(200) + client.close end context "Schema Registry without authentication" do @@ -102,14 +115,7 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) let(:config) { super().merge({ 'schema_uri' => full_schema_url }) } before do - client = Manticore::Client.new - response = client.post("#{schema_registry_url}/subjects/#{schema_subject}/versions", - headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, - body: { schema: test_schema_json }.to_json - ).call - puts "Schema registration response: #{response.code}" - expect(response.code).to eq(200) - client.close + register_schema(schema_registry_url, schema_subject, test_schema_json) end it "fetches and decodes schema from Schema Registry" do @@ -157,14 +163,8 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) let(:config) { super().merge({ 'schema_uri' => full_schema_url, 'username' => username, 'password' => password }) } before do - client_options = { auth: { user: username, password: password } } - client = Manticore::Client.new(client_options) - response = client.post("#{schema_registry_url}/subjects/#{schema_subject}/versions", - headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, - body: { schema: test_schema_json }.to_json - ).call - expect(response.code).to eq(200) - client.close + register_schema(schema_registry_url, schema_subject, test_schema_json, + username: username, password: password) end it "fetches schema with valid credentials" do @@ -191,14 +191,8 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) let(:full_schema_url) { "#{schema_registry_url}/subjects/#{schema_subject}/versions/latest" } before do - client_options = { auth: { user: username, password: password } } - client = Manticore::Client.new(client_options) - response = client.post("#{schema_registry_url}/subjects/#{schema_subject}/versions", - headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, - body: { schema: test_schema_json }.to_json - ).call - expect(response.code).to eq(200) - client.close + register_schema(schema_registry_url, schema_subject, test_schema_json, + username: username, password: password) end it "fails with invalid credentials" do @@ -210,7 +204,7 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) end end - context "Schema Registry with SSL/TLS" do + context "Schema Registry with truststore configuration" do let(:schema_registry_https_url) { "https://localhost:8083" } let(:truststore_path) { File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks") } let(:truststore_password) { "changeit" } @@ -255,13 +249,8 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) truststore_type: "jks", verify: :default } - client = Manticore::Client.new(ssl: ssl_options) - response = client.post("#{schema_registry_https_url}/subjects/#{schema_subject}/versions", - headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, - body: { schema: test_schema_json }.to_json - ).call - expect(response.code).to eq(200) - client.close + register_schema(schema_registry_https_url, schema_subject, test_schema_json, + ssl_options: ssl_options) end it "fetches schema using truststore" do @@ -285,13 +274,8 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) before do ssl_options = { ca_file: ca_cert_path, verify: :default } - client = Manticore::Client.new(ssl: ssl_options) - response = client.post("#{schema_registry_https_url}/subjects/#{schema_subject}/versions", - headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, - body: { schema: test_schema_json }.to_json - ).call - expect(response.code).to eq(200) - client.close + register_schema(schema_registry_https_url, schema_subject, test_schema_json, + ssl_options: ssl_options) end it "fetches schema using CA certificate" do @@ -350,17 +334,9 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) truststore_type: "jks", verify: :default } - client_options = { - auth: { user: username, password: password }, - ssl: ssl_options - } - client = Manticore::Client.new(client_options) - response = client.post("#{schema_registry_https_url}/subjects/#{schema_subject}/versions", - headers: { "Content-Type" => "application/vnd.schemaregistry.v1+json" }, - body: { schema: test_schema_json }.to_json - ).call - expect(response.code).to eq(200) - client.close + register_schema(schema_registry_https_url, schema_subject, test_schema_json, + username: username, password: password, + ssl_options: ssl_options) end it "fetches schema with both authentication and SSL" do @@ -382,4 +358,71 @@ def wait_for_schema_registry(url, username: nil, password: nil, ssl_options: {}) end end end + + context "Schema Registry with keystore configuration (mutual TLS)" do + let(:schema_registry_https_url) { "https://localhost:8083" } + + let(:truststore_path) { File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks") } + let(:truststore_password) { "changeit" } + let(:keystore_path) { File.join(INTEGRATION_DIR, "tls_repository", "schema_reg.jks") } + let(:keystore_password) { "changeit" } + + before(:all) do + run_integration_script("stop_schema_registry.sh") + run_integration_script("start_schema_registry_mutual.sh") + + ssl_options = { + keystore: File.join(INTEGRATION_DIR, "tls_repository", "schema_reg.jks"), + keystore_password: "changeit", + keystore_type: "jks", + truststore: File.join(INTEGRATION_DIR, "tls_repository", "clienttruststore.jks"), + truststore_password: "changeit", + truststore_type: "jks" + } + wait_for_schema_registry("https://localhost:8083", ssl_options: ssl_options) + end + + after(:all) do + run_integration_script("stop_schema_registry.sh") + end + + context "with keystore and truststore" do + let(:schema_subject) { "test-mutual-tls-#{Time.now.to_i}" } + let(:full_schema_url) { "#{schema_registry_https_url}/subjects/#{schema_subject}/versions/latest" } + + let(:config) do + super().merge({ + 'schema_uri' => full_schema_url, + 'ssl_enabled' => true, + 'ssl_keystore_path' => keystore_path, + 'ssl_keystore_password' => keystore_password, + 'ssl_keystore_type' => 'JKS', + 'ssl_truststore_path' => truststore_path, + 'ssl_truststore_password' => truststore_password, + 'ssl_truststore_type' => 'JKS', + 'ssl_verification_mode' => 'full' + }) + end + + before do + ssl_options = { + keystore: keystore_path, + keystore_password: keystore_password, + keystore_type: "jks", + truststore: truststore_path, + truststore_password: truststore_password, + truststore_type: "jks", + verify: :default + } + register_schema(schema_registry_https_url, schema_subject, test_schema_json, + ssl_options: ssl_options) + end + + it "fetches schema" do + encoded_data = encode_avro_data(test_schema_json, test_event_data) + events = decode_with_codec(codec, encoded_data) + expect_decoded_event_matches(events, test_event_data) + end + end + end end diff --git a/spec/integration/kafka_test_setup.sh b/spec/integration/kafka_test_setup.sh index b8e82e4..7d5548f 100755 --- a/spec/integration/kafka_test_setup.sh +++ b/spec/integration/kafka_test_setup.sh @@ -67,6 +67,11 @@ echo "ssl.keystore.location=`pwd`/tls_repository/schema_reg.jks" >> "build/confl echo "ssl.keystore.password=changeit" >> "build/confluent_platform/etc/schema-registry/schema-registry.properties" echo "ssl.key.password=changeit" >> "build/confluent_platform/etc/schema-registry/schema-registry.properties" +cp "build/confluent_platform/etc/schema-registry/schema-registry.properties" "build/confluent_platform/etc/schema-registry/schema-registry-mutual.properties" +echo "ssl.truststore.location=`pwd`/tls_repository/clienttruststore.jks" >> "build/confluent_platform/etc/schema-registry/schema-registry-mutual.properties" +echo "ssl.truststore.password=changeit" >> "build/confluent_platform/etc/schema-registry/schema-registry-mutual.properties" +echo "confluent.http.server.ssl.client.authentication=REQUIRED" >> "build/confluent_platform/etc/schema-registry/schema-registry-mutual.properties" + cp "build/confluent_platform/etc/schema-registry/schema-registry.properties" "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" echo "authentication.method=BASIC" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" echo "authentication.roles=admin,developer,user" >> "build/confluent_platform/etc/schema-registry/authed-schema-registry.properties" diff --git a/spec/integration/kafka_test_teardown.sh b/spec/integration/kafka_test_teardown.sh index 42419e9..619d66a 100755 --- a/spec/integration/kafka_test_teardown.sh +++ b/spec/integration/kafka_test_teardown.sh @@ -1,5 +1,4 @@ #!/bin/bash -# Setup Kafka and create test topics set -ex echo "Unregistering test topics" diff --git a/spec/integration/start_auth_schema_registry.sh b/spec/integration/start_auth_schema_registry.sh index a1d6497..c065558 100755 --- a/spec/integration/start_auth_schema_registry.sh +++ b/spec/integration/start_auth_schema_registry.sh @@ -1,5 +1,4 @@ #!/bin/bash -# Setup Kafka and create test topics set -ex echo "Starting authed SchemaRegistry" diff --git a/spec/integration/start_schema_registry.sh b/spec/integration/start_schema_registry.sh index b1d343c..9a86afc 100755 --- a/spec/integration/start_schema_registry.sh +++ b/spec/integration/start_schema_registry.sh @@ -1,5 +1,4 @@ #!/bin/bash -# Setup Kafka and create test topics set -ex echo "Starting SchemaRegistry" diff --git a/spec/integration/start_schema_registry_mutual.sh b/spec/integration/start_schema_registry_mutual.sh new file mode 100755 index 0000000..be9b6be --- /dev/null +++ b/spec/integration/start_schema_registry_mutual.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -ex + +echo "Starting SchemaRegistry" +build/confluent_platform/bin/schema-registry-start build/confluent_platform/etc/schema-registry/schema-registry-mutual.properties > /dev/null 2>&1 & diff --git a/spec/integration/stop_schema_registry.sh b/spec/integration/stop_schema_registry.sh index 4f255b0..7b402bd 100755 --- a/spec/integration/stop_schema_registry.sh +++ b/spec/integration/stop_schema_registry.sh @@ -1,5 +1,4 @@ #!/bin/bash -# Setup Kafka and create test topics set -ex echo "Stopping SchemaRegistry" From a272adfa838d63f882d3cbda224d7db9f18904ee Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Tue, 25 Nov 2025 10:13:42 -0800 Subject: [PATCH 14/19] Apply suggestions from code review Truststore and keystore formats lowercased. Logging improvements. Co-authored-by: kaisecheng <69120390+kaisecheng@users.noreply.github.com> --- .ci/run.sh | 1 - docs/index.asciidoc | 6 +++--- lib/logstash/codecs/avro.rb | 14 +++++++------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.ci/run.sh b/.ci/run.sh index 7b9ef93..9822cd1 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -10,7 +10,6 @@ if [[ "$INTEGRATION" != "true" ]]; then else # Define the Kafka:Confluent version pairs VERSIONS=( - # "3.9.1:7.4.0" "4.1.0:8.0.0" ) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 4d186e2..1b13d1e 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -190,7 +190,7 @@ ssl_certificate => "/path/to/client.crt" Path to PEM encoded CA certificate file(s) for server verification. This is an alternative to using <>. -You cannot use this setting and <> at the same time. +You may use this setting or <>, but not both simultaneously. *Example* [source,ruby] @@ -256,7 +256,7 @@ The password for the keystore file specified in <> * There is no default value for this setting. -The type of the keystore file. Can be either `JKS` or `PKCS12`. +The format of the keystore file. It must be either `jks` or `pkcs12`. [id="plugins-{type}s-{plugin}-ssl_supported_protocols"] ===== `ssl_supported_protocols` @@ -305,7 +305,7 @@ The password for the truststore file specified in <> * There is no default value for this setting. -The format of the truststore file. It must be either `JKS` or `PKCS12`. +The format of the truststore file. It must be either `jks` or `pkcs12`. [id="plugins-{type}s-{plugin}-ssl_verification_mode"] ===== `ssl_verification_mode` diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index 646750a..190797d 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -122,8 +122,8 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base # The keystore password config :ssl_keystore_password, :validate => :password - # Keystore type (JKS or PKCS12) - config :ssl_keystore_type, :validate => %w[JKS PKCS12] + # Keystore type (jks or pkcs12) + config :ssl_keystore_type, :validate => %w[pkcs12 jks] # The truststore path config :ssl_truststore_path, :validate => :path @@ -131,8 +131,8 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base # The truststore password config :ssl_truststore_password, :validate => :password - # Truststore type (JKS or PKCS12) - config :ssl_truststore_type, :validate => %w[JKS PKCS12] + # Truststore type (jks or pkcs12) + config :ssl_truststore_type, :validate => %w[jks pkcs12] # The list of cipher suites to use, listed by priorities. # Supported cipher suites vary depending on which version of Java is used. @@ -227,12 +227,12 @@ def fetch_remote_schema(uri_string) client = Manticore::Client.new(client_options) body = Stud.try(3.times, [Manticore::SocketException, Manticore::Timeout, StandardError]) do - @logger.debug("Fetching schema from #{uri_string}") if @logger + @logger.debug("Fetching schema from #{uri_string}") response = client.get(uri_string).call unless response.code == 200 - error_msg = "HTTP request failed: #{response.code} #{response.message}" - @logger.warn(error_msg) if @logger + error_msg = "Failed to fetch schema from #{uri_string}: #{response.code} - #{response.message}" + @logger.warn(error_msg) raise error_msg end From 3f5b63015d87c5f3e76d9ff3bd4fb0fcdc5c6832 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Tue, 25 Nov 2025 10:22:18 -0800 Subject: [PATCH 15/19] Update specs after making keystore and trustore type lowecase. --- spec/unit/avro_spec.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/spec/unit/avro_spec.rb b/spec/unit/avro_spec.rb index b875d33..264e1a0 100644 --- a/spec/unit/avro_spec.rb +++ b/spec/unit/avro_spec.rb @@ -448,7 +448,7 @@ 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', 'ssl_keystore_path' => paths[:test_path], 'ssl_keystore_password' => 'keystore_pass', - 'ssl_keystore_type' => 'JKS' + 'ssl_keystore_type' => 'jks' } end @@ -460,17 +460,17 @@ end end - context "with ssl_keystore_path and PKCS12 type" do + context "with ssl_keystore_path and pkcs12 type" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', 'ssl_keystore_path' => paths[:test_path], 'ssl_keystore_password' => 'keystore_pass', - 'ssl_keystore_type' => 'PKCS12' + 'ssl_keystore_type' => 'pkcs12' } end - it "configures PKCS12 keystore" do + it "configures pkcs12 keystore" do ssl_options = subject.send(:build_ssl_options) expect(ssl_options[:keystore_type]).to eq('pkcs12') end @@ -500,7 +500,7 @@ 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', 'ssl_truststore_path' => paths[:test_path], 'ssl_truststore_password' => 'truststore_pass', - 'ssl_truststore_type' => 'JKS' + 'ssl_truststore_type' => 'jks' } end @@ -512,17 +512,17 @@ end end - context "with ssl_truststore_path and PKCS12 type" do + context "with ssl_truststore_path and pkcs12 type" do let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', 'ssl_truststore_path' => paths[:test_path], 'ssl_truststore_password' => 'truststore_pass', - 'ssl_truststore_type' => 'PKCS12' + 'ssl_truststore_type' => 'pkcs12' } end - it "configures PKCS12 truststore" do + it "configures pkcs12 truststore" do ssl_options = subject.send(:build_ssl_options) expect(ssl_options[:truststore_type]).to eq('pkcs12') end @@ -772,7 +772,7 @@ let(:avro_config) do { 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_keystore_type' => 'JKS' + 'ssl_keystore_type' => 'jks' } end From 5f025fea2c4082b1a8f93354b219896144cb730a Mon Sep 17 00:00:00 2001 From: Mashhur Date: Tue, 25 Nov 2025 10:50:02 -0800 Subject: [PATCH 16/19] Allow truststore and keystore without using their password. --- lib/logstash/codecs/avro.rb | 4 +-- spec/integration/avro_integration_spec.rb | 15 ++++++----- spec/unit/avro_spec.rb | 32 ----------------------- 3 files changed, 10 insertions(+), 41 deletions(-) diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index 190797d..6caae87 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -123,7 +123,7 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base config :ssl_keystore_password, :validate => :password # Keystore type (jks or pkcs12) - config :ssl_keystore_type, :validate => %w[pkcs12 jks] + config :ssl_keystore_type, :validate => %w[jks pkcs12] # The truststore path config :ssl_truststore_path, :validate => :path @@ -281,7 +281,6 @@ def validate_ssl_settings! raise_config_error! "`ssl_key` is not allowed unless `ssl_certificate` is specified" if @ssl_key && !@ssl_certificate ensure_readable_and_non_writable! "ssl_key", @ssl_key if @ssl_key - raise_config_error! "`ssl_keystore_path` requires `ssl_keystore_password`" if @ssl_keystore_path && !@ssl_keystore_password raise_config_error! "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is specified" if @ssl_keystore_password && !@ssl_keystore_path raise_config_error! "`ssl_keystore_password` cannot be empty" if @ssl_keystore_password && @ssl_keystore_password.value.empty? raise_config_error! "`ssl_keystore_type` is not allowed unless `ssl_keystore_path` is specified" if @ssl_keystore_type && !@ssl_keystore_path @@ -297,7 +296,6 @@ def validate_ssl_settings! end raise_config_error! "`ssl_truststore_path` and `ssl_certificate_authorities` cannot be used together." if @ssl_truststore_path && @ssl_certificate_authorities - raise_config_error! "`ssl_truststore_path` requires `ssl_truststore_password`" if @ssl_truststore_path && !@ssl_truststore_password ensure_readable_and_non_writable! "ssl_truststore_path", @ssl_truststore_path if @ssl_truststore_path raise_config_error! "`ssl_truststore_password` is not allowed unless `ssl_truststore_path` is specified" if !@ssl_truststore_path && @ssl_truststore_password diff --git a/spec/integration/avro_integration_spec.rb b/spec/integration/avro_integration_spec.rb index efab6aa..2138cf8 100644 --- a/spec/integration/avro_integration_spec.rb +++ b/spec/integration/avro_integration_spec.rb @@ -198,8 +198,11 @@ def register_schema(base_url, subject, schema_json, username: nil, password: nil it "fails with invalid credentials" do expect { invalid_config = { 'schema_uri' => full_schema_url, 'username' => 'invalid', 'password' => 'wrong' } - LogStash::Codecs::Avro.new(invalid_config).tap { |c| c.register } - }.to raise_error(/401 Unauthorized/) + codec = LogStash::Codecs::Avro.new(invalid_config) + codec.register + }.to raise_error { |error| + expect(error.message).to include("401 - Unauthorized") + } end end end @@ -237,7 +240,7 @@ def register_schema(base_url, subject, schema_json, username: nil, password: nil 'ssl_enabled' => true, 'ssl_truststore_path' => truststore_path, 'ssl_truststore_password' => truststore_password, - 'ssl_truststore_type' => 'JKS', + 'ssl_truststore_type' => 'jks', 'ssl_verification_mode' => 'full' }) end @@ -322,7 +325,7 @@ def register_schema(base_url, subject, schema_json, username: nil, password: nil 'ssl_enabled' => true, 'ssl_truststore_path' => truststore_path, 'ssl_truststore_password' => truststore_password, - 'ssl_truststore_type' => 'JKS', + 'ssl_truststore_type' => 'jks', 'ssl_verification_mode' => 'full' }) end @@ -396,10 +399,10 @@ def register_schema(base_url, subject, schema_json, username: nil, password: nil 'ssl_enabled' => true, 'ssl_keystore_path' => keystore_path, 'ssl_keystore_password' => keystore_password, - 'ssl_keystore_type' => 'JKS', + 'ssl_keystore_type' => 'jks', 'ssl_truststore_path' => truststore_path, 'ssl_truststore_password' => truststore_password, - 'ssl_truststore_type' => 'JKS', + 'ssl_truststore_type' => 'jks', 'ssl_verification_mode' => 'full' }) end diff --git a/spec/unit/avro_spec.rb b/spec/unit/avro_spec.rb index 264e1a0..ffaad09 100644 --- a/spec/unit/avro_spec.rb +++ b/spec/unit/avro_spec.rb @@ -475,22 +475,6 @@ expect(ssl_options[:keystore_type]).to eq('pkcs12') end end - - context "with ssl_keystore_path but no password" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_keystore_path' => paths[:test_path], - } - end - - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_keystore_path` requires `ssl_keystore_password`/ - ) - end - end end context "truststore configuration" do @@ -527,22 +511,6 @@ expect(ssl_options[:truststore_type]).to eq('pkcs12') end end - - context "with ssl_truststore_path but no password" do - let(:avro_config) do - { - 'schema_uri' => 'https://schema-registry.example.com/schema.avsc', - 'ssl_truststore_path' => paths[:test_path] - } - end - - it "raises ConfigurationError" do - expect { subject.send(:validate_ssl_settings!) }.to raise_error( - LogStash::ConfigurationError, - /`ssl_truststore_path` requires `ssl_truststore_password`/ - ) - end - end end context "CA configuration" do From ded870c35a8801e04c8f6d3bcdb266d8eb19054c Mon Sep 17 00:00:00 2001 From: Mashhur Date: Tue, 25 Nov 2025 16:53:11 -0800 Subject: [PATCH 17/19] Retry mechanism test. --- lib/logstash/codecs/avro.rb | 40 +++++++++++++++++++++++++++---------- spec/unit/avro_spec.rb | 1 + 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index 6caae87..94098ae 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -4,7 +4,6 @@ require "avro" require "base64" require "json" -require "stud/try" require "logstash/codecs/base" require "logstash/event" require "logstash/timestamp" @@ -114,7 +113,7 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates; # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate. # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf) - config :ssl_verification_mode, :validate => %w[full none], :default => 'full' + config :ssl_verification_mode, :validate => %w[full none] # The keystore path config :ssl_keystore_path, :validate => :path @@ -139,7 +138,7 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base config :ssl_cipher_suites, :validate => :string, :list => true # SSL supported protocols - config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true + config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :list => true public def initialize(*params) @@ -182,7 +181,7 @@ def encode(event) @on_event.call(event, Base64.strict_encode64(buffer.string)) else @on_event.call(event, buffer.string) - end + end end private @@ -226,17 +225,36 @@ def fetch_remote_schema(uri_string) client = Manticore::Client.new(client_options) - body = Stud.try(3.times, [Manticore::SocketException, Manticore::Timeout, StandardError]) do - @logger.debug("Fetching schema from #{uri_string}") - response = client.get(uri_string).call + @logger.debug("Fetching schema from #{uri_string}") + + max_retries = 3 + retry_count = 0 + body = nil + begin + response = client.get(uri_string).call + unless response.code == 200 error_msg = "Failed to fetch schema from #{uri_string}: #{response.code} - #{response.message}" - @logger.warn(error_msg) - raise error_msg + @logger.error(error_msg) + raise StandardError, error_msg end - - response.body + body = response.body + rescue Manticore::ManticoreException => e + retry_count += 1 + if retry_count <= max_retries + backoff_time = 2 ** (retry_count - 1) # Exponential backoff: 1s, 2s, 4s + @logger.warn("Attempt #{retry_count}/#{max_retries} failed for #{uri_string}: #{e.class} - #{e.message}. Retrying in #{backoff_time}s...") + sleep(backoff_time) + retry + else + @logger.error("Failed to fetch schema from #{uri_string} after #{max_retries} attempts: #{e.class} - #{e.message}") + raise + end + rescue StandardError => e + # Don't retry HTTP errors (401, 404, etc.) or other non-transient errors + @logger.error("Failed to fetch schema from #{uri_string}: #{e.class} - #{e.message}") + raise end # Response may contain schema metadata, schema field is what we need diff --git a/spec/unit/avro_spec.rb b/spec/unit/avro_spec.rb index ffaad09..d3d8e83 100644 --- a/spec/unit/avro_spec.rb +++ b/spec/unit/avro_spec.rb @@ -436,6 +436,7 @@ end it "defaults to 'full'" do + subject.send(:validate_ssl_settings!) expect(subject.instance_variable_get(:@ssl_verification_mode)).to eq('full') end end From 86c5421e49f037c16ffd3efe495da0a807ef6c4a Mon Sep 17 00:00:00 2001 From: Mashhur Date: Wed, 26 Nov 2025 08:22:42 -0800 Subject: [PATCH 18/19] Introduce BadResponseCodeError class to handle non-retriable cases. --- lib/logstash/codecs/avro.rb | 54 +++++++++++++---------- spec/integration/avro_integration_spec.rb | 2 +- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index 94098ae..afadff8 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -52,6 +52,17 @@ # } # ---------------------------------- class LogStash::Codecs::Avro < LogStash::Codecs::Base + class BadResponseCodeError < LogStash::Error + attr_reader :code, :message, :uri + + def initialize(code, message, uri) + @code = code + @message = message + @uri = uri + super("HTTP #{code}: #{message} (#{uri})") + end + end + config_name "avro" include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1) @@ -233,14 +244,28 @@ def fetch_remote_schema(uri_string) begin response = client.get(uri_string).call - + unless response.code == 200 - error_msg = "Failed to fetch schema from #{uri_string}: #{response.code} - #{response.message}" - @logger.error(error_msg) - raise StandardError, error_msg + @logger.error("Failed to fetch schema from #{uri_string}: #{response.code} - #{response.message}") + raise BadResponseCodeError.new(response.code, response.message, uri_string) end + body = response.body - rescue Manticore::ManticoreException => e + + # Parse and extract schema + parsed = JSON.parse(body) + return parsed.is_a?(Hash) && parsed.has_key?('schema') ? parsed['schema'] : body + + rescue JSON::ParserError + return body + rescue Manticore::ManticoreException, BadResponseCodeError => e + # 4xx don't retry + if e.is_a?(BadResponseCodeError) && e.code >= 400 && e.code < 500 + @logger.error("Failed to fetch schema from #{uri_string}: #{e.code} - #{e.message}") + raise + end + + # retry block retry_count += 1 if retry_count <= max_retries backoff_time = 2 ** (retry_count - 1) # Exponential backoff: 1s, 2s, 4s @@ -248,27 +273,10 @@ def fetch_remote_schema(uri_string) sleep(backoff_time) retry else - @logger.error("Failed to fetch schema from #{uri_string} after #{max_retries} attempts: #{e.class} - #{e.message}") + @logger.error("Failed to fetch schema from #{uri_string} after #{max_retries + 1} attempts: #{e.class} - #{e.message}") raise end - rescue StandardError => e - # Don't retry HTTP errors (401, 404, etc.) or other non-transient errors - @logger.error("Failed to fetch schema from #{uri_string}: #{e.class} - #{e.message}") - raise - end - - # Response may contain schema metadata, schema field is what we need - # Example response: {"subject":"test-no-auth-1763597024","version":1,"id":1,"guid":"5c6c5f26-e876-e5ab-02b0-8d9bebbc90d7","schemaType":"AVRO","schema":"{"type":"record","name":"TestRecord","namespace":"com.example","fields":[{"name":"message","type":"string"},{"name":"timestamp","type":"long"}]}","ts":1763597024561,"deleted":false} - parsed = JSON.parse(body) - if parsed.is_a?(Hash) && parsed.has_key?('schema') - parsed['schema'] - else - # fallback to use the response as it is - body end - rescue JSON::ParserError - # Not JSON, return as-is (probably a direct schema) - body ensure client.close if client end diff --git a/spec/integration/avro_integration_spec.rb b/spec/integration/avro_integration_spec.rb index 2138cf8..660e21f 100644 --- a/spec/integration/avro_integration_spec.rb +++ b/spec/integration/avro_integration_spec.rb @@ -201,7 +201,7 @@ def register_schema(base_url, subject, schema_json, username: nil, password: nil codec = LogStash::Codecs::Avro.new(invalid_config) codec.register }.to raise_error { |error| - expect(error.message).to include("401 - Unauthorized") + expect(error.message).to include("Unauthorized") } end end From 3f610e115108e2a4a53ca22f5bff316b09e01e7d Mon Sep 17 00:00:00 2001 From: Mashhur Date: Wed, 26 Nov 2025 08:31:26 -0800 Subject: [PATCH 19/19] Update retry error message. --- lib/logstash/codecs/avro.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/logstash/codecs/avro.rb b/lib/logstash/codecs/avro.rb index afadff8..3eddff6 100644 --- a/lib/logstash/codecs/avro.rb +++ b/lib/logstash/codecs/avro.rb @@ -267,13 +267,13 @@ def fetch_remote_schema(uri_string) # retry block retry_count += 1 - if retry_count <= max_retries - backoff_time = 2 ** (retry_count - 1) # Exponential backoff: 1s, 2s, 4s + if retry_count < max_retries + backoff_time = 2 ** (retry_count) # Exponential backoff: 1s, 2s, 4s @logger.warn("Attempt #{retry_count}/#{max_retries} failed for #{uri_string}: #{e.class} - #{e.message}. Retrying in #{backoff_time}s...") sleep(backoff_time) retry else - @logger.error("Failed to fetch schema from #{uri_string} after #{max_retries + 1} attempts: #{e.class} - #{e.message}") + @logger.error("Failed to fetch schema from #{uri_string} after #{max_retries} attempts: #{e.class} - #{e.message}") raise end end