From fdf8ef3e97cb31cda39ee3a25265f04ba9f6d3ff Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Tue, 29 Nov 2022 11:12:19 +0530 Subject: [PATCH 1/7] add sticky bit (#265) --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 44e01f8..c182361 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -47,7 +47,8 @@ RUN yum update -y \ RUN groupadd -r $FLUENT_USER && \ useradd -r -g $FLUENT_USER $FLUENT_USER && \ mkdir -p /fluentd/log /fluentd/etc /fluentd/plugins &&\ - chown -R $FLUENT_USER /fluentd && chgrp -R $FLUENT_USER /fluentd + chown -R $FLUENT_USER /fluentd && chgrp -R $FLUENT_USER /fluentd && \ + chmod +t /tmp USER $FLUENT_USER CMD bundle exec fluentd -c /fluentd/etc/fluent.conf \ No newline at end of file From 3544e74537935a2dcc29a0e4499df5367be45c53 Mon Sep 17 00:00:00 2001 From: Christian Gibson Date: Mon, 9 Jan 2023 20:56:22 -0800 Subject: [PATCH 2/7] Add custom headers (#267) * [feat] Add support for @custom_headers --- README.md | 4 ++++ lib/fluent/plugin/out_splunk_hec.rb | 7 ++++++- test/fluent/plugin/out_splunk_hec_test.rb | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3b3612c..abe6f77 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,10 @@ Splunk app name using this plugin (default to `hec_plugin_gem`) The version of Splunk app using this this plugin (default to plugin version) +### custom_headers (Hash) (Optional) + +Hash of custom headers to be added to the HTTP request. Used to populate [`override_headers`](https://docs.seattlerb.org/net-http-persistent/Net/HTTP/Persistent.html#attribute-i-override_headers) attribute of the underlying `Net::HTTP::Persistent` connection. + #### When `data_type` is `event` In this case, parameters inside `` are used as indexed fields and removed from the original input events. Please see the "Add a "fields" property at the top JSON level" [here](http://dev.splunk.com/view/event-collector/SP-CAAAFB6) for details. Given we have configuration like diff --git a/lib/fluent/plugin/out_splunk_hec.rb b/lib/fluent/plugin/out_splunk_hec.rb index a88900d..ee14b14 100644 --- a/lib/fluent/plugin/out_splunk_hec.rb +++ b/lib/fluent/plugin/out_splunk_hec.rb @@ -134,6 +134,9 @@ class SplunkHecOutput < SplunkOutput DESC config_param :non_utf8_replacement_string, :string, :default => ' ' + desc 'Any custom headers to include alongside requests made to Splunk' + config_param :custom_headers, :hash, :default => {} + def initialize super @default_host = Socket.gethostname @@ -168,7 +171,9 @@ def start c.override_headers['Authorization'] = "Splunk #{@hec_token}" c.override_headers['__splunk_app_name'] = "#{@app_name}" c.override_headers['__splunk_app_version'] = "#{@app_version}" - + @custom_headers.each do |header, value| + c.override_headers[header] = value + end end end diff --git a/test/fluent/plugin/out_splunk_hec_test.rb b/test/fluent/plugin/out_splunk_hec_test.rb index 3ecfb69..6d8b018 100644 --- a/test/fluent/plugin/out_splunk_hec_test.rb +++ b/test/fluent/plugin/out_splunk_hec_test.rb @@ -66,6 +66,13 @@ it 'should support enabling gzip' do expect(create_hec_output_driver('hec_host hec_token', 'gzip_compression true').instance.gzip_compression).must_equal true end + it 'should define custom_headers as {} (hash) initially' do + assert_empty(create_hec_output_driver('hec_host hec_token').instance.custom_headers) + expect(create_hec_output_driver('hec_host hec_token').instance.custom_headers).is_a? Hash + end + it 'should allow setting custom_headers' do + assert_equal(create_hec_output_driver('hec_host hec_token', 'custom_headers {"custom":"header"}').instance.custom_headers, {"custom" => "header"}) + end end describe 'hec_host validation' do From 5c3a93dd2f2b6ae1a5d5208f5c723b78de4cf5b5 Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Tue, 24 Jan 2023 13:26:48 +0530 Subject: [PATCH 3/7] Post message for end of support (#268) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index abe6f77..ddb4067 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +# End of Support + +**Important:** The fluent-plugin-splunk-hec will reach End of Support on January 1, 2024. After that date, this repository will no longer receive updates from Splunk and will no longer be supported by Splunk. Until then, only critical security fixes and bug fixes will be provided. + # fluent-plugin-splunk-hec [Fluentd](https://fluentd.org/) output plugin to send events and metrics to [Splunk](https://www.splunk.com) in 2 modes:
From fc8cf7f816622809eb1a91810dba9c7e8afc69bc Mon Sep 17 00:00:00 2001 From: vihas-splunk <121151420+vihas-splunk@users.noreply.github.com> Date: Fri, 24 Mar 2023 17:18:48 +0530 Subject: [PATCH 4/7] Prep for 1.3.2 (#272) --- Gemfile.lock | 14 +++++++------- VERSION | 2 +- docker/Gemfile.lock | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6c5e066..51f9566 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-splunk-hec (1.3.1) + fluent-plugin-splunk-hec (1.3.2) fluentd (>= 1.5) json-jwt (~> 1.15.0) multi_json (~> 1.13) @@ -13,9 +13,9 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (7.0.4) - activesupport (= 7.0.4) - activesupport (7.0.4) + activemodel (7.0.4.3) + activesupport (= 7.0.4.3) + activesupport (7.0.4.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -25,7 +25,7 @@ GEM aes_key_wrap (1.1.0) attr_required (1.0.1) bindata (2.4.14) - concurrent-ruby (1.1.10) + concurrent-ruby (1.2.2) connection_pool (2.3.0) cool.io (1.7.1) crack (0.4.5) @@ -89,7 +89,7 @@ GEM power_assert (2.0.1) prometheus-client (4.0.0) public_suffix (4.0.6) - rack (3.0.1) + rack (3.0.6.1) rack-oauth2 (1.21.2) activesupport attr_required @@ -117,7 +117,7 @@ GEM test-unit (3.5.3) power_assert timeout (0.2.0) - tzinfo (2.0.5) + tzinfo (2.0.6) concurrent-ruby (~> 1.0) tzinfo-data (1.2022.6) tzinfo (>= 1.0.0) diff --git a/VERSION b/VERSION index 6261a05..d5e98f7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.1 \ No newline at end of file +1.3.2 \ No newline at end of file diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index 71835a4..0f608bc 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -12,7 +12,7 @@ GIT PATH remote: gem specs: - fluent-plugin-splunk-hec (1.3.1) + fluent-plugin-splunk-hec (1.3.2) fluentd (>= 1.5) json-jwt (~> 1.15.0) multi_json (~> 1.13) From 8f94144e781a8c505582a12c70ee00cd7c102c06 Mon Sep 17 00:00:00 2001 From: VihasMakwana <121151420+VihasMakwana@users.noreply.github.com> Date: Wed, 29 Mar 2023 16:04:42 +0530 Subject: [PATCH 5/7] Main -> Dev (#274) From 860c58e3a840e9e314580c4ae9a3c42b97344784 Mon Sep 17 00:00:00 2001 From: VihasMakwana <121151420+VihasMakwana@users.noreply.github.com> Date: Mon, 28 Aug 2023 20:05:20 +0530 Subject: [PATCH 6/7] fix: update ubi, fix cve, update ruby (#282) * fix: update ubi, fix cve, update ruby --- .github/workflows/ci_build_test.yaml | 8 ++-- .ruby-version | 2 +- Gemfile.lock | 10 ++-- ci_scripts/deploy_connector.sh | 5 +- ci_scripts/k8s-splunk.yml | 2 +- docker/Dockerfile | 6 +-- docker/Gemfile | 7 +-- docker/Gemfile.lock | 69 +++++++++++++++------------- 8 files changed, 57 insertions(+), 52 deletions(-) diff --git a/.github/workflows/ci_build_test.yaml b/.github/workflows/ci_build_test.yaml index 67be8fb..5209968 100644 --- a/.github/workflows/ci_build_test.yaml +++ b/.github/workflows/ci_build_test.yaml @@ -17,7 +17,7 @@ jobs: uses: ruby/setup-ruby@v1 with: bundler-cache: true - ruby-version: 2.7 + ruby-version: 3.1 - name: Install dependencies run: | @@ -63,12 +63,12 @@ jobs: CI_SPLUNK_PORT: 8089 CI_SPLUNK_USERNAME: admin CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959 - CI_SPLUNK_PASSWORD: helloworld + CI_SPLUNK_PASSWORD: changeme2 CI_INDEX_EVENTS: ci_events CI_INDEX_OBJECTS: ci_objects CI_INDEX_METRICS: ci_metrics KUBERNETES_VERSION: v1.23.2 - MINIKUBE_VERSION: v1.24.0 + MINIKUBE_VERSION: latest MINIKUBE_NODE_COUNTS: 2 GITHUB_ACTIONS: true @@ -158,6 +158,8 @@ jobs: curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable # Create new HEC token curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=a6b5e77f-d5f6-415a-bd43-930cecb12959&disabled=0&index=default-events&indexes=default-events,$CI_INDEX_METRICS,$CI_INDEX_OBJECTS,$CI_INDEX_EVENTS,ns-anno,pod-anno" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http + # lower the limit to 50MiB. Higher limits throws error 'Search not executed XXXX' + kubectl exec -it splunk -- bash -c 'echo -e "\n[diskUsage]\nminFreeSpace = 50" >> /opt/splunk/etc/system/local/server.conf' # Restart Splunk curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/server/control/restart -X POST diff --git a/.ruby-version b/.ruby-version index a4dd9db..ff365e0 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.7.4 +3.1.3 diff --git a/Gemfile.lock b/Gemfile.lock index 51f9566..fd872ef 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,9 +13,9 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (7.0.4.3) - activesupport (= 7.0.4.3) - activesupport (7.0.4.3) + activemodel (7.0.7.2) + activesupport (= 7.0.7.2) + activesupport (7.0.7.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -89,7 +89,7 @@ GEM power_assert (2.0.1) prometheus-client (4.0.0) public_suffix (4.0.6) - rack (3.0.6.1) + rack (3.0.8) rack-oauth2 (1.21.2) activesupport attr_required @@ -97,7 +97,7 @@ GEM json-jwt (>= 1.11.0) rack (>= 2.1.0) rake (13.0.6) - rexml (3.2.5) + rexml (3.2.6) ruby2_keywords (0.0.5) serverengine (2.3.0) sigdump (~> 0.2.2) diff --git a/ci_scripts/deploy_connector.sh b/ci_scripts/deploy_connector.sh index 25e7ab4..fff0e30 100755 --- a/ci_scripts/deploy_connector.sh +++ b/ci_scripts/deploy_connector.sh @@ -23,11 +23,12 @@ helm install ci-sck --set global.splunk.hec.token=$CI_SPLUNK_HEC_TOKEN \ --set splunk-kubernetes-logging.image.tag=recent \ --set splunk-kubernetes-logging.image.pullPolicy=IfNotPresent \ -f ci_scripts/sck_values.yml helm-chart/splunk-connect-for-kubernetes - +# kubectl get pod | grep "ci-sck-splunk-kubernetes-logging" | awk 'NR==1{print $1} kubectl get pod # wait for deployment to finish # metric and logging deamon set for each node + aggr + object + splunk PODS=$((MINIKUBE_NODE_COUNTS*2+2+1)) until kubectl get pod | grep Running | [[ $(wc -l) == $PODS ]]; do - sleep 1; + kubectl get pod + sleep 2; done diff --git a/ci_scripts/k8s-splunk.yml b/ci_scripts/k8s-splunk.yml index 543c6ca..e3c0883 100644 --- a/ci_scripts/k8s-splunk.yml +++ b/ci_scripts/k8s-splunk.yml @@ -26,6 +26,6 @@ spec: - name: SPLUNK_USER value: root - name: SPLUNK_PASSWORD - value: helloworld + value: changeme2 - name: SPLUNK_LAUNCH_CONF value: OPTIMISTIC_ABOUT_FILE_LOCKING=1 diff --git a/docker/Dockerfile b/docker/Dockerfile index c182361..42f0dd0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ruby:2.7.4-buster as builder +FROM ruby:3.1.4-buster as builder ADD ./ /app/ WORKDIR /app @@ -8,7 +8,7 @@ RUN bundle install RUN bundle exec rake build -t -v -FROM registry.access.redhat.com/ubi8/ruby-27 +FROM registry.access.redhat.com/ubi9/ruby-31 ARG VERSION @@ -33,8 +33,6 @@ COPY --from=builder /app/LICENSE /licenses/LICENSE RUN dnf install -y jq COPY --from=builder /app/docker/Gemfile* ./ -RUN gem update date cgi -RUN rm -f /usr/share/gems/specifications/default/cgi-0.1.0.gemspec /usr/share/gems/specifications/default/date-3.0.0.gemspec RUN yum update -y \ && yum remove -y nodejs npm \ && gem install bundler \ diff --git a/docker/Gemfile b/docker/Gemfile index 3ae7adc..4d7f4d3 100644 --- a/docker/Gemfile +++ b/docker/Gemfile @@ -3,7 +3,7 @@ source 'https://rubygems.org' # This is separate gemfile for building docker image that has all plugins # for kubernetes log collection agent # List all required gems here and install via bundler to resolve dependencies -gem "fluentd", ">=1.15" +gem "fluentd", "=1.15.3" gem "fluent-plugin-systemd", "=1.0.2" gem "fluent-plugin-concat", "=2.4.0" gem "fluent-plugin-prometheus", "=2.0.2" @@ -14,13 +14,14 @@ gem "oj", ">=3.11.2" gem 'multi_json', '~> 1.13' gem 'net-http-persistent', '~> 4.0' gem 'openid_connect', '~> 1.1.8' -gem 'prometheus-client', '>= 2.1.0' -gem 'activesupport', '~> 5.2.4.3' +gem 'prometheus-client', '=2.1.0' gem 'http_parser.rb', '=0.8.0' gem "rack", ">=3.0.0" gem "fluent-plugin-record-modifier", ">=2.1" gem 'json-jwt', '~> 1.15.0' gem 'rack-oauth2', '~> 1.19' +gem 'cgi', '~> 0.3.6' +gem 'date', '~> 3.3.3' gem 'fluent-plugin-splunk-hec', path: 'gem/' diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index 0f608bc..da3db6d 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -24,21 +24,23 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (5.2.4.6) - activesupport (= 5.2.4.6) - activesupport (5.2.4.6) + activemodel (7.0.7.2) + activesupport (= 7.0.7.2) + activesupport (7.0.7.2) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.5) + public_suffix (>= 2.0.2, < 6.0) aes_key_wrap (1.1.0) attr_required (1.0.1) - bindata (2.4.14) - concurrent-ruby (1.1.9) - connection_pool (2.2.5) - cool.io (1.7.1) + bindata (2.4.15) + cgi (0.3.6) + concurrent-ruby (1.2.2) + connection_pool (2.4.1) + cool.io (1.8.0) + date (3.3.3) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) ffi (1.15.5) @@ -80,36 +82,36 @@ GEM http-form_data (~> 2.2) http-parser (~> 1.2.0) http-accept (1.7.0) - http-cookie (1.0.4) + http-cookie (1.0.5) domain_name (~> 0.5) http-form_data (2.3.0) http-parser (1.2.3) ffi-compiler (>= 1.0, < 2.0) http_parser.rb (0.8.0) httpclient (2.8.3) - i18n (1.9.1) + i18n (1.14.1) concurrent-ruby (~> 1.0) json-jwt (1.15.3) activesupport (>= 4.2) aes_key_wrap bindata httpclient - jsonpath (1.1.0) + jsonpath (1.1.3) multi_json lru_redux (1.1.0) mail (2.7.1) mini_mime (>= 0.1.1) mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2022.0105) - mini_mime (1.1.2) - minitest (5.15.0) - msgpack (1.6.0) + mime-types-data (3.2023.0808) + mini_mime (1.1.5) + minitest (5.19.0) + msgpack (1.7.2) multi_json (1.15.0) - net-http-persistent (4.0.0) + net-http-persistent (4.0.2) connection_pool (~> 2.2) netrc (0.11.0) - oj (3.11.2) + oj (3.16.0) openid_connect (1.1.8) activemodel attr_required (>= 1.0.0) @@ -121,9 +123,9 @@ GEM validate_url webfinger (>= 1.0.1) prometheus-client (2.1.0) - public_suffix (4.0.6) - rack (3.0.1) - rack-oauth2 (1.19.0) + public_suffix (5.0.3) + rack (3.0.8) + rack-oauth2 (1.21.3) activesupport attr_required httpclient @@ -146,18 +148,18 @@ GEM httpclient (>= 2.4) systemd-journal (1.3.3) ffi (~> 1.9) - thread_safe (0.3.6) - tzinfo (1.2.10) - thread_safe (~> 0.1) - tzinfo-data (1.2022.6) + timeout (0.4.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2023.3) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext - unf_ext (0.0.8) + unf_ext (0.0.8.2) validate_email (0.1.6) activemodel (>= 3.0) mail (>= 2.2.5) - validate_url (1.0.13) + validate_url (1.0.15) activemodel (>= 3.0.0) public_suffix webfinger (1.2.0) @@ -170,7 +172,8 @@ PLATFORMS ruby DEPENDENCIES - activesupport (~> 5.2.4.3) + cgi (~> 0.3.6) + date (~> 3.3.3) fluent-plugin-concat (= 2.4.0) fluent-plugin-jq (= 0.5.1) fluent-plugin-kubernetes_metadata_filter (~> 3.1) @@ -178,7 +181,7 @@ DEPENDENCIES fluent-plugin-record-modifier (>= 2.1) fluent-plugin-splunk-hec! fluent-plugin-systemd (= 1.0.2) - fluentd (>= 1.15) + fluentd (= 1.15.3) http_parser.rb (= 0.8.0) json-jwt (~> 1.15.0) kubeclient! @@ -186,7 +189,7 @@ DEPENDENCIES net-http-persistent (~> 4.0) oj (>= 3.11.2) openid_connect (~> 1.1.8) - prometheus-client (>= 2.1.0) + prometheus-client (= 2.1.0) rack (>= 3.0.0) rack-oauth2 (~> 1.19) From db45e10444c93644f4308baa0f8486bd33d2c270 Mon Sep 17 00:00:00 2001 From: VihasMakwana <121151420+VihasMakwana@users.noreply.github.com> Date: Mon, 28 Aug 2023 20:31:51 +0530 Subject: [PATCH 7/7] update to 1.3.3 (#283) --- Gemfile.lock | 2 +- VERSION | 2 +- docker/Gemfile.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index fd872ef..48bbf19 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-splunk-hec (1.3.2) + fluent-plugin-splunk-hec (1.3.3) fluentd (>= 1.5) json-jwt (~> 1.15.0) multi_json (~> 1.13) diff --git a/VERSION b/VERSION index d5e98f7..785cda8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.2 \ No newline at end of file +1.3.3 \ No newline at end of file diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index da3db6d..2edc9f9 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -12,7 +12,7 @@ GIT PATH remote: gem specs: - fluent-plugin-splunk-hec (1.3.2) + fluent-plugin-splunk-hec (1.3.3) fluentd (>= 1.5) json-jwt (~> 1.15.0) multi_json (~> 1.13)