From 0f7dbd0c053c925227692b0fd87ed517505df3a3 Mon Sep 17 00:00:00 2001 From: tyxia Date: Wed, 2 Oct 2024 20:08:12 +0000 Subject: [PATCH 1/2] Handle encode metadata on recreated stream Signed-off-by: tyxia --- source/common/http/filter_manager.cc | 10 +++ source/common/http/filter_manager.h | 5 +- test/integration/BUILD | 2 + test/integration/filter_integration_test.cc | 80 +++++++++++++++++++ test/integration/filters/BUILD | 33 ++++++++ .../filters/add_encode_metadata_filter.cc | 41 ++++++++++ .../filters/encoder_recreate_stream_filter.cc | 55 +++++++++++++ 7 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 test/integration/filters/add_encode_metadata_filter.cc create mode 100644 test/integration/filters/encoder_recreate_stream_filter.cc diff --git a/source/common/http/filter_manager.cc b/source/common/http/filter_manager.cc index be2c9b7af44fb..d5ea8aa7a221d 100644 --- a/source/common/http/filter_manager.cc +++ b/source/common/http/filter_manager.cc @@ -1699,6 +1699,10 @@ bool ActiveStreamDecoderFilter::recreateStream(const ResponseHeaderMap* headers) return false; } + parent_.state_.decoder_filter_chain_aborted_ = true; + parent_.state_.encoder_filter_chain_aborted_ = true; + parent_.state_.recreated_stream_ = true; + parent_.streamInfo().setResponseCodeDetails( StreamInfo::ResponseCodeDetails::get().InternalRedirect); @@ -1765,6 +1769,12 @@ void ActiveStreamEncoderFilter::drainSavedResponseMetadata() { } void ActiveStreamEncoderFilter::handleMetadataAfterHeadersCallback() { + if (parent_.state_.recreated_stream_) { + // The stream has been recreated. In this case, there's no reason to encode saved metadata. + getSavedResponseMetadata()->clear(); + return; + } + // If we drain accumulated metadata, the iteration must start with the current filter. const bool saved_state = iterate_from_current_filter_; iterate_from_current_filter_ = true; diff --git a/source/common/http/filter_manager.h b/source/common/http/filter_manager.h index 82405f0cc4b2c..0e4a3cc680809 100644 --- a/source/common/http/filter_manager.h +++ b/source/common/http/filter_manager.h @@ -890,7 +890,8 @@ class FilterManager : public ScopeTrackedObject, has_1xx_headers_(false), created_filter_chain_(false), is_head_request_(false), is_grpc_request_(false), non_100_response_headers_encoded_(false), under_on_local_reply_(false), decoder_filter_chain_aborted_(false), - encoder_filter_chain_aborted_(false), saw_downstream_reset_(false) {} + encoder_filter_chain_aborted_(false), saw_downstream_reset_(false), + recreated_stream_(false) {} uint32_t filter_call_state_{0}; // Set after decoder filter chain has completed iteration. Prevents further calls to decoder @@ -928,6 +929,8 @@ class FilterManager : public ScopeTrackedObject, bool decoder_filter_chain_aborted_ : 1; bool encoder_filter_chain_aborted_ : 1; bool saw_downstream_reset_ : 1; + // True when the stream was recreated. + bool recreated_stream_ : 1; // The following 3 members are booleans rather than part of the space-saving bitfield as they // are passed as arguments to functions expecting bools. Extend State using the bitfield diff --git a/test/integration/BUILD b/test/integration/BUILD index fdfa59a6cf62f..32c769067243b 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -770,12 +770,14 @@ envoy_cc_test( "//source/common/http:header_map_lib", "//source/extensions/filters/http/buffer:config", "//test/integration/filters:add_body_filter_config_lib", + "//test/integration/filters:add_encode_metadata_filter_lib", "//test/integration/filters:add_invalid_data_filter_lib", "//test/integration/filters:assert_non_reentrant_filter_lib", "//test/integration/filters:buffer_continue_filter_lib", "//test/integration/filters:continue_after_local_reply_filter_lib", "//test/integration/filters:continue_headers_only_inject_body", "//test/integration/filters:encoder_decoder_buffer_filter_lib", + "//test/integration/filters:encoder_recreate_stream_filter_lib", "//test/integration/filters:invalid_header_filter_lib", "//test/integration/filters:local_reply_during_encoding_data_filter_lib", "//test/integration/filters:local_reply_during_encoding_filter_lib", diff --git a/test/integration/filter_integration_test.cc b/test/integration/filter_integration_test.cc index e83a9d231188c..5028b5eca5254 100644 --- a/test/integration/filter_integration_test.cc +++ b/test/integration/filter_integration_test.cc @@ -1575,5 +1575,85 @@ TEST_P(FilterIntegrationTest, FilterAddsDataToHeaderOnlyRequestWithIndependentHa testFilterAddsDataAndTrailersToHeaderOnlyRequest(); } +// Add metadata in the first filter before recreate the stream in the second filter, +// on response path. +TEST_P(FilterIntegrationTest, RecreateStreamAfterEncodeMetadata) { + // recreateStream is not supported in Upstream filter chain. + if (!testing_downstream_filter_) { + return; + } + + prependFilter("{ name: add-metadata-encode-headers-filter }"); + prependFilter("{ name: encoder-recreate-stream-filter }"); + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(default_response_headers_, true); + + // Second upstream request is triggered by recreateStream. + FakeStreamPtr upstream_request_2; + // Wait for the next stream on the upstream connection. + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_2)); + // Wait for the stream to be completely received. + ASSERT_TRUE(upstream_request_2->waitForEndStream(*dispatcher_)); + upstream_request_2->encodeHeaders(default_response_headers_, true); + + // Wait for the response to be completely received. + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + + // Verify the metadata is received. + std::set expected_metadata_keys = {"headers", "duplicate"}; + EXPECT_EQ(response->metadataMap().size(), expected_metadata_keys.size()); + for (const auto& key : expected_metadata_keys) { + // keys are the same as their corresponding values. + auto it = response->metadataMap().find(key); + ASSERT_FALSE(it == response->metadataMap().end()) << "key: " << key; + EXPECT_EQ(response->metadataMap().find(key)->second, key); + } +} + +// Add metadata in the first filter on local reply path. +TEST_P(FilterIntegrationTest, EncodeMetadataOnLocalReply) { + // Local replies are not seen by upstream HTTP filters. add-metadata-encode-headers-filter will + // not be invoked if it is installed in upstream filter chain. + // Thus, this test is only applicable to downstream filter chain. + if (!testing_downstream_filter_) { + return; + } + + prependFilter("{ name: local-reply-during-decode }"); + prependFilter("{ name: add-metadata-encode-headers-filter }"); + + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("500", response->headers().getStatusValue()); + + // Verify the metadata is received. + std::set expected_metadata_keys = {"headers", "duplicate"}; + EXPECT_EQ(response->metadataMap().size(), expected_metadata_keys.size()); + for (const auto& key : expected_metadata_keys) { + // keys are the same as their corresponding values. + auto it = response->metadataMap().find(key); + ASSERT_FALSE(it == response->metadataMap().end()) << "key: " << key; + EXPECT_EQ(response->metadataMap().find(key)->second, key); + } +} + } // namespace } // namespace Envoy diff --git a/test/integration/filters/BUILD b/test/integration/filters/BUILD index dfebb8f118cf7..819acf97b26eb 100644 --- a/test/integration/filters/BUILD +++ b/test/integration/filters/BUILD @@ -1013,3 +1013,36 @@ envoy_cc_test_library( "//test/extensions/filters/http/common:empty_http_filter_config_lib", ], ) + +envoy_cc_test_library( + name = "add_encode_metadata_filter_lib", + srcs = [ + "add_encode_metadata_filter.cc", + ], + deps = [ + ":common_lib", + "//envoy/event:timer_interface", + "//envoy/http:filter_interface", + "//envoy/registry", + "//envoy/server:filter_config_interface", + "//source/extensions/filters/http/common:pass_through_filter_lib", + "//test/extensions/filters/http/common:empty_http_filter_config_lib", + ], +) + +envoy_cc_test_library( + name = "encoder_recreate_stream_filter_lib", + srcs = [ + "encoder_recreate_stream_filter.cc", + ], + deps = [ + ":common_lib", + "//envoy/event:timer_interface", + "//envoy/http:filter_interface", + "//envoy/registry", + "//envoy/server:filter_config_interface", + "//source/common/router:string_accessor_lib", + "//source/extensions/filters/http/common:pass_through_filter_lib", + "//test/extensions/filters/http/common:empty_http_filter_config_lib", + ], +) diff --git a/test/integration/filters/add_encode_metadata_filter.cc b/test/integration/filters/add_encode_metadata_filter.cc new file mode 100644 index 0000000000000..4a840e2caaa53 --- /dev/null +++ b/test/integration/filters/add_encode_metadata_filter.cc @@ -0,0 +1,41 @@ +#include +#include + +#include "envoy/event/timer.h" +#include "envoy/http/filter.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/extensions/filters/http/common/pass_through_filter.h" + +#include "test/extensions/filters/http/common/empty_http_filter_config.h" +#include "test/integration/filters/common.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +// A filter add encoded metadata in encodeHeaders. +class AddEncodeMetadataFilter : public Http::PassThroughFilter { +public: + constexpr static char name[] = "add-metadata-encode-headers-filter"; + + Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap&, bool) override { + Http::MetadataMap metadata_map = {{"headers", "headers"}, {"duplicate", "duplicate"}}; + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); + return Http::FilterHeadersStatus::Continue; + } + + Http::FilterDataStatus encodeData(Buffer::Instance&, bool) override { + return Http::FilterDataStatus::Continue; + } +}; + +constexpr char AddEncodeMetadataFilter::name[]; +static Registry::RegisterFactory, + Server::Configuration::NamedHttpFilterConfigFactory> + register_; + +} // namespace Envoy \ No newline at end of file diff --git a/test/integration/filters/encoder_recreate_stream_filter.cc b/test/integration/filters/encoder_recreate_stream_filter.cc new file mode 100644 index 0000000000000..e1d2e16014a43 --- /dev/null +++ b/test/integration/filters/encoder_recreate_stream_filter.cc @@ -0,0 +1,55 @@ +#include +#include + +#include "envoy/event/timer.h" +#include "envoy/http/filter.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/extensions/filters/http/common/pass_through_filter.h" + +#include "test/extensions/filters/http/common/empty_http_filter_config.h" +#include "source/common/router/string_accessor_impl.h" +#include "test/integration/filters/common.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +class EncoderRecreateStreamFilter : public Http::PassThroughFilter { +public: + constexpr static char name[] = "encoder-recreate-stream-filter"; + + Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap&, bool) override { + const auto* filter_state = + decoder_callbacks_->streamInfo().filterState()->getDataReadOnly( + "test_key"); + + if (filter_state != nullptr) { + return ::Envoy::Http::FilterHeadersStatus::Continue; + } + + decoder_callbacks_->streamInfo().filterState()->setData( + "test_key", std::make_unique("test_value"), + StreamInfo::FilterState::StateType::ReadOnly, StreamInfo::FilterState::LifeSpan::Request); + + if (decoder_callbacks_->recreateStream(nullptr)) { + return ::Envoy::Http::FilterHeadersStatus::StopIteration; + } + + return ::Envoy::Http::FilterHeadersStatus::Continue; + } + + void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override { + decoder_callbacks_ = &callbacks; + } +}; + +// perform static registration +constexpr char EncoderRecreateStreamFilter::name[]; +static Registry::RegisterFactory, + Server::Configuration::NamedHttpFilterConfigFactory> + register_; + +} // namespace Envoy From e1d1509db0a9add8f34f6a3549c0ffc6cc740807 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 20:12:29 +0000 Subject: [PATCH 2/2] build(deps): bump slack-sdk from 3.31.0 to 3.33.1 in /tools/base Bumps [slack-sdk](https://github.com/slackapi/python-slack-sdk) from 3.31.0 to 3.33.1. - [Release notes](https://github.com/slackapi/python-slack-sdk/releases) - [Commits](https://github.com/slackapi/python-slack-sdk/compare/v3.31.0...v3.33.1) --- updated-dependencies: - dependency-name: slack-sdk dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- tools/base/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 7fbe642f7f6a6..06bb4ce3976de 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -1279,9 +1279,9 @@ six==1.16.0 \ # pyu2f # sphinxcontrib-httpdomain # thrift -slack-sdk==3.31.0 \ - --hash=sha256:740d2f9c49cbfcbd46fca56b4be9d527934c225312aac18fd2c0fca0ef6bc935 \ - --hash=sha256:a120cc461e8ebb7d9175f171dbe0ded37a6878d9f7b96b28e4bad1227399047b +slack-sdk==3.33.1 \ + --hash=sha256:e328bb661d95db5f66b993b1d64288ac7c72201a745b4c7cf8848dafb7b74e40 \ + --hash=sha256:ef93beec3ce9c8f64da02fd487598a05ec4bc9c92ceed58f122dbe632691cbe2 # via -r requirements.in smmap==5.0.1 \ --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \