diff --git a/pkg/objstore/s3store/BUILD.bazel b/pkg/objstore/s3store/BUILD.bazel index 3921b110e5ba0..200559ec9b0db 100644 --- a/pkg/objstore/s3store/BUILD.bazel +++ b/pkg/objstore/s3store/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "s3store", srcs = [ "client.go", + "gcs_s3_signer.go", "interface.go", "ks3.go", "logger.go", @@ -28,6 +29,7 @@ go_library( "@com_github_aws_aws_sdk_go_v2//aws", "@com_github_aws_aws_sdk_go_v2//aws/ratelimit", "@com_github_aws_aws_sdk_go_v2//aws/retry", + "@com_github_aws_aws_sdk_go_v2//aws/signer/v4:signer", "@com_github_aws_aws_sdk_go_v2_config//:config", "@com_github_aws_aws_sdk_go_v2_credentials//:credentials", "@com_github_aws_aws_sdk_go_v2_credentials//stscreds", @@ -57,6 +59,7 @@ go_test( timeout = "short", srcs = [ "client_test.go", + "gcs_s3_test.go", "main_test.go", "retry_test.go", "s3_flags_test.go", diff --git a/pkg/objstore/s3store/gcs_s3_signer.go b/pkg/objstore/s3store/gcs_s3_signer.go new file mode 100644 index 0000000000000..047f9fe5f1c6b --- /dev/null +++ b/pkg/objstore/s3store/gcs_s3_signer.go @@ -0,0 +1,59 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3store + +import ( + "context" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/signer/v4" +) + +type gcsS3CompatibleSigner struct { + signer *v4.Signer +} + +func newGCSS3CompatibleSigner() *gcsS3CompatibleSigner { + return &gcsS3CompatibleSigner{signer: v4.NewSigner()} +} + +func (s *gcsS3CompatibleSigner) SignHTTP( + ctx context.Context, + credentials aws.Credentials, + r *http.Request, + payloadHash string, + service string, + region string, + signingTime time.Time, + optFns ...func(*v4.SignerOptions), +) error { + // GCS S3 interoperability can reject signatures that include Accept-Encoding. + // See https://github.com/aws/aws-sdk-go-v2/issues/1816#issuecomment-1232780084. + // Keep sending the header, but exclude it from the canonical request. + savedHeaders := http.Header{} + for _, key := range []string{"Accept-Encoding"} { + if values, ok := r.Header[key]; ok { + savedHeaders[key] = append([]string(nil), values...) + r.Header.Del(key) + } + } + err := s.signer.SignHTTP(ctx, credentials, r, payloadHash, service, region, signingTime, optFns...) + for key, values := range savedHeaders { + r.Header[key] = values + } + return err +} diff --git a/pkg/objstore/s3store/gcs_s3_test.go b/pkg/objstore/s3store/gcs_s3_test.go new file mode 100644 index 0000000000000..25ea7f37953a2 --- /dev/null +++ b/pkg/objstore/s3store/gcs_s3_test.go @@ -0,0 +1,214 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3store + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + backuppb "github.com/pingcap/kvproto/pkg/brpb" + "github.com/pingcap/tidb/pkg/objstore/storeapi" + "github.com/stretchr/testify/require" +) + +func TestIsGCSS3Compatible(t *testing.T) { + require.True(t, isGCSS3Compatible(&backuppb.S3{ + Provider: "gcs", + Endpoint: "http://127.0.0.1:9000", + })) + require.True(t, isGCSS3Compatible(&backuppb.S3{ + Provider: "ceph", + Endpoint: "https://storage.googleapis.com", + })) + require.True(t, isGCSS3Compatible(&backuppb.S3{ + Endpoint: "https://storage.googleapis.com/", + })) + require.True(t, isGCSS3Compatible(&backuppb.S3{ + Endpoint: "https://bucket.storage.googleapis.com", + })) + require.False(t, isGCSS3Compatible(&backuppb.S3{ + Provider: "ceph", + Endpoint: "https://s3.example.com", + })) + require.False(t, isGCSS3Compatible(&backuppb.S3{ + Endpoint: "://bad-endpoint", + })) +} + +func TestGCSS3CompatibleSignerSkipsAcceptEncoding(t *testing.T) { + const listObjectsV2Response = ` + + bucket + + 0 + 1 + false +` + + type requestInfo struct { + method string + signedHeaders string + listType string + writeErr error + } + + testCases := []struct { + name string + provider string + endpoint string + httpClient func(string) *http.Client + }{ + { + name: "provider_gcs", + provider: "gcs", + }, + { + name: "endpoint_only", + endpoint: "https://storage.googleapis.com", + httpClient: newRewriteHostHTTPClient, + }, + { + name: "aws_provider_gcs_endpoint", + provider: "aws", + endpoint: "https://storage.googleapis.com", + httpClient: newRewriteHostHTTPClient, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var ( + mu sync.Mutex + requests []requestInfo + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + info := requestInfo{ + method: r.Method, + signedHeaders: getSignedHeaders(r.Header.Get("Authorization")), + listType: r.URL.Query().Get("list-type"), + } + defer func() { + mu.Lock() + requests = append(requests, info) + mu.Unlock() + }() + + switch r.Method { + case http.MethodHead: + w.WriteHeader(http.StatusOK) + case http.MethodGet: + w.Header().Set("Content-Type", "application/xml") + _, info.writeErr = w.Write([]byte(listObjectsV2Response)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + endpoint := tc.endpoint + if endpoint == "" { + endpoint = server.URL + } + opts := &storeapi.Options{ + CheckPermissions: []storeapi.Permission{storeapi.AccessBuckets, storeapi.ListObjects}, + } + if tc.httpClient != nil { + opts.HTTPClient = tc.httpClient(server.URL) + } + storage, err := NewS3Storage(context.Background(), &backuppb.S3{ + Bucket: "bucket", + Endpoint: endpoint, + Provider: tc.provider, + ForcePathStyle: true, + AccessKey: "access-key", + SecretAccessKey: "secret-access-key", + }, opts) + require.NoError(t, err) + require.NotNil(t, storage) + + mu.Lock() + observedRequests := append([]requestInfo(nil), requests...) + mu.Unlock() + require.Len(t, observedRequests, 2) + + var headSeen, listSeen bool + for _, req := range observedRequests { + require.NoError(t, req.writeErr) + require.NotEmpty(t, req.signedHeaders) + require.NotContains(t, req.signedHeaders, "accept-encoding") + require.Contains(t, req.signedHeaders, "amz-sdk-invocation-id") + require.Contains(t, req.signedHeaders, "amz-sdk-request") + require.Contains(t, req.signedHeaders, "host") + require.Contains(t, req.signedHeaders, "x-amz-content-sha256") + require.Contains(t, req.signedHeaders, "x-amz-date") + + switch req.method { + case http.MethodHead: + headSeen = true + case http.MethodGet: + listSeen = true + require.Equal(t, "2", req.listType) + default: + require.Failf(t, "unexpected request method", "method: %s", req.method) + } + } + require.True(t, headSeen) + require.True(t, listSeen) + }) + } +} + +type rewriteHostTransport struct { + scheme string + host string + base http.RoundTripper +} + +func newRewriteHostHTTPClient(target string) *http.Client { + return &http.Client{ + Transport: &rewriteHostTransport{ + scheme: "http", + host: strings.TrimPrefix(target, "http://"), + base: http.DefaultTransport, + }, + } +} + +func (t *rewriteHostTransport) RoundTrip(r *http.Request) (*http.Response, error) { + req := r.Clone(r.Context()) + if req.Host == "" { + req.Host = r.URL.Host + } + req.URL.Scheme = t.scheme + req.URL.Host = t.host + if t.base == nil { + return http.DefaultTransport.RoundTrip(req) + } + return t.base.RoundTrip(req) +} + +func getSignedHeaders(authorization string) string { + for _, part := range strings.Split(authorization, ",") { + part = strings.TrimSpace(part) + if strings.HasPrefix(part, "SignedHeaders=") { + return strings.TrimPrefix(part, "SignedHeaders=") + } + } + return "" +} diff --git a/pkg/objstore/s3store/store.go b/pkg/objstore/s3store/store.go index 9e66b12ea49e0..3d5becbd6e5c6 100644 --- a/pkg/objstore/s3store/store.go +++ b/pkg/objstore/s3store/store.go @@ -17,6 +17,7 @@ package s3store import ( "context" "fmt" + "net/url" "strings" alicred "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials" @@ -43,6 +44,12 @@ import ( const ( defaultRegion = "us-east-1" + gcsProvider = "gcs" + // GCS S3 interoperability documents storage.googleapis.com as the XML API + // endpoint for S3-compatible tools. + // See https://cloud.google.com/storage/docs/interoperability and + // https://cloud.google.com/storage/docs/request-endpoints. + gcsEndpoint = "storage.googleapis.com" // to check the cloud type by endpoint tag. domainAliyun = "aliyuncs.com" ) @@ -50,6 +57,7 @@ const ( // NewS3Storage initialize a new s3 storage for metadata. func NewS3Storage(ctx context.Context, backend *backuppb.S3, opts *storeapi.Options) (obj *s3like.Storage, errRet error) { qs := *backend + gcsS3Compatible := isGCSS3Compatible(&qs) // Start with default configuration loading var configOpts []func(*config.LoadOptions) error @@ -125,6 +133,11 @@ func NewS3Storage(ctx context.Context, backend *backuppb.S3, opts *storeapi.Opti // These logs will be printed when log level is `DEBUG`. o.ClientLogMode |= aws.LogRetries | aws.LogRequest | aws.LogResponse | aws.LogDeprecatedUsage }) + if gcsS3Compatible { + s3Opts = append(s3Opts, func(o *s3.Options) { + o.HTTPSignerV4 = newGCSS3CompatibleSigner() + }) + } // ⚠️ Do NOT set a global endpoint in the AWS config. // Setting a global endpoint will break AssumeRoleWithWebIdentity, @@ -227,7 +240,11 @@ func NewS3Storage(ctx context.Context, backend *backuppb.S3, opts *storeapi.Opti // Perform region detection and validation var detectedRegion string - officialS3 := len(qs.Provider) == 0 || qs.Provider == "aws" + awsProvider := len(qs.Provider) == 0 || qs.Provider == "aws" + // GCS S3-compatible endpoints must skip AWS bucket-region discovery: + // GCS interoperability can reject the HeadBucket request before normal + // object access starts, and the configured region is only used for signing. + officialS3 := awsProvider && !gcsS3Compatible if officialS3 { // For AWS provider, detect the actual bucket region // In AWS SDK v2, GetBucketRegion has a simpler signature @@ -300,6 +317,21 @@ func NewS3Storage(ctx context.Context, backend *backuppb.S3, opts *storeapi.Opti return s3Storage, nil } +func isGCSS3Compatible(qs *backuppb.S3) bool { + if strings.EqualFold(qs.Provider, gcsProvider) { + return true + } + if qs.Endpoint == "" { + return false + } + u, err := url.Parse(qs.Endpoint) + if err != nil { + return false + } + host := strings.ToLower(u.Hostname()) + return host == gcsEndpoint || strings.HasSuffix(host, "."+gcsEndpoint) +} + // IsObjectLockEnabled checks whether the S3 bucket has Object Lock enabled. func IsObjectLockEnabled(svc S3API, options *backuppb.S3) bool { input := &s3.GetObjectLockConfigurationInput{