objstore: support GCS S3-compatible signing

pkg/objstore/s3store/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "s3store",
     srcs = [
         "client.go",
+        "gcs_s3_signer.go",
         "interface.go",
         "ks3.go",
         "logger.go",
@@ -28,6 +29,7 @@ go_library(
         "@com_github_aws_aws_sdk_go_v2//aws",
         "@com_github_aws_aws_sdk_go_v2//aws/ratelimit",
         "@com_github_aws_aws_sdk_go_v2//aws/retry",
+        "@com_github_aws_aws_sdk_go_v2//aws/signer/v4:signer",
         "@com_github_aws_aws_sdk_go_v2_config//:config",
         "@com_github_aws_aws_sdk_go_v2_credentials//:credentials",
         "@com_github_aws_aws_sdk_go_v2_credentials//stscreds",
@@ -57,6 +59,7 @@ go_test(
     timeout = "short",
     srcs = [
         "client_test.go",
+        "gcs_s3_test.go",
         "main_test.go",
         "retry_test.go",
         "s3_flags_test.go",

pkg/objstore/s3store/gcs_s3_signer.go

@@ -0,0 +1,59 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3store
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+)
+
+type gcsS3CompatibleSigner struct {
+	signer *v4.Signer
+}
+
+func newGCSS3CompatibleSigner() *gcsS3CompatibleSigner {
+	return &gcsS3CompatibleSigner{signer: v4.NewSigner()}
+}
+
+func (s *gcsS3CompatibleSigner) SignHTTP(
+	ctx context.Context,
+	credentials aws.Credentials,
+	r *http.Request,
+	payloadHash string,
+	service string,
+	region string,
+	signingTime time.Time,
+	optFns ...func(*v4.SignerOptions),
+) error {
+	// GCS S3 interoperability can reject signatures that include Accept-Encoding.
+	// See https://github.com/aws/aws-sdk-go-v2/issues/1816#issuecomment-1232780084.
+	// Keep sending the header, but exclude it from the canonical request.
+	savedHeaders := http.Header{}
+	for _, key := range []string{"Accept-Encoding"} {
+		if values, ok := r.Header[key]; ok {
+			savedHeaders[key] = append([]string(nil), values...)
+			r.Header.Del(key)
+		}
+	}
+	err := s.signer.SignHTTP(ctx, credentials, r, payloadHash, service, region, signingTime, optFns...)
+	for key, values := range savedHeaders {
+		r.Header[key] = values
+	}
+	return err
+}

pkg/objstore/s3store/gcs_s3_test.go

@@ -0,0 +1,214 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3store
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+
+	backuppb "github.com/pingcap/kvproto/pkg/brpb"
+	"github.com/pingcap/tidb/pkg/objstore/storeapi"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsGCSS3Compatible(t *testing.T) {
+	require.True(t, isGCSS3Compatible(&backuppb.S3{
+		Provider: "gcs",
+		Endpoint: "http://127.0.0.1:9000",
+	}))
+	require.True(t, isGCSS3Compatible(&backuppb.S3{
+		Provider: "ceph",
+		Endpoint: "https://storage.googleapis.com",
+	}))
+	require.True(t, isGCSS3Compatible(&backuppb.S3{
+		Endpoint: "https://storage.googleapis.com/",
+	}))
+	require.True(t, isGCSS3Compatible(&backuppb.S3{
+		Endpoint: "https://bucket.storage.googleapis.com",
+	}))
+	require.False(t, isGCSS3Compatible(&backuppb.S3{
+		Provider: "ceph",
+		Endpoint: "https://s3.example.com",
+	}))
+	require.False(t, isGCSS3Compatible(&backuppb.S3{
+		Endpoint: "://bad-endpoint",
+	}))
+}
+
+func TestGCSS3CompatibleSignerSkipsAcceptEncoding(t *testing.T) {
+	const listObjectsV2Response = `<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Name>bucket</Name>
+  <Prefix></Prefix>
+  <KeyCount>0</KeyCount>
+  <MaxKeys>1</MaxKeys>
+  <IsTruncated>false</IsTruncated>
+</ListBucketResult>`
+
+	type requestInfo struct {
+		method        string
+		signedHeaders string
+		listType      string
+		writeErr      error
+	}
+
+	testCases := []struct {
+		name       string
+		provider   string
+		endpoint   string
+		httpClient func(string) *http.Client
+	}{
+		{
+			name:     "provider_gcs",
+			provider: "gcs",
+		},
+		{
+			name:       "endpoint_only",
+			endpoint:   "https://storage.googleapis.com",
+			httpClient: newRewriteHostHTTPClient,
+		},
+		{
+			name:       "aws_provider_gcs_endpoint",
+			provider:   "aws",
+			endpoint:   "https://storage.googleapis.com",
+			httpClient: newRewriteHostHTTPClient,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var (
+				mu       sync.Mutex
+				requests []requestInfo
+			)
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				info := requestInfo{
+					method:        r.Method,
+					signedHeaders: getSignedHeaders(r.Header.Get("Authorization")),
+					listType:      r.URL.Query().Get("list-type"),
+				}
+				defer func() {
+					mu.Lock()
+					requests = append(requests, info)
+					mu.Unlock()
+				}()
+
+				switch r.Method {
+				case http.MethodHead:
+					w.WriteHeader(http.StatusOK)
+				case http.MethodGet:
+					w.Header().Set("Content-Type", "application/xml")
+					_, info.writeErr = w.Write([]byte(listObjectsV2Response))
+				default:
+					w.WriteHeader(http.StatusNotFound)
+				}
+			}))
+			defer server.Close()
+
+			endpoint := tc.endpoint
+			if endpoint == "" {
+				endpoint = server.URL
+			}
+			opts := &storeapi.Options{
+				CheckPermissions: []storeapi.Permission{storeapi.AccessBuckets, storeapi.ListObjects},
+			}
+			if tc.httpClient != nil {
+				opts.HTTPClient = tc.httpClient(server.URL)
+			}
+			storage, err := NewS3Storage(context.Background(), &backuppb.S3{
+				Bucket:          "bucket",
+				Endpoint:        endpoint,
+				Provider:        tc.provider,
+				ForcePathStyle:  true,
+				AccessKey:       "access-key",
+				SecretAccessKey: "secret-access-key",
+			}, opts)
+			require.NoError(t, err)
+			require.NotNil(t, storage)
+
+			mu.Lock()
+			observedRequests := append([]requestInfo(nil), requests...)
+			mu.Unlock()
+			require.Len(t, observedRequests, 2)
+
+			var headSeen, listSeen bool
+			for _, req := range observedRequests {
+				require.NoError(t, req.writeErr)
+				require.NotEmpty(t, req.signedHeaders)
+				require.NotContains(t, req.signedHeaders, "accept-encoding")
+				require.Contains(t, req.signedHeaders, "amz-sdk-invocation-id")
+				require.Contains(t, req.signedHeaders, "amz-sdk-request")
+				require.Contains(t, req.signedHeaders, "host")
+				require.Contains(t, req.signedHeaders, "x-amz-content-sha256")
+				require.Contains(t, req.signedHeaders, "x-amz-date")
+
+				switch req.method {
+				case http.MethodHead:
+					headSeen = true
+				case http.MethodGet:
+					listSeen = true
+					require.Equal(t, "2", req.listType)
+				default:
+					require.Failf(t, "unexpected request method", "method: %s", req.method)
+				}
+			}
+			require.True(t, headSeen)
+			require.True(t, listSeen)
+		})
+	}
+}
+
+type rewriteHostTransport struct {
+	scheme string
+	host   string
+	base   http.RoundTripper
+}
+
+func newRewriteHostHTTPClient(target string) *http.Client {
+	return &http.Client{
+		Transport: &rewriteHostTransport{
+			scheme: "http",
+			host:   strings.TrimPrefix(target, "http://"),
+			base:   http.DefaultTransport,
+		},
+	}
+}
+
+func (t *rewriteHostTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	req := r.Clone(r.Context())
+	if req.Host == "" {
+		req.Host = r.URL.Host
+	}
+	req.URL.Scheme = t.scheme
+	req.URL.Host = t.host
+	if t.base == nil {
+		return http.DefaultTransport.RoundTrip(req)
+	}
+	return t.base.RoundTrip(req)
+}
+
+func getSignedHeaders(authorization string) string {
+	for _, part := range strings.Split(authorization, ",") {
+		part = strings.TrimSpace(part)
+		if strings.HasPrefix(part, "SignedHeaders=") {
+			return strings.TrimPrefix(part, "SignedHeaders=")
+		}
+	}
+	return ""
+}

pkg/objstore/s3store/store.go

@@ -17,6 +17,7 @@ package s3store
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"strings"
 
 	alicred "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials"
@@ -43,13 +44,20 @@ import (
 
 const (
 	defaultRegion = "us-east-1"
+	gcsProvider   = "gcs"
+	// GCS S3 interoperability documents storage.googleapis.com as the XML API
+	// endpoint for S3-compatible tools.
+	// See https://cloud.google.com/storage/docs/interoperability and
+	// https://cloud.google.com/storage/docs/request-endpoints.
+	gcsEndpoint = "storage.googleapis.com"
 	// to check the cloud type by endpoint tag.
 	domainAliyun = "aliyuncs.com"
 )
 
 // NewS3Storage initialize a new s3 storage for metadata.
 func NewS3Storage(ctx context.Context, backend *backuppb.S3, opts *storeapi.Options) (obj *s3like.Storage, errRet error) {
 	qs := *backend
+	gcsS3Compatible := isGCSS3Compatible(&qs)
 
 	// Start with default configuration loading
 	var configOpts []func(*config.LoadOptions) error
@@ -125,6 +133,11 @@ func NewS3Storage(ctx context.Context, backend *backuppb.S3, opts *storeapi.Opti
 		// These logs will be printed when log level is `DEBUG`.
 		o.ClientLogMode |= aws.LogRetries | aws.LogRequest | aws.LogResponse | aws.LogDeprecatedUsage
 	})
+	if gcsS3Compatible {
+		s3Opts = append(s3Opts, func(o *s3.Options) {
+			o.HTTPSignerV4 = newGCSS3CompatibleSigner()
+		})
+	}
 
 	// ⚠️ Do NOT set a global endpoint in the AWS config.
 	// Setting a global endpoint will break AssumeRoleWithWebIdentity,
@@ -227,7 +240,11 @@ func NewS3Storage(ctx context.Context, backend *backuppb.S3, opts *storeapi.Opti
 
 	// Perform region detection and validation
 	var detectedRegion string
-	officialS3 := len(qs.Provider) == 0 || qs.Provider == "aws"
+	awsProvider := len(qs.Provider) == 0 || qs.Provider == "aws"
+	// GCS S3-compatible endpoints must skip AWS bucket-region discovery:
+	// GCS interoperability can reject the HeadBucket request before normal
+	// object access starts, and the configured region is only used for signing.
+	officialS3 := awsProvider && !gcsS3Compatible
 	if officialS3 {
 		// For AWS provider, detect the actual bucket region
 		// In AWS SDK v2, GetBucketRegion has a simpler signature
@@ -300,6 +317,21 @@ func NewS3Storage(ctx context.Context, backend *backuppb.S3, opts *storeapi.Opti
 	return s3Storage, nil
 }
 
+func isGCSS3Compatible(qs *backuppb.S3) bool {
+	if strings.EqualFold(qs.Provider, gcsProvider) {
+		return true
+	}
+	if qs.Endpoint == "" {
+		return false
+	}
+	u, err := url.Parse(qs.Endpoint)
+	if err != nil {
+		return false
+	}
+	host := strings.ToLower(u.Hostname())
+	return host == gcsEndpoint || strings.HasSuffix(host, "."+gcsEndpoint)
+}
+
 // IsObjectLockEnabled checks whether the S3 bucket has Object Lock enabled.
 func IsObjectLockEnabled(svc S3API, options *backuppb.S3) bool {
 	input := &s3.GetObjectLockConfigurationInput{