From e55d792c943b9a768d1e12e05ae49afdd8adb7ca Mon Sep 17 00:00:00 2001 From: Vincenzo Comito Date: Mon, 13 Apr 2026 07:54:53 -0700 Subject: [PATCH] 887: add .netrc support for remote asset fetches --- README.md | 9 + server/BUILD.bazel | 2 + server/grpc.go | 2 + server/grpc_asset.go | 5 +- server/grpc_asset_netrc_test.go | 323 ++++++++++++++++++++++++++++++++ server/grpc_asset_test.go | 24 ++- server/netrc.go | 269 ++++++++++++++++++++++++++ 7 files changed, 631 insertions(+), 3 deletions(-) create mode 100644 server/grpc_asset_netrc_test.go create mode 100644 server/netrc.go diff --git a/README.md b/README.md index c168a8b94..76c2a1649 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,15 @@ which can be enabled with the `--experimental_remote_asset_api` flag. To use this with Bazel, specify [--experimental_remote_downloader=grpc://replace-with-your.host:port](https://docs.bazel.build/versions/master/command-line-reference.html#flag--experimental_remote_downloader). +#### HTTP authentication for remote asset + +When fetching `http` or `https` URIs, bazel-remote can use credentials from +the `.netrc` file in the home directory of the bazel-remote process. Set the +`NETRC` environment variable to use a different file. Credentials from `.netrc` +are only used when the request does not already include an `Authorization` +header and the URI does not include user info. `default` entries in `.netrc` +are ignored, since they might mistakenly exfiltrate credentials. + ### Byte Stream compressed-blobs This version of bazel-remote supports the diff --git a/server/BUILD.bazel b/server/BUILD.bazel index 9411df57c..c14733102 100644 --- a/server/BUILD.bazel +++ b/server/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "grpc_cas.go", "grpc_idle_timeout.go", "http.go", + "netrc.go", ], importpath = "github.com/buchgr/bazel-remote/v2/server", visibility = ["//visibility:public"], @@ -49,6 +50,7 @@ go_library( go_test( name = "go_default_test", srcs = [ + "grpc_asset_netrc_test.go", "grpc_asset_test.go", "grpc_cas_spliceblobs_test.go", "grpc_test.go", diff --git a/server/grpc.go b/server/grpc.go index c4300683e..04617305c 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -43,6 +43,7 @@ type grpcServer struct { depsCheck bool mangleACKeys bool maxCasBlobSizeBytes int64 + netrcCredentials netrcFileCredentials } var readOnlyMethods = map[string]struct{}{ @@ -94,6 +95,7 @@ func ServeGRPC(l net.Listener, srv *grpc.Server, pb.RegisterContentAddressableStorageServer(srv, s) bytestream.RegisterByteStreamServer(srv, s) if enableRemoteAssetAPI { + s.netrcCredentials = loadNetrcCredentials(e) asset.RegisterFetchServer(srv, s) } diff --git a/server/grpc_asset.go b/server/grpc_asset.go index dad682314..fb67c3411 100644 --- a/server/grpc_asset.go +++ b/server/grpc_asset.go @@ -85,7 +85,9 @@ func (s *grpcServer) FetchBlob(ctx context.Context, req *asset.FetchBlobRequest) if strings.HasPrefix(q.Name, QualifierHTTPHeaderPrefix) { key := q.Name[len(QualifierHTTPHeaderPrefix):] - globalHeader[key] = strings.Split(q.Value, ",") + for _, value := range strings.Split(q.Value, ",") { + globalHeader.Add(key, value) + } continue } else if strings.HasPrefix(q.Name, QualifierHTTPHeaderUrlPrefix) { idxAndKey := q.Name[len(QualifierHTTPHeaderUrlPrefix):] @@ -212,6 +214,7 @@ func (s *grpcServer) fetchItem(ctx context.Context, uri string, headers http.Hea } req.Header = headers + applyNetrcCredentials(req, s.netrcCredentials) resp, err := http.DefaultClient.Do(req) if err != nil { diff --git a/server/grpc_asset_netrc_test.go b/server/grpc_asset_netrc_test.go new file mode 100644 index 000000000..1986ad32a --- /dev/null +++ b/server/grpc_asset_netrc_test.go @@ -0,0 +1,323 @@ +package server + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/buchgr/bazel-remote/v2/cache/disk" + asset "github.com/buchgr/bazel-remote/v2/genproto/build/bazel/remote/asset/v1" + + "google.golang.org/grpc/codes" + + testutils "github.com/buchgr/bazel-remote/v2/utils" +) + +func TestAssetFetchBlobUsesNetrcCredentials(t *testing.T) { + blobDir := t.TempDir() + diskCache, err := disk.New(blobDir, 1024*1024, disk.WithAccessLogger(testutils.NewSilentLogger())) + if err != nil { + t.Fatal(err) + } + + ts := newAuthenticatedTestGetServer("alice", "secret") + defer ts.srv.Close() + + netrc := fmt.Sprintf("machine %s login alice password secret\n", strings.Split(strings.TrimPrefix(ts.srv.URL, "http://"), ":")[0]) + writeNetrcFile(t, netrc) + srv := newAssetFetchTestGRPCServer(diskCache) + srv.netrcCredentials = loadNetrcCredentials(testutils.NewSilentLogger()) + + resp, err := srv.FetchBlob(ctx, &asset.FetchBlobRequest{ + Uris: []string{ts.srv.URL + "/" + ts.path}, + }) + if err != nil { + t.Fatal(err) + } + + if resp.Status.GetCode() != int32(codes.OK) { + t.Fatalf("expected successful fetch, got status %d", resp.Status.GetCode()) + } +} + +func TestAssetFetchBlobDoesNotUseNetrcWhenAuthorizationQualifierExists(t *testing.T) { + blobDir := t.TempDir() + diskCache, err := disk.New(blobDir, 1024*1024, disk.WithAccessLogger(testutils.NewSilentLogger())) + if err != nil { + t.Fatal(err) + } + + var gotAuthorization string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuthorization = r.Header.Get("Authorization") + if gotAuthorization != "Bearer token" { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + _, _ = w.Write([]byte("blob")) + })) + defer ts.Close() + + netrc := fmt.Sprintf("machine %s login alice password secret\n", strings.Split(strings.TrimPrefix(ts.URL, "http://"), ":")[0]) + writeNetrcFile(t, netrc) + srv := newAssetFetchTestGRPCServer(diskCache) + srv.netrcCredentials = loadNetrcCredentials(testutils.NewSilentLogger()) + + resp, err := srv.FetchBlob(ctx, &asset.FetchBlobRequest{ + Uris: []string{ts.URL + "/blob"}, + Qualifiers: []*asset.Qualifier{ + { + Name: "http_header:authorization", + Value: "Bearer token", + }, + }, + }) + if err != nil { + t.Fatal(err) + } + + if resp.Status.GetCode() != int32(codes.OK) { + t.Fatalf("expected successful fetch, got status %d", resp.Status.GetCode()) + } + if gotAuthorization != "Bearer token" { + t.Fatalf("Authorization header = %q, want %q", gotAuthorization, "Bearer token") + } +} + +func TestLookupNetrcCredentialsIgnoresDefault(t *testing.T) { + netrc := strings.Join([]string{ + "machine Example.COM login alice password secret", + "default login guest password guest-secret", + "", + }, "\n") + writeNetrcFile(t, netrc) + + logger := &recordingLogger{} + loadedCredentials := loadNetrcCredentials(logger) + creds := lookupNetrcCredentials("example.com", loadedCredentials) + if creds == nil { + t.Fatal("expected credentials for matching host") + } + if creds.login != "alice" || creds.password != "secret" { + t.Fatalf("lookupNetrcCredentials returned %+v, want login/password alice/secret", creds) + } + + if creds := lookupNetrcCredentials("missing.example", loadedCredentials); creds != nil { + t.Fatalf("lookupNetrcCredentials returned %+v for missing host, want nil credentials", creds) + } + + if got := logger.joinedMessages(); !strings.Contains(got, ".netrc default entry found; explicitly ignoring default credentials") { + t.Fatalf("logger messages = %q, want warning about ignored default credentials", got) + } +} + +func TestLookupNetrcCredentialsParsesTokensSplitAcrossLines(t *testing.T) { + netrc := strings.Join([]string{ + "machine repo.example", + "login alice", + "password secret", + "", + }, "\n") + writeNetrcFile(t, netrc) + + loadedCredentials := loadNetrcCredentials(testutils.NewSilentLogger()) + creds := lookupNetrcCredentials("repo.example", loadedCredentials) + if creds == nil { + t.Fatal("expected credentials") + } + if creds.login != "alice" || creds.password != "secret" { + t.Fatalf("lookupNetrcCredentials returned %+v, want login/password alice/secret", creds) + } +} + +func TestLookupNetrcCredentialsKeepsCompleteEntryAtEOF(t *testing.T) { + netrc := strings.Join([]string{ + "machine repo.example", + "login alice", + "password secret", + }, "\n") + writeNetrcFile(t, netrc) + + loadedCredentials := loadNetrcCredentials(testutils.NewSilentLogger()) + creds := lookupNetrcCredentials("repo.example", loadedCredentials) + if creds == nil { + t.Fatal("expected credentials") + } + if creds.login != "alice" || creds.password != "secret" { + t.Fatalf("lookupNetrcCredentials returned %+v, want login/password alice/secret", creds) + } +} + +func TestLookupNetrcCredentialsUsesLastDuplicateMachineEntry(t *testing.T) { + netrc := strings.Join([]string{ + "machine repo.example login alice password old", + "machine repo.example login bob password new", + "", + }, "\n") + writeNetrcFile(t, netrc) + + loadedCredentials := loadNetrcCredentials(testutils.NewSilentLogger()) + creds := lookupNetrcCredentials("repo.example", loadedCredentials) + if creds == nil { + t.Fatal("expected credentials") + } + if creds.login != "bob" || creds.password != "new" { + t.Fatalf("lookupNetrcCredentials returned %+v, want login/password bob/new", creds) + } +} + +func TestLookupNetrcCredentialsIgnoresTokensInsideMacdef(t *testing.T) { + netrc := strings.Join([]string{ + "machine repo.example login alice password secret", + "macdef init", + "machine ignored.example login mallory password secret", + "", + "default login guest password guest-secret", + "", + }, "\n") + writeNetrcFile(t, netrc) + + loadedCredentials := loadNetrcCredentials(testutils.NewSilentLogger()) + creds := lookupNetrcCredentials("ignored.example", loadedCredentials) + if creds != nil { + t.Fatalf("lookupNetrcCredentials returned %+v, want nil credentials", creds) + } +} + +func TestLookupNetrcCredentialsIgnoresComments(t *testing.T) { + netrc := strings.Join([]string{ + "# machine ignored.example login mallory password secret", + " # machine also-ignored.example login mallory password secret", + "machine repo.example login alice password secret", + "", + }, "\n") + writeNetrcFile(t, netrc) + + loadedCredentials := loadNetrcCredentials(testutils.NewSilentLogger()) + creds := lookupNetrcCredentials("repo.example", loadedCredentials) + if creds == nil { + t.Fatal("expected credentials") + } + if creds.login != "alice" || creds.password != "secret" { + t.Fatalf("lookupNetrcCredentials returned %+v, want login/password alice/secret", creds) + } + + creds = lookupNetrcCredentials("ignored.example", loadedCredentials) + if creds != nil { + t.Fatalf("lookupNetrcCredentials returned %+v, want nil credentials", creds) + } + + creds = lookupNetrcCredentials("also-ignored.example", loadedCredentials) + if creds != nil { + t.Fatalf("lookupNetrcCredentials returned %+v, want nil credentials", creds) + } +} + +func TestLookupNetrcCredentialsPreservesHashInsidePassword(t *testing.T) { + netrc := "machine repo.example login alice password #secret\n" + writeNetrcFile(t, netrc) + + loadedCredentials := loadNetrcCredentials(testutils.NewSilentLogger()) + creds := lookupNetrcCredentials("repo.example", loadedCredentials) + if creds == nil { + t.Fatal("expected credentials") + } + if creds.login != "alice" || creds.password != "#secret" { + t.Fatalf("lookupNetrcCredentials returned %+v, want login/password alice/#secret", creds) + } +} + +func TestLookupNetrcCredentialsLogsMalformedEntriesAndKeepsValidEntries(t *testing.T) { + netrc := strings.Join([]string{ + "machine broken.example login", + "machine repo.example login alice password secret", + "", + }, "\n") + writeNetrcFile(t, netrc) + + logger := &recordingLogger{} + loadedCredentials := loadNetrcCredentials(logger) + creds := lookupNetrcCredentials("repo.example", loadedCredentials) + if creds == nil { + t.Fatal("expected credentials") + } + if creds.login != "alice" || creds.password != "secret" { + t.Fatalf("lookupNetrcCredentials returned %+v, want login/password alice/secret", creds) + } + if got := logger.joinedMessages(); !strings.Contains(got, "malformed .netrc: missing login value on line 1") { + t.Fatalf("logger messages = %q, want warning about malformed entry", got) + } +} + +func TestApplyNetrcCredentialsDoesNotOverrideAuthorizationHeader(t *testing.T) { + netrc := "machine repo.example login alice password secret\n" + writeNetrcFile(t, netrc) + + loadedCredentials := loadNetrcCredentials(testutils.NewSilentLogger()) + req, err := http.NewRequest(http.MethodGet, "https://repo.example/file", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer token") + + applyNetrcCredentials(req, loadedCredentials) + + if got := req.Header.Get("Authorization"); got != "Bearer token" { + t.Fatalf("Authorization header = %q, want %q", got, "Bearer token") + } +} + +func TestApplyNetrcCredentialsDoesNotOverrideURLUserinfo(t *testing.T) { + netrc := "machine repo.example login alice password secret\n" + writeNetrcFile(t, netrc) + + loadedCredentials := loadNetrcCredentials(testutils.NewSilentLogger()) + req, err := http.NewRequest(http.MethodGet, "https://bob:manual@repo.example/file", nil) + if err != nil { + t.Fatal(err) + } + + applyNetrcCredentials(req, loadedCredentials) + + if got := req.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization header = %q, want empty header", got) + } +} + +func writeNetrcFile(t *testing.T, contents string) string { + t.Helper() + + netrcFile := filepath.Join(t.TempDir(), ".netrc") + if err := os.WriteFile(netrcFile, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + + t.Setenv("NETRC", netrcFile) + + return netrcFile +} + +func newAssetFetchTestGRPCServer(diskCache disk.Cache) *grpcServer { + return &grpcServer{ + cache: diskCache, + accessLogger: testutils.NewSilentLogger(), + errorLogger: testutils.NewSilentLogger(), + } +} + +type recordingLogger struct { + messages []string +} + +func (l *recordingLogger) Printf(format string, v ...interface{}) { + l.messages = append(l.messages, fmt.Sprintf(format, v...)) +} + +func (l *recordingLogger) joinedMessages() string { + return strings.Join(l.messages, "\n") +} diff --git a/server/grpc_asset_test.go b/server/grpc_asset_test.go index d2e20304f..207734b38 100644 --- a/server/grpc_asset_test.go +++ b/server/grpc_asset_test.go @@ -66,8 +66,11 @@ func TestAssetFetchBlob(t *testing.T) { type testGetServer struct { srv *httptest.Server - blob []byte - path string + blob []byte + path string + username string + password string + requireBasicAuth bool } func (s *testGetServer) handler(w http.ResponseWriter, r *http.Request) { @@ -82,6 +85,15 @@ func (s *testGetServer) handler(w http.ResponseWriter, r *http.Request) { return } + if s.requireBasicAuth { + username, password, ok := r.BasicAuth() + if !ok || username != s.username || password != s.password { + w.Header().Set("WWW-Authenticate", `Basic realm="test"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + } + w.WriteHeader(http.StatusOK) if r.Method == http.MethodHead { @@ -104,3 +116,11 @@ func newTestGetServer() *testGetServer { return &ts } + +func newAuthenticatedTestGetServer(username, password string) *testGetServer { + ts := newTestGetServer() + ts.username = username + ts.password = password + ts.requireBasicAuth = true + return ts +} diff --git a/server/netrc.go b/server/netrc.go new file mode 100644 index 000000000..1bff34493 --- /dev/null +++ b/server/netrc.go @@ -0,0 +1,269 @@ +package server + +import ( + "bufio" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/buchgr/bazel-remote/v2/cache" +) + +type netrcCredentials struct { + login string + password string +} + +type netrcEntry struct { + machine string + login string + password string +} + +type netrcFileCredentials struct { + hosts map[string]netrcCredentials +} + +func applyNetrcCredentials(req *http.Request, loadedCredentials netrcFileCredentials) { + if req.URL == nil { + return + } + + if hasAuthorizationHeader(req.Header) { + return + } + + if req.URL.User != nil { + return + } + + creds := lookupNetrcCredentials(req.URL.Hostname(), loadedCredentials) + if creds == nil { + return + } + + req.SetBasicAuth(creds.login, creds.password) +} + +func hasAuthorizationHeader(header http.Header) bool { + for key, values := range header { + if !strings.EqualFold(key, "Authorization") { + continue + } + + for _, value := range values { + if value != "" { + return true + } + } + } + + return false +} + +func loadNetrcCredentials(logger cache.Logger) netrcFileCredentials { + path, err := netrcPath() + if err != nil { + if logger != nil { + logger.Printf("failed to read .netrc credentials: %v", err) + } + return emptyNetrcFileCredentials() + } + if path == "" { + return emptyNetrcFileCredentials() + } + + return loadNetrcCredentialsForPath(path, logger) +} + +func lookupNetrcCredentials(host string, credsByPath netrcFileCredentials) *netrcCredentials { + return credsByPath.lookup(host) +} + +func loadNetrcCredentialsForPath(path string, logger cache.Logger) netrcFileCredentials { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return emptyNetrcFileCredentials() + } + if logger != nil { + logger.Printf("failed to read .netrc credentials: %v", err) + } + return emptyNetrcFileCredentials() + } + + entries := parseNetrcEntries(string(data), logger) + + return newNetrcFileCredentials(entries) +} + +func netrcPath() (string, error) { + if path := os.Getenv("NETRC"); path != "" { + return path, nil + } + + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + if home == "" { + return "", nil + } + + return filepath.Join(home, ".netrc"), nil +} + +func parseNetrcEntries(data string, logger cache.Logger) []netrcEntry { + scanner := bufio.NewScanner(strings.NewReader(data)) + + var ( + entries []netrcEntry + entry netrcEntry + inMacro bool + lineNum int + ) + + commitEntry := func() { + if entry.machine == "" || entry.login == "" || entry.password == "" { + return + } + + entries = append(entries, entry) + entry = netrcEntry{} + } + + for scanner.Scan() { + lineNum++ + line := scanner.Text() + + if inMacro { + if strings.TrimSpace(line) == "" { + inMacro = false + } + continue + } + + fields := netrcFields(line) + for i := 0; i < len(fields); { + switch fields[i] { + case "machine": + if i+1 >= len(fields) { + logMalformedNetrc(logger, "missing machine name", lineNum) + i++ + continue + } + + entry = netrcEntry{machine: fields[i+1]} + i += 2 + case "default": + if logger != nil { + logger.Printf(".netrc default entry found; explicitly ignoring default credentials") + } + entry = netrcEntry{} + i++ + case "login": + if i+1 >= len(fields) { + logMalformedNetrc(logger, "missing login value", lineNum) + i++ + continue + } + + entry.login = fields[i+1] + i += 2 + case "password": + if i+1 >= len(fields) { + logMalformedNetrc(logger, "missing password value", lineNum) + i++ + continue + } + + entry.password = fields[i+1] + i += 2 + case "account": + if i+1 >= len(fields) { + logMalformedNetrc(logger, "missing account value", lineNum) + i++ + continue + } + + i += 2 + case "macdef": + if i+1 >= len(fields) { + logMalformedNetrc(logger, "missing macro name", lineNum) + i++ + continue + } + + i += 2 + inMacro = true + default: + if i+1 < len(fields) { + i += 2 + } else { + i++ + } + } + + commitEntry() + if inMacro { + break + } + } + } + + commitEntry() + + if err := scanner.Err(); err != nil { + if logger != nil { + logger.Printf("failed to read .netrc credentials: %v", err) + } + } + + return entries +} + +func emptyNetrcFileCredentials() netrcFileCredentials { + return netrcFileCredentials{hosts: make(map[string]netrcCredentials)} +} + +func netrcFields(line string) []string { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + return nil + } + + return strings.Fields(line) +} + +func logMalformedNetrc(logger cache.Logger, msg string, lineNum int) { + if logger != nil { + logger.Printf("malformed .netrc: %s on line %d", msg, lineNum) + } +} + +func newNetrcFileCredentials(entries []netrcEntry) netrcFileCredentials { + credsByHost := netrcFileCredentials{ + hosts: make(map[string]netrcCredentials, len(entries)), + } + + for _, entry := range entries { + creds := netrcCredentials{ + login: entry.login, + password: entry.password, + } + + key := strings.ToLower(entry.machine) + credsByHost.hosts[key] = creds + } + + return credsByHost +} + +func (c netrcFileCredentials) lookup(host string) *netrcCredentials { + if creds, ok := c.hosts[strings.ToLower(host)]; ok { + hostCreds := creds + return &hostCreds + } + + return nil +}