Fix races in AWS compose up

.github/workflows/go.yml

@@ -33,7 +33,7 @@ jobs:
         run: echo "name=$(echo '${{ github.ref_name }}' | tr '/' '-')" >> "$GITHUB_OUTPUT"
 
       - name: Run Go unit tests
-        run: go test -test.short -v -coverprofile=coverage.out -covermode=atomic ./...
+        run: go test -race -test.short -v -coverprofile=coverage.out -covermode=atomic ./...
         working-directory: src
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # to avoid GH rate limits
@@ -205,7 +205,7 @@ jobs:
   #         cache-dependency-path: src/go.sum
 
   #     - name: Run sanity tests
-  #       run: go run ./cmd/cli -C testdata/sanity compose up --debug
+  #       run: go run -race ./cmd/cli -C testdata/sanity compose up --debug
   #       working-directory: src
 
   go-playground-test:
@@ -223,27 +223,27 @@ jobs:
           cache-dependency-path: src/go.sum
 
       - name: Login using GitHub token
-        run: go run ./cmd/cli login --debug
+        run: go run -race ./cmd/cli login --debug
         working-directory: src
 
       - name: Add dummy config
         id: add-dummy-config
-        run: echo blah | go run ./cmd/cli -C testdata/sanity config set --provider=defang -n dummy --debug
+        run: echo blah | go run -race ./cmd/cli -C testdata/sanity config set --provider=defang -n dummy --debug
         working-directory: src
 
       - name: Run sanity tests UP
         continue-on-error: true # until we have multi-project support in playground
-        run: go run ./cmd/cli -C testdata/sanity compose up --provider=defang --debug
+        run: go run -race ./cmd/cli -C testdata/sanity compose up --provider=defang --debug
         working-directory: src
 
       - name: Run sanity tests DOWN
         continue-on-error: true # until we have multi-project support in playground
-        run: go run ./cmd/cli -C testdata/sanity compose down --provider=defang --detach --debug
+        run: go run -race ./cmd/cli -C testdata/sanity compose down --provider=defang --detach --debug
         working-directory: src
 
       - name: Remove dummy config
         if: steps.add-dummy-config.conclusion == 'success' # only clean up if the config was added
-        run: go run ./cmd/cli -C testdata/sanity config rm --provider=defang -n dummy --debug
+        run: go run -race ./cmd/cli -C testdata/sanity config rm --provider=defang -n dummy --debug
         working-directory: src
 
   build-and-sign:

src/Makefile

@@ -72,11 +72,11 @@ install: $(BINARY_NAME)
 
 .PHONY: test
 test: tidy $(GO) $(PROTOS)
-	set -o pipefail && go test -short ./... | $(FAIL-RED)
+	set -o pipefail && go test -race -short ./... | $(FAIL-RED)
 
 .PHONY: integ
 integ: $(GO) $(PROTOS)
-	set -o pipefail && go test -v -tags=integration ./... | $(FAIL-RED)
+	set -o pipefail && go test -race -v -tags=integration ./... | $(FAIL-RED)
 
 .PHONY: linux-amd64
 linux-amd64: $(GO) test

src/pkg/auth/auth_test.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 )
@@ -76,9 +77,9 @@ func TestPoll(t *testing.T) {
 	})
 
 	t.Run("5xx retries until context cancelled", func(t *testing.T) {
-		calls := 0
+		var calls atomic.Int32
 		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			calls++
+			calls.Add(1)
 			http.Error(w, "internal error", http.StatusInternalServerError)
 		}))
 		t.Cleanup(server.Close)
@@ -94,15 +95,15 @@ func TestPoll(t *testing.T) {
 		if err == nil {
 			t.Error("expected error after context cancellation")
 		}
-		if calls < 2 {
+		if calls.Load() < 2 {
 			t.Error("expected server to be called at least twice")
 		}
 	})
 
 	t.Run("408 retries until context cancelled", func(t *testing.T) {
-		calls := 0
+		var calls atomic.Int32
 		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			calls++
+			calls.Add(1)
 			w.WriteHeader(http.StatusRequestTimeout)
 		}))
 		t.Cleanup(server.Close)
@@ -118,15 +119,15 @@ func TestPoll(t *testing.T) {
 		if err == nil {
 			t.Error("expected error after context cancellation")
 		}
-		if calls < 2 {
-			t.Errorf("expected at least 2 calls for timeout retries, got %d", calls)
+		if calls.Load() < 2 {
+			t.Errorf("expected at least 2 calls for timeout retries, got %d", calls.Load())
 		}
 	})
 
 	t.Run("4xx does not retry", func(t *testing.T) {
-		calls := 0
+		var calls atomic.Int32
 		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			calls++
+			calls.Add(1)
 			http.Error(w, "bad request", http.StatusBadRequest)
 		}))
 		t.Cleanup(server.Close)
@@ -139,8 +140,8 @@ func TestPoll(t *testing.T) {
 		if err == nil {
 			t.Error("expected error for 4xx response")
 		}
-		if calls != 1 {
-			t.Errorf("expected exactly 1 call (no retry for 4xx), got %d", calls)
+		if calls.Load() != 1 {
+			t.Errorf("expected exactly 1 call (no retry for 4xx), got %d", calls.Load())
 		}
 	})
 }
@@ -226,9 +227,9 @@ func TestPollForAuthCode(t *testing.T) {
 func TestPoll_ContextDone(t *testing.T) {
 	for _, name := range []string{"context.Canceled", "context.DeadlineExceeded"} {
 		t.Run(name, func(t *testing.T) {
-			calls := 0
+			var calls atomic.Int32
 			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				calls++
+				calls.Add(1)
 				http.Error(w, "server error", http.StatusInternalServerError)
 			}))
 			t.Cleanup(server.Close)
@@ -253,8 +254,8 @@ func TestPoll_ContextDone(t *testing.T) {
 			if err == nil {
 				t.Fatal("expected error for done context")
 			}
-			if calls > 1 {
-				t.Errorf("Poll must not retry with done context, got %d server calls", calls)
+			if calls.Load() > 1 {
+				t.Errorf("Poll must not retry with done context, got %d server calls", calls.Load())
 			}
 		})
 	}

src/pkg/cli/client/byoc/aws/byoc.go

@@ -835,7 +835,7 @@ func (b *ByocAws) queryOrTailLogs(ctx context.Context, cwClient cw.LogsClient, r
 }
 
 func (b *ByocAws) makeLogGroupARN(name string) string {
-	return b.driver.MakeARN("logs", "log-group:"+name)
+	return b.driver.MakeRegionalARN("logs", "log-group:"+name)
 }
 
 func (b *ByocAws) getLogGroupInputs(etag types.ETag, projectName, service, filter string, logType logs.LogType) []cw.LogGroupInput {

src/pkg/cli/client/byoc/aws/list.go

@@ -10,7 +10,6 @@ import (
 	"github.com/DefangLabs/defang/src/pkg/cli/client/byoc"
 	"github.com/DefangLabs/defang/src/pkg/cli/client/byoc/state"
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws"
-	"github.com/DefangLabs/defang/src/pkg/clouds/aws/region"
 	"github.com/DefangLabs/defang/src/pkg/term"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -135,7 +134,7 @@ func (b *ByocAws) listPulumiStacksAllRegions(ctx context.Context, s3client S3Cli
 			// GetBucketLocation returns empty string for us-east-1 buckets
 			bucketRegion := aws.Region(locationOutput.LocationConstraint)
 			if bucketRegion == "" {
-				bucketRegion = region.USEast1
+				bucketRegion = "us-east-1"
 			}
 
 			wg.Add(1)

src/pkg/cli/client/byoc/baseclient.go

@@ -73,10 +73,6 @@ func (b *ByocBaseClient) GetStackName() string {
 	return b.PulumiStack
 }
 
-func (b *ByocBaseClient) Debug(context.Context, *defangv1.DebugRequest) (*defangv1.DebugResponse, error) {
-	return nil, client.ErrNotImplemented("AI debugging is not yet supported for BYOC")
-}
-
 func (b *ByocBaseClient) SetCanIUseConfig(quotas *defangv1.CanIUseResponse) {
 	b.CanIUseConfig.AllowGPU = quotas.Gpu
 	b.CanIUseConfig.AllowScaling = quotas.AllowScaling

src/pkg/cli/client/byoc/gcp/byoc.go

@@ -123,7 +123,6 @@ type ByocGcp struct {
 
 	bucket               string
 	cdServiceAccount     string
-	setupDone            bool
 	uploadServiceAccount string
 	delegateDomainZone   string
 
@@ -170,7 +169,7 @@ func (*ByocGcp) Driver() string {
 }
 
 func (b *ByocGcp) SetUpCD(ctx context.Context, force bool) error {
-	if b.setupDone {
+	if b.SetupDone {
 		return nil
 	}
 	// TODO: Handle project creation flow
@@ -286,7 +285,7 @@ func (b *ByocGcp) SetUpCD(ctx context.Context, force bool) error {
 	// 5. Setup Cloud Run Job
 	term.Debugf("Using CD image: %q", b.CDImage)
 
-	b.setupDone = true
+	b.SetupDone = true
 	return nil
 }
 

src/pkg/cli/client/state.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 
 	"github.com/google/uuid"
@@ -15,6 +16,7 @@ var (
 	StateDir  = filepath.Join(stateDir, "defang")
 	statePath = filepath.Join(StateDir, "state.json")
 	state     State
+	stateOnce sync.Once
 )
 
 type State struct {
@@ -32,6 +34,12 @@ func initState(path string) State {
 	return state
 }
 
+func ensureState() {
+	stateOnce.Do(func() {
+		state = initState(statePath)
+	})
+}
+
 func (state State) write(path string) error {
 	if bytes, err := json.MarshalIndent(state, "", "  "); err != nil {
 		return err
@@ -52,14 +60,16 @@ func (state State) termsAccepted() bool {
 }
 
 func GetAnonID() string {
-	state = initState(statePath)
+	ensureState()
 	return state.AnonID
 }
 
 func AcceptTerms() error {
+	ensureState()
 	return state.acceptTerms()
 }
 
 func TermsAccepted() bool {
+	ensureState()
 	return state.termsAccepted()
 }

src/pkg/cli/tail.go

@@ -100,10 +100,18 @@ var originalLocal = time.Local
 
 // SetUTCMode sets the local time zone to UTC.
 func SetUTCMode(enable bool) {
+	// The conditional guards avoid writing to time.Local when it already has
+	// the correct value; this prevents a data race with stdlib reads (e.g.
+	// time.Now) that we cannot synchronize because the read side is a plain
+	// load inside the Go runtime, eg. net/http.(*conn).setState()
 	if enable {
-		time.Local = time.UTC
+		if time.Local != time.UTC {
+			time.Local = time.UTC
+		}
 	} else {
-		time.Local = originalLocal
+		if time.Local != originalLocal {
+			time.Local = originalLocal
+		}
 	}
 }
 

src/pkg/clouds/aws/codebuild/cfn/setup.go

@@ -7,11 +7,11 @@ import (
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
-	common "github.com/DefangLabs/defang/src/pkg/clouds/aws"
+	"github.com/DefangLabs/defang/src/pkg/clouds/aws"
 	awscodebuild "github.com/DefangLabs/defang/src/pkg/clouds/aws/codebuild"
-	"github.com/DefangLabs/defang/src/pkg/clouds/aws/region"
 	"github.com/DefangLabs/defang/src/pkg/term"
 	"github.com/aws/aws-sdk-go-v2/service/cloudformation"
 	cfnTypes "github.com/aws/aws-sdk-go-v2/service/cloudformation/types"
@@ -22,18 +22,20 @@ import (
 type AwsCfn struct {
 	awscodebuild.AwsCodeBuild
 	stackName string
+	fillOnce  sync.Once
+	fillErr   error
 }
 
 const stackTimeout = time.Minute * 3
 
-func New(stack string, region region.Region) *AwsCfn {
+func New(stack string, region aws.Region) *AwsCfn {
 	if stack == "" {
 		panic("stack must be set")
 	}
 	return &AwsCfn{
 		stackName: stack,
 		AwsCodeBuild: awscodebuild.AwsCodeBuild{
-			Aws:          common.Aws{Region: region},
+			Aws:          aws.Aws{Region: region},
 			RetainBucket: true,
 		},
 	}
@@ -183,6 +185,13 @@ func (a *AwsCfn) upsertStackAndWait(ctx context.Context, templateBody []byte, fo
 type ErrStackNotFoundException = cfnTypes.StackNotFoundException
 
 func (a *AwsCfn) FillOutputs(ctx context.Context) error {
+	a.fillOnce.Do(func() {
+		a.fillErr = a.describeAndFillOutputs(ctx)
+	})
+	return a.fillErr
+}
+
+func (a *AwsCfn) describeAndFillOutputs(ctx context.Context) error {
 	cfn, err := a.newClient(ctx)
 	if err != nil {
 		return err
@@ -227,7 +236,7 @@ func (a *AwsCfn) fillWithOutputs(dso *cloudformation.DescribeStacksOutput) error
 	}
 
 	if a.AccountID == "" && a.LogGroupARN != "" {
-		a.AccountID = common.GetAccountID(a.LogGroupARN)
+		a.AccountID = aws.GetAccountID(a.LogGroupARN)
 	}
 	return nil
 }

src/pkg/clouds/aws/codebuild/cfn/setup_test.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 
 	"github.com/DefangLabs/defang/src/pkg"
-	"github.com/DefangLabs/defang/src/pkg/clouds/aws/region"
 )
 
 func TestCloudFormation(t *testing.T) {
@@ -15,7 +14,7 @@ func TestCloudFormation(t *testing.T) {
 	}
 
 	user := pkg.GetCurrentUser() // avoid conflict with other users in the same account
-	aws := New("crun-test-"+user, region.Region("us-west-2"))
+	aws := New("crun-test-"+user, "us-west-2")
 	if aws == nil {
 		t.Fatal("aws is nil")
 	}

src/pkg/clouds/aws/codebuild/common.go

@@ -1,8 +1,6 @@
 package codebuild
 
 import (
-	"strings"
-
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws"
 )
 
@@ -20,14 +18,3 @@ type AwsCodeBuild struct {
 	ProjectName  string // CodeBuild project name
 	RetainBucket bool   // CloudFormation template input parameter
 }
-
-func (a *AwsCodeBuild) MakeARN(service, resource string) string {
-	return strings.Join([]string{
-		"arn",
-		"aws",
-		service,
-		string(a.Region),
-		a.AccountID,
-		resource,
-	}, ":")
-}