diff --git a/src/cmd/cli/command/compose.go b/src/cmd/cli/command/compose.go index 95ab0a572..ec2f2836f 100644 --- a/src/cmd/cli/command/compose.go +++ b/src/cmd/cli/command/compose.go @@ -502,7 +502,7 @@ func newTailOptionsForDown(stack, deployment string, since time.Time) cli.TailOp return nil // keep tailing logs }, Verbose: global.Verbose, - LogType: logs.LogTypeAll, + LogType: logs.LogTypeCD, } } diff --git a/src/pkg/cli/cd.go b/src/pkg/cli/cd.go index b4de0e109..9204cab24 100644 --- a/src/pkg/cli/cd.go +++ b/src/pkg/cli/cd.go @@ -31,7 +31,7 @@ func CdCommand(ctx context.Context, projectName string, provider client.Provider } var statesUrl, eventsUrl string - if _, ok := provider.(*client.PlaygroundProvider); !ok { // Do not need upload URLs for Playground + if _, ok := provider.(*client.PlaygroundProvider); !ok && command != client.CdCommandList { // Do not need upload URLs for Playground/List var err error statesUrl, eventsUrl, err = GetStatesAndEventsUploadUrls(ctx, projectName, provider, fabric) if err != nil { @@ -97,7 +97,7 @@ func CdCommandAndTail(ctx context.Context, provider client.Provider, projectName options := TailOptions{ Deployment: etag, - LogType: logs.LogTypeBuild, + LogType: logs.LogTypeCD, Since: since, Verbose: verbose, Stack: provider.GetStackName(), diff --git a/src/pkg/cli/client/byoc/aws/byoc.go b/src/pkg/cli/client/byoc/aws/byoc.go index bb7a5b4c2..4683c1093 100644 --- a/src/pkg/cli/client/byoc/aws/byoc.go +++ b/src/pkg/cli/client/byoc/aws/byoc.go @@ -55,7 +55,7 @@ type ByocAws struct { cdEtag types.ETag cdStart time.Time - cdTaskArn ecs.TaskArn + cdTaskArn ecs.TaskArn // for GetDeploymentStatus needDockerHubCreds bool } @@ -699,22 +699,27 @@ func (b *ByocAws) QueryLogs(ctx context.Context, req *defangv1.TailRequest) (ite return nil, AnnotateAwsError(err) } + parser := &logEventParser{ + etag: req.Etag, + services: req.Services, + } + // How to tail multiple tasks/services at once? - // * Etag is invalid: treat Etag as CD task ID and tail only that task's logs - // * No Etag, no services: tail all tasks/services - // * No Etag, service: tail all tasks/services with that service name - // * Etag, no services: tail all tasks/services with that Etag - // * Etag, service: tail that task/service + // * Etag matches CD task: tail only that CD task's logs + // * Etag is invalid: treat Etag as a task ID and tail only that task's logs + // * No Etag, no services: tail all tasks/services + // * No Etag, service: tail all tasks/services with that service name + // * Valid Etag, no services: tail all tasks/services with that Etag + // * Valid Etag, service: tail that task/service var logSeq iter.Seq2[cw.LogEvent, error] - etag, err := types.ParseEtag(req.Etag) - if err != nil && req.Etag != "" { - // Assume invalid "etag" is the task ID of the CD task - cdSeq, err := b.queryOrTailCdLogs(ctx, cwClient, req) + if taskID := b.deriveTaskID(req.Etag); taskID != "" && logs.LogType(req.LogType) == logs.LogTypeCD { + cdSeq, err := b.queryOrTailLogsByTaskID(ctx, cwClient, req, taskID) if err != nil { return nil, AnnotateAwsError(err) } logSeq = cw.Flatten(cdSeq) // No need to filter events by etag because we only show logs from the specified task ID + parser.etag = "" // disable etag filtering } else { logSeq, err = b.queryOrTailLogs(ctx, cwClient, req) if err != nil { @@ -722,10 +727,6 @@ func (b *ByocAws) QueryLogs(ctx context.Context, req *defangv1.TailRequest) (ite } } - parser := &logEventParser{ - etag: etag, - services: req.Services, - } return func(yield func(*defangv1.TailResponse, error) bool) { for event, err := range logSeq { if err != nil { @@ -750,21 +751,34 @@ func (b *ByocAws) QueryLogs(ctx context.Context, req *defangv1.TailRequest) (ite }, nil } -func (b *ByocAws) queryOrTailCdLogs(ctx context.Context, cwClient cw.LogsClient, req *defangv1.TailRequest) (iter.Seq2[[]cw.LogEvent, error], error) { - var err error - b.cdTaskArn, err = b.driver.GetTaskArn(req.Etag) // only fails on missing task ID - if err != nil { - return nil, err +func (b *ByocAws) queryOrTailLogsByTaskID(ctx context.Context, cwClient cw.LogsClient, req *defangv1.TailRequest, taskID string) (iter.Seq2[[]cw.LogEvent, error], error) { + if b.cdTaskArn == nil { + var err error + b.cdTaskArn, err = b.driver.GetTaskArn(taskID) // only fails on missing task ID + if err != nil { + return nil, err + } } if req.Follow { - return b.driver.TailTaskID(ctx, cwClient, req.Etag) + return b.driver.TailTaskID(ctx, cwClient, taskID) } else { start := timeutils.AsTime(req.Since, time.Time{}) end := timeutils.AsTime(req.Until, time.Time{}) - return b.driver.QueryTaskID(ctx, cwClient, req.Etag, start, end, req.Limit) + return b.driver.QueryTaskID(ctx, cwClient, taskID, start, end, req.Limit) } } +// deriveTaskID returns the CD task ID if the etag refers to a CD task, or empty string otherwise. +func (b *ByocAws) deriveTaskID(reqEtag string) string { + if b.cdTaskArn != nil && b.cdEtag == reqEtag { + return ecs.GetTaskID(b.cdTaskArn) + } + if _, err := types.ParseEtag(reqEtag); err != nil { + return reqEtag // legacy: assume invalid etag is a task ID + } + return "" +} + func (b *ByocAws) queryOrTailLogs(ctx context.Context, cwClient cw.LogsClient, req *defangv1.TailRequest) (iter.Seq2[cw.LogEvent, error], error) { start := timeutils.AsTime(req.Since, time.Time{}) end := timeutils.AsTime(req.Until, time.Time{}) @@ -832,7 +846,7 @@ func (b *ByocAws) getLogGroupInputs(etag types.ETag, projectName, service, filte var groups []cw.LogGroupInput // Tail CD and builds - if logType.Has(logs.LogTypeBuild) { + if logType.Has(logs.LogTypeCD) { if b.driver.LogGroupARN == "" { term.Debug("CD stack LogGroupARN is not set; skipping CD logs") } else { @@ -844,6 +858,8 @@ func (b *ByocAws) getLogGroupInputs(etag types.ETag, projectName, service, filte groups = append(groups, cdTail) term.Debug("Query CD logs", cdTail.LogGroupARN, cdTail.LogStreamNames, filter) } + } + if logType.Has(logs.LogTypeBuild) && projectName != "" { buildsTail := cw.LogGroupInput{LogGroupARN: b.makeLogGroupARN(b.StackDir(projectName, "builds")), LogEventFilterPattern: pattern} // must match logic in ecs/common.ts; TODO: filter by etag/service term.Debug("Query builds logs", buildsTail.LogGroupARN, filter) groups = append(groups, buildsTail) @@ -852,7 +868,7 @@ func (b *ByocAws) getLogGroupInputs(etag types.ETag, projectName, service, filte groups = append(groups, ecsTail) } // Tail services - if logType.Has(logs.LogTypeRun) { + if logType.Has(logs.LogTypeRun) && projectName != "" { servicesTail := cw.LogGroupInput{LogGroupARN: b.makeLogGroupARN(b.StackDir(projectName, "logs")), LogEventFilterPattern: pattern} // must match logic in ecs/common.ts if service != "" && etag != "" { servicesTail.LogStreamNamePrefix = service + "/" + service + "_" + etag diff --git a/src/pkg/cli/client/byoc/aws/byoc_test.go b/src/pkg/cli/client/byoc/aws/byoc_test.go index be50d6618..a397d8827 100644 --- a/src/pkg/cli/client/byoc/aws/byoc_test.go +++ b/src/pkg/cli/client/byoc/aws/byoc_test.go @@ -17,6 +17,7 @@ import ( "github.com/DefangLabs/defang/src/pkg/cli/client/byoc" "github.com/DefangLabs/defang/src/pkg/clouds/aws" "github.com/DefangLabs/defang/src/pkg/clouds/aws/cw" + "github.com/DefangLabs/defang/src/pkg/clouds/aws/ecs" "github.com/DefangLabs/defang/src/pkg/clouds/aws/ecs/cfn" "github.com/DefangLabs/defang/src/pkg/dns" "github.com/DefangLabs/defang/src/pkg/logs" @@ -26,6 +27,7 @@ import ( awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" cwTypes "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" + "github.com/aws/smithy-go/ptr" composeTypes "github.com/compose-spec/compose-go/v2/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -143,8 +145,8 @@ func TestSubscribe(t *testing.T) { line := lines.Text() cwEvents = append(cwEvents, cw.LogEvent{ LogGroupIdentifier: &ecsLogGroup, - LogStreamName: awssdk.String("some-stream"), - Message: awssdk.String(line), + LogStreamName: ptr.String("some-stream"), + Message: ptr.String(line), Timestamp: &ts, }) } @@ -427,7 +429,7 @@ func (m *mockCWClient) FilterLogEvents(ctx context.Context, input *cloudwatchlog func (m *mockCWClient) StartLiveTail(ctx context.Context, input *cloudwatchlogs.StartLiveTailInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.StartLiveTailOutput, error) { return nil, &cwTypes.ResourceNotFoundException{ - Message: awssdk.String("mock: log group does not exist"), + Message: ptr.String("mock: log group does not exist"), } } @@ -438,9 +440,9 @@ func makeMockEvents(n int, service, etag string) []cwTypes.FilteredLogEvent { for i := range events { ts := int64((i + 1) * 1000) // 1000, 2000, 3000, ... events[i] = cwTypes.FilteredLogEvent{ - Message: awssdk.String(fmt.Sprintf("log message %d", i+1)), + Message: ptr.String(fmt.Sprintf("log message %d", i+1)), Timestamp: &ts, - LogStreamName: awssdk.String(fmt.Sprintf("%s/%s_%s/task%d", service, service, etag, i)), + LogStreamName: ptr.String(fmt.Sprintf("%s/%s_%s/task%d", service, service, etag, i)), } } return events @@ -689,7 +691,7 @@ func TestQueryCdLogs(t *testing.T) { events: makeMockEvents(tt.numEvents, "crun", ""), } - batchSeq, err := b.queryOrTailCdLogs(t.Context(), mock, tt.req) + batchSeq, err := b.queryOrTailLogsByTaskID(t.Context(), mock, tt.req, tt.req.Etag) require.NoError(t, err) // Flatten and collect @@ -706,3 +708,47 @@ func TestQueryCdLogs(t *testing.T) { func TestQueryCdLogs_FollowMode(t *testing.T) { t.Skip("requires ECS API mock for getTaskStatus") } + +func TestDeriveTaskID(t *testing.T) { + validEtag := types.NewEtag() + + tests := []struct { + name string + cdTaskArn ecs.TaskArn + cdEtag string + reqEtag string + wantTaskID string + }{ + { + name: "matching cd etag returns task ID from ARN", + cdTaskArn: ptr.String("arn:aws:ecs:us-west-2:123456789012:task/cluster/abc123def456"), + cdEtag: validEtag, + reqEtag: validEtag, + wantTaskID: "abc123def456", + }, + { + name: "invalid etag treated as legacy task ID", + reqEtag: "some-task-id", + wantTaskID: "some-task-id", + }, + { + name: "valid etag not matching cd returns empty", + cdEtag: "aaaaaaaaaaaa", + reqEtag: "bbbbbbbbbbbb", + }, + { + name: "empty etag returns empty", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := newTestByocAws() + b.cdTaskArn = tt.cdTaskArn + b.cdEtag = tt.cdEtag + + got := b.deriveTaskID(tt.reqEtag) + assert.Equal(t, tt.wantTaskID, got) + }) + } +} diff --git a/src/pkg/cli/client/byoc/aws/domain.go b/src/pkg/cli/client/byoc/aws/domain.go index ea500978f..254595d43 100644 --- a/src/pkg/cli/client/byoc/aws/domain.go +++ b/src/pkg/cli/client/byoc/aws/domain.go @@ -125,7 +125,7 @@ func createUsableDelegationSet(ctx context.Context, domain string, r53Client aws return delegationSet, nil } } - return nil, errors.New("failed to create a usable delegation set without conflicting NS records after multiple attempts") + return nil, errors.New("could not find a usable set of DNS servers; if you are on a VPN, disable it and retry, as it may intercept DNS lookups") } func nameServersHasConflict(ctx context.Context, nameServers []string, domains []string, resolverAt func(string) dns.Resolver) (bool, error) { diff --git a/src/pkg/cli/client/byoc/gcp/byoc.go b/src/pkg/cli/client/byoc/gcp/byoc.go index 044c1035b..3416e2685 100644 --- a/src/pkg/cli/client/byoc/gcp/byoc.go +++ b/src/pkg/cli/client/byoc/gcp/byoc.go @@ -609,7 +609,7 @@ func (b *ByocGcp) getLogStream(ctx context.Context, gcpLogsClient GcpLogsClient, logStream.AddUntil(req.Until.AsTime()) } etag := req.Etag - if logs.LogType(req.LogType).Has(logs.LogTypeBuild) { + if logs.LogType(req.LogType).Has(logs.LogTypeBuild) || logs.LogType(req.LogType).Has(logs.LogTypeCD) { logStream.AddCloudBuildLog(b.PulumiStack, req.Project, etag, req.Services) // CD logs and CloudBuild logs } if logs.LogType(req.LogType).Has(logs.LogTypeRun) { diff --git a/src/pkg/cli/estimate.go b/src/pkg/cli/estimate.go index b11a77a6d..30d03c23c 100644 --- a/src/pkg/cli/estimate.go +++ b/src/pkg/cli/estimate.go @@ -71,7 +71,7 @@ func GeneratePreview(ctx context.Context, project *compose.Project, client clien var pulumiPreviewLogLines []string tailOptions := TailOptions{ Deployment: resp.Etag, - LogType: logs.LogTypeBuild, + LogType: logs.LogTypeCD, Since: since, Verbose: true, } diff --git a/src/pkg/cli/getServices_test.go b/src/pkg/cli/getServices_test.go index 8b45b26ec..c73bc9121 100644 --- a/src/pkg/cli/getServices_test.go +++ b/src/pkg/cli/getServices_test.go @@ -388,7 +388,7 @@ func TestRunHealthcheck(t *testing.T) { }, { name: "Invalid endpoint", - endpoint: "http://invalid-endpoint", + endpoint: "https://invalid-endpoint", healthcheckPath: "/healthy", expectedStatus: "", expectedErr: true, diff --git a/src/pkg/cli/preview.go b/src/pkg/cli/preview.go index 4dfd3230e..0314cc57c 100644 --- a/src/pkg/cli/preview.go +++ b/src/pkg/cli/preview.go @@ -16,7 +16,7 @@ func Preview(ctx context.Context, project *compose.Project, fabric client.Fabric options := TailOptions{ Deployment: resp.Etag, - LogType: logs.LogTypeBuild, + LogType: logs.LogTypeBuild | logs.LogTypeCD, Verbose: true, Stack: provider.GetStackName(), } diff --git a/src/pkg/logs/log_type.go b/src/pkg/logs/log_type.go index 4a497f257..2068eb79a 100644 --- a/src/pkg/logs/log_type.go +++ b/src/pkg/logs/log_type.go @@ -2,6 +2,7 @@ package logs import ( "fmt" + "math" "strings" ) @@ -16,14 +17,15 @@ func (e ErrInvalidLogType) Error() string { } const ( - _LogTypeUnused LogType = 1 << iota + LogTypeCD LogType = 1 << iota LogTypeRun LogTypeBuild LogTypeUnspecified LogType = 0 - LogTypeAll LogType = 0xFFFFFFFF + LogTypeAll LogType = math.MaxUint32 ) var AllLogTypes = []LogType{ + LogTypeCD, LogTypeRun, LogTypeBuild, LogTypeAll, @@ -32,12 +34,14 @@ var AllLogTypes = []LogType{ var ( logType_name = map[LogType]string{ LogTypeUnspecified: "UNSPECIFIED", + LogTypeCD: "CD", LogTypeRun: "RUN", LogTypeBuild: "BUILD", LogTypeAll: "ALL", } logType_value = map[string]LogType{ "UNSPECIFIED": LogTypeUnspecified, + "CD": LogTypeCD, "RUN": LogTypeRun, "BUILD": LogTypeBuild, "ALL": LogTypeAll, diff --git a/src/pkg/logs/log_type_test.go b/src/pkg/logs/log_type_test.go index 9c6fa0021..18bbc90ed 100644 --- a/src/pkg/logs/log_type_test.go +++ b/src/pkg/logs/log_type_test.go @@ -37,6 +37,7 @@ func TestLogTypeString(t *testing.T) { want string }{ {"unspecified", LogTypeUnspecified, "UNSPECIFIED"}, + {"cd", LogTypeCD, "CD"}, {"run", LogTypeRun, "RUN"}, {"build", LogTypeBuild, "BUILD"}, {"run and build", LogTypeRun | LogTypeBuild, "RUN,BUILD"},