CodeBuild event streaming

pkgs/npm/README.md

@@ -10,7 +10,7 @@ The Defang Command-Line Interface [(CLI)](https://docs.defang.io/docs/getting-st
 
 - Read our [Getting Started](https://docs.defang.io/docs/getting-started) page
 - Follow the installation instructions from the [Installing](https://docs.defang.io/docs/getting-started/installing) page
-- Take a look at our [Samples folder](https://github.com/DefangLabs/defang/tree/main/samples) for example projects in various programming languages.
+- Take a look at our [Samples](https://github.com/DefangLabs/samples) in Golang, Python, and Node.js that show how to accomplish various tasks and deploy them
 - Try the AI integration by running `defang generate`
 - Start your new service with `defang compose up`
 

src/README.md

@@ -10,7 +10,7 @@ The Defang Command-Line Interface [(CLI)](https://docs.defang.io/docs/getting-st
 
 - Read our [Getting Started](https://docs.defang.io/docs/getting-started) page
 - Follow the installation instructions from the [Installing](https://docs.defang.io/docs/getting-started/installing) page
-- Take a look at our [Samples folder](https://github.com/DefangLabs/defang/tree/main/samples) for example projects in various programming languages.
+- Take a look at our [Samples](https://github.com/DefangLabs/samples) in Golang, Python, and Node.js that show how to accomplish various tasks and deploy them
 - Try the AI integration by running `defang generate`
 - Start your new service with `defang compose up`
 

src/pkg/cli/client/byoc/aws/byoc.go

@@ -21,6 +21,7 @@ import (
 	"github.com/DefangLabs/defang/src/pkg/cli/compose"
 	"github.com/DefangLabs/defang/src/pkg/clouds"
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws"
+	"github.com/DefangLabs/defang/src/pkg/clouds/aws/codebuild"
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws/cw"
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws/ecs"
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws/ecs/cfn"
@@ -54,11 +55,12 @@ type ByocAws struct {
 
 	driver *cfn.AwsEcsCfn // TODO: ecs is stateful, contains the output of the cd cfn stack after SetUpCD
 
-	ecsEventHandlers []ECSEventHandler
-	handlersLock     sync.RWMutex
-	cdEtag           types.ETag
-	cdStart          time.Time
-	cdTaskArn        ecs.TaskArn
+	ecsEventHandlers       []ECSEventHandler
+	codebuildEventHandlers []CodebuildEventHandler
+	handlersLock           sync.RWMutex
+	cdEtag                 types.ETag
+	cdStart                time.Time
+	cdTaskArn              ecs.TaskArn
 
 	needDockerHubCreds bool
 }
@@ -709,7 +711,7 @@ func (b *ByocAws) QueryLogs(ctx context.Context, req *defangv1.TailRequest) (cli
 	if err != nil {
 		return nil, AnnotateAwsError(err)
 	}
-	return newByocServerStream(tailStream, etag, req.Services, b), nil
+	return newByocServerStream(tailStream, etag, req.Services, b, b), nil
 }
 
 func (b *ByocAws) queryCdLogs(ctx context.Context, cwClient *cloudwatchlogs.Client, req *defangv1.TailRequest) (cw.LiveTailStream, error) {
@@ -890,6 +892,10 @@ type ECSEventHandler interface {
 	HandleECSEvent(evt ecs.Event)
 }
 
+type CodebuildEventHandler interface {
+	HandleCodebuildEvent(evt codebuild.Event)
+}
+
 func (b *ByocAws) Subscribe(ctx context.Context, req *defangv1.SubscribeRequest) (client.ServerStream[defangv1.SubscribeResponse], error) {
 	s := &byocSubscribeServerStream{
 		services: req.Services,
@@ -899,6 +905,7 @@ func (b *ByocAws) Subscribe(ctx context.Context, req *defangv1.SubscribeRequest)
 		done: make(chan struct{}),
 	}
 	b.AddEcsEventHandler(s)
+	b.AddCodebuildEventHandler(s)
 	return s, nil
 }
 
@@ -910,12 +917,26 @@ func (b *ByocAws) HandleECSEvent(evt ecs.Event) {
 	}
 }
 
+func (b *ByocAws) HandleCodebuildEvent(evt codebuild.Event) {
+	b.handlersLock.RLock()
+	defer b.handlersLock.RUnlock()
+	for _, handler := range b.codebuildEventHandlers {
+		handler.HandleCodebuildEvent(evt)
+	}
+}
+
 func (b *ByocAws) AddEcsEventHandler(handler ECSEventHandler) {
 	b.handlersLock.Lock()
 	defer b.handlersLock.Unlock()
 	b.ecsEventHandlers = append(b.ecsEventHandlers, handler)
 }
 
+func (b *ByocAws) AddCodebuildEventHandler(handler CodebuildEventHandler) {
+	b.handlersLock.Lock()
+	defer b.handlersLock.Unlock()
+	b.codebuildEventHandlers = append(b.codebuildEventHandlers, handler)
+}
+
 func (b *ByocAws) GetPrivateDomain(projectName string) string {
 	return b.GetProjectLabel(projectName) + ".internal"
 }

src/pkg/cli/client/byoc/aws/stream.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/DefangLabs/defang/src/pkg/cli/client"
 	"github.com/DefangLabs/defang/src/pkg/cli/client/byoc"
+	"github.com/DefangLabs/defang/src/pkg/clouds/aws/codebuild"
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws/cw"
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws/ecs"
 	"github.com/DefangLabs/defang/src/pkg/logs"
@@ -29,16 +30,18 @@ type byocServerStream struct {
 	services []string
 	stream   cw.LiveTailStream
 
-	ecsEventsHandler ECSEventHandler
+	ecsEventsHandler      ECSEventHandler
+	codebuildEventHandler CodebuildEventHandler
 }
 
-func newByocServerStream(stream cw.LiveTailStream, etag string, services []string, ecsEventHandler ECSEventHandler) *byocServerStream {
+func newByocServerStream(stream cw.LiveTailStream, etag string, services []string, ecsEventHandler ECSEventHandler, codebuildEventHandler CodebuildEventHandler) *byocServerStream {
 	return &byocServerStream{
 		etag:     etag,
 		stream:   stream,
 		services: services,
 
-		ecsEventsHandler: ecsEventHandler,
+		ecsEventsHandler:      ecsEventHandler,
+		codebuildEventHandler: codebuildEventHandler,
 	}
 }
 
@@ -169,7 +172,9 @@ func (bs *byocServerStream) parseEvents(events []cw.LogEvent) *defangv1.TailResp
 			if err != nil {
 				term.Debugf("error parsing ECS event, output raw event log: %v", err)
 			} else {
-				bs.ecsEventsHandler.HandleECSEvent(evt)
+				if bs.ecsEventsHandler != nil {
+					bs.ecsEventsHandler.HandleECSEvent(evt)
+				}
 				entry.Service = evt.Service()
 				entry.Etag = evt.Etag()
 				entry.Host = evt.Host()
@@ -179,6 +184,10 @@ func (bs *byocServerStream) parseEvents(events []cw.LogEvent) *defangv1.TailResp
 			entry.Service = response.Service
 			entry.Etag = response.Etag
 			entry.Host = response.Host
+			evt := codebuild.ParseCodebuildEvent(entry)
+			if bs.codebuildEventHandler != nil && evt.State() != defangv1.ServiceState_NOT_SPECIFIED {
+				bs.codebuildEventHandler.HandleCodebuildEvent(evt)
+			}
 		} else if (response.Service == "cd") && (strings.HasPrefix(entry.Message, logs.ErrorPrefix) || strings.Contains(strings.ToLower(entry.Message), "error:")) {
 			entry.Stderr = true
 		}

src/pkg/cli/client/byoc/aws/stream_test.go

@@ -143,7 +143,7 @@ func TestStreamToLogEvent(t *testing.T) {
 		},
 	}
 
-	var byocServiceStream = newByocServerStream(nil, testEtag, []string{"cd", "app", "django", "django-image"}, nil)
+	var byocServiceStream = newByocServerStream(nil, testEtag, []string{"cd", "app", "django", "django-image"}, nil, nil)
 
 	for _, td := range testdata {
 		tailResp := byocServiceStream.parseEvents([]cw.LogEvent{*td.event})

src/pkg/cli/client/byoc/aws/subscribe.go

@@ -2,7 +2,9 @@ package aws
 
 import (
 	"slices"
+	"strings"
 
+	"github.com/DefangLabs/defang/src/pkg/clouds/aws/codebuild"
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws/ecs"
 	"github.com/DefangLabs/defang/src/pkg/types"
 	defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1"
@@ -18,6 +20,25 @@ type byocSubscribeServerStream struct {
 	done chan struct{}
 }
 
+func (s *byocSubscribeServerStream) HandleCodebuildEvent(evt codebuild.Event) {
+	if etag := evt.Etag(); etag == "" || etag != s.etag {
+		return
+	}
+	service := strings.TrimSuffix(evt.Service(), "-image")
+	if len(s.services) > 0 && !slices.Contains(s.services, service) {
+		return
+	}
+	resp := defangv1.SubscribeResponse{
+		Name:   evt.Service(),
+		Status: evt.Status(),
+		State:  evt.State(),
+	}
+	select {
+	case s.ch <- &resp:
+	case <-s.done:
+	}
+}
+
 func (s *byocSubscribeServerStream) HandleECSEvent(evt ecs.Event) {
 	if etag := evt.Etag(); etag == "" || etag != s.etag {
 		return

src/pkg/clouds/aws/codebuild/event.go

@@ -0,0 +1,92 @@
+package codebuild
+
+import (
+	"strings"
+	"time"
+
+	defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1"
+)
+
+type Event interface {
+	Service() string
+	Etag() string
+	Host() string
+	Status() string
+	State() defangv1.ServiceState
+}
+
+type eventCommonFields struct {
+	Account    string
+	DetailType string
+	Id         string
+	Region     string
+	Resources  []string
+	Source     string
+	Time       time.Time
+	Version    string
+}
+
+type CodebuildEvent struct {
+	eventCommonFields
+	message string
+	service string
+	etag    string
+	host    string
+	state   defangv1.ServiceState
+}
+
+func ParseCodebuildEvent(entry *defangv1.LogEntry) Event {
+	message := entry.Message
+	state := parseCodebuildMessage(message)
+
+	return &CodebuildEvent{
+		message: message,
+		service: entry.Service,
+		etag:    entry.Etag,
+		host:    entry.Host,
+		state:   state,
+	}
+}
+
+func (e *CodebuildEvent) State() defangv1.ServiceState {
+	return e.state
+}
+
+func (e *CodebuildEvent) Service() string {
+	return e.service
+}
+
+func (e *CodebuildEvent) Etag() string {
+	return e.etag
+}
+
+func (e *CodebuildEvent) Host() string {
+	return "codebuild"
+}
+
+func (e *CodebuildEvent) Status() string {
+	return ""
+}
+
+func parseCodebuildMessage(message string) defangv1.ServiceState {
+	switch {
+	case strings.Contains(message, "Phase complete: ") && strings.Contains(message, "State: FAILED"):
+		return defangv1.ServiceState_BUILD_FAILED
+	case strings.Contains(message, "Running on CodeBuild"):
+		return defangv1.ServiceState_BUILD_ACTIVATING
+	case strings.Contains(message, "Phase is DOWNLOAD_SOURCE"):
+		return defangv1.ServiceState_BUILD_RUNNING
+	case strings.Contains(message, "Entering phase INSTALL"):
+		return defangv1.ServiceState_BUILD_RUNNING
+	case strings.Contains(message, "Entering phase PRE_BUILD"):
+		return defangv1.ServiceState_BUILD_RUNNING
+	case strings.Contains(message, "Entering phase BUILD"):
+		return defangv1.ServiceState_BUILD_RUNNING
+	case strings.Contains(message, "Entering phase POST_BUILD"):
+		return defangv1.ServiceState_BUILD_STOPPING
+	case strings.Contains(message, "Phase complete: UPLOAD_ARTIFACTS State: SUCCEEDED"):
+		return defangv1.ServiceState_DEPLOYMENT_PENDING
+	default:
+		return defangv1.ServiceState_NOT_SPECIFIED
+	}
+}