Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions internal/apperr/apperr.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,43 @@ func (c *ContextError) Unwrap() error {
return c.Err
}

// AbortedError marks a child result that did not fail on its own merits but
// was cut short because a sibling running under the same fail-fast
// orchestration failed first (e.g. its context was canceled, killing an
// in-flight subprocess). Callers such as printUserFriendly must render these
// distinctly from genuine failures and must not attach failure hints to them.
type AbortedError struct {
// ContextName is the context (e.g. Docker context/host) that was aborted.
ContextName string
// Err is the underlying error observed on the aborted child, if any
// (typically context.Canceled or a wrapped form of it).
Err error
}

func (a *AbortedError) Error() string {
if a == nil {
return "aborted: another context failed"
}
if a.ContextName != "" {
return fmt.Sprintf("context %s: aborted: another context failed", a.ContextName)
}
return "aborted: another context failed"
}

func (a *AbortedError) Unwrap() error {
if a == nil {
return nil
}
return a.Err
}

// IsAborted reports whether err is (or wraps) an *AbortedError, i.e. a child
// that was cut short by a sibling's failure rather than failing on its own.
func IsAborted(err error) bool {
var aborted *AbortedError
return errors.As(err, &aborted)
}

// MultiError groups multiple errors and supports errors.Is/errors.As traversal.
type MultiError struct {
Errors []error
Expand Down
24 changes: 22 additions & 2 deletions internal/cli/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -300,12 +300,32 @@ func printUserFriendly(err error) {
// printMultiErrorDetail prints, for each child error in a MultiError, its
// deepest underlying message (e.g. captured compose/docker stderr) along with
// a per-failure actionable hint when one of the known patterns matches.
//
// Children that were only aborted because a sibling context failed first
// (apperr.AbortedError, e.g. a killed in-flight subprocess from fail-fast
// cancellation) are rendered distinctly as "aborted: another context failed"
// and never receive a failure hint, so they cannot be mistaken for genuine
// failures in the summary.
func printMultiErrorDetail(multi *apperr.MultiError) {
for _, child := range multi.Errors {
detail := apperr.DeepestMessage(child)
contextName := ""
var ctxErr *apperr.ContextError
if errors.As(child, &ctxErr) {
fmt.Fprintf(os.Stderr, "context %s: %s\n", ctxErr.ContextName, detail)
contextName = ctxErr.ContextName
}

if apperr.IsAborted(child) {
if contextName != "" {
fmt.Fprintf(os.Stderr, "context %s: aborted: another context failed\n", contextName)
} else {
fmt.Fprintln(os.Stderr, "aborted: another context failed")
}
continue
}

detail := apperr.DeepestMessage(child)
if contextName != "" {
fmt.Fprintf(os.Stderr, "context %s: %s\n", contextName, detail)
} else {
fmt.Fprintf(os.Stderr, "%s\n", detail)
}
Expand Down
7 changes: 5 additions & 2 deletions internal/planner/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,11 @@ func (p *Planner) ApplyWithPlan(ctx context.Context, cfg manifest.Config, plan *
"stacks", len(allStacks),
"filesets", len(allFilesets))

// Process each context (parallel by default, sequential with --sequential)
err := p.ExecuteAcrossContexts(ctx, &cfg, func(ctx context.Context, contextName string) error {
// Process each context (parallel by default, sequential with --sequential).
// Apply mutates state (compose up, volume/network creation), so contexts
// always run to completion: a failure on one host must never cancel an
// in-flight compose up on another, healthy host.
err := p.ExecuteAcrossContextsMode(ctx, &cfg, RunToCompletion, func(ctx context.Context, contextName string) error {
contextConfig := cfg.Contexts[contextName]

// Get Docker client for this context
Expand Down
10 changes: 8 additions & 2 deletions internal/planner/destroy.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,10 @@ func (p *Planner) BuildDestroyPlan(ctx context.Context, cfg manifest.Config) (*P

var mu sync.Mutex

err := p.ExecuteAcrossContexts(ctx, &cfg, func(ctx context.Context, contextName string) error {
// BuildDestroyPlan only discovers/lists resources (no mutation), so it is
// safe to fail fast: canceling a sibling's in-flight discovery call loses
// nothing.
err := p.ExecuteAcrossContextsMode(ctx, &cfg, FailFast, func(ctx context.Context, contextName string) error {
client := p.getClientForContext(contextName, &cfg)
if client == nil {
return apperr.New("planner.BuildDestroyPlan", apperr.Precondition, "docker client not available for context %s", contextName)
Expand Down Expand Up @@ -208,7 +211,10 @@ func (p *Planner) DestroyWithOptions(ctx context.Context, cfg manifest.Config, o
}
scope := newDestroyScope(&cfg)

err := p.ExecuteAcrossContexts(ctx, &cfg, func(ctx context.Context, contextName string) error {
// Destroy mutates state (removes containers/networks/volumes), so contexts
// always run to completion: a failure on one host must never cancel
// in-flight teardown work on another host.
err := p.ExecuteAcrossContextsMode(ctx, &cfg, RunToCompletion, func(ctx context.Context, contextName string) error {
client := p.getClientForContext(contextName, &cfg)
if client == nil {
return apperr.New("planner.Destroy", apperr.Precondition, "docker client not available for context %s", contextName)
Expand Down
121 changes: 110 additions & 11 deletions internal/planner/orchestrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,33 @@ type ContextResult struct {
Err error
}

// ExecutionMode controls how executeParallel reacts to a child error.
type ExecutionMode int

const (
// FailFast cancels sibling goroutines as soon as one context errors.
// Appropriate for read-only/discovery operations where an in-flight
// docker command can be safely interrupted. Any sibling error caused by
// that cancellation is classified as aborted, not as a genuine failure.
FailFast ExecutionMode = iota
// RunToCompletion lets every context run to completion regardless of
// sibling errors. Required for mutating operations (apply/destroy/prune)
// where killing an in-flight `compose up`/`compose down` on an unrelated,
// healthy host is worse than waiting for it to finish.
RunToCompletion
)

// ExecuteAcrossContexts runs fn for each context, either in parallel or sequentially
// based on the planner's parallel flag.
// based on the planner's parallel flag. Mutating operations (apply/destroy/prune)
// must run in RunToCompletion mode so a failure on one context never kills an
// in-flight mutation on another; read-only/discovery operations may use FailFast.
func (p *Planner) ExecuteAcrossContexts(ctx context.Context, cfg *manifest.Config, fn func(ctx context.Context, contextName string) error) error {
return p.ExecuteAcrossContextsMode(ctx, cfg, RunToCompletion, fn)
}

// ExecuteAcrossContextsMode is like ExecuteAcrossContexts but lets the caller
// pick the orchestration mode explicitly.
func (p *Planner) ExecuteAcrossContextsMode(ctx context.Context, cfg *manifest.Config, mode ExecutionMode, fn func(ctx context.Context, contextName string) error) error {
contextNames := sortedKeys(cfg.Contexts)
if len(contextNames) == 0 {
return nil
Expand All @@ -28,7 +52,7 @@ func (p *Planner) ExecuteAcrossContexts(ctx context.Context, cfg *manifest.Confi
if !p.parallel || len(contextNames) == 1 {
return executeSequential(ctx, contextNames, fn)
}
return executeParallel(ctx, contextNames, fn)
return executeParallel(ctx, contextNames, mode, fn)
}

func executeSequential(ctx context.Context, contextNames []string, fn func(ctx context.Context, contextName string) error) error {
Expand All @@ -43,23 +67,38 @@ func executeSequential(ctx context.Context, contextNames []string, fn func(ctx c
return nil
}

func executeParallel(ctx context.Context, contextNames []string, fn func(ctx context.Context, contextName string) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
func executeParallel(parentCtx context.Context, contextNames []string, mode ExecutionMode, fn func(ctx context.Context, contextName string) error) error {
runCtx := parentCtx
var cancel context.CancelFunc
if mode == FailFast {
runCtx, cancel = context.WithCancel(parentCtx)
defer cancel()
}

var mu sync.Mutex
var errs []ContextResult
var wg sync.WaitGroup
// firstFailure records which context triggered the cancellation, so that
// exactly that one is never classified as aborted even though, by the
// time we get around to sorting/classifying, runCtx.Err() is already set.
firstFailure := ""
cancelled := false

for _, name := range contextNames {
wg.Add(1)
go func(contextName string) {
defer wg.Done()
if err := fn(ctx, contextName); err != nil {
if err := fn(runCtx, contextName); err != nil {
mu.Lock()
errs = append(errs, ContextResult{ContextName: contextName, Err: err})
if mode == FailFast && !cancelled {
cancelled = true
firstFailure = contextName
}
mu.Unlock()
cancel() // signal other goroutines to stop
if mode == FailFast && cancel != nil {
cancel() // signal other goroutines to stop
}
}
}(name)
}
Expand All @@ -69,17 +108,45 @@ func executeParallel(ctx context.Context, contextNames []string, fn func(ctx con
if len(errs) == 0 {
return nil
}

// Sort errors by context name for deterministic output
sort.Slice(errs, func(i, j int) bool { return errs[i].ContextName < errs[j].ContextName })

if mode == FailFast {
// Only classify children as aborted when the cancellation was OURS:
// the parent context is still alive while the run context is
// canceled, which can only mean our fail-fast cancel() fired in
// response to firstFailure. If the parent itself was canceled (user
// Ctrl-C, upstream deadline), no sibling caused anything — leave all
// errors as-is; upstream already renders user-canceled runs.
if parentCtx.Err() == nil && runCtx.Err() != nil {
errs = classifyAborted(errs, firstFailure)
}
}

if len(errs) == 1 {
return errs[0].Err
}

// Sort errors by context name for deterministic output
sort.Slice(errs, func(i, j int) bool { return errs[i].ContextName < errs[j].ContextName })
// Build the summary message so aborted contexts (cut short only because a
// sibling failed first) are never worded as failures alongside the
// genuine one(s) — design requirement: the aggregate must clearly
// distinguish the two and never present an aborted context as failed.
var msgs []string
wrapped := make([]error, 0, len(errs))
failureCount := 0
for _, r := range errs {
msgs = append(msgs, fmt.Sprintf("context %s: %v", r.ContextName, r.Err))
wrapped = append(wrapped, &apperr.ContextError{ContextName: r.ContextName, Err: r.Err})
if apperr.IsAborted(r.Err) {
msgs = append(msgs, fmt.Sprintf("context %s: aborted: another context failed", r.ContextName))
continue
}
failureCount++
msgs = append(msgs, fmt.Sprintf("context %s: %v", r.ContextName, r.Err))
}
summary := fmt.Sprintf("%d context(s) failed", failureCount)
if failureCount != len(errs) {
summary = fmt.Sprintf("%s (%d aborted after a sibling failure)", summary, len(errs)-failureCount)
}
// Preserve each child's underlying cause (rather than pre-stringifying with
// %v) so the deepest error detail, e.g. captured compose stderr, survives
Expand All @@ -88,6 +155,38 @@ func executeParallel(ctx context.Context, contextNames []string, fn func(ctx con
Op: "planner.ExecuteAcrossContexts",
Kind: apperr.External,
Err: &apperr.MultiError{Errors: wrapped},
Msg: fmt.Sprintf("multiple context errors:\n %s", strings.Join(msgs, "\n ")),
Msg: fmt.Sprintf("%s:\n %s", summary, strings.Join(msgs, "\n ")),
}
}

// classifyAborted rewrites child errors that only failed because our
// fail-fast cancel() fired in response to firstFailure's genuine failure.
// It must only be called when that cancellation is known to be ours (the
// caller checks that the parent context is still alive while the run
// context is canceled) — otherwise a user Ctrl-C or an upstream deadline
// would mislabel every child as "aborted: another context failed" when no
// sibling failed at all.
//
// Under that gate every non-first child error is cancellation-induced by
// construction, regardless of its shape: some chain context.Canceled (a
// docker call that observes ctx.Done() before starting its own work), while
// others are opaque — exec.CommandContext SIGKILLs an in-flight `docker
// compose` subprocess and the resulting "signal: killed" *exec.ExitError
// carries no context.Canceled anywhere in its chain, so errors.Is matching
// alone could never recognize it.
//
// firstFailure is the context name that actually triggered cancel(); it is
// never classified as aborted — even when its error happens to chain
// context.Canceled — because it is the genuine failure by construction.
func classifyAborted(errs []ContextResult, firstFailure string) []ContextResult {
out := make([]ContextResult, 0, len(errs))
for _, r := range errs {
if r.ContextName == firstFailure {
out = append(out, r)
continue
}
out = append(out, ContextResult{ContextName: r.ContextName, Err: &apperr.AbortedError{ContextName: r.ContextName, Err: r.Err}})
}
return out
}

Loading
Loading