From 82b64ba421e1228a681709f0f63172c6e2fb7ea2 Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 15 Jul 2026 19:42:16 +0800 Subject: [PATCH 1/9] Add startup timing and Docker pull progress --- cmd/ephemeral-action-runner/start.go | 20 +- go.mod | 36 ++- go.sum | 77 ++++++ internal/pool/docker_pull.go | 354 +++++++++++++++++++++++++++ internal/pool/host_trust_test.go | 3 + internal/pool/image.go | 18 +- internal/pool/image_manifest.go | 30 ++- internal/pool/image_manifest_test.go | 6 + internal/pool/image_test.go | 17 ++ internal/pool/manager.go | 106 ++++---- internal/pool/startup_timing.go | 203 +++++++++++++++ 11 files changed, 816 insertions(+), 54 deletions(-) create mode 100644 go.sum create mode 100644 internal/pool/docker_pull.go create mode 100644 internal/pool/startup_timing.go diff --git a/cmd/ephemeral-action-runner/start.go b/cmd/ephemeral-action-runner/start.go index 2f5da31..78604ce 100644 --- a/cmd/ephemeral-action-runner/start.go +++ b/cmd/ephemeral-action-runner/start.go @@ -23,6 +23,11 @@ type hostTrustLockingStarterManager interface { AcquireHostTrustControllerLock() (io.Closer, error) } +type startupTimingStarterManager interface { + StartStartupTiming() (string, error) + FinishStartupTiming(error) +} + type starterManagerFactory func(configPath, projectRoot string, dryRun bool, githubEnabled bool) (starterManager, error) var newStarterManager starterManagerFactory = func(configPath, projectRoot string, dryRun bool, githubEnabled bool) (starterManager, error) { @@ -74,7 +79,7 @@ func runStart(args []string) error { }) } -func runStartWithOptions(opts startOptions) error { +func runStartWithOptions(opts startOptions) (err error) { if opts.Context == nil { opts.Context = context.Background() } @@ -107,6 +112,14 @@ func runStartWithOptions(opts startOptions) error { if err != nil { return err } + if timingManager, ok := manager.(startupTimingStarterManager); ok { + if _, err := timingManager.StartStartupTiming(); err != nil { + return fmt.Errorf("start startup timing log: %w", err) + } + defer func() { + timingManager.FinishStartupTiming(err) + }() + } hostTrustLockHeld := false if lockingManager, ok := manager.(hostTrustLockingStarterManager); ok { controllerLock, err := lockingManager.AcquireHostTrustControllerLock() @@ -119,7 +132,7 @@ func runStartWithOptions(opts startOptions) error { } } fmt.Fprintf(opts.Out, "Ensuring runner image is current for %s\n", configPath) - if err := manager.EnsureImage(opts.Context); err != nil { + if err = manager.EnsureImage(opts.Context); err != nil { return err } if opts.Instances > 0 { @@ -127,7 +140,7 @@ func runStartWithOptions(opts startOptions) error { } else { fmt.Fprintf(opts.Out, "Starting EPAR pool using pool.instances from config. Press Ctrl-C to stop; cleanup is enabled by default.\n") } - return manager.RunPool(opts.Context, pool.RunOptions{ + err = manager.RunPool(opts.Context, pool.RunOptions{ Instances: opts.Instances, Register: opts.Register, KeepOnExit: opts.KeepOnExit, @@ -135,6 +148,7 @@ func runStartWithOptions(opts startOptions) error { MonitorInterval: opts.MonitorInterval, HostTrustLockHeld: hostTrustLockHeld, }) + return err } func ensureConfigForStart(opts startOptions) (string, error) { diff --git a/go.mod b/go.mod index db6210b..f46ede3 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,37 @@ module github.com/solutionforest/ephemeral-action-runner -go 1.25 +go 1.25.0 + +require ( + github.com/google/go-containerregistry v0.21.7 + github.com/moby/moby/api v1.55.0 + github.com/moby/moby/client v0.5.0 + github.com/opencontainers/image-spec v1.1.1 + golang.org/x/term v0.45.0 +) + +require ( + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/cli v29.5.3+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.3 // indirect + github.com/docker/go-connections v0.7.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.47.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..fc23c8e --- /dev/null +++ b/go.sum @@ -0,0 +1,77 @@ +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/cli v29.5.3+incompatible h1:nbEFfz774vBwQ5KRYv7c/AghjReqnGISvrRhzjV0evs= +github.com/docker/cli v29.5.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.21.7 h1:/vPFuVXDjtFREsVArW+0h1CIl5urnOhzei4X2DMW9IU= +github.com/google/go-containerregistry v0.21.7/go.mod h1:kjSbt7/zMsKLWfnHrIvKvhXHUw91jbe9DNjPPJ32gXE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc= +github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/internal/pool/docker_pull.go b/internal/pool/docker_pull.go new file mode 100644 index 0000000..3b3ad71 --- /dev/null +++ b/internal/pool/docker_pull.go @@ -0,0 +1,354 @@ +package pool + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + gcrv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/moby/moby/api/types/jsonstream" + "github.com/moby/moby/api/types/registry" + "github.com/moby/moby/client" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/term" +) + +const dockerPullProgressInterval = 250 * time.Millisecond + +type dockerSourcePullOptions struct { + Image string + Platform string + LogPath string + AnnounceRemoteSize bool +} + +type dockerLayerProgress struct { + current int64 + total int64 + completed bool +} + +// pullDockerSourceCommand is kept as a small seam for existing command-based +// image preparation tests. Production always uses Manager.pullDockerSource. +var pullDockerSourceCommand = (*Manager).pullDockerSource + +func (m *Manager) pullDockerSource(ctx context.Context, opts dockerSourcePullOptions) error { + cli, err := client.New(client.FromEnv) + if err != nil { + return m.pullDockerSourceWithCLI(ctx, opts, fmt.Errorf("initialize Docker Engine client: %w", err)) + } + if _, err := cli.Ping(ctx, client.PingOptions{}); err != nil { + return m.pullDockerSourceWithCLI(ctx, opts, fmt.Errorf("connect to Docker Engine: %w", err)) + } + + platform, err := m.resolveDockerPullPlatform(ctx, cli, opts.Platform) + if err != nil { + return m.pullDockerSourceWithCLI(ctx, opts, err) + } + if opts.AnnounceRemoteSize { + if size, err := remoteCompressedLayerSize(opts.Image, platform); err != nil { + m.writeDockerPullNotice(opts.LogPath, "warning: could not determine remote compressed layer size: "+sanitizeTimingError(err)) + } else { + m.writeDockerPullNotice(opts.LogPath, fmt.Sprintf("Remote compressed layers: %s; actual transfer may be lower when Docker reuses layers.", formatDockerPullBytes(size))) + } + } + + registryAuth, err := dockerRegistryAuth(opts.Image) + if err != nil { + m.writeDockerPullNotice(opts.LogPath, "warning: could not load Docker registry credentials; continuing without explicit credentials: "+sanitizeTimingError(err)) + } + response, err := cli.ImagePull(ctx, opts.Image, client.ImagePullOptions{ + RegistryAuth: registryAuth, + Platforms: []ocispec.Platform{platform}, + }) + if err != nil { + return fmt.Errorf("Docker Engine pull %s: %w", opts.Image, err) + } + if err := renderDockerPullProgress(ctx, response, opts.LogPath); err != nil { + return fmt.Errorf("Docker Engine pull %s: %w", opts.Image, err) + } + m.writeDockerPullNotice(opts.LogPath, "Docker source pull complete: "+opts.Image) + return nil +} + +func (m *Manager) pullDockerSourceWithCLI(ctx context.Context, opts dockerSourcePullOptions, apiErr error) error { + fmt.Printf("warning: %v; falling back to docker pull CLI\n", apiErr) + m.writeDockerPullNotice(opts.LogPath, "warning: "+sanitizeTimingError(apiErr)+"; falling back to docker pull CLI") + args := []string{"pull"} + if opts.Platform != "" { + args = append(args, "--platform", opts.Platform) + } + args = append(args, opts.Image) + return runHostLoggedCommand(ctx, opts.LogPath, "docker", args...) +} + +func (m *Manager) resolveDockerPullPlatform(ctx context.Context, cli *client.Client, configured string) (ocispec.Platform, error) { + if platform, ok := normalizedDockerPlatform(configured, ""); ok { + return platform, nil + } + if platform, ok := normalizedDockerPlatform(m.Config.Provider.Platform, ""); ok { + return platform, nil + } + info, err := cli.Info(ctx, client.InfoOptions{}) + if err != nil { + return ocispec.Platform{}, fmt.Errorf("inspect Docker Engine platform: %w", err) + } + if platform, ok := normalizedDockerPlatform(info.Info.OSType+"/"+info.Info.Architecture, ""); ok { + return platform, nil + } + if platform, ok := normalizedDockerPlatform(runtime.GOOS+"/"+runtime.GOARCH, ""); ok { + return platform, nil + } + return ocispec.Platform{}, fmt.Errorf("Docker Engine did not report a usable platform") +} + +func normalizedDockerPlatform(value, fallbackOS string) (ocispec.Platform, bool) { + parts := strings.Split(strings.Trim(strings.ToLower(value), "/"), "/") + if len(parts) == 0 || len(parts) > 3 || parts[0] == "" { + return ocispec.Platform{}, false + } + platform := ocispec.Platform{OS: fallbackOS} + if len(parts) == 1 { + platform.Architecture = normalizeDockerArchitecture(parts[0]) + } else { + platform.OS = parts[0] + platform.Architecture = normalizeDockerArchitecture(parts[1]) + if len(parts) == 3 { + platform.Variant = parts[2] + } + } + if platform.OS == "" { + platform.OS = "linux" + } + if platform.Architecture == "" { + return ocispec.Platform{}, false + } + return platform, true +} + +func normalizeDockerArchitecture(architecture string) string { + switch architecture { + case "x86_64", "x64": + return "amd64" + case "aarch64": + return "arm64" + default: + return architecture + } +} + +func remoteCompressedLayerSize(image string, platform ocispec.Platform) (int64, error) { + ref, authenticator, err := dockerImageReferenceAndAuth(image) + if err != nil { + return 0, err + } + remoteImage, err := remote.Image(ref, remote.WithAuth(authenticator), remote.WithPlatform(gcrv1.Platform{ + OS: platform.OS, + Architecture: platform.Architecture, + Variant: platform.Variant, + })) + if err != nil { + return 0, err + } + layers, err := remoteImage.Layers() + if err != nil { + return 0, err + } + var total int64 + for _, layer := range layers { + size, err := layer.Size() + if err != nil { + return 0, err + } + total += size + } + return total, nil +} + +func dockerRegistryAuth(image string) (string, error) { + ref, authenticator, err := dockerImageReferenceAndAuth(image) + if err != nil { + return "", err + } + credentials, err := authenticator.Authorization() + if err != nil { + return "", err + } + content, err := json.Marshal(registry.AuthConfig{ + Username: credentials.Username, + Password: credentials.Password, + Auth: credentials.Auth, + ServerAddress: ref.Context().RegistryStr(), + IdentityToken: credentials.IdentityToken, + RegistryToken: credentials.RegistryToken, + }) + if err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(content), nil +} + +func dockerImageReferenceAndAuth(image string) (name.Reference, authn.Authenticator, error) { + ref, err := name.ParseReference(image) + if err != nil { + return nil, nil, err + } + authenticator, err := authn.DefaultKeychain.Resolve(ref.Context().Registry) + if err != nil { + return nil, nil, err + } + return ref, authenticator, nil +} + +func renderDockerPullProgress(ctx context.Context, response client.ImagePullResponse, logPath string) error { + logFile, err := openDockerPullLog(logPath) + if err != nil { + return err + } + if logFile != nil { + defer logFile.Close() + } + interactive := term.IsTerminal(int(os.Stdout.Fd())) + layers := map[string]dockerLayerProgress{} + lastRender := time.Time{} + rendered := false + for message, streamErr := range response.JSONMessages(ctx) { + if streamErr != nil { + return streamErr + } + writeDockerPullEvent(logFile, message.ID, message.Status, message.Progress, message.Stream, message.Error) + if message.Error != nil { + return message.Error + } + if message.ID != "" { + layer := layers[message.ID] + if message.Progress != nil { + layer.current = message.Progress.Current + if message.Progress.Total > 0 { + layer.total = message.Progress.Total + } + } + if dockerPullLayerComplete(message.Status) { + layer.completed = true + if layer.total > 0 { + layer.current = layer.total + } + } + layers[message.ID] = layer + } + if time.Since(lastRender) >= dockerPullProgressInterval { + writeDockerPullSummary(os.Stdout, interactive, layers) + lastRender = time.Now() + rendered = true + } + } + if rendered { + writeDockerPullSummary(os.Stdout, interactive, layers) + if interactive { + fmt.Fprintln(os.Stdout) + } + } + return nil +} + +func openDockerPullLog(path string) (*os.File, error) { + if path == "" { + return nil, nil + } + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return nil, err + } + return os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) +} + +func (m *Manager) writeDockerPullNotice(logPath, message string) { + fmt.Println(message) + logFile, err := openDockerPullLog(logPath) + if err != nil { + return + } + if logFile != nil { + defer logFile.Close() + fmt.Fprintf(logFile, "%s %s\n", time.Now().UTC().Format(time.RFC3339Nano), message) + } +} + +func writeDockerPullEvent(logFile io.Writer, id, status string, progress *jsonstream.Progress, stream string, pullErr error) { + if logFile == nil { + return + } + parts := make([]string, 0, 4) + if id != "" { + parts = append(parts, id) + } + if status != "" { + parts = append(parts, status) + } + if progress != nil { + parts = append(parts, fmt.Sprintf("progress=%d/%d", progress.Current, progress.Total)) + } + if stream != "" { + parts = append(parts, strings.TrimSpace(stream)) + } + if pullErr != nil { + parts = append(parts, "error="+sanitizeTimingError(pullErr)) + } + fmt.Fprintf(logFile, "%s %s\n", time.Now().UTC().Format(time.RFC3339Nano), strings.Join(parts, " ")) +} + +func dockerPullLayerComplete(status string) bool { + status = strings.ToLower(strings.TrimSpace(status)) + return status == "pull complete" || status == "already exists" || status == "exists" +} + +func writeDockerPullSummary(w io.Writer, interactive bool, layers map[string]dockerLayerProgress) { + var complete, known int + var currentBytes, totalBytes int64 + for _, layer := range layers { + if layer.completed { + complete++ + } + if layer.total > 0 { + known++ + totalBytes += layer.total + currentBytes += min(layer.current, layer.total) + } + } + line := fmt.Sprintf("Docker source pull: %d/%d layers complete; %s/%s", complete, len(layers), formatDockerPullBytes(currentBytes), formatDockerPullBytes(totalBytes)) + if totalBytes > 0 { + line += fmt.Sprintf(" (%.0f%%)", float64(currentBytes)*100/float64(totalBytes)) + } + if known < len(layers) { + line += fmt.Sprintf("; %d layer(s) size pending", len(layers)-known) + } + if interactive { + fmt.Fprintf(w, "\r\033[2K%s", line) + return + } + fmt.Fprintf(w, "%s %s\n", time.Now().UTC().Format(time.RFC3339Nano), line) +} + +func formatDockerPullBytes(value int64) string { + const unit = 1024 + if value < unit { + return fmt.Sprintf("%d B", value) + } + units := []string{"KiB", "MiB", "GiB", "TiB"} + size := float64(value) + index := -1 + for size >= unit && index+1 < len(units) { + size /= unit + index++ + } + return fmt.Sprintf("%.1f %s", size, units[index]) +} diff --git a/internal/pool/host_trust_test.go b/internal/pool/host_trust_test.go index 9982985..c3b2fbc 100644 --- a/internal/pool/host_trust_test.go +++ b/internal/pool/host_trust_test.go @@ -338,11 +338,13 @@ func TestHostTrustImageBuildRetriesChangedGenerationBeforePublishing(t *testing. oldOutput := runHostOutputCommand oldQuiet := runHostQuietCommand oldRun := runHostCommand + oldPull := pullDockerSourceCommand t.Cleanup(func() { runHostLoggedCommand = oldLogged runHostOutputCommand = oldOutput runHostQuietCommand = oldQuiet runHostCommand = oldRun + pullDockerSourceCommand = oldPull }) builds := 0 tagged := false @@ -356,6 +358,7 @@ func TestHostTrustImageBuildRetriesChangedGenerationBeforePublishing(t *testing. return `["source@sha256:1234"]`, nil } runHostQuietCommand = func(context.Context, string, ...string) error { return nil } + pullDockerSourceCommand = func(*Manager, context.Context, dockerSourcePullOptions) error { return nil } runHostCommand = func(_ context.Context, name string, args ...string) error { if name == "docker" && len(args) >= 4 && args[0] == "image" && args[1] == "tag" { tagged = true diff --git a/internal/pool/image.go b/internal/pool/image.go index 1f0802c..e540f72 100644 --- a/internal/pool/image.go +++ b/internal/pool/image.go @@ -101,6 +101,12 @@ func (m *Manager) BuildImage(ctx context.Context, opts ImageBuildOptions) error } func (m *Manager) buildDockerDindImage(ctx context.Context, opts ImageBuildOptions, upstreamDir string) error { + return m.timeStartupStage("dind_image_build", func() error { + return m.buildDockerDindImageUntimed(ctx, opts, upstreamDir) + }) +} + +func (m *Manager) buildDockerDindImageUntimed(ctx context.Context, opts ImageBuildOptions, upstreamDir string) error { buildLogPath := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), imageLogStem(m.Config.Image.OutputImage)+".docker-build.log") if err := resetLogs(buildLogPath); err != nil { return err @@ -288,6 +294,12 @@ func (m *Manager) buildTartImage(ctx context.Context, opts ImageBuildOptions, up } func (m *Manager) buildWSLImage(ctx context.Context, opts ImageBuildOptions, upstreamDir string) error { + return m.timeStartupStage("wsl_image_build", func() error { + return m.buildWSLImageUntimed(ctx, opts, upstreamDir) + }) +} + +func (m *Manager) buildWSLImageUntimed(ctx context.Context, opts ImageBuildOptions, upstreamDir string) error { exporter, ok := m.Provider.(wslExporter) if !ok { return fmt.Errorf("provider.type=wsl requires provider export support") @@ -517,7 +529,11 @@ func (m *Manager) prepareWSLDockerSourceRootfs(ctx context.Context, outputPath, return "", "", err } fmt.Printf("preparing WSL source rootfs from Docker image %s\n", image) - if err := runHostLoggedCommand(ctx, buildLogPath, "docker", pullArgs...); err != nil { + if err := pullDockerSourceCommand(m, ctx, dockerSourcePullOptions{ + Image: image, + Platform: platform, + LogPath: buildLogPath, + }); err != nil { return "", "", fmt.Errorf("wsl image.sourceType=docker-image requires Docker Desktop, Docker Engine, or another reachable Docker daemon; alternatively set image.sourceType=rootfs-tar and provide a prepared rootfs tar: %w", err) } if err := runHostLoggedCommand(ctx, buildLogPath, "docker", createArgs...); err != nil { diff --git a/internal/pool/image_manifest.go b/internal/pool/image_manifest.go index c34e805..283e1dd 100644 --- a/internal/pool/image_manifest.go +++ b/internal/pool/image_manifest.go @@ -252,6 +252,16 @@ func (m *Manager) desiredImageManifestWithHostTrust(ctx context.Context, snapsho } func (m *Manager) refreshDockerSourceDigest(ctx context.Context) (string, error) { + var digest string + err := m.timeStartupStage("source_image_pull", func() error { + var err error + digest, err = m.refreshDockerSourceDigestUntimed(ctx) + return err + }) + return digest, err +} + +func (m *Manager) refreshDockerSourceDigestUntimed(ctx context.Context) (string, error) { if m.DryRun { return "dry-run", nil } @@ -260,14 +270,14 @@ func (m *Manager) refreshDockerSourceDigest(ctx context.Context) (string, error) return "", fmt.Errorf("image.sourceImage is required when image.sourceType=docker-image") } platform := strings.TrimSpace(m.Config.Image.SourcePlatform) - pullArgs := []string{"pull"} - if platform != "" { - pullArgs = append(pullArgs, "--platform", platform) - } - pullArgs = append(pullArgs, image) logPath := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), imageLogStem(m.Config.Image.OutputImage)+".source.log") fmt.Printf("refreshing Docker source image %s\n", image) - if err := runHostLoggedCommand(ctx, logPath, "docker", pullArgs...); err != nil { + if err := pullDockerSourceCommand(m, ctx, dockerSourcePullOptions{ + Image: image, + Platform: platform, + LogPath: logPath, + AnnounceRemoteSize: true, + }); err != nil { return "", fmt.Errorf("refresh Docker source image %s: %w", image, err) } digestsJSON, err := runHostOutputCommand(ctx, "docker", "image", "inspect", "--format", "{{json .RepoDigests}}", image) @@ -280,13 +290,17 @@ func (m *Manager) refreshDockerSourceDigest(ctx context.Context) (string, error) } sort.Strings(digests) if len(digests) > 0 { - return digests[0], nil + digest := digests[0] + m.writeDockerPullNotice(logPath, "Docker source image digest: "+digest) + return digest, nil } imageID, err := runHostOutputCommand(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", image) if err != nil { return "", err } - return strings.TrimSpace(imageID), nil + digest := strings.TrimSpace(imageID) + m.writeDockerPullNotice(logPath, "Docker source image ID: "+digest) + return digest, nil } func (m *Manager) eparScriptDigests() ([]fileDigest, error) { diff --git a/internal/pool/image_manifest_test.go b/internal/pool/image_manifest_test.go index a979291..a68c4ef 100644 --- a/internal/pool/image_manifest_test.go +++ b/internal/pool/image_manifest_test.go @@ -187,10 +187,12 @@ func TestPrepareWSLDockerSourceRootfsInvalidatesMismatchedCache(t *testing.T) { oldLogged := runHostLoggedCommand oldOutput := runHostOutputCommand oldQuiet := runHostQuietCommand + oldPull := pullDockerSourceCommand t.Cleanup(func() { runHostLoggedCommand = oldLogged runHostOutputCommand = oldOutput runHostQuietCommand = oldQuiet + pullDockerSourceCommand = oldPull }) root := t.TempDir() outputPath := filepath.Join(root, "work", "images", "epar-wsl-catthehacker-ubuntu.tar") @@ -229,6 +231,10 @@ func TestPrepareWSLDockerSourceRootfsInvalidatesMismatchedCache(t *testing.T) { runHostQuietCommand = func(context.Context, string, ...string) error { return nil } + pullDockerSourceCommand = func(_ *Manager, _ context.Context, opts dockerSourcePullOptions) error { + calls = append(calls, "docker pull "+opts.Image) + return nil + } manager := Manager{ Config: config.Config{ diff --git a/internal/pool/image_test.go b/internal/pool/image_test.go index 6578fa8..53a78aa 100644 --- a/internal/pool/image_test.go +++ b/internal/pool/image_test.go @@ -293,10 +293,12 @@ func TestPrepareWSLDockerSourceRootfsExportsContainerFilesystem(t *testing.T) { oldLogged := runHostLoggedCommand oldOutput := runHostOutputCommand oldQuiet := runHostQuietCommand + oldPull := pullDockerSourceCommand defer func() { runHostLoggedCommand = oldLogged runHostOutputCommand = oldOutput runHostQuietCommand = oldQuiet + pullDockerSourceCommand = oldPull }() var calls []string @@ -322,6 +324,15 @@ func TestPrepareWSLDockerSourceRootfsExportsContainerFilesystem(t *testing.T) { calls = append(calls, name+" "+strings.Join(args, " ")) return nil } + pullDockerSourceCommand = func(_ *Manager, _ context.Context, opts dockerSourcePullOptions) error { + args := []string{"pull"} + if opts.Platform != "" { + args = append(args, "--platform", opts.Platform) + } + args = append(args, opts.Image) + calls = append(calls, "docker "+strings.Join(args, " ")) + return nil + } manager := Manager{ Config: config.Config{ @@ -373,11 +384,17 @@ func TestPrepareWSLDockerSourceRootfsUsesCachedRootfs(t *testing.T) { origLogged := runHostLoggedCommand origOutput := runHostOutputCommand origQuiet := runHostQuietCommand + origPull := pullDockerSourceCommand defer func() { runHostLoggedCommand = origLogged runHostOutputCommand = origOutput runHostQuietCommand = origQuiet + pullDockerSourceCommand = origPull }() + pullDockerSourceCommand = func(*Manager, context.Context, dockerSourcePullOptions) error { + t.Fatal("docker pull should not run when cached rootfs and env metadata exist") + return nil + } runHostLoggedCommand = func(context.Context, string, string, ...string) error { t.Fatal("docker command should not run when cached rootfs and env metadata exist") return nil diff --git a/internal/pool/manager.go b/internal/pool/manager.go index cd50bbd..1cdfed0 100644 --- a/internal/pool/manager.go +++ b/internal/pool/manager.go @@ -18,12 +18,13 @@ import ( ) type Manager struct { - Config config.Config - Provider provider.Provider - GitHub GitHubClient - ProjectRoot string - ConfigPath string - DryRun bool + Config config.Config + Provider provider.Provider + GitHub GitHubClient + ProjectRoot string + ConfigPath string + DryRun bool + startupTiming *startupTiming hostTrustResolver func(context.Context) (hosttrust.Snapshot, error) hostTrustImageEnsurer func(context.Context) error @@ -458,11 +459,16 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register return vm, err } fmt.Printf("[%s] cloning %s\n", name, m.Config.Provider.SourceImage) - if err := m.Provider.Clone(ctx, m.Config.Provider.SourceImage, name); err != nil { + if err := m.timeFirstInstanceStage(name, "instance_container_create", func() error { + return m.Provider.Clone(ctx, m.Config.Provider.SourceImage, name) + }); err != nil { return vm, err } fmt.Printf("[%s] starting instance\n", name) - if _, err := m.Provider.Start(ctx, name, m.startOptions(logPath)); err != nil { + if err := m.timeFirstInstanceStage(name, m.startupInstanceStartStage(), func() error { + _, err := m.Provider.Start(ctx, name, m.startOptions(logPath)) + return err + }); err != nil { return vm, err } ip, err := m.Provider.IP(ctx, name, m.Config.Timeouts.BootSeconds) @@ -471,11 +477,13 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register } vm.IP = ip fmt.Printf("[%s] reachable at %s\n", name, ip) - if err := m.configureDockerRegistryMirrors(ctx, name); err != nil { - return vm, err - } fmt.Printf("[%s] validating runner runtime\n", name) - if err := m.validateRuntimeWithRetry(ctx, name, guestLogPath); err != nil { + if err := m.timeFirstInstanceStage(name, "runtime_validation", func() error { + if err := m.configureDockerRegistryMirrors(ctx, name); err != nil { + return err + } + return m.validateRuntimeWithRetry(ctx, name, guestLogPath) + }); err != nil { return vm, err } fmt.Printf("[%s] runtime validation passed\n", name) @@ -508,44 +516,60 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register } return vm, fmt.Errorf("github client is required for registration") } - fmt.Printf("[%s] requesting GitHub registration token\n", name) - token, err := m.GitHub.RegistrationToken(ctx) - if err != nil { - return vm, err - } - env := map[string]string{ - "RUNNER_URL": m.GitHub.OrganizationURL(), - "RUNNER_TOKEN": token.Token, - "RUNNER_NAME": name, - "RUNNER_LABELS": strings.Join(m.Config.Runner.Labels, ","), - "RUNNER_EPHEMERAL": fmt.Sprintf("%t", m.Config.Runner.Ephemeral), - "RUNNER_GROUP": m.Config.Runner.Group, - "RUNNER_NO_DEFAULT_LABELS": fmt.Sprintf("%t", m.Config.Runner.NoDefaultLabels), - } - if _, err := m.execGuest(ctx, name, []string{"sudo", "-E", "bash", "/opt/epar/configure-runner.sh"}, provider.ExecOptions{Env: env}); err != nil { - return vm, err - } - fmt.Printf("[%s] starting runner service\n", name) - if _, err := m.execGuest(ctx, name, []string{"sudo", "bash", "/opt/epar/run-runner.sh"}, provider.ExecOptions{}); err != nil { - m.captureRunnerReadinessDiagnostics(name, guestLogPath) - return vm, err - } - readiness := "online/idle" - if allowBusy { - readiness = "online" - } - fmt.Printf("[%s] waiting for GitHub %s\n", name, readiness) - runner, err := m.waitRunnerReadyAndHealthy(ctx, vm, time.Duration(m.Config.Timeouts.GitHubOnlineSeconds)*time.Second, allowBusy) - if err != nil { + var ( + token gh.RegistrationToken + runner gh.Runner + readiness = "online/idle" + ) + if err := m.timeFirstInstanceStage(name, "github_registration_and_online_idle", func() error { + fmt.Printf("[%s] requesting GitHub registration token\n", name) + var err error + token, err = m.GitHub.RegistrationToken(ctx) + if err != nil { + return err + } + env := map[string]string{ + "RUNNER_URL": m.GitHub.OrganizationURL(), + "RUNNER_TOKEN": token.Token, + "RUNNER_NAME": name, + "RUNNER_LABELS": strings.Join(m.Config.Runner.Labels, ","), + "RUNNER_EPHEMERAL": fmt.Sprintf("%t", m.Config.Runner.Ephemeral), + "RUNNER_GROUP": m.Config.Runner.Group, + "RUNNER_NO_DEFAULT_LABELS": fmt.Sprintf("%t", m.Config.Runner.NoDefaultLabels), + } + if _, err := m.execGuest(ctx, name, []string{"sudo", "-E", "bash", "/opt/epar/configure-runner.sh"}, provider.ExecOptions{Env: env}); err != nil { + return err + } + fmt.Printf("[%s] starting runner service\n", name) + if _, err := m.execGuest(ctx, name, []string{"sudo", "bash", "/opt/epar/run-runner.sh"}, provider.ExecOptions{}); err != nil { + return err + } + if allowBusy { + readiness = "online" + } + fmt.Printf("[%s] waiting for GitHub %s\n", name, readiness) + runner, err = m.waitRunnerReadyAndHealthy(ctx, vm, time.Duration(m.Config.Timeouts.GitHubOnlineSeconds)*time.Second, allowBusy) + return err + }); err != nil { m.captureRunnerReadinessDiagnostics(name, guestLogPath) return vm, err } vm.RunnerID = runner.ID fmt.Printf("[%s] GitHub runner %s id=%d busy=%t\n", name, readiness, runner.ID, runner.Busy) + m.finishFirstRunnerReady(name) + } else { + m.finishFirstRunnerReady(name) } return vm, nil } +func (m *Manager) startupInstanceStartStage() string { + if m.Config.Provider.Type == "docker-dind" { + return "instance_start_and_inner_docker_ready" + } + return "instance_start_and_provider_ready" +} + type runnerReadinessResult struct { runner gh.Runner err error diff --git a/internal/pool/startup_timing.go b/internal/pool/startup_timing.go new file mode 100644 index 0000000..2ddcff2 --- /dev/null +++ b/internal/pool/startup_timing.go @@ -0,0 +1,203 @@ +package pool + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "github.com/solutionforest/ephemeral-action-runner/internal/config" +) + +var timingSecretPattern = regexp.MustCompile(`(?i)((?:runner_)?token|password|secret|private[_-]?key)=\S+`) + +type startupTimingEvent struct { + Timestamp string `json:"timestamp"` + Provider string `json:"provider"` + Stage string `json:"stage"` + Outcome string `json:"outcome"` + ElapsedMS int64 `json:"elapsedMs"` + Error string `json:"error,omitempty"` +} + +type startupTimingStage struct { + name string + elapsed time.Duration +} + +type startupTiming struct { + mu sync.Mutex + file *os.File + encoder *json.Encoder + path string + provider string + startedAt time.Time + stages []startupTimingStage + firstInstance string + closed bool +} + +// StartStartupTiming records the initial Docker-DinD or WSL start path. +func (m *Manager) StartStartupTiming() (string, error) { + provider := m.Config.Provider.Type + if !supportsStartupTiming(provider) { + return "", nil + } + dir := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), "benchmarks") + if err := os.MkdirAll(dir, 0755); err != nil { + return "", err + } + startedAt := time.Now().UTC() + path := filepath.Join(dir, startedAt.Format("20060102T150405.000000000Z")+"-"+provider+".jsonl") + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) + if err != nil { + return "", err + } + timing := &startupTiming{ + file: file, + encoder: json.NewEncoder(file), + path: path, + provider: provider, + startedAt: startedAt, + } + m.startupTiming = timing + timing.eventLocked("total_startup", "started", 0, nil) + return path, nil +} + +// FinishStartupTiming records a terminal error before the first runner is ready. +func (m *Manager) FinishStartupTiming(err error) { + timing := m.startupTiming + if timing == nil { + return + } + timing.finish(err) +} + +func (m *Manager) timeStartupStage(stage string, fn func() error) error { + timing := m.startupTiming + if timing == nil { + return fn() + } + return timing.measure(stage, fn) +} + +func (m *Manager) timeFirstInstanceStage(name, stage string, fn func() error) error { + timing := m.startupTiming + if timing == nil || !timing.isFirstInstance(name) { + return fn() + } + return timing.measure(stage, fn) +} + +func (m *Manager) finishFirstRunnerReady(name string) { + timing := m.startupTiming + if timing == nil || !timing.isFirstInstance(name) { + return + } + timing.finish(nil) +} + +func (t *startupTiming) isFirstInstance(name string) bool { + t.mu.Lock() + defer t.mu.Unlock() + if t.closed { + return false + } + if t.firstInstance == "" { + t.firstInstance = name + } + return t.firstInstance == name +} + +func (t *startupTiming) measure(stage string, fn func() error) error { + startedAt := time.Now() + t.mu.Lock() + if t.closed { + t.mu.Unlock() + return fn() + } + t.eventLocked(stage, "started", 0, nil) + t.mu.Unlock() + + err := fn() + elapsed := time.Since(startedAt) + t.mu.Lock() + defer t.mu.Unlock() + if t.closed { + return err + } + if err != nil { + t.eventLocked(stage, "failed", elapsed, err) + return err + } + t.stages = append(t.stages, startupTimingStage{name: stage, elapsed: elapsed}) + t.eventLocked(stage, "completed", elapsed, nil) + return nil +} + +func (t *startupTiming) finish(err error) { + t.mu.Lock() + defer t.mu.Unlock() + if t.closed { + return + } + elapsed := time.Since(t.startedAt) + if err != nil { + t.eventLocked("total_startup", "failed", elapsed, err) + } else { + t.stages = append(t.stages, startupTimingStage{name: "total_startup", elapsed: elapsed}) + t.eventLocked("total_startup", "completed", elapsed, nil) + fmt.Printf("%s startup timing (first runner online/idle):\n", startupTimingLabel(t.provider)) + for _, stage := range t.stages { + fmt.Printf(" %s: %s\n", stage.name, stage.elapsed.Round(time.Millisecond)) + } + fmt.Printf(" timing log: %s\n", t.path) + } + t.closed = true + _ = t.file.Close() +} + +func (t *startupTiming) eventLocked(stage, outcome string, elapsed time.Duration, err error) { + event := startupTimingEvent{ + Timestamp: time.Now().UTC().Format(time.RFC3339Nano), + Provider: t.provider, + Stage: stage, + Outcome: outcome, + ElapsedMS: elapsed.Milliseconds(), + } + if err != nil { + event.Error = sanitizeTimingError(err) + } + _ = t.encoder.Encode(event) +} + +func supportsStartupTiming(provider string) bool { + return provider == "docker-dind" || provider == "wsl" +} + +func startupTimingLabel(provider string) string { + switch provider { + case "docker-dind": + return "DinD" + case "wsl": + return "WSL" + default: + return provider + } +} + +func sanitizeTimingError(err error) string { + text := strings.Join(strings.Fields(err.Error()), " ") + text = timingSecretPattern.ReplaceAllStringFunc(text, func(match string) string { + return match[:strings.Index(match, "=")+1] + "[REDACTED]" + }) + if len(text) > 500 { + return text[:500] + "..." + } + return text +} From a44cf904926cf63123347d0102e7002755fd5a46 Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 16 Jul 2026 00:29:36 +0800 Subject: [PATCH 2/9] Add structured logging and retention --- cmd/ephemeral-action-runner/init.go | 40 +- cmd/ephemeral-action-runner/init_test.go | 10 + cmd/ephemeral-action-runner/logging_test.go | 139 +++++ cmd/ephemeral-action-runner/main.go | 203 ++++++- cmd/ephemeral-action-runner/start.go | 7 + configs/docker-dind.act.example.yml | 20 +- configs/docker-dind.core.example.yml | 20 +- configs/docker-dind.example.yml | 20 +- configs/docker-dind.web-e2e.example.yml | 20 +- configs/tart.example.yml | 20 +- configs/tart.web-e2e.example.yml | 20 +- configs/wsl.example.yml | 20 +- configs/wsl.lean.example.yml | 20 +- configs/wsl.web-e2e.example.yml | 20 +- docs/advanced/macos-startup.md | 4 +- docs/configuration.md | 5 +- docs/logging.md | 82 +++ docs/operations.md | 10 +- docs/troubleshooting.md | 8 +- docs/usage.md | 2 +- examples/macos/start-epar.command | 2 +- examples/observability/README.md | 5 + examples/observability/kubernetes-console.yml | 18 + examples/observability/local-file.yml | 19 + .../observability/otel-collector-filelog.yaml | 30 + go.mod | 1 + go.sum | 2 + internal/config/config.go | 310 ++++++++++- internal/config/config_test.go | 189 +++++++ internal/filelock/filelock.go | 53 ++ internal/filelock/filelock_test.go | 59 ++ internal/filelock/filelock_unix.go | 25 + internal/filelock/filelock_unsupported.go | 18 + internal/filelock/filelock_windows.go | 43 ++ internal/logging/handler.go | 155 ++++++ internal/logging/human_handler.go | 148 +++++ internal/logging/pathcase_other.go | 5 + internal/logging/pathcase_windows.go | 7 + internal/logging/paths.go | 119 ++++ internal/logging/paths_test.go | 84 +++ internal/logging/recognition.go | 116 ++++ internal/logging/registry.go | 315 +++++++++++ internal/logging/reparse_other.go | 7 + internal/logging/reparse_windows.go | 16 + internal/logging/reparse_windows_test.go | 16 + internal/logging/replace_other.go | 7 + internal/logging/replace_windows.go | 18 + internal/logging/retention.go | 463 ++++++++++++++++ internal/logging/retention_test.go | 318 +++++++++++ internal/logging/runtime.go | 155 ++++++ internal/logging/runtime_test.go | 520 ++++++++++++++++++ internal/logging/transcript.go | 225 ++++++++ internal/logging/types.go | 222 ++++++++ internal/pool/docker_pull.go | 58 +- internal/pool/host_trust.go | 20 +- internal/pool/host_trust_test.go | 9 +- internal/pool/image.go | 277 +++++----- internal/pool/image_manifest.go | 13 +- internal/pool/image_manifest_test.go | 3 +- internal/pool/image_test.go | 10 +- internal/pool/log_paths.go | 19 + internal/pool/logging.go | 162 ++++++ internal/pool/manager.go | 166 ++++-- internal/pool/manager_test.go | 148 ++++- internal/pool/startup_timing.go | 12 +- internal/pool/trusted_ca.go | 2 +- internal/provider/dockerdind/docker_dind.go | 50 +- .../provider/dockerdind/docker_dind_test.go | 31 +- internal/provider/provider.go | 5 + internal/provider/tart/tart.go | 69 +-- internal/provider/tart/tart_test.go | 11 + internal/provider/wsl/wsl.go | 80 ++- internal/provider/wsl/wsl_test.go | 29 +- scripts/ci/core-runner-controller.sh | 20 +- scripts/ci/core-runner-controller_test.sh | 2 +- 75 files changed, 5151 insertions(+), 425 deletions(-) create mode 100644 cmd/ephemeral-action-runner/logging_test.go create mode 100644 docs/logging.md create mode 100644 examples/observability/README.md create mode 100644 examples/observability/kubernetes-console.yml create mode 100644 examples/observability/local-file.yml create mode 100644 examples/observability/otel-collector-filelog.yaml create mode 100644 internal/filelock/filelock.go create mode 100644 internal/filelock/filelock_test.go create mode 100644 internal/filelock/filelock_unix.go create mode 100644 internal/filelock/filelock_unsupported.go create mode 100644 internal/filelock/filelock_windows.go create mode 100644 internal/logging/handler.go create mode 100644 internal/logging/human_handler.go create mode 100644 internal/logging/pathcase_other.go create mode 100644 internal/logging/pathcase_windows.go create mode 100644 internal/logging/paths.go create mode 100644 internal/logging/paths_test.go create mode 100644 internal/logging/recognition.go create mode 100644 internal/logging/registry.go create mode 100644 internal/logging/reparse_other.go create mode 100644 internal/logging/reparse_windows.go create mode 100644 internal/logging/reparse_windows_test.go create mode 100644 internal/logging/replace_other.go create mode 100644 internal/logging/replace_windows.go create mode 100644 internal/logging/retention.go create mode 100644 internal/logging/retention_test.go create mode 100644 internal/logging/runtime.go create mode 100644 internal/logging/runtime_test.go create mode 100644 internal/logging/transcript.go create mode 100644 internal/logging/types.go create mode 100644 internal/pool/log_paths.go create mode 100644 internal/pool/logging.go diff --git a/cmd/ephemeral-action-runner/init.go b/cmd/ephemeral-action-runner/init.go index c8b80c8..4302ef3 100644 --- a/cmd/ephemeral-action-runner/init.go +++ b/cmd/ephemeral-action-runner/init.go @@ -481,7 +481,25 @@ image: pool: instances: 1 namePrefix: %s - logDir: work/logs +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 runner: labels: [self-hosted, linux, epar-docker-dind-catthehacker-ubuntu] @@ -527,7 +545,25 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: %s - logDir: work/logs +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 runner: labels: [self-hosted, linux, X64, epar-wsl-catthehacker-ubuntu] diff --git a/cmd/ephemeral-action-runner/init_test.go b/cmd/ephemeral-action-runner/init_test.go index daa09ef..b5bd13f 100644 --- a/cmd/ephemeral-action-runner/init_test.go +++ b/cmd/ephemeral-action-runner/init_test.go @@ -64,6 +64,16 @@ func TestInitCreatesDefaultDockerDindConfig(t *testing.T) { if got, want := cfg.Pool.NamePrefix, "build-box-01-a4f9c2"; got != want { t.Fatalf("pool.namePrefix = %q, want %q", got, want) } + if got, want := cfg.Logging.Directory, "work/logs"; got != want { + t.Fatalf("logging.directory = %q, want %q", got, want) + } + configText, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(configText), "logging:\n directory: work/logs\n managerSinks: [console]\n") { + t.Fatalf("generated config did not include logging schema:\n%s", configText) + } if got := strings.Join(cfg.Runner.Labels, ","); !strings.Contains(got, "epar-docker-dind-catthehacker-ubuntu") { t.Fatalf("runner labels = %q", got) } diff --git a/cmd/ephemeral-action-runner/logging_test.go b/cmd/ephemeral-action-runner/logging_test.go new file mode 100644 index 0000000..0885958 --- /dev/null +++ b/cmd/ephemeral-action-runner/logging_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestLogsPathListAndPrune(t *testing.T) { + root := t.TempDir() + configPath := writeLoggingCommandConfig(t, root, "artifact-logs") + logRoot := filepath.Join(root, "artifact-logs") + instanceDir := filepath.Join(logRoot, "instances") + if err := os.MkdirAll(instanceDir, 0o755); err != nil { + t.Fatal(err) + } + oldPath := filepath.Join(instanceDir, "runner-1.guest.log") + if err := os.WriteFile(oldPath, []byte("old transcript\n"), 0o644); err != nil { + t.Fatal(err) + } + oldTime := time.Now().Add(-48 * time.Hour) + if err := os.Chtimes(oldPath, oldTime, oldTime); err != nil { + t.Fatal(err) + } + + pathOutput, err := captureStdout(t, func() error { + return run([]string{"logs", "path", "--project-root", root, "--config", configPath}) + }) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(pathOutput) != logRoot { + t.Fatalf("logs path output = %q, want %q", pathOutput, logRoot) + } + + listOutput, err := captureStdout(t, func() error { + return run([]string{"logs", "list", "--project-root", root, "--config", configPath}) + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.ToLower(listOutput), strings.ToLower(oldPath)) { + t.Fatalf("logs list output missing %s:\n%s", oldPath, listOutput) + } + + if _, err := captureStdout(t, func() error { + return run([]string{"logs", "prune", "--dry-run", "--project-root", root, "--config", configPath}) + }); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(oldPath); err != nil { + t.Fatalf("dry-run removed transcript: %v", err) + } + if _, err := captureStdout(t, func() error { + return run([]string{"logs", "prune", "--project-root", root, "--config", configPath}) + }); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(oldPath); !os.IsNotExist(err) { + t.Fatalf("prune left old transcript, stat error = %v", err) + } +} + +func TestCLIConfigLoadWarnsForLegacyPoolLogDir(t *testing.T) { + root := t.TempDir() + configPath := filepath.Join(root, "legacy.yml") + if err := os.WriteFile(configPath, []byte("pool:\n logDir: legacy/logs\n"), 0o644); err != nil { + t.Fatal(err) + } + stderr, err := captureStderr(t, func() error { + _, cfg, resolved, err := loadLoggingConfig(root, configPath) + if err != nil { + return err + } + if cfg.Logging.Directory != "legacy/logs" || resolved != filepath.Join(root, "legacy", "logs") { + t.Fatalf("legacy directory migration = %q, resolved = %q", cfg.Logging.Directory, resolved) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(stderr, "warning:") || !strings.Contains(stderr, "pool.logDir is deprecated") { + t.Fatalf("stderr = %q, want migration warning", stderr) + } +} + +func TestWriteLastErrorReportUsesConfiguredDirectoryAndFallback(t *testing.T) { + root := t.TempDir() + configPath := writeLoggingCommandConfig(t, root, "custom-logs") + path := writeLastErrorReport([]string{"start", "--project-root", root, "--config", configPath}, errors.New("configured failure")) + if path != filepath.Join(root, "custom-logs", "epar-last-error.log") { + t.Fatalf("configured report path = %q", path) + } + archives, err := filepath.Glob(filepath.Join(root, "custom-logs", "errors", "epar-*-error.log")) + if err != nil || len(archives) != 1 { + t.Fatalf("configured error archives = %v, err = %v", archives, err) + } + + fallbackRoot := t.TempDir() + fallback := writeLastErrorReport([]string{"start", "--project-root", fallbackRoot, "--config", "missing.yml"}, errors.New("fallback failure")) + if fallback != filepath.Join(fallbackRoot, "work", "logs", "epar-last-error.log") { + t.Fatalf("fallback report path = %q", fallback) + } +} + +func writeLoggingCommandConfig(t *testing.T, root, directory string) string { + t.Helper() + path := filepath.Join(root, "config.yml") + content := "logging:\n directory: " + directory + "\n instanceMaxAgeDays: 1\n" + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func captureStderr(t *testing.T, fn func() error) (string, error) { + t.Helper() + oldStderr := os.Stderr + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stderr = writer + defer func() { os.Stderr = oldStderr }() + fnErr := fn() + if err := writer.Close(); err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + return string(data), fnErr +} diff --git a/cmd/ephemeral-action-runner/main.go b/cmd/ephemeral-action-runner/main.go index 834ac48..e03a181 100644 --- a/cmd/ephemeral-action-runner/main.go +++ b/cmd/ephemeral-action-runner/main.go @@ -7,11 +7,13 @@ import ( "os" "os/signal" "path/filepath" + "strings" "syscall" "time" "github.com/solutionforest/ephemeral-action-runner/internal/config" gh "github.com/solutionforest/ephemeral-action-runner/internal/github" + "github.com/solutionforest/ephemeral-action-runner/internal/logging" "github.com/solutionforest/ephemeral-action-runner/internal/pool" "github.com/solutionforest/ephemeral-action-runner/internal/provider" dockerdindprovider "github.com/solutionforest/ephemeral-action-runner/internal/provider/dockerdind" @@ -24,22 +26,31 @@ const binaryName = "ephemeral-action-runner" func main() { if err := run(os.Args[1:]); err != nil { fmt.Fprintln(os.Stderr, binaryName+":", err) - if reportPath := writeLastErrorReport(err); reportPath != "" { + if reportPath := writeLastErrorReport(os.Args[1:], err); reportPath != "" { fmt.Fprintln(os.Stderr, "error report:", reportPath) } os.Exit(1) } } -func writeLastErrorReport(runErr error) string { +func writeLastErrorReport(args []string, runErr error) string { cwd, err := os.Getwd() if err != nil { return "" } - logDir := filepath.Join(cwd, "work", "logs") + projectRoot, explicitConfig := errorReportFlags(args, cwd) + logDir := filepath.Join(projectRoot, "work", "logs") + if resolvedConfig, resolveErr := resolveConfigPath(projectRoot, explicitConfig); resolveErr == nil && resolvedConfig != "" { + if cfg, loadErr := config.Load(resolvedConfig); loadErr == nil && config.ValidateLogging(cfg.Logging) == nil { + logDir = config.ProjectPath(projectRoot, cfg.Logging.Directory) + } + } if err := os.MkdirAll(logDir, 0755); err != nil { return "" } + if err := os.MkdirAll(filepath.Join(logDir, "errors"), 0755); err != nil { + return "" + } now := time.Now().UTC() content := fmt.Sprintf(`EPAR failed time: %s @@ -50,15 +61,39 @@ command: %q error: %v `, now.Format(time.RFC3339), cwd, os.Args, versionString(), runErr) - lastPath := filepath.Join(logDir, "epar-last-error.log") + lastPath := logging.LastErrorPath(logDir) if err := os.WriteFile(lastPath, []byte(content), 0644); err != nil { return "" } - stampedPath := filepath.Join(logDir, fmt.Sprintf("epar-%s-error.log", now.Format("20060102-150405"))) + stampedPath := logging.ErrorPath(logDir, now) _ = os.WriteFile(stampedPath, []byte(content), 0644) return lastPath } +func errorReportFlags(args []string, fallbackRoot string) (string, string) { + projectRoot := fallbackRoot + configPath := "" + for index := 0; index < len(args); index++ { + value := args[index] + switch { + case value == "--project-root" && index+1 < len(args): + index++ + projectRoot = args[index] + case strings.HasPrefix(value, "--project-root="): + projectRoot = strings.TrimPrefix(value, "--project-root=") + case value == "--config" && index+1 < len(args): + index++ + configPath = args[index] + case strings.HasPrefix(value, "--config="): + configPath = strings.TrimPrefix(value, "--config=") + } + } + if absolute, absErr := filepath.Abs(projectRoot); absErr == nil { + projectRoot = absolute + } + return projectRoot, configPath +} + func run(args []string) error { if len(args) == 0 { return runStart(nil) @@ -76,6 +111,8 @@ func run(args []string) error { return runCleanup(args[1:]) case "status": return runStatus(args[1:]) + case "logs": + return runLogs(args[1:]) case "version": printVersion(os.Stdout) return nil @@ -87,6 +124,98 @@ func run(args []string) error { } } +func runLogs(args []string) error { + if len(args) == 0 { + return fmt.Errorf("logs requires subcommand: path, list, or prune") + } + fs := flag.NewFlagSet("logs "+args[0], flag.ExitOnError) + common := addCommonFlags(fs) + if err := fs.Parse(args[1:]); err != nil { + return err + } + if fs.NArg() != 0 { + return fmt.Errorf("logs %s does not accept positional arguments", args[0]) + } + projectRoot, cfg, root, err := loadLoggingConfig(*common.projectRoot, *common.configPath) + if err != nil { + return err + } + _ = projectRoot + switch args[0] { + case "path": + fmt.Fprintln(os.Stdout, root) + return nil + case "list": + report, err := logging.ListRetention(root, retentionPolicy(cfg.Logging)) + if err != nil { + return err + } + for _, entry := range report.Entries { + if entry.Recognized { + fmt.Fprintf(os.Stdout, "%s\t%s\t%d\t%s\n", entry.Category, entry.Action, entry.Size, entry.Path) + } + } + fmt.Fprintln(os.Stdout, report.Summary()) + return nil + case "prune": + report, err := logging.PruneRetention(root, retentionPolicy(cfg.Logging), *common.dryRun) + if err != nil { + return err + } + for _, entry := range report.Entries { + if entry.Action == logging.RetentionWouldDelete || entry.Action == logging.RetentionDeleted || entry.Action == logging.RetentionSkipped { + fmt.Fprintf(os.Stdout, "%s\t%s\t%s\n", entry.Action, entry.Reason, entry.Path) + } + } + fmt.Fprintln(os.Stdout, report.Summary()) + for _, warning := range report.Warnings { + fmt.Fprintln(os.Stderr, "warning:", warning) + } + return nil + default: + return fmt.Errorf("unknown logs subcommand %q", args[0]) + } +} + +func loadLoggingConfig(projectRoot, configPath string) (string, config.Config, string, error) { + projectRoot, err := filepath.Abs(projectRoot) + if err != nil { + return "", config.Config{}, "", err + } + resolvedConfigPath, err := resolveConfigPath(projectRoot, configPath) + if err != nil { + return "", config.Config{}, "", err + } + if resolvedConfigPath == "" { + return "", config.Config{}, "", fmt.Errorf("no config found; run %s init from the EPAR directory to create .local/config.yml", binaryName) + } + cfg, err := config.Load(resolvedConfigPath) + if err != nil { + return "", config.Config{}, "", err + } + printConfigWarnings(cfg) + if err := config.Validate(cfg); err != nil { + return "", config.Config{}, "", err + } + root, err := filepath.Abs(config.ProjectPath(projectRoot, cfg.Logging.Directory)) + if err != nil { + return "", config.Config{}, "", err + } + return projectRoot, cfg, root, nil +} + +func retentionPolicy(cfg config.LoggingConfig) logging.RetentionPolicy { + days := func(value int) time.Duration { return time.Duration(value) * 24 * time.Hour } + return logging.RetentionPolicy{ + MaxTotalBytes: int64(cfg.RetentionMaxTotalMiB) * 1024 * 1024, + ManagerMaxAge: days(cfg.ManagerMaxAgeDays), + InstanceMaxAge: days(cfg.InstanceMaxAgeDays), + BuildMaxAge: days(cfg.BuildMaxAgeDays), + ErrorMaxAge: days(cfg.ErrorMaxAgeDays), + BenchmarkMaxAge: days(cfg.BenchmarkMaxAgeDays), + } +} + func runImage(args []string) error { if len(args) == 0 { return fmt.Errorf("image requires subcommand: update-upstream or build") @@ -102,6 +231,7 @@ func runImage(args []string) error { if err != nil { return err } + defer m.Close() return m.UpdateUpstream(context.Background()) case "build": fs := flag.NewFlagSet("image build", flag.ExitOnError) @@ -116,11 +246,13 @@ func runImage(args []string) error { if err != nil { return err } + defer m.Close() ctx := interruptContext() controllerLock, err := m.AcquireHostTrustControllerLock() if err != nil { return err } + defer m.Close() if controllerLock != nil { defer controllerLock.Close() } @@ -140,6 +272,7 @@ func runImage(args []string) error { if err != nil { return err } + defer m.Close() return m.RefreshScripts(interruptContext()) default: return fmt.Errorf("unknown image subcommand %q", args[0]) @@ -167,6 +300,7 @@ func runPool(args []string) error { if err != nil { return err } + defer m.Close() return m.Verify(interruptContext(), pool.VerifyOptions{Instances: *instances, RegisterOnly: *registerOnly, Cleanup: *cleanup}) case "up": fs := flag.NewFlagSet("pool up", flag.ExitOnError) @@ -211,6 +345,7 @@ func runCleanup(args []string) error { if err != nil { return err } + defer m.Close() return m.Cleanup(context.Background()) } @@ -225,6 +360,7 @@ func runStatus(args []string) error { if err != nil { return err } + defer m.Close() status, err := m.Status(context.Background()) if err != nil { return err @@ -274,6 +410,7 @@ func newManager(configPath, projectRoot string, dryRun bool, githubEnabled bool) if err != nil { return nil, err } + printConfigWarnings(cfg) if err := config.Validate(cfg); err != nil { return nil, err } @@ -288,14 +425,63 @@ func newManager(configPath, projectRoot string, dryRun bool, githubEnabled bool) if err != nil { return nil, err } - return &pool.Manager{ + runtime, err := logging.NewRuntime(logging.Options{ + Directory: config.ProjectPath(projectRoot, cfg.Logging.Directory), + ManagerSinks: loggingSinks(cfg.Logging.ManagerSinks), + ManagerConsoleFormat: logging.Format(cfg.Logging.ManagerConsoleFormat), + ManagerConsoleTextFormat: cfg.Logging.ManagerConsoleTextFormat, + ManagerFileFormat: logging.Format(cfg.Logging.ManagerFileFormat), + TranscriptSinks: loggingSinks(cfg.Logging.TranscriptSinks), + TranscriptConsoleFormat: logging.Format(cfg.Logging.TranscriptConsoleFormat), + TranscriptConsoleTextFormat: cfg.Logging.TranscriptConsoleTextFormat, + Rotation: logging.Rotation{ + MaxSizeMiB: cfg.Logging.MaxFileSizeMiB, + MaxBackups: cfg.Logging.MaxBackups, + Compress: cfg.Logging.CompressBackups, + }, + }) + if err != nil { + return nil, err + } + manager := &pool.Manager{ Config: cfg, Provider: provider, GitHub: client, ProjectRoot: projectRoot, ConfigPath: resolvedConfigPath, DryRun: dryRun, - }, nil + Logging: runtime, + } + if cfg.Logging.RetentionEnabled { + report, pruneErr := manager.PruneLogs(false) + if pruneErr != nil { + runtime.Logger().Warn("log retention failed", "operation", "logs-prune", "error", pruneErr) + } else { + for _, warning := range report.Warnings { + runtime.Logger().Warn("log retention skipped candidate", "operation", "logs-prune", "warning", warning) + } + } + } + return manager, nil +} + +func loggingSinks(values []string) logging.Sinks { + var sinks logging.Sinks + for _, value := range values { + switch value { + case "console": + sinks |= logging.SinkConsole + case "file": + sinks |= logging.SinkFile + } + } + return sinks +} + +func printConfigWarnings(cfg config.Config) { + for _, warning := range cfg.Warnings() { + fmt.Fprintln(os.Stderr, "warning:", warning) + } } func resolveConfigPath(projectRoot, explicit string) (string, error) { @@ -365,6 +551,9 @@ Commands: ephemeral-action-runner pool down ephemeral-action-runner cleanup ephemeral-action-runner status + ephemeral-action-runner logs path + ephemeral-action-runner logs list + ephemeral-action-runner logs prune [--dry-run] ephemeral-action-runner version `) } diff --git a/cmd/ephemeral-action-runner/start.go b/cmd/ephemeral-action-runner/start.go index 78604ce..fba9142 100644 --- a/cmd/ephemeral-action-runner/start.go +++ b/cmd/ephemeral-action-runner/start.go @@ -28,6 +28,10 @@ type startupTimingStarterManager interface { FinishStartupTiming(error) } +type closingStarterManager interface { + Close() error +} + type starterManagerFactory func(configPath, projectRoot string, dryRun bool, githubEnabled bool) (starterManager, error) var newStarterManager starterManagerFactory = func(configPath, projectRoot string, dryRun bool, githubEnabled bool) (starterManager, error) { @@ -112,6 +116,9 @@ func runStartWithOptions(opts startOptions) (err error) { if err != nil { return err } + if closingManager, ok := manager.(closingStarterManager); ok { + defer closingManager.Close() + } if timingManager, ok := manager.(startupTimingStarterManager); ok { if _, err := timingManager.StartStartupTiming(); err != nil { return fmt.Errorf("start startup timing log: %w", err) diff --git a/configs/docker-dind.act.example.yml b/configs/docker-dind.act.example.yml index b1110c8..9c51164 100644 --- a/configs/docker-dind.act.example.yml +++ b/configs/docker-dind.act.example.yml @@ -22,7 +22,25 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix - logDir: work/logs +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 runner: labels: [self-hosted, linux, epar-docker-dind-catthehacker-act] diff --git a/configs/docker-dind.core.example.yml b/configs/docker-dind.core.example.yml index f558101..2ec64d6 100644 --- a/configs/docker-dind.core.example.yml +++ b/configs/docker-dind.core.example.yml @@ -22,7 +22,25 @@ image: pool: instances: 1 namePrefix: epar-ci-core - logDir: work/logs +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 runner: group: epar-ci-canary diff --git a/configs/docker-dind.example.yml b/configs/docker-dind.example.yml index 3e998ee..fea1ad1 100644 --- a/configs/docker-dind.example.yml +++ b/configs/docker-dind.example.yml @@ -22,7 +22,25 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix - logDir: work/logs +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 runner: labels: [self-hosted, linux, epar-docker-dind-catthehacker-ubuntu] diff --git a/configs/docker-dind.web-e2e.example.yml b/configs/docker-dind.web-e2e.example.yml index c1d7547..abbd6ec 100644 --- a/configs/docker-dind.web-e2e.example.yml +++ b/configs/docker-dind.web-e2e.example.yml @@ -23,7 +23,25 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix - logDir: work/logs +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 runner: labels: [self-hosted, linux, epar-docker-dind-catthehacker-ubuntu-web-e2e] diff --git a/configs/tart.example.yml b/configs/tart.example.yml index 7b0acae..d182ec4 100644 --- a/configs/tart.example.yml +++ b/configs/tart.example.yml @@ -18,7 +18,25 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix - logDir: work/logs +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 runner: labels: [self-hosted, linux, ARM64, epar-tart-ubuntu-24.04-base] diff --git a/configs/tart.web-e2e.example.yml b/configs/tart.web-e2e.example.yml index 0c96cb6..22bd36e 100644 --- a/configs/tart.web-e2e.example.yml +++ b/configs/tart.web-e2e.example.yml @@ -18,7 +18,25 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix - logDir: work/logs +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 runner: labels: [self-hosted, linux, ARM64, epar-tart-ubuntu-24.04-web-e2e, epar-tart-rosetta-amd64] diff --git a/configs/wsl.example.yml b/configs/wsl.example.yml index 85df66f..c39457e 100644 --- a/configs/wsl.example.yml +++ b/configs/wsl.example.yml @@ -20,7 +20,25 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix - logDir: work/logs +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 runner: labels: [self-hosted, linux, X64, epar-wsl-catthehacker-ubuntu] diff --git a/configs/wsl.lean.example.yml b/configs/wsl.lean.example.yml index 30765a6..612105c 100644 --- a/configs/wsl.lean.example.yml +++ b/configs/wsl.lean.example.yml @@ -19,7 +19,25 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix - logDir: work/logs +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 runner: labels: [self-hosted, linux, X64, epar-wsl-ubuntu-24.04-base] diff --git a/configs/wsl.web-e2e.example.yml b/configs/wsl.web-e2e.example.yml index 44f36ac..e0576c5 100644 --- a/configs/wsl.web-e2e.example.yml +++ b/configs/wsl.web-e2e.example.yml @@ -19,7 +19,25 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix - logDir: work/logs +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 runner: labels: [self-hosted, linux, X64, epar-wsl-ubuntu-24.04-web-e2e] diff --git a/docs/advanced/macos-startup.md b/docs/advanced/macos-startup.md index b305b82..f29a858 100644 --- a/docs/advanced/macos-startup.md +++ b/docs/advanced/macos-startup.md @@ -98,10 +98,10 @@ Example `~/Library/LaunchAgents/com.example.epar.plist`: /path/to/ephemeral-action-runner StandardOutPath - /path/to/ephemeral-action-runner/work/logs/launchd.out.log + /path/to/ephemeral-action-runner/work/state/launchd.out.log StandardErrorPath - /path/to/ephemeral-action-runner/work/logs/launchd.err.log + /path/to/ephemeral-action-runner/work/state/launchd.err.log ``` diff --git a/docs/configuration.md b/docs/configuration.md index e8970dd..148f91d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,7 +20,8 @@ EPAR looks for config in this order: | `github` | GitHub App ID, organization, private key path, and optional GitHub API/web URLs. | | `provider` | How EPAR creates disposable runners: `docker-dind`, `wsl`, or `tart`. | | `image` | Source image/rootfs, output image, runner version, and optional install scripts. | -| `pool` | Runner count, instance name prefix, and log directory. | +| `pool` | Runner count and instance name prefix. | +| `logging` | Manager and transcript sinks, formats, rotation, retention, and log directory. | | `runner` | GitHub Actions labels, runner group, default-label policy, and whether to add the host-machine label. | | `docker` | Optional Docker registry mirrors and Docker-DinD daemon proxy settings. | | `timeouts` | Boot, GitHub online, and command timeout values in seconds. | @@ -82,6 +83,8 @@ Use a different config file: go run ./cmd/ephemeral-action-runner start --config .local/wsl.yml ``` +Configure logging and retention in the top-level `logging` section. The complete schema and local/Kubernetes examples are in [Logging](logging.md). Unknown configuration keys are rejected. For compatibility, a legacy `pool.logDir` value is used as `logging.directory` with a migration warning when the new key is absent; the file is not rewritten automatically. A configuration containing both keys is rejected as ambiguous. + ### Host trust inheritance Docker-DinD runners can inherit the host's trusted TLS root anchors: diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 0000000..4fea523 --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,82 @@ +# Logging + +EPAR uses `work/logs` under the project root by default. Set `logging.directory` to an absolute path to store logs elsewhere, or to another relative path to resolve it from the project root. + +```text +work/logs/ +├── epar.log # when managerSinks includes file +├── epar-last-error.log +├── errors/ +├── instances/ +├── builds/ +└── benchmarks/ +``` + +Manager events and command transcripts have independent sinks. `console` and `file` may be selected separately or together. Console manager events route debug and info to stdout and warnings and errors to stderr. File manager events go to `epar.log`. File transcripts remain raw command output; console transcripts frame every completed stdout or stderr line with timestamp, instance, component, and stream context. JSON console mode emits one JSON object per line. + +The local default keeps manager events on the console and command transcripts in files: + +```yaml +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 +``` + +The default manager console line is compact and human-readable: + +```text +2026-07-16T00:14:58.108+08:00 [INFO] cloning instance +``` + +When `managerConsoleFormat` is `text`, `managerConsoleTextFormat` accepts `{time}`, `{level}`, `{message}`, and `{attributes}`. `{message}` is required. The default deliberately omits structured attributes for concise local output. To include them, use `"{time} [{level}] {message}{attributes}"`; `{attributes}` expands to structured fields with its own leading space when fields exist. + +When `transcriptConsoleFormat` is `text`, the optional `transcriptConsoleTextFormat` accepts `{time}`, `{instance}`, `{component}`, `{stream}`, `{message}`, `{session}`, `{category}`, `{provider}`, and `{attributes}`. Its default is `"{time} {stream} {instance} {component} {message}{attributes}"`. + +Custom text formats are rejected when the corresponding console format is `json`. JSON records keep their fixed structured schema so downstream parsers and log shippers can rely on it. + +For Kubernetes, use console sinks so the container runtime can collect stdout and stderr: + +```yaml +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: json + managerFileFormat: json + transcriptSinks: [console] + transcriptConsoleFormat: json + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 +``` + +Benchmark JSONL and error reports remain file artifacts in every sink mode. EPAR rotates active manager and transcript files at the configured size, retains the configured number of gzip-compressed backups, and applies category age limits before the aggregate size budget. It protects active files across EPAR processes, does not follow links or reparse points, ignores unknown files, and never removes `epar-last-error.log`. + +Wrapper control files and command result files are state rather than logs. New wrappers place them under `work/state`, outside retention scope. + +Use `ephemeral-action-runner logs path` to find the resolved root, `ephemeral-action-runner logs list` to inspect recognized artifacts, and `ephemeral-action-runner logs prune --dry-run` to preview retention. Remove `--dry-run` to prune immediately. + +EPAR intentionally does not embed OTLP or vendor-specific clients. To ship file artifacts, use an external agent such as the OpenTelemetry Collector `filelog` receiver. See [`examples/observability`](../examples/observability/README.md). diff --git a/docs/operations.md b/docs/operations.md index 80564c2..e998250 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -2,14 +2,12 @@ ## Logs -Host-side provider logs go under `work/logs` by default. Runner logs inside the Ubuntu guest are under: +Host-side logs go under `work/logs` by default. Run `ephemeral-action-runner logs path` to print the resolved directory. When `managerSinks` includes `file`, manager events use `work/logs/epar.log`; the default manager sink is console-only. Instance transcripts use `work/logs/instances/`, build/source transcripts use `work/logs/builds/`, startup timing JSONL uses `work/logs/benchmarks/`, and timestamped error reports use `work/logs/errors/`. See [Logging](logging.md) for sink, rotation, retention, and shipping configuration. Runner logs inside the Ubuntu guest are under: - `/var/log/actions-runner/run.log` - `/opt/actions-runner/_diag` -Guest provisioning command output is streamed to -`work/logs/.guest.log`. If runner launch or GitHub online -readiness fails, EPAR first appends bounded diagnostics to that host guest log: +Guest provisioning command output is streamed to `work/logs/instances/.guest.log`. If runner launch or GitHub online readiness fails, EPAR first appends bounded diagnostics to that host guest log: runner PID/process state, tails from `run.log` and the latest `Runner_*.log`, and the Docker-DinD daemon log when present. Diagnostic collection is best-effort and does not replace the original readiness error. @@ -50,7 +48,7 @@ This section is a compact checklist. For symptom-first diagnostics with host/pro - If a Docker/browser or web/E2E image build fails before package installation, run `image update-upstream`. - If an image build fails with `E: You don't have enough free space in /var/cache/apt/archives/.`, check the Docker daemon or VM storage with `docker system df` and `docker run --rm ghcr.io/catthehacker/ubuntu:full-latest df -h /`. On Windows Docker Desktop with WSL2, the container-visible disk can be much smaller than Windows Explorer free space; see [Windows Docker Desktop WSL2 Disk Is Smaller Than Expected](troubleshooting.md#windows-docker-desktop-wsl2-disk-is-smaller-than-expected). -- If Docker validation fails for a Docker-enabled image, inspect `work/logs/.guest.log`. +- If Docker validation fails for a Docker-enabled image, inspect `work/logs/builds/.guest.log`. - If browser validation fails on ARM64, confirm `epar-browser` exists inside the guest and inspect `/opt/epar/browser`. - If a Docker Compose job uses an amd64-only runtime image on an ARM64 Tart runner and fails with `exec format error` or repeated container exits such as status `139`, use a runner label that supports that image instead of changing application runtime settings only for runner compatibility. Suitable targets include Docker-DinD with verified `linux/amd64` emulation, WSL x64, an x64 Linux host, or a Tart image with Rosetta enabled and validated. - If a workflow uses fixed Compose project names, fixed container names, or fixed ports, Docker-DinD is often a better fit than a shared host Docker socket because each runner gets a private inner Docker daemon. Verify by starting two unregistered instances, running the same compose stack in both, and confirming host Docker only shows the outer EPAR runner containers. @@ -60,7 +58,7 @@ This section is a compact checklist. For symptom-first diagnostics with host/pro - If stale runners remain, run `ephemeral-action-runner cleanup`. - If using Tart `softnet`, verify the host has the privileges Tart requires. - If default WSL image build fails before import, confirm Docker Desktop, Docker Engine, or another Docker daemon is reachable so EPAR can export `ghcr.io/catthehacker/ubuntu:full-latest` into a rootfs tar. For lean WSL configs, confirm the clean Ubuntu rootfs was exported from an Ubuntu 24.04 WSL distro. -- If WSL image build fails before systemd is ready, confirm WSL2 is enabled and inspect `work/logs/.guest.log`. +- If WSL image build fails before systemd is ready, confirm WSL2 is enabled and inspect `work/logs/builds/.guest.log`. - If Docker-DinD startup fails, confirm the host Docker runtime supports privileged containers and inspect `/var/log/epar-dockerd.log` inside the runner container. - If Docker-DinD `docker run` fails with nested overlay mount errors, keep the default `EPAR_DOCKERD_STORAGE_DRIVER=vfs`. Only switch to `overlay2` or `auto` in a derived image after proving that storage driver works on the exact host runtime. - If the default WSL or Docker-DinD build cannot validate Docker, confirm the source image still provides `docker`, `dockerd`, Compose, Buildx, and `iptables`. If you intentionally use a clean Ubuntu source image instead of Catthehacker's runner image, run `image update-upstream` first and use a config that installs Docker from EPAR's pinned `actions/runner-images` Docker install harness. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index d4bd1a5..f5b6f5e 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -17,8 +17,8 @@ work/logs/epar-last-error.log Image build logs use provider-specific names, for example: ```text -work/logs/epar-docker-dind-catthehacker-ubuntu.docker-build.log -work/logs/epar-wsl-catthehacker-ubuntu.wsl-build.log +work/logs/builds/epar-docker-dind-catthehacker-ubuntu.docker-build.log +work/logs/builds/epar-wsl-catthehacker-ubuntu.wsl-build.log ``` Check the EPAR version and selected config: @@ -339,8 +339,8 @@ For persistent `0x8000FFFF` or `E_UNEXPECTED` failures, follow If the WSL image build fails after import but before systemd is ready, inspect: ```text -work/logs/.wsl-build.log -work/logs/.guest.log +work/logs/builds/.wsl-build.log +work/logs/builds/.guest.log ``` ## GitHub Runner Registration Fails diff --git a/docs/usage.md b/docs/usage.md index 1fe499a..1aadd30 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -210,7 +210,7 @@ Docker-DinD output is a Docker image tag, such as `epar-docker-dind-catthehacker docker image ls epar-docker-dind-catthehacker-ubuntu ``` -Build logs are written under `work/logs`. +Build logs are written under `work/logs/builds` by default. Run `ephemeral-action-runner logs path` to resolve a customized logging root and see [Logging](logging.md) for rotation and retention. ## Customize The Image diff --git a/examples/macos/start-epar.command b/examples/macos/start-epar.command index ac5f25d..3eb52fc 100755 --- a/examples/macos/start-epar.command +++ b/examples/macos/start-epar.command @@ -33,7 +33,7 @@ WAIT_FOR_DOCKER="${EPAR_WAIT_FOR_DOCKER:-1}" DOCKER_WAIT_ATTEMPTS="${EPAR_DOCKER_WAIT_ATTEMPTS:-120}" cd "${EPAR_ROOT}" -mkdir -p work/logs +mkdir -p work/state if [[ "${WAIT_FOR_DOCKER}" != "0" ]]; then echo "[$(date '+%Y-%m-%d %H:%M:%S')] waiting for Docker to become ready..." diff --git a/examples/observability/README.md b/examples/observability/README.md new file mode 100644 index 0000000..3f19cc2 --- /dev/null +++ b/examples/observability/README.md @@ -0,0 +1,5 @@ +# Observability Examples + +`local-file.yml` shows the normal workstation configuration with manager and transcript files enabled. `kubernetes-console.yml` shows cloud-native JSON console logging. These fragments contain the complete strict `logging` section and can replace that section in an EPAR config. + +`otel-collector-filelog.yaml` shows an OpenTelemetry Collector Contrib `filelog` receiver for an EPAR log volume. Mount the same log directory into the Collector at `/var/log/epar`, configure a production exporter, and persist the Collector storage directory. The example does not delete source files; EPAR remains responsible for retention. diff --git a/examples/observability/kubernetes-console.yml b/examples/observability/kubernetes-console.yml new file mode 100644 index 0000000..40c60f0 --- /dev/null +++ b/examples/observability/kubernetes-console.yml @@ -0,0 +1,18 @@ +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: json + managerFileFormat: json + transcriptSinks: [console] + transcriptConsoleFormat: json + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 diff --git a/examples/observability/local-file.yml b/examples/observability/local-file.yml new file mode 100644 index 0000000..4fbf217 --- /dev/null +++ b/examples/observability/local-file.yml @@ -0,0 +1,19 @@ +logging: + directory: work/logs + managerSinks: [console, file] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 diff --git a/examples/observability/otel-collector-filelog.yaml b/examples/observability/otel-collector-filelog.yaml new file mode 100644 index 0000000..c9c8351 --- /dev/null +++ b/examples/observability/otel-collector-filelog.yaml @@ -0,0 +1,30 @@ +extensions: + file_storage: + directory: /var/lib/otelcol/file-storage + +receivers: + filelog/epar: + include: + - /var/log/epar/epar.log + - /var/log/epar/errors/*.log + - /var/log/epar/instances/*.log + - /var/log/epar/builds/*.log + - /var/log/epar/benchmarks/*.jsonl + exclude: + - /var/log/epar/epar-last-error.log + - /var/log/epar/.epar-control/** + start_at: end + include_file_name: true + include_file_path: true + storage: file_storage + +exporters: + debug: + verbosity: basic + +service: + extensions: [file_storage] + pipelines: + logs: + receivers: [filelog/epar] + exporters: [debug] diff --git a/go.mod b/go.mod index f46ede3..2e92fbe 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/moby/moby/client v0.5.0 github.com/opencontainers/image-spec v1.1.1 golang.org/x/term v0.45.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) require ( diff --git a/go.sum b/go.sum index fc23c8e..7e4991a 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,8 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= diff --git a/internal/config/config.go b/internal/config/config.go index 43bc471..fdb0256 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -15,10 +15,17 @@ type Config struct { GitHub GitHubConfig Image ImageConfig Pool PoolConfig + Logging LoggingConfig Runner RunnerConfig Provider ProviderConfig Docker DockerConfig Timeouts TimeoutConfig + warnings []string +} + +// Warnings returns non-fatal configuration migration notices discovered while loading. +func (cfg Config) Warnings() []string { + return append([]string(nil), cfg.warnings...) } type GitHubConfig struct { @@ -54,7 +61,28 @@ const ( type PoolConfig struct { Instances int NamePrefix string - LogDir string +} + +type LoggingConfig struct { + Directory string + ManagerSinks []string + ManagerConsoleFormat string + ManagerConsoleTextFormat string + ManagerFileFormat string + TranscriptSinks []string + TranscriptConsoleFormat string + TranscriptConsoleTextFormat string + MaxFileSizeMiB int + MaxBackups int + CompressBackups bool + RetentionEnabled bool + RetentionMaxTotalMiB int + ManagerMaxAgeDays int + InstanceMaxAgeDays int + BuildMaxAgeDays int + ErrorMaxAgeDays int + BenchmarkMaxAgeDays int + RetentionIntervalMinutes int } type RunnerConfig struct { @@ -114,7 +142,25 @@ func Default() Config { Pool: PoolConfig{ Instances: 1, NamePrefix: "epar", - LogDir: "work/logs", + }, + Logging: LoggingConfig{ + Directory: "work/logs", + ManagerSinks: []string{"console"}, + ManagerConsoleFormat: "text", + ManagerFileFormat: "json", + TranscriptSinks: []string{"file"}, + TranscriptConsoleFormat: "text", + MaxFileSizeMiB: 100, + MaxBackups: 3, + CompressBackups: true, + RetentionEnabled: true, + RetentionMaxTotalMiB: 1024, + ManagerMaxAgeDays: 14, + InstanceMaxAgeDays: 14, + BuildMaxAgeDays: 14, + ErrorMaxAgeDays: 30, + BenchmarkMaxAgeDays: 90, + RetentionIntervalMinutes: 60, }, Runner: RunnerConfig{ Labels: []string{"self-hosted", "linux", "ARM64", "epar-tart-ubuntu-24.04-base"}, @@ -152,6 +198,8 @@ func Load(path string) (Config, error) { lineNo := 0 var pendingList *pendingListKey explicit := map[string]bool{} + legacyLogDir := "" + legacyLogDirLine := 0 for scanner.Scan() { lineNo++ raw := strings.TrimRight(scanner.Text(), " \t") @@ -180,12 +228,24 @@ func Load(path string) (Config, error) { key = strings.TrimSpace(key) value = strings.TrimSpace(value) if indent == 0 && value == "" { + if !isKnownSection(key) { + return cfg, fmt.Errorf("%s:%d: unknown section %q", path, lineNo, key) + } section = key continue } + if indent == 0 { + return cfg, fmt.Errorf("%s:%d: section %q must not have a scalar value", path, lineNo, key) + } if section == "" { return cfg, fmt.Errorf("%s:%d: key %q must be under a section", path, lineNo, key) } + if section == "pool" && key == "logDir" { + legacyLogDir = trimQuotes(value) + legacyLogDirLine = lineNo + explicit["pool.logDir"] = true + continue + } if value == "" && isListKey(section, key) { if err := setListValue(&cfg, section, key, nil); err != nil { return cfg, fmt.Errorf("%s:%d: %w", path, lineNo, err) @@ -202,6 +262,13 @@ func Load(path string) (Config, error) { if err := scanner.Err(); err != nil { return cfg, err } + if explicit["pool.logDir"] { + if explicit["logging.directory"] { + return cfg, fmt.Errorf("%s:%d: pool.logDir cannot be used with logging.directory; remove pool.logDir", path, legacyLogDirLine) + } + cfg.Logging.Directory = legacyLogDir + cfg.warnings = append(cfg.warnings, fmt.Sprintf("%s:%d: pool.logDir is deprecated; using its value as logging.directory (move it to the top-level logging section)", path, legacyLogDirLine)) + } applyProviderDefaults(&cfg, explicit) applyRunnerHostLabel(&cfg) cfg.GitHub.PrivateKeyPath = expandHome(cfg.GitHub.PrivateKeyPath) @@ -236,6 +303,8 @@ func apply(cfg *Config, section, key, value string) error { cfg.GitHub.APIBaseURL = strings.TrimRight(value, "/") case "webBaseUrl": cfg.GitHub.WebBaseURL = strings.TrimRight(value, "/") + default: + return unknownKey(section, key) } case "image": switch key { @@ -263,6 +332,8 @@ func apply(cfg *Config, section, key, value string) error { cfg.Image.HostTrustMode = strings.ToLower(value) case "hostTrustScopes": return setListValue(cfg, section, key, parseList(value)) + default: + return unknownKey(section, key) } case "pool": switch key { @@ -275,7 +346,77 @@ func apply(cfg *Config, section, key, value string) error { case "namePrefix", "vmPrefix": cfg.Pool.NamePrefix = value case "logDir": - cfg.Pool.LogDir = value + return fmt.Errorf("pool.logDir is deprecated; use logging.directory") + default: + return unknownKey(section, key) + } + case "logging": + switch key { + case "directory": + cfg.Logging.Directory = value + case "managerSinks", "transcriptSinks": + return setListValue(cfg, section, key, parseList(value)) + case "managerConsoleFormat": + cfg.Logging.ManagerConsoleFormat = strings.ToLower(value) + case "managerConsoleTextFormat": + cfg.Logging.ManagerConsoleTextFormat = value + case "managerFileFormat": + cfg.Logging.ManagerFileFormat = strings.ToLower(value) + case "transcriptConsoleFormat": + cfg.Logging.TranscriptConsoleFormat = strings.ToLower(value) + case "transcriptConsoleTextFormat": + cfg.Logging.TranscriptConsoleTextFormat = value + case "maxFileSizeMiB": + v, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("invalid logging.maxFileSizeMiB: %w", err) + } + cfg.Logging.MaxFileSizeMiB = v + case "maxBackups": + v, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("invalid logging.maxBackups: %w", err) + } + cfg.Logging.MaxBackups = v + case "compressBackups": + v, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("invalid logging.compressBackups: %w", err) + } + cfg.Logging.CompressBackups = v + case "retentionEnabled": + v, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("invalid logging.retentionEnabled: %w", err) + } + cfg.Logging.RetentionEnabled = v + case "retentionMaxTotalMiB": + v, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("invalid logging.retentionMaxTotalMiB: %w", err) + } + cfg.Logging.RetentionMaxTotalMiB = v + case "managerMaxAgeDays", "instanceMaxAgeDays", "buildMaxAgeDays", "errorMaxAgeDays", "benchmarkMaxAgeDays", "retentionIntervalMinutes": + v, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("invalid logging.%s: %w", key, err) + } + switch key { + case "managerMaxAgeDays": + cfg.Logging.ManagerMaxAgeDays = v + case "instanceMaxAgeDays": + cfg.Logging.InstanceMaxAgeDays = v + case "buildMaxAgeDays": + cfg.Logging.BuildMaxAgeDays = v + case "errorMaxAgeDays": + cfg.Logging.ErrorMaxAgeDays = v + case "benchmarkMaxAgeDays": + cfg.Logging.BenchmarkMaxAgeDays = v + case "retentionIntervalMinutes": + cfg.Logging.RetentionIntervalMinutes = v + } + default: + return unknownKey(section, key) } case "runner": switch key { @@ -301,6 +442,8 @@ func apply(cfg *Config, section, key, value string) error { return fmt.Errorf("invalid runner.noDefaultLabels: %w", err) } cfg.Runner.NoDefaultLabels = v + default: + return unknownKey(section, key) } case "provider": switch key { @@ -316,6 +459,8 @@ func apply(cfg *Config, section, key, value string) error { cfg.Provider.InstallRoot = value case "platform": cfg.Provider.Platform = value + default: + return unknownKey(section, key) } case "docker": switch key { @@ -327,6 +472,8 @@ func apply(cfg *Config, section, key, value string) error { cfg.Docker.HTTPSProxy = value case "noProxy": cfg.Docker.NoProxy = value + default: + return unknownKey(section, key) } case "timeouts": v, err := strconv.Atoi(value) @@ -340,6 +487,8 @@ func apply(cfg *Config, section, key, value string) error { cfg.Timeouts.GitHubOnlineSeconds = v case "commandSeconds": cfg.Timeouts.CommandSeconds = v + default: + return unknownKey(section, key) } default: return fmt.Errorf("unknown section %q", section) @@ -347,6 +496,19 @@ func apply(cfg *Config, section, key, value string) error { return nil } +func unknownKey(section, key string) error { + return fmt.Errorf("unknown key %s.%s", section, key) +} + +func isKnownSection(section string) bool { + switch section { + case "github", "image", "pool", "logging", "runner", "provider", "docker", "timeouts": + return true + default: + return false + } +} + func applyProviderDefaults(cfg *Config, explicit map[string]bool) { switch cfg.Provider.Type { case "wsl": @@ -506,6 +668,8 @@ func isListKey(section, key string) bool { return key == "labels" case "docker": return key == "registryMirrors" + case "logging": + return key == "managerSinks" || key == "transcriptSinks" default: return false } @@ -535,6 +699,15 @@ func setListValue(cfg *Config, section, key string, values []string) error { cfg.Docker.RegistryMirrors = values return nil } + case "logging": + switch key { + case "managerSinks": + cfg.Logging.ManagerSinks = values + return nil + case "transcriptSinks": + cfg.Logging.TranscriptSinks = values + return nil + } } return fmt.Errorf("unsupported list key %s.%s", section, key) } @@ -567,11 +740,23 @@ func appendListValue(cfg *Config, section, key, value string) error { cfg.Docker.RegistryMirrors = append(cfg.Docker.RegistryMirrors, item) return nil } + case "logging": + switch key { + case "managerSinks": + cfg.Logging.ManagerSinks = append(cfg.Logging.ManagerSinks, item) + return nil + case "transcriptSinks": + cfg.Logging.TranscriptSinks = append(cfg.Logging.TranscriptSinks, item) + return nil + } } return fmt.Errorf("unsupported list key %s.%s", section, key) } func Validate(cfg Config) error { + if err := ValidateLogging(cfg.Logging); err != nil { + return err + } if cfg.Provider.Type == "" { return fmt.Errorf("provider.type is required") } @@ -658,6 +843,125 @@ func Validate(cfg Config) error { return nil } +func ValidateLogging(logging LoggingConfig) error { + if strings.TrimSpace(logging.Directory) == "" { + return fmt.Errorf("logging.directory is required") + } + if err := validateLoggingSinks("managerSinks", logging.ManagerSinks); err != nil { + return err + } + if err := validateLoggingSinks("transcriptSinks", logging.TranscriptSinks); err != nil { + return err + } + if err := validateLoggingFormat("managerConsoleFormat", logging.ManagerConsoleFormat); err != nil { + return err + } + if err := validateLoggingFormat("managerFileFormat", logging.ManagerFileFormat); err != nil { + return err + } + if err := validateLoggingFormat("transcriptConsoleFormat", logging.TranscriptConsoleFormat); err != nil { + return err + } + if err := validateConsoleTextFormat("managerConsoleTextFormat", logging.ManagerConsoleTextFormat, logging.ManagerConsoleFormat, []string{"time", "level", "message", "attributes"}); err != nil { + return err + } + if err := validateConsoleTextFormat("transcriptConsoleTextFormat", logging.TranscriptConsoleTextFormat, logging.TranscriptConsoleFormat, []string{"time", "instance", "component", "stream", "message", "session", "category", "provider", "attributes"}); err != nil { + return err + } + if logging.MaxFileSizeMiB < 1 { + return fmt.Errorf("logging.maxFileSizeMiB must be 1 or greater") + } + if logging.MaxBackups < 1 { + return fmt.Errorf("logging.maxBackups must be 1 or greater") + } + if logging.RetentionMaxTotalMiB < 1 { + return fmt.Errorf("logging.retentionMaxTotalMiB must be 1 or greater") + } + for key, value := range map[string]int{ + "managerMaxAgeDays": logging.ManagerMaxAgeDays, + "instanceMaxAgeDays": logging.InstanceMaxAgeDays, + "buildMaxAgeDays": logging.BuildMaxAgeDays, + "errorMaxAgeDays": logging.ErrorMaxAgeDays, + "benchmarkMaxAgeDays": logging.BenchmarkMaxAgeDays, + "retentionIntervalMinutes": logging.RetentionIntervalMinutes, + } { + if value < 1 { + return fmt.Errorf("logging.%s must be 1 or greater", key) + } + } + return nil +} + +func validateLoggingSinks(key string, sinks []string) error { + if len(sinks) == 0 { + return fmt.Errorf("logging.%s must not be empty", key) + } + seen := make(map[string]struct{}, len(sinks)) + for _, sink := range sinks { + if sink != "console" && sink != "file" { + return fmt.Errorf("unsupported logging.%s value %q", key, sink) + } + if _, exists := seen[sink]; exists { + return fmt.Errorf("logging.%s must not contain duplicate sink %q", key, sink) + } + seen[sink] = struct{}{} + } + return nil +} + +func validateLoggingFormat(key, format string) error { + switch format { + case "text", "json": + return nil + default: + return fmt.Errorf("unsupported logging.%s %q; supported values are text and json", key, format) + } +} + +func validateConsoleTextFormat(key, template, outputFormat string, allowed []string) error { + if template == "" { + return nil + } + if outputFormat != "text" { + return fmt.Errorf("logging.%s is supported only when the corresponding console format is text", key) + } + if strings.ContainsAny(template, "\r\n") { + return fmt.Errorf("logging.%s must be a single line", key) + } + allowedSet := make(map[string]struct{}, len(allowed)) + for _, placeholder := range allowed { + allowedSet[placeholder] = struct{}{} + } + foundMessage := false + remaining := template + for { + open := strings.IndexByte(remaining, '{') + if open < 0 { + if strings.ContainsRune(remaining, '}') { + return fmt.Errorf("logging.%s contains an unmatched closing brace", key) + } + break + } + if strings.ContainsRune(remaining[:open], '}') { + return fmt.Errorf("logging.%s contains an unmatched closing brace", key) + } + closeOffset := strings.IndexByte(remaining[open+1:], '}') + if closeOffset < 0 { + return fmt.Errorf("logging.%s contains an unmatched opening brace", key) + } + placeholder := remaining[open+1 : open+1+closeOffset] + if _, ok := allowedSet[placeholder]; !ok { + return fmt.Errorf("logging.%s contains unsupported placeholder {%s}", key, placeholder) + } + foundMessage = foundMessage || placeholder == "message" + remaining = remaining[open+closeOffset+2:] + } + if !foundMessage { + return fmt.Errorf("logging.%s must contain {message}", key) + } + return nil +} + // ValidateHostTrust keeps host trust inheritance deliberately limited to the // ephemeral Docker-in-Docker image path. Other providers do not have a // portable, unambiguous host trust boundary. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d9b1f2b..03d67a7 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -99,6 +99,195 @@ func TestRunnerRegistrationControlsDefaultToDisabled(t *testing.T) { } } +func TestLoggingDefaults(t *testing.T) { + got := Default().Logging + if got.Directory != "work/logs" || !slices.Equal(got.ManagerSinks, []string{"console"}) || got.ManagerConsoleFormat != "text" || got.ManagerFileFormat != "json" || !slices.Equal(got.TranscriptSinks, []string{"file"}) || got.TranscriptConsoleFormat != "text" { + t.Fatalf("unexpected logging destination defaults: %+v", got) + } + if got.MaxFileSizeMiB != 100 || got.MaxBackups != 3 || !got.CompressBackups || !got.RetentionEnabled || got.RetentionMaxTotalMiB != 1024 { + t.Fatalf("unexpected logging retention defaults: %+v", got) + } + if got.ManagerMaxAgeDays != 14 || got.InstanceMaxAgeDays != 14 || got.BuildMaxAgeDays != 14 || got.ErrorMaxAgeDays != 30 || got.BenchmarkMaxAgeDays != 90 || got.RetentionIntervalMinutes != 60 { + t.Fatalf("unexpected logging age defaults: %+v", got) + } +} + +func TestCheckedInExampleConfigurationsValidate(t *testing.T) { + patterns := []string{ + filepath.Join("..", "..", "configs", "*.yml"), + filepath.Join("..", "..", "examples", "observability", "*.yml"), + } + for _, pattern := range patterns { + paths, err := filepath.Glob(pattern) + if err != nil { + t.Fatal(err) + } + if len(paths) == 0 { + t.Fatalf("no examples matched %s", pattern) + } + for _, path := range paths { + t.Run(filepath.Base(path), func(t *testing.T) { + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load(%s): %v", path, err) + } + if err := Validate(cfg); err != nil { + t.Fatalf("Validate(%s): %v", path, err) + } + }) + } + } +} + +func TestLoadLoggingConfiguration(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yml") + if err := os.WriteFile(path, []byte(` +logging: + directory: custom/logs + managerSinks: + - console + - file + managerConsoleFormat: json + managerFileFormat: text + transcriptSinks: [file, console] + transcriptConsoleFormat: json + maxFileSizeMiB: 64 + maxBackups: 5 + compressBackups: false + retentionEnabled: false + retentionMaxTotalMiB: 2048 + managerMaxAgeDays: 7 + instanceMaxAgeDays: 8 + buildMaxAgeDays: 9 + errorMaxAgeDays: 10 + benchmarkMaxAgeDays: 11 + retentionIntervalMinutes: 12 +`), 0644); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + got := cfg.Logging + if got.Directory != "custom/logs" || !slices.Equal(got.ManagerSinks, []string{"console", "file"}) || !slices.Equal(got.TranscriptSinks, []string{"file", "console"}) || got.ManagerConsoleFormat != "json" || got.ManagerFileFormat != "text" || got.TranscriptConsoleFormat != "json" || got.MaxFileSizeMiB != 64 || got.MaxBackups != 5 || got.CompressBackups || got.RetentionEnabled || got.RetentionMaxTotalMiB != 2048 || got.ManagerMaxAgeDays != 7 || got.InstanceMaxAgeDays != 8 || got.BuildMaxAgeDays != 9 || got.ErrorMaxAgeDays != 10 || got.BenchmarkMaxAgeDays != 11 || got.RetentionIntervalMinutes != 12 { + t.Fatalf("unexpected logging config: %+v", got) + } + if err := Validate(cfg); err != nil { + t.Fatal(err) + } +} + +func TestLoadCustomConsoleTextFormats(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yml") + content := "logging:\n managerConsoleFormat: text\n managerConsoleTextFormat: '[{level}] {message}{attributes}'\n transcriptConsoleFormat: text\n transcriptConsoleTextFormat: '{stream} {instance}: {message}'\n" + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if got, want := cfg.Logging.ManagerConsoleTextFormat, "[{level}] {message}{attributes}"; got != want { + t.Fatalf("managerConsoleTextFormat = %q, want %q", got, want) + } + if got, want := cfg.Logging.TranscriptConsoleTextFormat, "{stream} {instance}: {message}"; got != want { + t.Fatalf("transcriptConsoleTextFormat = %q, want %q", got, want) + } + if err := Validate(cfg); err != nil { + t.Fatal(err) + } +} + +func TestLoadMigratesPoolLogDirInMemoryWithWarning(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yml") + if err := os.WriteFile(path, []byte("pool:\n logDir: custom/logs\n"), 0644); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if got, want := cfg.Logging.Directory, "custom/logs"; got != want { + t.Fatalf("logging.directory = %q, want legacy value %q", got, want) + } + warnings := cfg.Warnings() + if len(warnings) != 1 || !strings.Contains(warnings[0], "pool.logDir is deprecated") || !strings.Contains(warnings[0], "logging.directory") { + t.Fatalf("Warnings() = %#v, want migration warning", warnings) + } + if err := Validate(cfg); err != nil { + t.Fatal(err) + } +} + +func TestLoadRejectsAmbiguousLegacyAndNewLogDirectories(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yml") + content := "logging:\n directory: new/logs\npool:\n logDir: old/logs\n" + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil || !strings.Contains(err.Error(), "pool.logDir cannot be used with logging.directory") { + t.Fatalf("Load() error = %v, want ambiguous-directory error", err) + } +} + +func TestLoadRejectsUnknownSectionsAndKeys(t *testing.T) { + for _, text := range []string{ + "unknown:\n value: true\n", + "github:\n unknown: value\n", + "image:\n unknown: value\n", + "pool:\n unknown: value\n", + "logging:\n unknown: value\n", + "runner:\n unknown: value\n", + "provider:\n unknown: value\n", + "docker:\n unknown: value\n", + "timeouts:\n unknown: 1\n", + } { + dir := t.TempDir() + path := filepath.Join(dir, "config.yml") + if err := os.WriteFile(path, []byte(text), 0644); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil || !strings.Contains(err.Error(), "unknown") { + t.Fatalf("Load(%q) error = %v, want unknown-key error", text, err) + } + } +} + +func TestValidateLoggingRejectsInvalidValues(t *testing.T) { + for _, mutate := range []func(*LoggingConfig){ + func(logging *LoggingConfig) { logging.Directory = " " }, + func(logging *LoggingConfig) { logging.ManagerSinks = nil }, + func(logging *LoggingConfig) { logging.TranscriptSinks = []string{"syslog"} }, + func(logging *LoggingConfig) { logging.ManagerSinks = []string{"Console"} }, + func(logging *LoggingConfig) { logging.ManagerFileFormat = "yaml" }, + func(logging *LoggingConfig) { logging.ManagerConsoleFormat = "JSON" }, + func(logging *LoggingConfig) { + logging.ManagerConsoleFormat = "json" + logging.ManagerConsoleTextFormat = "{level} {message}" + }, + func(logging *LoggingConfig) { logging.ManagerConsoleTextFormat = "{unknown} {message}" }, + func(logging *LoggingConfig) { logging.ManagerConsoleTextFormat = "{level}" }, + func(logging *LoggingConfig) { logging.TranscriptConsoleTextFormat = "{level} {message}" }, + func(logging *LoggingConfig) { logging.MaxFileSizeMiB = 0 }, + func(logging *LoggingConfig) { logging.MaxBackups = -1 }, + func(logging *LoggingConfig) { logging.MaxBackups = 0 }, + func(logging *LoggingConfig) { logging.RetentionMaxTotalMiB = 0 }, + func(logging *LoggingConfig) { logging.ErrorMaxAgeDays = 0 }, + func(logging *LoggingConfig) { logging.RetentionIntervalMinutes = 0 }, + } { + cfg := Default() + mutate(&cfg.Logging) + if err := ValidateLogging(cfg.Logging); err == nil { + t.Fatal("ValidateLogging accepted invalid config") + } + } +} + func TestTrustedCACertificatePathsDefaultToEmpty(t *testing.T) { cfg := Default() if len(cfg.Image.TrustedCACertificatePaths) != 0 { diff --git a/internal/filelock/filelock.go b/internal/filelock/filelock.go new file mode 100644 index 0000000..40bd1b2 --- /dev/null +++ b/internal/filelock/filelock.go @@ -0,0 +1,53 @@ +// Package filelock provides non-blocking, advisory, cross-process file locks. +package filelock + +import ( + "errors" + "fmt" + "os" + "sync" +) + +// ErrLocked means another process or file descriptor currently owns the lock. +var ErrLocked = errors.New("file lock is already held") + +// Lock is an exclusive advisory lock. Close releases it and is idempotent. +type Lock struct { + file *os.File + once sync.Once + err error +} + +// Acquire opens path, creating it when necessary, and attempts to acquire an +// exclusive lock without waiting. +func Acquire(path string) (*Lock, error) { + if path == "" { + return nil, errors.New("lock path is empty") + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("open lock file %s: %w", path, err) + } + if err := lockFile(file); err != nil { + _ = file.Close() + if errors.Is(err, errPlatformLocked) { + return nil, fmt.Errorf("%w: %s", ErrLocked, path) + } + return nil, fmt.Errorf("lock file %s: %w", path, err) + } + return &Lock{file: file}, nil +} + +// Close releases the lock. +func (lock *Lock) Close() error { + if lock == nil { + return nil + } + lock.once.Do(func() { + lock.err = unlockFile(lock.file) + if err := lock.file.Close(); lock.err == nil { + lock.err = err + } + }) + return lock.err +} diff --git a/internal/filelock/filelock_test.go b/internal/filelock/filelock_test.go new file mode 100644 index 0000000..94c4efb --- /dev/null +++ b/internal/filelock/filelock_test.go @@ -0,0 +1,59 @@ +package filelock + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestAcquireExcludesAnotherProcess(t *testing.T) { + if os.Getenv("EPAR_FILELOCK_HELPER") == "1" { + lock, err := Acquire(os.Getenv("EPAR_FILELOCK_PATH")) + if errors.Is(err, ErrLocked) { + os.Exit(23) + } + if err != nil { + os.Exit(24) + } + _ = lock.Close() + os.Exit(0) + } + + path := filepath.Join(t.TempDir(), "active.lock") + lock, err := Acquire(path) + if err != nil { + t.Fatalf("Acquire: %v", err) + } + defer lock.Close() + + command := exec.Command(os.Args[0], "-test.run=^TestAcquireExcludesAnotherProcess$") + command.Env = append(os.Environ(), "EPAR_FILELOCK_HELPER=1", "EPAR_FILELOCK_PATH="+path) + err = command.Run() + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 23 { + t.Fatalf("child result = %v, want ErrLocked exit code", err) + } +} + +func TestCloseAllowsReacquireAndIsIdempotent(t *testing.T) { + path := filepath.Join(t.TempDir(), "active.lock") + first, err := Acquire(path) + if err != nil { + t.Fatalf("Acquire first: %v", err) + } + if err := first.Close(); err != nil { + t.Fatalf("Close first: %v", err) + } + if err := first.Close(); err != nil { + t.Fatalf("Close first again: %v", err) + } + second, err := Acquire(path) + if err != nil { + t.Fatalf("Acquire second: %v", err) + } + if err := second.Close(); err != nil { + t.Fatalf("Close second: %v", err) + } +} diff --git a/internal/filelock/filelock_unix.go b/internal/filelock/filelock_unix.go new file mode 100644 index 0000000..8ad69aa --- /dev/null +++ b/internal/filelock/filelock_unix.go @@ -0,0 +1,25 @@ +//go:build linux || darwin + +package filelock + +import ( + "errors" + "os" + "syscall" +) + +var errPlatformLocked = errors.New("platform file lock is already held") + +func lockFile(file *os.File) error { + if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return errPlatformLocked + } + return err + } + return nil +} + +func unlockFile(file *os.File) error { + return syscall.Flock(int(file.Fd()), syscall.LOCK_UN) +} diff --git a/internal/filelock/filelock_unsupported.go b/internal/filelock/filelock_unsupported.go new file mode 100644 index 0000000..046ddb8 --- /dev/null +++ b/internal/filelock/filelock_unsupported.go @@ -0,0 +1,18 @@ +//go:build !windows && !linux && !darwin + +package filelock + +import ( + "errors" + "os" +) + +var errPlatformLocked = errors.New("file locks are unsupported on this platform") + +func lockFile(_ *os.File) error { + return errPlatformLocked +} + +func unlockFile(_ *os.File) error { + return nil +} diff --git a/internal/filelock/filelock_windows.go b/internal/filelock/filelock_windows.go new file mode 100644 index 0000000..e9db7b8 --- /dev/null +++ b/internal/filelock/filelock_windows.go @@ -0,0 +1,43 @@ +//go:build windows + +package filelock + +import ( + "errors" + "os" + "syscall" + "unsafe" +) + +const ( + lockfileFailImmediately = 0x00000001 + lockfileExclusiveLock = 0x00000002 +) + +var ( + errPlatformLocked = errors.New("platform file lock is already held") + kernel32 = syscall.NewLazyDLL("kernel32.dll") + procLockFileEx = kernel32.NewProc("LockFileEx") + procUnlockFileEx = kernel32.NewProc("UnlockFileEx") +) + +func lockFile(file *os.File) error { + var overlapped syscall.Overlapped + result, _, callErr := procLockFileEx.Call(file.Fd(), lockfileExclusiveLock|lockfileFailImmediately, 0, 1, 0, uintptr(unsafe.Pointer(&overlapped))) + if result != 0 { + return nil + } + if callErr == syscall.Errno(33) || callErr == syscall.Errno(158) { + return errPlatformLocked + } + return callErr +} + +func unlockFile(file *os.File) error { + var overlapped syscall.Overlapped + result, _, callErr := procUnlockFileEx.Call(file.Fd(), 0, 1, 0, uintptr(unsafe.Pointer(&overlapped))) + if result != 0 { + return nil + } + return callErr +} diff --git a/internal/logging/handler.go b/internal/logging/handler.go new file mode 100644 index 0000000..56e3d60 --- /dev/null +++ b/internal/logging/handler.go @@ -0,0 +1,155 @@ +package logging + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "sync" +) + +type fanoutHandler []slog.Handler + +func (handler fanoutHandler) Enabled(ctx context.Context, level slog.Level) bool { + for _, child := range handler { + if child.Enabled(ctx, level) { + return true + } + } + return false +} + +func (handler fanoutHandler) Handle(ctx context.Context, record slog.Record) error { + var handlerErrors []error + for _, child := range handler { + if child.Enabled(ctx, record.Level) { + if err := child.Handle(ctx, record.Clone()); err != nil { + handlerErrors = append(handlerErrors, err) + } + } + } + return errors.Join(handlerErrors...) +} + +func (handler fanoutHandler) WithAttrs(attributes []slog.Attr) slog.Handler { + children := make(fanoutHandler, len(handler)) + for index, child := range handler { + children[index] = child.WithAttrs(attributes) + } + return children +} + +func (handler fanoutHandler) WithGroup(name string) slog.Handler { + children := make(fanoutHandler, len(handler)) + for index, child := range handler { + children[index] = child.WithGroup(name) + } + return children +} + +type levelRangeHandler struct { + handler slog.Handler + min slog.Leveler + max *slog.Level +} + +func (handler levelRangeHandler) Enabled(ctx context.Context, level slog.Level) bool { + if level < handler.min.Level() || handler.max != nil && level >= *handler.max { + return false + } + return handler.handler.Enabled(ctx, level) +} + +func (handler levelRangeHandler) Handle(ctx context.Context, record slog.Record) error { + if !handler.Enabled(ctx, record.Level) { + return nil + } + return handler.handler.Handle(ctx, record) +} + +func (handler levelRangeHandler) WithAttrs(attributes []slog.Attr) slog.Handler { + handler.handler = handler.handler.WithAttrs(attributes) + return handler +} + +func (handler levelRangeHandler) WithGroup(name string) slog.Handler { + handler.handler = handler.handler.WithGroup(name) + return handler +} + +type discardHandler struct{} + +func (discardHandler) Enabled(context.Context, slog.Level) bool { return false } +func (discardHandler) Handle(context.Context, slog.Record) error { return nil } +func (discardHandler) WithAttrs([]slog.Attr) slog.Handler { return discardHandler{} } +func (discardHandler) WithGroup(string) slog.Handler { return discardHandler{} } + +type warningHandler struct { + handler slog.Handler + state *warningState +} + +type warningState struct { + once sync.Once + writer io.Writer +} + +func (handler warningHandler) Enabled(ctx context.Context, level slog.Level) bool { + return handler.handler.Enabled(ctx, level) +} + +func (handler warningHandler) Handle(ctx context.Context, record slog.Record) error { + err := handler.handler.Handle(ctx, record) + if err != nil { + handler.state.once.Do(func() { + _, _ = fmt.Fprintf(handler.state.writer, "warning: logging sink write failed: %v\n", err) + }) + } + return err +} + +func (handler warningHandler) WithAttrs(attributes []slog.Attr) slog.Handler { + handler.handler = handler.handler.WithAttrs(attributes) + return handler +} + +func (handler warningHandler) WithGroup(name string) slog.Handler { + handler.handler = handler.handler.WithGroup(name) + return handler +} + +func formattedHandler(writer io.Writer, format Format, level slog.Leveler) slog.Handler { + options := &slog.HandlerOptions{Level: level} + if format == FormatJSON { + return slog.NewJSONHandler(writer, options) + } + return slog.NewTextHandler(writer, options) +} + +func managerHandler(options Options, file io.Writer) slog.Handler { + children := make(fanoutHandler, 0, 3) + if options.ManagerSinks.Has(SinkConsole) { + warning := slog.LevelWarn + stdout := managerConsoleHandler(options.Stdout, options) + stderr := managerConsoleHandler(options.Stderr, options) + children = append(children, + levelRangeHandler{handler: stdout, min: options.Level, max: &warning}, + levelRangeHandler{handler: stderr, min: warning}, + ) + } + if options.ManagerSinks.Has(SinkFile) { + children = append(children, formattedHandler(file, options.ManagerFileFormat, options.Level)) + } + if len(children) == 0 { + return discardHandler{} + } + return warningHandler{handler: children, state: &warningState{writer: options.Stderr}} +} + +func managerConsoleHandler(writer io.Writer, options Options) slog.Handler { + if options.ManagerConsoleFormat == FormatJSON { + return formattedHandler(writer, FormatJSON, options.Level) + } + return newHumanTextHandler(writer, options.Level, options.ManagerConsoleTextFormat) +} diff --git a/internal/logging/human_handler.go b/internal/logging/human_handler.go new file mode 100644 index 0000000..417efe2 --- /dev/null +++ b/internal/logging/human_handler.go @@ -0,0 +1,148 @@ +package logging + +import ( + "context" + "fmt" + "io" + "log/slog" + "strconv" + "strings" + "sync" + "time" + "unicode" +) + +const humanTimestampLayout = "2006-01-02T15:04:05.000Z07:00" + +type boundAttribute struct { + groups []string + attr slog.Attr +} + +type humanTextHandler struct { + writer io.Writer + level slog.Leveler + template string + groups []string + attributes []boundAttribute + mu *sync.Mutex +} + +func newHumanTextHandler(writer io.Writer, level slog.Leveler, template string) slog.Handler { + return &humanTextHandler{writer: writer, level: level, template: template, mu: &sync.Mutex{}} +} + +func (handler *humanTextHandler) Enabled(_ context.Context, level slog.Level) bool { + return level >= handler.level.Level() +} + +func (handler *humanTextHandler) Handle(_ context.Context, record slog.Record) error { + attributes := append([]boundAttribute(nil), handler.attributes...) + record.Attrs(func(attribute slog.Attr) bool { + attributes = append(attributes, boundAttribute{groups: append([]string(nil), handler.groups...), attr: attribute}) + return true + }) + renderedAttributes := renderHumanAttributes(attributes) + if renderedAttributes != "" { + renderedAttributes = " " + renderedAttributes + } + line := strings.NewReplacer( + "{time}", record.Time.Format(humanTimestampLayout), + "{level}", record.Level.String(), + "{message}", singleLine(record.Message), + "{attributes}", renderedAttributes, + ).Replace(handler.template) + handler.mu.Lock() + defer handler.mu.Unlock() + _, err := io.WriteString(handler.writer, line+"\n") + return err +} + +func (handler *humanTextHandler) WithAttrs(attributes []slog.Attr) slog.Handler { + clone := handler.clone() + for _, attribute := range attributes { + clone.attributes = append(clone.attributes, boundAttribute{groups: append([]string(nil), handler.groups...), attr: attribute}) + } + return clone +} + +func (handler *humanTextHandler) WithGroup(name string) slog.Handler { + if name == "" { + return handler + } + clone := handler.clone() + clone.groups = append(clone.groups, name) + return clone +} + +func (handler *humanTextHandler) clone() *humanTextHandler { + clone := *handler + clone.groups = append([]string(nil), handler.groups...) + clone.attributes = append([]boundAttribute(nil), handler.attributes...) + return &clone +} + +func renderHumanAttributes(attributes []boundAttribute) string { + parts := make([]string, 0, len(attributes)) + for _, attribute := range attributes { + appendHumanAttribute(&parts, attribute.groups, attribute.attr) + } + return strings.Join(parts, " ") +} + +func appendHumanAttribute(parts *[]string, groups []string, attribute slog.Attr) { + value := attribute.Value.Resolve() + if value.Kind() == slog.KindGroup { + nestedGroups := groups + if attribute.Key != "" { + nestedGroups = append(append([]string(nil), groups...), attribute.Key) + } + for _, nested := range value.Group() { + appendHumanAttribute(parts, nestedGroups, nested) + } + return + } + keyParts := append([]string(nil), groups...) + if attribute.Key != "" { + keyParts = append(keyParts, attribute.Key) + } + if len(keyParts) == 0 { + return + } + *parts = append(*parts, strings.Join(keyParts, ".")+"="+formatHumanValue(value)) +} + +func formatHumanValue(value slog.Value) string { + switch value.Kind() { + case slog.KindString: + return quoteHumanString(value.String()) + case slog.KindInt64: + return strconv.FormatInt(value.Int64(), 10) + case slog.KindUint64: + return strconv.FormatUint(value.Uint64(), 10) + case slog.KindFloat64: + return strconv.FormatFloat(value.Float64(), 'g', -1, 64) + case slog.KindBool: + return strconv.FormatBool(value.Bool()) + case slog.KindDuration: + return value.Duration().String() + case slog.KindTime: + return value.Time().Format(time.RFC3339Nano) + case slog.KindAny: + return quoteHumanString(fmt.Sprint(value.Any())) + default: + return quoteHumanString(value.String()) + } +} + +func quoteHumanString(value string) string { + if value == "" || strings.IndexFunc(value, unicode.IsSpace) >= 0 || strings.ContainsAny(value, `="\`) { + return strconv.Quote(singleLine(value)) + } + return singleLine(value) +} + +func singleLine(value string) string { + value = strings.ReplaceAll(value, "\r", `\r`) + return strings.ReplaceAll(value, "\n", `\n`) +} diff --git a/internal/logging/pathcase_other.go b/internal/logging/pathcase_other.go new file mode 100644 index 0000000..bb9d7c3 --- /dev/null +++ b/internal/logging/pathcase_other.go @@ -0,0 +1,5 @@ +//go:build !windows + +package logging + +func canonicalCase(path string) string { return path } diff --git a/internal/logging/pathcase_windows.go b/internal/logging/pathcase_windows.go new file mode 100644 index 0000000..dd23068 --- /dev/null +++ b/internal/logging/pathcase_windows.go @@ -0,0 +1,7 @@ +//go:build windows + +package logging + +import "strings" + +func canonicalCase(path string) string { return strings.ToLower(path) } diff --git a/internal/logging/paths.go b/internal/logging/paths.go new file mode 100644 index 0000000..668a97a --- /dev/null +++ b/internal/logging/paths.go @@ -0,0 +1,119 @@ +package logging + +import ( + "errors" + "fmt" + "path/filepath" + "regexp" + "strings" + "time" +) + +var safeComponentPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`) + +// ManagerPath returns the active manager log path. +func ManagerPath(root string) string { return filepath.Join(root, ManagerFilename) } + +// LastErrorPath returns the stable latest-error report path. +func LastErrorPath(root string) string { return filepath.Join(root, LastErrorFilename) } + +// CategoryDirectory returns a supported shallow category directory. +func CategoryDirectory(root string, category Category) (string, error) { + if !category.validTranscriptCategory() { + return "", fmt.Errorf("unsupported transcript category %q", category) + } + return filepath.Join(root, string(category)), nil +} + +// InstancePath returns instances/..log. Component is a +// provider name or "guest". +func InstancePath(root, instance, component string) (string, error) { + if err := validateComponent("instance", instance); err != nil { + return "", err + } + if err := validateComponent("instance component", component); err != nil { + return "", err + } + if !validInstanceComponent(component) { + return "", fmt.Errorf("unsupported instance component %q", component) + } + directory, _ := CategoryDirectory(root, CategoryInstances) + return filepath.Join(directory, instance+"."+component+".log"), nil +} + +// BuildPath returns builds/..log. Component must be a +// recognized build transcript suffix such as "docker-build" or "refresh". +func BuildPath(root, imageStem, component string) (string, error) { + if err := validateComponent("image stem", imageStem); err != nil { + return "", err + } + if !validBuildComponent(component) { + return "", fmt.Errorf("unsupported build component %q", component) + } + directory, _ := CategoryDirectory(root, CategoryBuilds) + return filepath.Join(directory, imageStem+"."+component+".log"), nil +} + +// ErrorPath returns errors/epar-YYYYMMDD-HHMMSS-error.log. +func ErrorPath(root string, timestamp time.Time) string { + directory, _ := CategoryDirectory(root, CategoryErrors) + return filepath.Join(directory, "epar-"+timestamp.UTC().Format("20060102-150405")+"-error.log") +} + +// BenchmarkPath returns benchmarks/-.jsonl. +func BenchmarkPath(root string, timestamp time.Time, provider string) (string, error) { + if err := validateComponent("benchmark provider", provider); err != nil { + return "", err + } + directory, _ := CategoryDirectory(root, CategoryBenchmarks) + return filepath.Join(directory, timestamp.UTC().Format("20060102T150405.000000000Z")+"-"+provider+".jsonl"), nil +} + +func validateComponent(label, value string) error { + if value == "" { + return fmt.Errorf("%s is empty", label) + } + if value == "." || value == ".." || strings.ContainsAny(value, `/\\`) || !safeComponentPattern.MatchString(value) { + return fmt.Errorf("%s %q is not a safe path component", label, value) + } + return nil +} + +func validBuildComponent(component string) bool { + switch component { + case "docker-build", "wsl-build", "build", "source", "refresh", "wsl-refresh", "guest": + return true + default: + return false + } +} + +func validInstanceComponent(component string) bool { + switch component { + case "guest", "docker-dind", "wsl", "tart": + return true + default: + return false + } +} + +func canonicalPath(path string) (string, error) { + if path == "" { + return "", errors.New("path is empty") + } + absolute, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("resolve absolute path: %w", err) + } + absolute = filepath.Clean(absolute) + parent, err := filepath.EvalSymlinks(filepath.Dir(absolute)) + if err == nil { + absolute = filepath.Join(parent, filepath.Base(absolute)) + } + return canonicalCase(absolute), nil +} + +func pathWithin(root, path string) bool { + relative, err := filepath.Rel(root, path) + return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) && !filepath.IsAbs(relative) +} diff --git a/internal/logging/paths_test.go b/internal/logging/paths_test.go new file mode 100644 index 0000000..4221055 --- /dev/null +++ b/internal/logging/paths_test.go @@ -0,0 +1,84 @@ +package logging + +import ( + "path/filepath" + "testing" + "time" +) + +func TestPathHelpersAndRecognition(t *testing.T) { + root := t.TempDir() + timestamp := time.Date(2026, 7, 15, 1, 2, 3, 4, time.UTC) + tests := []struct { + path string + category Category + }{ + {mustPath(InstancePath(root, "runner-1", "docker-dind")), CategoryInstances}, + {mustPath(InstancePath(root, "runner-1", "guest")), CategoryInstances}, + {mustPath(BuildPath(root, "ubuntu-24.04", "docker-build")), CategoryBuilds}, + {mustPath(BuildPath(root, "ubuntu-24.04", "guest")), CategoryBuilds}, + {ErrorPath(root, timestamp), CategoryErrors}, + {mustPath(BenchmarkPath(root, timestamp, "docker-dind")), CategoryBenchmarks}, + } + for _, test := range tests { + recognized, ok := recognizePath(root, test.path) + if !ok || recognized.category != test.category || recognized.current || recognized.backup { + t.Errorf("recognizePath(%s) = %#v, %v", test.path, recognized, ok) + } + canonicalRoot, _ := canonicalPath(root) + canonicalTestPath, _ := canonicalPath(test.path) + if canonicalRecognized, canonicalOK := recognizePath(canonicalRoot, canonicalTestPath); !canonicalOK || canonicalRecognized.category != test.category { + t.Errorf("canonical recognizePath(%s) = %#v, %v", canonicalTestPath, canonicalRecognized, canonicalOK) + } + name := filepath.Base(test.path) + extension := filepath.Ext(name) + rotated := test.path[:len(test.path)-len(extension)] + "-2026-07-15T01-02-03.004" + extension + ".gz" + recognized, ok = recognizePath(root, rotated) + if !ok || recognized.category != test.category || !recognized.backup { + t.Errorf("rotated recognizePath(%s) = %#v, %v", rotated, recognized, ok) + } + } + if _, err := InstancePath(root, "../escape", "guest"); err == nil { + t.Fatal("unsafe instance path accepted") + } + if _, err := BuildPath(root, "image", "unknown"); err == nil { + t.Fatal("unknown build component accepted") + } + if _, err := InstancePath(root, "runner", "unknown"); err == nil { + t.Fatal("unknown instance component accepted") + } +} + +func TestLegacyFlatRecognitionIsConstrained(t *testing.T) { + root := t.TempDir() + tests := []struct { + name string + category Category + }{ + {"epar-pool-20260715-010203-007.guest.log", CategoryInstances}, + {"epar-pool-20260715-010203-007.docker-dind.log", CategoryInstances}, + {"ubuntu.docker-build.log", CategoryBuilds}, + {"ubuntu.source.log", CategoryBuilds}, + {"epar-20260715-010203-error.log", CategoryErrors}, + {"20260715T010203.000000123Z-docker-dind.jsonl", CategoryBenchmarks}, + {"epar-2026-07-15T01-02-03.004.log.gz", CategoryManager}, + } + for _, test := range tests { + recognized, ok := recognizePath(root, filepath.Join(root, test.name)) + if !ok || recognized.category != test.category { + t.Errorf("legacy %s = %#v, %v", test.name, recognized, ok) + } + } + for _, name := range []string{"notes.user.log", "arbitrary.guest.log", "runner.foo.log", "almost-20260715-010203-01.guest.log"} { + if recognized, ok := recognizePath(root, filepath.Join(root, name)); ok { + t.Errorf("unknown legacy name %s recognized as %#v", name, recognized) + } + } +} + +func mustPath(path string, err error) string { + if err != nil { + panic(err) + } + return path +} diff --git a/internal/logging/recognition.go b/internal/logging/recognition.go new file mode 100644 index 0000000..d1b5cab --- /dev/null +++ b/internal/logging/recognition.go @@ -0,0 +1,116 @@ +package logging + +import ( + "path/filepath" + "regexp" + "strings" +) + +var ( + lumberjackSuffixPattern = regexp.MustCompile(`-\d{4}-\d{2}-\d{2}[Tt]\d{2}-\d{2}-\d{2}\.\d{3}(\.log|\.jsonl)$`) + errorNamePattern = regexp.MustCompile(`^epar-\d{8}-\d{6}-error\.log$`) + instanceNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*\.(guest|docker-dind|wsl|tart)\.log$`) + legacyInstancePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*-\d{8}-\d{6}-\d{3}\.(guest|docker-dind|wsl|tart)\.log$`) + buildNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*\.(docker-build|wsl-build|build|source|refresh|wsl-refresh|guest)\.log$`) + legacyBuildNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*\.(docker-build|wsl-build|build|source|refresh|wsl-refresh)\.log$`) + benchmarkNamePattern = regexp.MustCompile(`^\d{8}[Tt]\d{6}\.\d{9}[Zz]-[A-Za-z0-9][A-Za-z0-9_-]*\.jsonl$`) +) + +type recognizedFile struct { + category Category + current bool + backup bool +} + +func recognizePath(root, path string) (recognizedFile, bool) { + relative, err := filepath.Rel(root, path) + if err != nil || filepath.IsAbs(relative) || relative == "." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return recognizedFile{}, false + } + parts := strings.Split(filepath.Clean(relative), string(filepath.Separator)) + if len(parts) == 1 { + return recognizeName(parts[0], "") + } + if len(parts) != 2 { + return recognizedFile{}, false + } + category := Category(parts[0]) + if !category.validTranscriptCategory() { + return recognizedFile{}, false + } + recognized, ok := recognizeName(parts[1], category) + if !ok || recognized.category != category { + return recognizedFile{}, false + } + return recognized, true +} + +func recognizeName(name string, expected Category) (recognizedFile, bool) { + primary, backup := stripLumberjackSuffix(name) + if expected == "" || expected == CategoryManager { + if primary == ManagerFilename { + return recognizedFile{category: CategoryManager, current: !backup, backup: backup}, true + } + if primary == LastErrorFilename && !backup { + return recognizedFile{category: CategoryErrors, current: true}, true + } + } + if expected == "" || expected == CategoryErrors { + if errorNamePattern.MatchString(primary) { + return recognizedFile{category: CategoryErrors, backup: backup}, true + } + } + if expected == "" || expected == CategoryBuilds { + pattern := buildNamePattern + if expected == "" { + pattern = legacyBuildNamePattern + } + if pattern.MatchString(primary) { + return recognizedFile{category: CategoryBuilds, backup: backup}, true + } + } + if expected == "" || expected == CategoryBenchmarks { + if benchmarkNamePattern.MatchString(primary) { + return recognizedFile{category: CategoryBenchmarks, backup: backup}, true + } + } + if expected == "" || expected == CategoryInstances { + pattern := instanceNamePattern + if expected == "" { + pattern = legacyInstancePattern + } + if pattern.MatchString(primary) { + return recognizedFile{category: CategoryInstances, backup: backup}, true + } + } + return recognizedFile{}, false +} + +func stripLumberjackSuffix(name string) (string, bool) { + original := name + gzipped := strings.HasSuffix(name, ".gz") + if gzipped { + name = strings.TrimSuffix(name, ".gz") + } + location := lumberjackSuffixPattern.FindStringIndex(name) + if location == nil || location[1] != len(name) { + return original, false + } + suffix := name[location[0]:] + extension := ".log" + if strings.HasSuffix(suffix, ".jsonl") { + extension = ".jsonl" + } + return name[:location[0]] + extension, true +} + +func activeLockPathForRecognized(path string, recognized recognizedFile) string { + if !recognized.backup { + return path + } + primary, ok := stripLumberjackSuffix(filepath.Base(path)) + if !ok { + return path + } + return filepath.Join(filepath.Dir(path), primary) +} diff --git a/internal/logging/registry.go b/internal/logging/registry.go new file mode 100644 index 0000000..7d9ce9e --- /dev/null +++ b/internal/logging/registry.go @@ -0,0 +1,315 @@ +package logging + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "sync" + "time" + + "github.com/solutionforest/ephemeral-action-runner/internal/filelock" + "gopkg.in/natefinch/lumberjack.v2" +) + +const controlDirectoryName = ".epar-control" + +var ( + registryMu sync.Mutex + registry = make(map[string]*rotationEntry) +) + +type rotationEntry struct { + path string + root string + rotation Rotation + writer *lumberjack.Logger + lock *filelock.Lock + lockPath string + metadataPath string + ownerToken string + startedAt time.Time + refs int + sessions map[string]TranscriptMetadata + writeMu sync.Mutex +} + +type rotationHandle struct { + entry *rotationEntry + id string + mu sync.Mutex + close error + closed bool +} + +type activeState struct { + Version int `json:"version"` + Path string `json:"path"` + PID int `json:"pid"` + StartedAt time.Time `json:"startedAt"` + OwnerToken string `json:"ownerToken"` + Sessions []TranscriptMetadata `json:"sessions"` +} + +func openRotating(root, path string, rotation Rotation, metadata TranscriptMetadata, now time.Time) (*rotationHandle, error) { + canonicalRoot, err := canonicalPath(root) + if err != nil { + return nil, fmt.Errorf("canonicalize logging root: %w", err) + } + canonical, err := canonicalPath(path) + if err != nil { + return nil, fmt.Errorf("canonicalize log path: %w", err) + } + if !pathWithin(canonicalRoot, canonical) { + return nil, fmt.Errorf("log path %s is outside logging root %s", canonical, canonicalRoot) + } + if info, statErr := os.Lstat(canonical); statErr == nil && isLinkOrReparse(info) { + return nil, fmt.Errorf("refuse symlink or reparse-point log path %s", canonical) + } else if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { + return nil, fmt.Errorf("inspect log path %s: %w", canonical, statErr) + } + if err := ensureSafeDirectory(filepath.Dir(canonical), 0o755); err != nil { + return nil, fmt.Errorf("create log directory: %w", err) + } + + handleID, err := randomToken() + if err != nil { + return nil, fmt.Errorf("create log handle token: %w", err) + } + if metadata.SessionID == "" { + metadata.SessionID = handleID + } + + registryMu.Lock() + defer registryMu.Unlock() + if entry := registry[canonical]; entry != nil { + if entry.rotation != rotation { + return nil, fmt.Errorf("log path %s is already open with different rotation settings", canonical) + } + entry.refs++ + entry.sessions[handleID] = cloneMetadata(metadata) + if err := writeActiveState(entry); err != nil { + delete(entry.sessions, handleID) + entry.refs-- + return nil, err + } + return &rotationHandle{entry: entry, id: handleID}, nil + } + + control := filepath.Join(canonicalRoot, controlDirectoryName) + lockDir := filepath.Join(control, "locks") + activeDir := filepath.Join(control, "active") + if err := ensureSafeDirectory(control, 0o700); err != nil { + return nil, fmt.Errorf("create log control directory: %w", err) + } + if err := ensureSafeDirectory(lockDir, 0o700); err != nil { + return nil, fmt.Errorf("create log lock directory: %w", err) + } + if err := ensureSafeDirectory(activeDir, 0o700); err != nil { + return nil, fmt.Errorf("create active log directory: %w", err) + } + hash := pathHash(canonical) + lockPath := filepath.Join(lockDir, hash+".lock") + lock, err := filelock.Acquire(lockPath) + if err != nil { + if errors.Is(err, filelock.ErrLocked) { + return nil, fmt.Errorf("log path is active in another process: %w", err) + } + return nil, err + } + probe, err := os.OpenFile(canonical, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + _ = lock.Close() + return nil, fmt.Errorf("initialize rotating log file %s: %w", canonical, err) + } + if err := probe.Close(); err != nil { + _ = lock.Close() + return nil, fmt.Errorf("close initialized rotating log file %s: %w", canonical, err) + } + ownerToken, err := randomToken() + if err != nil { + _ = lock.Close() + return nil, fmt.Errorf("create active log owner token: %w", err) + } + entry := &rotationEntry{ + path: canonical, + root: canonicalRoot, + rotation: rotation, + writer: &lumberjack.Logger{Filename: canonical, MaxSize: rotation.MaxSizeMiB, MaxBackups: rotation.MaxBackups, Compress: rotation.Compress, LocalTime: false}, + lock: lock, + lockPath: lockPath, + metadataPath: filepath.Join(activeDir, hash+".json"), + ownerToken: ownerToken, + startedAt: now.UTC(), + refs: 1, + sessions: map[string]TranscriptMetadata{handleID: cloneMetadata(metadata)}, + } + if err := writeActiveState(entry); err != nil { + _ = lock.Close() + return nil, err + } + registry[canonical] = entry + return &rotationHandle{entry: entry, id: handleID}, nil +} + +func (handle *rotationHandle) Write(data []byte) (int, error) { + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.closed { + return 0, os.ErrClosed + } + handle.entry.writeMu.Lock() + defer handle.entry.writeMu.Unlock() + return handle.entry.writer.Write(data) +} + +func (handle *rotationHandle) Close() error { + if handle == nil { + return nil + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.closed { + return handle.close + } + handle.closed = true + + registryMu.Lock() + defer registryMu.Unlock() + entry := handle.entry + delete(entry.sessions, handle.id) + entry.refs-- + if entry.refs > 0 { + handle.close = writeActiveState(entry) + return handle.close + } + delete(registry, entry.path) + entry.writeMu.Lock() + writerErr := entry.writer.Close() + entry.writeMu.Unlock() + metadataErr := removeOwnedActiveState(entry) + lockErr := entry.lock.Close() + handle.close = errors.Join(writerErr, metadataErr, lockErr) + return handle.close +} + +func writeActiveState(entry *rotationEntry) error { + sessionKeys := make([]string, 0, len(entry.sessions)) + for key := range entry.sessions { + sessionKeys = append(sessionKeys, key) + } + sort.Strings(sessionKeys) + sessions := make([]TranscriptMetadata, 0, len(sessionKeys)) + for _, key := range sessionKeys { + sessions = append(sessions, cloneMetadata(entry.sessions[key])) + } + state := activeState{Version: 1, Path: entry.path, PID: os.Getpid(), StartedAt: entry.startedAt, OwnerToken: entry.ownerToken, Sessions: sessions} + data, err := json.MarshalIndent(state, "", " ") + if err != nil { + return fmt.Errorf("encode active log metadata: %w", err) + } + data = append(data, '\n') + temporary, err := os.CreateTemp(filepath.Dir(entry.metadataPath), ".active-*.tmp") + if err != nil { + return fmt.Errorf("create active log metadata: %w", err) + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return fmt.Errorf("secure active log metadata: %w", err) + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return fmt.Errorf("write active log metadata: %w", err) + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return fmt.Errorf("sync active log metadata: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close active log metadata: %w", err) + } + if err := replaceFile(temporaryPath, entry.metadataPath); err != nil { + return fmt.Errorf("publish active log metadata: %w", err) + } + return nil +} + +func removeOwnedActiveState(entry *rotationEntry) error { + data, err := os.ReadFile(entry.metadataPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read active log metadata during close: %w", err) + } + var state activeState + if err := json.Unmarshal(data, &state); err != nil { + return fmt.Errorf("decode active log metadata during close: %w", err) + } + if state.OwnerToken != entry.ownerToken { + return fmt.Errorf("active log metadata owner changed for %s", entry.path) + } + if err := os.Remove(entry.metadataPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove active log metadata: %w", err) + } + return nil +} + +func activeRegistryPaths() map[string]struct{} { + registryMu.Lock() + defer registryMu.Unlock() + paths := make(map[string]struct{}, len(registry)) + for path := range registry { + paths[path] = struct{}{} + } + return paths +} + +func pathHash(path string) string { + sum := sha256.Sum256([]byte(path)) + return hex.EncodeToString(sum[:]) +} + +func randomToken() (string, error) { + bytes := make([]byte, 16) + if _, err := io.ReadFull(rand.Reader, bytes); err != nil { + return "", err + } + return hex.EncodeToString(bytes), nil +} + +func cloneMetadata(metadata TranscriptMetadata) TranscriptMetadata { + copy := metadata + if metadata.Attributes != nil { + copy.Attributes = make(map[string]string, len(metadata.Attributes)) + for key, value := range metadata.Attributes { + copy.Attributes[key] = value + } + } + return copy +} + +func ensureSafeDirectory(path string, mode os.FileMode) error { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + if err := os.Mkdir(path, mode); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + info, err = os.Lstat(path) + } + if err != nil { + return err + } + if !info.IsDir() || isLinkOrReparse(info) { + return fmt.Errorf("%s is not a safe directory", path) + } + return nil +} diff --git a/internal/logging/reparse_other.go b/internal/logging/reparse_other.go new file mode 100644 index 0000000..06cb130 --- /dev/null +++ b/internal/logging/reparse_other.go @@ -0,0 +1,7 @@ +//go:build !windows + +package logging + +import "os" + +func isLinkOrReparse(info os.FileInfo) bool { return info.Mode()&os.ModeSymlink != 0 } diff --git a/internal/logging/reparse_windows.go b/internal/logging/reparse_windows.go new file mode 100644 index 0000000..71de430 --- /dev/null +++ b/internal/logging/reparse_windows.go @@ -0,0 +1,16 @@ +//go:build windows + +package logging + +import ( + "os" + "syscall" +) + +func isLinkOrReparse(info os.FileInfo) bool { + if info.Mode()&os.ModeSymlink != 0 { + return true + } + data, ok := info.Sys().(*syscall.Win32FileAttributeData) + return ok && data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0 +} diff --git a/internal/logging/reparse_windows_test.go b/internal/logging/reparse_windows_test.go new file mode 100644 index 0000000..3f1e802 --- /dev/null +++ b/internal/logging/reparse_windows_test.go @@ -0,0 +1,16 @@ +//go:build windows + +package logging + +import ( + "os" + "syscall" + "testing" +) + +func TestWindowsReparsePointDetection(t *testing.T) { + info := fakeFileInfo{name: "junction", mode: os.ModeDir, sys: &syscall.Win32FileAttributeData{FileAttributes: syscall.FILE_ATTRIBUTE_REPARSE_POINT}} + if !isLinkOrReparse(info) { + t.Fatal("Windows reparse point was not detected") + } +} diff --git a/internal/logging/replace_other.go b/internal/logging/replace_other.go new file mode 100644 index 0000000..8327171 --- /dev/null +++ b/internal/logging/replace_other.go @@ -0,0 +1,7 @@ +//go:build !windows + +package logging + +import "os" + +func replaceFile(source, destination string) error { return os.Rename(source, destination) } diff --git a/internal/logging/replace_windows.go b/internal/logging/replace_windows.go new file mode 100644 index 0000000..503531f --- /dev/null +++ b/internal/logging/replace_windows.go @@ -0,0 +1,18 @@ +//go:build windows + +package logging + +import ( + "errors" + "os" +) + +func replaceFile(source, destination string) error { + if err := os.Rename(source, destination); err == nil { + return nil + } + if err := os.Remove(destination); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return os.Rename(source, destination) +} diff --git a/internal/logging/retention.go b/internal/logging/retention.go new file mode 100644 index 0000000..8ece1f8 --- /dev/null +++ b/internal/logging/retention.go @@ -0,0 +1,463 @@ +package logging + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/solutionforest/ephemeral-action-runner/internal/filelock" +) + +// RetentionPolicy applies category ages first, then an aggregate total-byte +// budget to the remaining recognized files. A zero value disables that limit. +type RetentionPolicy struct { + MaxTotalBytes int64 + ManagerMaxAge time.Duration + InstanceMaxAge time.Duration + BuildMaxAge time.Duration + ErrorMaxAge time.Duration + BenchmarkMaxAge time.Duration + Now func() time.Time +} + +func (policy RetentionPolicy) maxAge(category Category) time.Duration { + switch category { + case CategoryManager: + return policy.ManagerMaxAge + case CategoryInstances: + return policy.InstanceMaxAge + case CategoryBuilds: + return policy.BuildMaxAge + case CategoryErrors: + return policy.ErrorMaxAge + case CategoryBenchmarks: + return policy.BenchmarkMaxAge + default: + return 0 + } +} + +// RetentionAction describes the outcome or protection applied to one entry. +type RetentionAction string + +const ( + RetentionKeep RetentionAction = "keep" + RetentionProtected RetentionAction = "protected" + RetentionWouldDelete RetentionAction = "would-delete" + RetentionDeleted RetentionAction = "deleted" + RetentionSkipped RetentionAction = "skipped" +) + +// RetentionEntry is a shallow, evidence-bearing retention decision. +type RetentionEntry struct { + Path string `json:"path"` + Category Category `json:"category,omitempty"` + Size int64 `json:"size"` + ModTime time.Time `json:"modTime"` + Recognized bool `json:"recognized"` + Active bool `json:"active,omitempty"` + Action RetentionAction `json:"action"` + Reason string `json:"reason"` + info os.FileInfo +} + +// RetentionReport summarizes a list, dry-run, or deletion pass. +type RetentionReport struct { + DryRun bool `json:"dryRun"` + Scanned int `json:"scanned"` + Recognized int `json:"recognized"` + Protected int `json:"protected"` + WouldDelete int `json:"wouldDelete"` + Deleted int `json:"deleted"` + ReclaimedBytes int64 `json:"reclaimedBytes"` + TotalBytesBefore int64 `json:"totalBytesBefore"` + TotalBytesAfter int64 `json:"totalBytesAfter"` + Warnings []string `json:"warnings,omitempty"` + Entries []RetentionEntry `json:"entries"` +} + +// Summary returns a compact stable summary suitable for list/dry-run output. +func (report RetentionReport) Summary() string { + return fmt.Sprintf("scanned=%d recognized=%d protected=%d would_delete=%d deleted=%d reclaimed_bytes=%d total_before=%d total_after=%d warnings=%d", report.Scanned, report.Recognized, report.Protected, report.WouldDelete, report.Deleted, report.ReclaimedBytes, report.TotalBytesBefore, report.TotalBytesAfter, len(report.Warnings)) +} + +// ListRetention performs the same classification and planning as a dry-run +// prune without modifying the filesystem. +func ListRetention(root string, policy RetentionPolicy) (RetentionReport, error) { + return pruneRetention(root, policy, true) +} + +// PruneRetention removes planned entries unless dryRun is true. +func PruneRetention(root string, policy RetentionPolicy, dryRun bool) (RetentionReport, error) { + return pruneRetention(root, policy, dryRun) +} + +// ListRetention lists this runtime's logging root. +func (runtime *Runtime) ListRetention(policy RetentionPolicy) (RetentionReport, error) { + return ListRetention(runtime.root, policy) +} + +// PruneRetention prunes this runtime's logging root. +func (runtime *Runtime) PruneRetention(policy RetentionPolicy, dryRun bool) (RetentionReport, error) { + return PruneRetention(runtime.root, policy, dryRun) +} + +func pruneRetention(root string, policy RetentionPolicy, dryRun bool) (RetentionReport, error) { + canonicalRoot, err := canonicalPath(root) + if err != nil { + return RetentionReport{}, err + } + rootInfo, err := os.Lstat(canonicalRoot) + if errors.Is(err, os.ErrNotExist) { + return RetentionReport{DryRun: dryRun}, nil + } + if err != nil { + return RetentionReport{}, fmt.Errorf("inspect logging root: %w", err) + } + if !rootInfo.IsDir() || isLinkOrReparse(rootInfo) { + return RetentionReport{}, fmt.Errorf("logging root %s is not a safe directory", canonicalRoot) + } + if policy.MaxTotalBytes < 0 { + return RetentionReport{}, errors.New("retention max total bytes must not be negative") + } + for _, age := range []time.Duration{policy.ManagerMaxAge, policy.InstanceMaxAge, policy.BuildMaxAge, policy.ErrorMaxAge, policy.BenchmarkMaxAge} { + if age < 0 { + return RetentionReport{}, errors.New("retention max ages must not be negative") + } + } + now := time.Now().UTC() + if policy.Now != nil { + now = policy.Now().UTC() + } + report := RetentionReport{DryRun: dryRun} + active, uncertain := discoverActivePaths(canonicalRoot, &report) + entries, err := scanRetentionEntries(canonicalRoot, active, uncertain, &report) + if err != nil { + return RetentionReport{}, err + } + report.Entries = entries + + deletions := make(map[int]string) + for index := range report.Entries { + entry := &report.Entries[index] + if entry.Action == RetentionProtected { + report.Protected++ + } + if !entry.Recognized { + continue + } + report.Recognized++ + report.TotalBytesBefore += entry.Size + if entry.Action == RetentionProtected { + continue + } + maxAge := policy.maxAge(entry.Category) + if maxAge > 0 && !entry.ModTime.After(now.Add(-maxAge)) { + deletions[index] = "category-age" + } + } + + remaining := report.TotalBytesBefore + for index := range deletions { + remaining -= report.Entries[index].Size + } + if policy.MaxTotalBytes > 0 && remaining > policy.MaxTotalBytes { + eligible := make([]int, 0, len(report.Entries)) + for index, entry := range report.Entries { + if entry.Recognized && entry.Action != RetentionProtected { + if _, selected := deletions[index]; !selected { + eligible = append(eligible, index) + } + } + } + sort.Slice(eligible, func(i, j int) bool { + left, right := report.Entries[eligible[i]], report.Entries[eligible[j]] + if left.ModTime.Equal(right.ModTime) { + return left.Path < right.Path + } + return left.ModTime.Before(right.ModTime) + }) + for _, index := range eligible { + if remaining <= policy.MaxTotalBytes { + break + } + deletions[index] = "aggregate-budget" + remaining -= report.Entries[index].Size + } + if remaining > policy.MaxTotalBytes { + report.Warnings = append(report.Warnings, fmt.Sprintf("retention aggregate budget remains exceeded: projected_bytes=%d budget_bytes=%d protected_or_active_bytes=%d", remaining, policy.MaxTotalBytes, remaining)) + } + } + + for index, reason := range deletions { + entry := &report.Entries[index] + entry.Reason = reason + entry.Action = RetentionWouldDelete + report.WouldDelete++ + if dryRun { + continue + } + if warning := deleteRetentionEntry(canonicalRoot, entry); warning != "" { + entry.Action = RetentionSkipped + entry.Reason = "safe-skip" + report.Warnings = append(report.Warnings, warning) + continue + } + entry.Action = RetentionDeleted + report.Deleted++ + report.ReclaimedBytes += entry.Size + } + if dryRun { + var projectedBytes int64 + for index := range deletions { + projectedBytes += report.Entries[index].Size + } + report.TotalBytesAfter = report.TotalBytesBefore - projectedBytes + } else { + report.TotalBytesAfter = report.TotalBytesBefore - report.ReclaimedBytes + } + sort.Slice(report.Entries, func(i, j int) bool { return report.Entries[i].Path < report.Entries[j].Path }) + return report, nil +} + +func scanRetentionEntries(root string, active map[string]struct{}, uncertain bool, report *RetentionReport) ([]RetentionEntry, error) { + rootEntries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read logging root: %w", err) + } + var entries []RetentionEntry + for _, directoryEntry := range rootEntries { + path := filepath.Join(root, directoryEntry.Name()) + if directoryEntry.Name() == controlDirectoryName { + entries = append(entries, protectedEntry(path, "control")) + continue + } + category := Category(directoryEntry.Name()) + if directoryEntry.IsDir() && category.validTranscriptCategory() { + info, infoErr := directoryEntry.Info() + if infoErr != nil { + report.Warnings = append(report.Warnings, fmt.Sprintf("safe skip %s: %v", path, infoErr)) + entries = append(entries, protectedEntry(path, "inspection-error")) + continue + } + if isLinkOrReparse(info) { + entries = append(entries, protectedEntry(path, "symlink-or-reparse")) + continue + } + children, readErr := os.ReadDir(path) + if readErr != nil { + report.Warnings = append(report.Warnings, fmt.Sprintf("safe skip category %s: %v", path, readErr)) + entries = append(entries, protectedEntry(path, "inspection-error")) + continue + } + for _, child := range children { + entries = append(entries, classifyRetentionEntry(root, filepath.Join(path, child.Name()), child, active, uncertain, report)) + } + continue + } + entries = append(entries, classifyRetentionEntry(root, path, directoryEntry, active, uncertain, report)) + } + return entries, nil +} + +func classifyRetentionEntry(root, path string, directoryEntry os.DirEntry, active map[string]struct{}, uncertain bool, report *RetentionReport) RetentionEntry { + report.Scanned++ + info, err := directoryEntry.Info() + if err != nil { + report.Warnings = append(report.Warnings, fmt.Sprintf("safe skip %s: %v", path, err)) + return protectedEntry(path, "inspection-error") + } + entry := RetentionEntry{Path: path, Size: info.Size(), ModTime: info.ModTime().UTC(), Action: RetentionProtected, Reason: "unknown", info: info} + if !info.Mode().IsRegular() || isLinkOrReparse(info) { + entry.Reason = "non-regular-or-reparse" + return entry + } + recognized, ok := recognizePath(root, path) + if !ok { + return entry + } + entry.Recognized = true + entry.Category = recognized.category + if recognized.current { + entry.Reason = "current" + return entry + } + canonical, err := canonicalPath(path) + if err != nil { + entry.Reason = "canonicalization-error" + report.Warnings = append(report.Warnings, fmt.Sprintf("safe skip %s: %v", path, err)) + return entry + } + if _, ok := active[canonical]; ok { + entry.Active = true + entry.Reason = "active" + return entry + } + if recognized.backup { + activeBase, baseErr := canonicalPath(activeLockPathForRecognized(canonical, recognized)) + if baseErr != nil { + entry.Reason = "canonicalization-error" + report.Warnings = append(report.Warnings, fmt.Sprintf("safe skip backup base %s: %v", path, baseErr)) + return entry + } + if _, ok := active[activeBase]; ok { + entry.Active = true + entry.Reason = "active" + return entry + } + } + if uncertain { + entry.Reason = "active-state-uncertain" + return entry + } + entry.Action = RetentionKeep + entry.Reason = "within-policy" + return entry +} + +func protectedEntry(path, reason string) RetentionEntry { + return RetentionEntry{Path: path, Action: RetentionProtected, Reason: reason} +} + +func deleteRetentionEntry(root string, entry *RetentionEntry) string { + canonical, err := canonicalPath(entry.Path) + if err != nil || !pathWithin(root, canonical) { + return fmt.Sprintf("safe skip %s: path no longer resolves inside logging root", entry.Path) + } + if _, active := activeRegistryPaths()[canonical]; active { + return fmt.Sprintf("safe skip %s: path became active", entry.Path) + } + lockDir := filepath.Join(root, controlDirectoryName, "locks") + if err := ensureSafeDirectory(filepath.Join(root, controlDirectoryName), 0o700); err != nil { + return fmt.Sprintf("safe skip %s: create retention control directory: %v", entry.Path, err) + } + if err := ensureSafeDirectory(lockDir, 0o700); err != nil { + return fmt.Sprintf("safe skip %s: create retention lock directory: %v", entry.Path, err) + } + recognized, ok := recognizePath(root, canonical) + if !ok { + return fmt.Sprintf("safe skip %s: filename is no longer recognized", entry.Path) + } + lockTarget, err := canonicalPath(activeLockPathForRecognized(canonical, recognized)) + if err != nil { + return fmt.Sprintf("safe skip %s: resolve active lock target: %v", entry.Path, err) + } + lock, err := filelock.Acquire(filepath.Join(lockDir, pathHash(lockTarget)+".lock")) + if err != nil { + return fmt.Sprintf("safe skip %s: acquire deletion lock: %v", entry.Path, err) + } + defer lock.Close() + info, err := os.Lstat(canonical) + if err != nil { + return fmt.Sprintf("safe skip %s: re-inspect before delete: %v", entry.Path, err) + } + if !info.Mode().IsRegular() || isLinkOrReparse(info) || entry.info == nil || !os.SameFile(entry.info, info) || info.Size() != entry.Size || !info.ModTime().Equal(entry.info.ModTime()) { + return fmt.Sprintf("safe skip %s: file identity or metadata changed", entry.Path) + } + if err := os.Remove(canonical); err != nil { + return fmt.Sprintf("safe skip %s: remove failed: %v", entry.Path, err) + } + return "" +} + +func discoverActivePaths(root string, report *RetentionReport) (map[string]struct{}, bool) { + active := activeRegistryPaths() + control := filepath.Join(root, controlDirectoryName) + lockDir := filepath.Join(control, "locks") + activeDir := filepath.Join(control, "active") + if info, err := os.Lstat(control); err == nil { + if !info.IsDir() || isLinkOrReparse(info) { + report.Warnings = append(report.Warnings, fmt.Sprintf("active log control path %s is not a safe directory", control)) + return active, true + } + } else if errors.Is(err, os.ErrNotExist) { + return active, false + } else { + report.Warnings = append(report.Warnings, fmt.Sprintf("cannot inspect active log control path %s: %v", control, err)) + return active, true + } + hashes := make(map[string]struct{}) + uncertain := false + collect := func(directory, extension string) { + info, statErr := os.Lstat(directory) + if errors.Is(statErr, os.ErrNotExist) { + return + } + if statErr != nil || !info.IsDir() || isLinkOrReparse(info) { + report.Warnings = append(report.Warnings, fmt.Sprintf("cannot safely inspect active log state %s", directory)) + uncertain = true + return + } + entries, err := os.ReadDir(directory) + if err != nil { + report.Warnings = append(report.Warnings, fmt.Sprintf("cannot inspect active log state %s: %v", directory, err)) + uncertain = true + return + } + for _, entry := range entries { + if entry.Type().IsRegular() && strings.HasSuffix(entry.Name(), extension) { + hashes[strings.TrimSuffix(entry.Name(), extension)] = struct{}{} + } + } + } + collect(lockDir, ".lock") + collect(activeDir, ".json") + for hash := range hashes { + if len(hash) != 64 { + report.Warnings = append(report.Warnings, fmt.Sprintf("ignored malformed active-state key %q", hash)) + uncertain = true + continue + } + lockPath := filepath.Join(lockDir, hash+".lock") + if err := ensureSafeDirectory(lockDir, 0o700); err != nil { + report.Warnings = append(report.Warnings, fmt.Sprintf("cannot create active-state lock directory: %v", err)) + uncertain = true + break + } + lock, err := filelock.Acquire(lockPath) + if err == nil { + metadataPath := filepath.Join(activeDir, hash+".json") + if removeErr := os.Remove(metadataPath); removeErr == nil { + report.Warnings = append(report.Warnings, fmt.Sprintf("recovered stale active log metadata %s", metadataPath)) + } else if !errors.Is(removeErr, os.ErrNotExist) { + report.Warnings = append(report.Warnings, fmt.Sprintf("cannot remove stale active log metadata %s: %v", metadataPath, removeErr)) + } + _ = lock.Close() + continue + } + if !errors.Is(err, filelock.ErrLocked) { + report.Warnings = append(report.Warnings, fmt.Sprintf("cannot determine active log state for %s: %v", hash, err)) + uncertain = true + continue + } + metadataPath := filepath.Join(activeDir, hash+".json") + data, readErr := os.ReadFile(metadataPath) + if readErr != nil { + report.Warnings = append(report.Warnings, fmt.Sprintf("active lock %s has no readable metadata: %v", hash, readErr)) + uncertain = true + continue + } + var state activeState + if decodeErr := json.Unmarshal(data, &state); decodeErr != nil { + report.Warnings = append(report.Warnings, fmt.Sprintf("active lock %s has invalid metadata: %v", hash, decodeErr)) + uncertain = true + continue + } + canonical, canonicalErr := canonicalPath(state.Path) + if canonicalErr != nil || pathHash(canonical) != hash || !pathWithin(root, canonical) { + report.Warnings = append(report.Warnings, fmt.Sprintf("active lock %s has unsafe path metadata", hash)) + uncertain = true + continue + } + active[canonical] = struct{}{} + } + return active, uncertain +} diff --git a/internal/logging/retention_test.go b/internal/logging/retention_test.go new file mode 100644 index 0000000..5f53e15 --- /dev/null +++ b/internal/logging/retention_test.go @@ -0,0 +1,318 @@ +package logging + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/solutionforest/ephemeral-action-runner/internal/filelock" +) + +func TestRetentionAppliesAgeThenAggregateBudgetAndProtectsUnknownCurrent(t *testing.T) { + root := t.TempDir() + now := time.Date(2026, 7, 15, 12, 0, 0, 0, time.UTC) + old := now.Add(-72 * time.Hour) + recentOldest := now.Add(-3 * time.Hour) + recentNewest := now.Add(-2 * time.Hour) + errorPath := ErrorPath(root, old) + instancePath, _ := InstancePath(root, "runner-1", "guest") + buildPath, _ := BuildPath(root, "ubuntu", "build") + for _, file := range []struct { + path string + when time.Time + }{ + {errorPath, old}, + {instancePath, recentOldest}, + {buildPath, recentNewest}, + {ManagerPath(root), recentNewest}, + } { + writeSizedFile(t, file.path, 8, file.when) + } + unknown := filepath.Join(root, "instances", "runner-1.ready") + writeSizedFile(t, unknown, 100, old) + unknownLog := filepath.Join(root, "instances", "notes.user.log") + writeSizedFile(t, unknownLog, 100, old) + unknownFlatGuest := filepath.Join(root, "arbitrary.guest.log") + writeSizedFile(t, unknownFlatGuest, 100, old) + + policy := RetentionPolicy{MaxTotalBytes: 10, ErrorMaxAge: 24 * time.Hour, Now: func() time.Time { return now }} + report, err := ListRetention(root, policy) + if err != nil { + t.Fatalf("ListRetention: %v", err) + } + if report.WouldDelete != 3 || report.TotalBytesBefore != 32 || report.TotalBytesAfter != 8 { + t.Fatalf("dry-run report = %#v", report) + } + assertRetentionReason(t, report, unknown, RetentionProtected, "unknown") + assertRetentionReason(t, report, unknownLog, RetentionProtected, "unknown") + assertRetentionReason(t, report, unknownFlatGuest, RetentionProtected, "unknown") + assertRetentionReason(t, report, ManagerPath(root), RetentionProtected, "current") + + report, err = PruneRetention(root, policy, false) + if err != nil { + t.Fatalf("PruneRetention: %v", err) + } + if report.Deleted != 3 || report.ReclaimedBytes != 24 || report.TotalBytesAfter != 8 { + t.Fatalf("prune report = %#v", report) + } + for _, path := range []string{errorPath, instancePath, buildPath} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("deleted path %s still exists: %v", path, err) + } + } + for _, path := range []string{unknown, unknownLog, unknownFlatGuest, ManagerPath(root)} { + if _, err := os.Stat(path); err != nil { + t.Errorf("protected path %s: %v", path, err) + } + } +} + +func TestRetentionProtectsActiveTranscriptThenDeletesAfterClose(t *testing.T) { + root := t.TempDir() + now := time.Now().UTC() + runtime, err := NewRuntime(Options{Directory: root, TranscriptSinks: SinkFile}) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + path, _ := InstancePath(root, "runner-active", "guest") + transcript, err := runtime.OpenTranscript(TranscriptMetadata{Category: CategoryInstances, Instance: "runner-active", Component: "guest"}, path) + if err != nil { + t.Fatalf("OpenTranscript: %v", err) + } + _, _ = transcript.Stdout.Write([]byte("active\n")) + old := now.Add(-48 * time.Hour) + if err := os.Chtimes(path, old, old); err != nil { + t.Fatalf("Chtimes: %v", err) + } + backupPath := strings.TrimSuffix(path, ".log") + "-2026-07-14T01-02-03.004.log.gz" + writeSizedFile(t, backupPath, 8, old) + recognizedBackup, ok := recognizePath(mustCanonicalPath(t, root), mustCanonicalPath(t, backupPath)) + if !ok || !recognizedBackup.backup { + t.Fatalf("backup not recognized: %#v %v", recognizedBackup, ok) + } + if got, want := mustCanonicalPath(t, activeLockPathForRecognized(mustCanonicalPath(t, backupPath), recognizedBackup)), mustCanonicalPath(t, path); got != want { + t.Fatalf("backup active lock path = %s, want %s", got, want) + } + policy := RetentionPolicy{InstanceMaxAge: time.Hour, Now: func() time.Time { return now }} + report, err := ListRetention(root, policy) + if err != nil { + t.Fatalf("ListRetention: %v", err) + } + assertRetentionReason(t, report, path, RetentionProtected, "active") + assertRetentionReason(t, report, backupPath, RetentionProtected, "active") + if err := transcript.Close(); err != nil { + t.Fatalf("Close transcript: %v", err) + } + if err := runtime.Close(); err != nil { + t.Fatalf("Close runtime: %v", err) + } + report, err = PruneRetention(root, policy, false) + if err != nil { + t.Fatalf("PruneRetention: %v", err) + } + if report.Deleted != 2 { + t.Fatalf("deleted = %d, report=%#v", report.Deleted, report) + } +} + +func TestRetentionRecoversStaleActiveMetadata(t *testing.T) { + root, err := canonicalPath(t.TempDir()) + if err != nil { + t.Fatal(err) + } + path, _ := InstancePath(root, "runner-stale", "guest") + canonical, _ := canonicalPath(path) + hash := pathHash(canonical) + activeDir := filepath.Join(root, controlDirectoryName, "active") + if err := os.MkdirAll(activeDir, 0o700); err != nil { + t.Fatal(err) + } + metadataPath := filepath.Join(activeDir, hash+".json") + data, err := json.Marshal(activeState{Version: 1, Path: canonical, PID: 999999}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(metadataPath, data, 0o600); err != nil { + t.Fatal(err) + } + report, err := ListRetention(root, RetentionPolicy{}) + if err != nil { + t.Fatalf("ListRetention: %v", err) + } + if _, err := os.Stat(metadataPath); !os.IsNotExist(err) { + t.Fatalf("stale metadata still exists: %v", err) + } + found := false + for _, warning := range report.Warnings { + found = found || strings.Contains(warning, "recovered stale") + } + if !found { + t.Fatalf("warnings = %#v", report.Warnings) + } +} + +func TestRetentionWarnsWhenProtectedFilesAloneExceedBudget(t *testing.T) { + root := t.TempDir() + writeSizedFile(t, ManagerPath(root), 20, time.Now().UTC()) + report, err := ListRetention(root, RetentionPolicy{MaxTotalBytes: 10}) + if err != nil { + t.Fatal(err) + } + if report.WouldDelete != 0 || report.TotalBytesAfter != 20 { + t.Fatalf("report = %#v", report) + } + found := false + for _, warning := range report.Warnings { + found = found || strings.Contains(warning, "aggregate budget remains exceeded") + } + if !found { + t.Fatalf("warnings = %#v", report.Warnings) + } +} + +func TestRetentionDryRunAndPruneSelectTheSameFiles(t *testing.T) { + root := t.TempDir() + now := time.Now().UTC() + paths := []string{ErrorPath(root, now.Add(-48*time.Hour)), mustPath(InstancePath(root, "runner-old", "wsl")), mustPath(BuildPath(root, "image-old", "source"))} + for index, path := range paths { + writeSizedFile(t, path, index+1, now.Add(-time.Duration(48+index)*time.Hour)) + } + policy := RetentionPolicy{InstanceMaxAge: time.Hour, BuildMaxAge: time.Hour, ErrorMaxAge: time.Hour, Now: func() time.Time { return now }} + dryRun, err := ListRetention(root, policy) + if err != nil { + t.Fatal(err) + } + planned := retentionPathsWithAction(dryRun, RetentionWouldDelete) + pruned, err := PruneRetention(root, policy, false) + if err != nil { + t.Fatal(err) + } + deleted := retentionPathsWithAction(pruned, RetentionDeleted) + if !reflect.DeepEqual(planned, deleted) { + t.Fatalf("planned=%v deleted=%v", planned, deleted) + } +} + +func TestRetentionSafeSkipsSymlinkEntry(t *testing.T) { + root := t.TempDir() + path, _ := InstancePath(root, "runner-link", "guest") + report := RetentionReport{} + entry := classifyRetentionEntry(root, path, fakeDirEntry{info: fakeFileInfo{name: filepath.Base(path), mode: os.ModeSymlink}}, map[string]struct{}{}, false, &report) + if entry.Action != RetentionProtected || entry.Reason != "non-regular-or-reparse" { + t.Fatalf("entry = %#v", entry) + } +} + +func TestRetentionProtectsRealSymlinkWhenAvailable(t *testing.T) { + root := t.TempDir() + target := filepath.Join(t.TempDir(), "target.log") + writeSizedFile(t, target, 8, time.Now().Add(-48*time.Hour)) + link, _ := InstancePath(root, "runner-link", "guest") + if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlink creation unavailable: %v", err) + } + report, err := ListRetention(root, RetentionPolicy{InstanceMaxAge: time.Hour}) + if err != nil { + t.Fatal(err) + } + assertRetentionReason(t, report, link, RetentionProtected, "non-regular-or-reparse") +} + +func TestDeleteRetentionEntrySafeSkipsLockedPath(t *testing.T) { + root, _ := canonicalPath(t.TempDir()) + path, _ := InstancePath(root, "runner-locked", "tart") + writeSizedFile(t, path, 8, time.Now().Add(-48*time.Hour)) + directoryEntries, err := os.ReadDir(filepath.Dir(path)) + if err != nil { + t.Fatal(err) + } + report := RetentionReport{} + entry := classifyRetentionEntry(root, path, directoryEntries[0], map[string]struct{}{}, false, &report) + control := filepath.Join(root, controlDirectoryName) + lockDir := filepath.Join(control, "locks") + if err := ensureSafeDirectory(control, 0o700); err != nil { + t.Fatal(err) + } + if err := ensureSafeDirectory(lockDir, 0o700); err != nil { + t.Fatal(err) + } + lock, err := filelock.Acquire(filepath.Join(lockDir, pathHash(mustCanonicalPath(t, path))+".lock")) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + warning := deleteRetentionEntry(root, &entry) + if !strings.Contains(warning, "acquire deletion lock") { + t.Fatalf("warning = %q", warning) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("locked file was removed: %v", err) + } +} + +func retentionPathsWithAction(report RetentionReport, action RetentionAction) []string { + var paths []string + for _, entry := range report.Entries { + if entry.Action == action { + paths = append(paths, entry.Path) + } + } + sort.Strings(paths) + return paths +} + +type fakeDirEntry struct{ info fakeFileInfo } + +func (entry fakeDirEntry) Name() string { return entry.info.Name() } +func (entry fakeDirEntry) IsDir() bool { return entry.info.IsDir() } +func (entry fakeDirEntry) Type() os.FileMode { return entry.info.Mode().Type() } +func (entry fakeDirEntry) Info() (os.FileInfo, error) { return entry.info, nil } + +type fakeFileInfo struct { + name string + mode os.FileMode + sys any +} + +func (info fakeFileInfo) Name() string { return info.name } +func (info fakeFileInfo) Size() int64 { return 0 } +func (info fakeFileInfo) Mode() os.FileMode { return info.mode } +func (info fakeFileInfo) ModTime() time.Time { return time.Time{} } +func (info fakeFileInfo) IsDir() bool { return info.mode.IsDir() } +func (info fakeFileInfo) Sys() any { return info.sys } + +func writeSizedFile(t *testing.T, path string, size int, timestamp time.Time) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(strings.Repeat("x", size)), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, timestamp, timestamp); err != nil { + t.Fatal(err) + } +} + +func assertRetentionReason(t *testing.T, report RetentionReport, path string, action RetentionAction, reason string) { + t.Helper() + want, _ := canonicalPath(path) + for _, entry := range report.Entries { + got, _ := canonicalPath(entry.Path) + if got == want { + if entry.Action != action || entry.Reason != reason { + t.Fatalf("entry %s = action %s reason %s, want %s %s", path, entry.Action, entry.Reason, action, reason) + } + return + } + } + t.Fatalf("entry %s not found", path) +} diff --git a/internal/logging/runtime.go b/internal/logging/runtime.go new file mode 100644 index 0000000..c0579ca --- /dev/null +++ b/internal/logging/runtime.go @@ -0,0 +1,155 @@ +package logging + +import ( + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "sync" +) + +// Runtime owns the manager logger and all transcripts opened through it. +type Runtime struct { + options Options + root string + manager *slog.Logger + managerFile *rotationHandle + mu sync.Mutex + closed bool + transcripts map[*Transcript]struct{} + warnings []error +} + +// NewRuntime constructs a manager logger and, when configured, opens epar.log. +func NewRuntime(options Options) (*Runtime, error) { + options, err := options.normalized() + if err != nil { + return nil, err + } + root, err := canonicalPath(options.Directory) + if err != nil { + return nil, fmt.Errorf("canonicalize logging directory: %w", err) + } + if err := os.MkdirAll(root, 0o755); err != nil { + return nil, fmt.Errorf("create logging directory: %w", err) + } + if err := ensureSafeDirectory(root, 0o755); err != nil { + return nil, fmt.Errorf("validate logging directory: %w", err) + } + runtime := &Runtime{options: options, root: root, transcripts: make(map[*Transcript]struct{})} + var managerFile *rotationHandle + if options.ManagerSinks.Has(SinkFile) { + managerFile, err = openRotating(root, filepath.Join(root, ManagerFilename), options.Rotation, TranscriptMetadata{SessionID: "manager", Category: CategoryManager, Component: "manager"}, options.Now()) + if err != nil { + err = fmt.Errorf("open manager log: %w", err) + if !options.ManagerSinks.Has(SinkConsole) { + return nil, err + } + runtime.warnings = append(runtime.warnings, err) + writeInitializationWarning(options.Stderr, err) + options.ManagerSinks &^= SinkFile + runtime.options.ManagerSinks &^= SinkFile + } else { + runtime.managerFile = managerFile + } + } + runtime.manager = slog.New(managerHandler(options, managerFile)) + return runtime, nil +} + +// Manager returns the structured manager logger. +func (runtime *Runtime) Manager() *slog.Logger { return runtime.manager } + +// Logger is an alias for Manager for call sites that only use one logger. +func (runtime *Runtime) Logger() *slog.Logger { return runtime.manager } + +// Directory returns the canonical logging root. +func (runtime *Runtime) Directory() string { return runtime.root } + +// InitializationWarnings returns defensive sink fallbacks encountered by the +// runtime. The returned slice is a copy. +func (runtime *Runtime) InitializationWarnings() []error { + runtime.mu.Lock() + defer runtime.mu.Unlock() + return append([]error(nil), runtime.warnings...) +} + +// OpenTranscript opens a coalesced raw transcript and separate contextual +// stdout/stderr writers. The path must match the metadata category and one of +// the recognized shallow current-file patterns. +func (runtime *Runtime) OpenTranscript(metadata TranscriptMetadata, path string) (*Transcript, error) { + if !metadata.Category.validTranscriptCategory() { + return nil, fmt.Errorf("unsupported transcript category %q", metadata.Category) + } + canonical, err := canonicalPath(path) + if err != nil { + return nil, err + } + recognized, ok := recognizePath(runtime.root, canonical) + if !ok || recognized.category != metadata.Category || recognized.backup || recognized.current { + return nil, fmt.Errorf("transcript path %s does not match category %q or a recognized current-file pattern", canonical, metadata.Category) + } + + runtime.mu.Lock() + if runtime.closed { + runtime.mu.Unlock() + return nil, os.ErrClosed + } + transcript, warning, err := newTranscript(runtime.options, runtime.root, canonical, metadata) + if err != nil { + runtime.mu.Unlock() + return nil, err + } + if warning != nil { + runtime.warnings = append(runtime.warnings, warning) + writeInitializationWarning(runtime.options.Stderr, warning) + } + transcript.onClose = runtime.removeTranscript + runtime.transcripts[transcript] = struct{}{} + runtime.mu.Unlock() + return transcript, nil +} + +func writeInitializationWarning(writer io.Writer, err error) { + _, _ = fmt.Fprintf(writer, "warning: logging file sink disabled: %v\n", err) +} + +func (runtime *Runtime) removeTranscript(transcript *Transcript) { + runtime.mu.Lock() + delete(runtime.transcripts, transcript) + runtime.mu.Unlock() +} + +// Close flushes partial transcript lines, closes every open transcript, and +// releases the manager log's active-path lock. +func (runtime *Runtime) Close() error { + if runtime == nil { + return nil + } + runtime.mu.Lock() + if runtime.closed { + runtime.mu.Unlock() + return nil + } + runtime.closed = true + transcripts := make([]*Transcript, 0, len(runtime.transcripts)) + for transcript := range runtime.transcripts { + transcripts = append(transcripts, transcript) + } + runtime.mu.Unlock() + + errorsFound := make([]error, 0, len(transcripts)+1) + for _, transcript := range transcripts { + if err := transcript.Close(); err != nil { + errorsFound = append(errorsFound, err) + } + } + if runtime.managerFile != nil { + if err := runtime.managerFile.Close(); err != nil { + errorsFound = append(errorsFound, err) + } + } + return errors.Join(errorsFound...) +} diff --git a/internal/logging/runtime_test.go b/internal/logging/runtime_test.go new file mode 100644 index 0000000..7e0f8fc --- /dev/null +++ b/internal/logging/runtime_test.go @@ -0,0 +1,520 @@ +package logging + +import ( + "bufio" + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func TestManagerFansOutByLevelAndWritesJSONFile(t *testing.T) { + root := t.TempDir() + var stdout, stderr bytes.Buffer + runtime, err := NewRuntime(Options{ + Directory: root, + ManagerSinks: SinkBoth, + ManagerConsoleFormat: FormatText, + ManagerFileFormat: FormatJSON, + TranscriptSinks: SinkNone, + Level: slog.LevelDebug, + Stdout: &stdout, + Stderr: &stderr, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + runtime.Manager().Debug("debug message") + runtime.Manager().Info("info message") + runtime.Manager().Warn("warn message") + runtime.Manager().Error("error message") + if err := runtime.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + if got := stdout.String(); !strings.Contains(got, "debug message") || !strings.Contains(got, "info message") || strings.Contains(got, "warn message") { + t.Fatalf("stdout routing = %q", got) + } + if got := stderr.String(); !strings.Contains(got, "warn message") || !strings.Contains(got, "error message") || strings.Contains(got, "info message") { + t.Fatalf("stderr routing = %q", got) + } + data, err := os.ReadFile(ManagerPath(root)) + if err != nil { + t.Fatalf("ReadFile manager: %v", err) + } + scanner := bufio.NewScanner(bytes.NewReader(data)) + var messages []string + for scanner.Scan() { + var record map[string]any + if err := json.Unmarshal(scanner.Bytes(), &record); err != nil { + t.Fatalf("manager line is not JSON: %v", err) + } + messages = append(messages, record["msg"].(string)) + } + if got := strings.Join(messages, ","); got != "debug message,info message,warn message,error message" { + t.Fatalf("manager messages = %q", got) + } +} + +func TestDefaultManagerTextFormatIsHumanReadable(t *testing.T) { + var stdout bytes.Buffer + runtime, err := NewRuntime(Options{Directory: t.TempDir(), ManagerSinks: SinkConsole, Stdout: &stdout}) + if err != nil { + t.Fatal(err) + } + runtime.Manager().Info("runner ready", "provider", "docker-dind", "attempt", 2) + if err := runtime.Close(); err != nil { + t.Fatal(err) + } + parts := strings.SplitN(strings.TrimSpace(stdout.String()), " ", 3) + if len(parts) != 3 { + t.Fatalf("console line = %q", stdout.String()) + } + if _, err := time.Parse(humanTimestampLayout, parts[0]); err != nil { + t.Fatalf("timestamp %q is not human RFC3339 milliseconds: %v", parts[0], err) + } + if parts[1] != "[INFO]" || parts[2] != "runner ready" { + t.Fatalf("console line = %q", stdout.String()) + } +} + +func TestCustomManagerAndTranscriptTextFormats(t *testing.T) { + var stdout bytes.Buffer + runtime, err := NewRuntime(Options{ + Directory: t.TempDir(), + ManagerSinks: SinkConsole, + ManagerConsoleTextFormat: "[{level}] {message}{attributes}", + TranscriptSinks: SinkConsole, + TranscriptConsoleTextFormat: "[{stream}] {instance}/{component}: {message}{attributes}", + Stdout: &stdout, + }) + if err != nil { + t.Fatal(err) + } + runtime.Manager().Info("manager ready", "provider", "wsl") + path, _ := InstancePath(runtime.Directory(), "runner-1", "guest") + transcript, err := runtime.OpenTranscript(TranscriptMetadata{Category: CategoryInstances, Instance: "runner-1", Component: "guest", Attributes: map[string]string{"attempt": "2"}}, path) + if err != nil { + t.Fatal(err) + } + if _, err := transcript.Stdout.Write([]byte("guest ready\n")); err != nil { + t.Fatal(err) + } + if err := runtime.Close(); err != nil { + t.Fatal(err) + } + want := "[INFO] manager ready provider=wsl\n[stdout] runner-1/guest: guest ready attempt=2\n" + if got := stdout.String(); got != want { + t.Fatalf("custom text output = %q, want %q", got, want) + } +} + +func TestJSONConsoleRejectsCustomTextFormat(t *testing.T) { + _, err := NewRuntime(Options{Directory: t.TempDir(), ManagerSinks: SinkConsole, ManagerConsoleFormat: FormatJSON, ManagerConsoleTextFormat: "{level} {message}"}) + if err == nil || !strings.Contains(err.Error(), "only with text output") { + t.Fatalf("NewRuntime() error = %v, want text-only validation", err) + } +} + +func TestManagerFileInitializationFallbackRequiresConsole(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(ManagerPath(root), 0o755); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + runtime, err := NewRuntime(Options{Directory: root, ManagerSinks: SinkBoth, Stdout: &stdout, Stderr: &stderr}) + if err != nil { + t.Fatalf("NewRuntime with console fallback: %v", err) + } + if len(runtime.InitializationWarnings()) != 1 || strings.Count(stderr.String(), "warning: logging file sink disabled:") != 1 { + t.Fatalf("warnings = %#v, stderr = %q", runtime.InitializationWarnings(), stderr.String()) + } + runtime.Manager().Info("console survived") + if !strings.Contains(stdout.String(), "console survived") { + t.Fatalf("stdout = %q", stdout.String()) + } + if err := runtime.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if _, err := NewRuntime(Options{Directory: root, ManagerSinks: SinkFile}); err == nil { + t.Fatal("sole manager file sink unexpectedly fell back") + } +} + +func TestManagerFanoutIsolatesSinkFailureAndWarnsOnce(t *testing.T) { + root := t.TempDir() + var stderr bytes.Buffer + runtime, err := NewRuntime(Options{Directory: root, ManagerSinks: SinkBoth, Stdout: failingWriter{}, Stderr: &stderr}) + if err != nil { + t.Fatal(err) + } + runtime.Manager().Info("first survives") + runtime.Manager().Info("second survives") + if err := runtime.Close(); err != nil { + t.Fatal(err) + } + if count := strings.Count(stderr.String(), "warning: logging sink write failed:"); count != 1 { + t.Fatalf("warning count = %d, stderr=%q", count, stderr.String()) + } + data, err := os.ReadFile(ManagerPath(root)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "first survives") || !strings.Contains(string(data), "second survives") { + t.Fatalf("surviving file sink = %q", data) + } +} + +func TestTranscriptFileInitializationFallbackRequiresConsole(t *testing.T) { + root := t.TempDir() + path, _ := InstancePath(root, "runner-fallback", "guest") + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + runtime, err := NewRuntime(Options{Directory: root, TranscriptSinks: SinkBoth, Stdout: &stdout, Stderr: &stderr}) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + transcript, err := runtime.OpenTranscript(TranscriptMetadata{Category: CategoryInstances, Instance: "runner-fallback", Component: "guest"}, path) + if err != nil { + t.Fatalf("OpenTranscript with console fallback: %v", err) + } + _, _ = transcript.Stdout.Write([]byte("console only\n")) + if !strings.Contains(stdout.String(), "console only") || strings.Count(stderr.String(), "warning: logging file sink disabled:") != 1 { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + _ = transcript.Close() + _ = runtime.Close() + + fileOnly, err := NewRuntime(Options{Directory: root, TranscriptSinks: SinkFile}) + if err != nil { + t.Fatal(err) + } + if _, err := fileOnly.OpenTranscript(TranscriptMetadata{Category: CategoryInstances, Instance: "runner-fallback", Component: "guest"}, path); err == nil { + t.Fatal("sole transcript file sink unexpectedly fell back") + } + _ = fileOnly.Close() +} + +func TestTranscriptCoalescesRawWritesAndFlushesContextualPartialLines(t *testing.T) { + root := t.TempDir() + var stdout, stderr bytes.Buffer + fixed := time.Date(2026, 7, 15, 12, 0, 0, 123, time.UTC) + runtime, err := NewRuntime(Options{ + Directory: root, + TranscriptSinks: SinkBoth, + TranscriptConsoleFormat: FormatJSON, + Stdout: &stdout, + Stderr: &stderr, + Now: func() time.Time { return fixed }, + }) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + path, err := InstancePath(root, "runner-1", "docker-dind") + if err != nil { + t.Fatalf("InstancePath: %v", err) + } + transcript, err := runtime.OpenTranscript(TranscriptMetadata{SessionID: "session-1", Category: CategoryInstances, Instance: "runner-1", Component: "provider", Provider: "docker-dind"}, path) + if err != nil { + t.Fatalf("OpenTranscript: %v", err) + } + if _, err := transcript.Stdout.Write([]byte("one")); err != nil { + t.Fatalf("write stdout 1: %v", err) + } + if _, err := transcript.Stderr.Write([]byte("err\n")); err != nil { + t.Fatalf("write stderr: %v", err) + } + if _, err := transcript.File.Write([]byte("raw-event\n")); err != nil { + t.Fatalf("write raw file: %v", err) + } + if _, err := transcript.Stdout.Write([]byte(" two\npartial")); err != nil { + t.Fatalf("write stdout 2: %v", err) + } + if err := transcript.Close(); err != nil { + t.Fatalf("Close transcript: %v", err) + } + if err := runtime.Close(); err != nil { + t.Fatalf("Close runtime: %v", err) + } + + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile transcript: %v", err) + } + if got, want := string(raw), "oneerr\nraw-event\n two\npartial"; got != want { + t.Fatalf("raw transcript = %q, want %q", got, want) + } + stdoutEnvelopes := decodeEnvelopeLines(t, stdout.Bytes()) + if len(stdoutEnvelopes) != 2 || stdoutEnvelopes[0].Message != "one two" || stdoutEnvelopes[1].Message != "partial" { + t.Fatalf("stdout envelopes = %#v", stdoutEnvelopes) + } + if stdoutEnvelopes[0].Instance != "runner-1" || stdoutEnvelopes[0].Component != "provider" || stdoutEnvelopes[0].Stream != "stdout" || stdoutEnvelopes[0].Timestamp != fixed.Format(time.RFC3339Nano) { + t.Fatalf("stdout context = %#v", stdoutEnvelopes[0]) + } + stderrEnvelopes := decodeEnvelopeLines(t, stderr.Bytes()) + if len(stderrEnvelopes) != 1 || stderrEnvelopes[0].Message != "err" || stderrEnvelopes[0].Stream != "stderr" { + t.Fatalf("stderr envelopes = %#v", stderrEnvelopes) + } +} + +func TestTranscriptFanoutStillFramesConsoleWhenFileWriteFails(t *testing.T) { + root := t.TempDir() + var stdout bytes.Buffer + runtime, err := NewRuntime(Options{Directory: root, TranscriptSinks: SinkBoth, TranscriptConsoleFormat: FormatJSON, Stdout: &stdout}) + if err != nil { + t.Fatal(err) + } + path, _ := InstancePath(root, "runner-failure", "guest") + transcript, err := runtime.OpenTranscript(TranscriptMetadata{Category: CategoryInstances, Instance: "runner-failure", Component: "guest"}, path) + if err != nil { + t.Fatal(err) + } + if err := transcript.file.Close(); err != nil { + t.Fatal(err) + } + written, err := transcript.Stdout.Write([]byte("console survives\n")) + if err == nil || written != 0 { + t.Fatalf("write = %d, %v; want file error", written, err) + } + envelopes := decodeEnvelopeLines(t, stdout.Bytes()) + if len(envelopes) != 1 || envelopes[0].Message != "console survives" { + t.Fatalf("console envelopes = %#v", envelopes) + } + _ = transcript.Close() + _ = runtime.Close() +} + +func TestTranscriptConcurrentWritesRemainWhole(t *testing.T) { + root := t.TempDir() + runtime, err := NewRuntime(Options{Directory: root, TranscriptSinks: SinkFile}) + if err != nil { + t.Fatalf("NewRuntime: %v", err) + } + path, _ := InstancePath(root, "runner-race", "guest") + transcript, err := runtime.OpenTranscript(TranscriptMetadata{Category: CategoryInstances, Instance: "runner-race", Component: "guest"}, path) + if err != nil { + t.Fatalf("OpenTranscript: %v", err) + } + const writers = 8 + const linesPerWriter = 40 + var wait sync.WaitGroup + for writer := 0; writer < writers; writer++ { + wait.Add(1) + go func(writer int) { + defer wait.Done() + target := transcript.Stdout + if writer%2 == 1 { + target = transcript.Stderr + } + for line := 0; line < linesPerWriter; line++ { + _, _ = fmt.Fprintf(target, "writer-%02d-line-%02d\n", writer, line) + } + }(writer) + } + wait.Wait() + if err := transcript.Close(); err != nil { + t.Fatalf("Close transcript: %v", err) + } + _ = runtime.Close() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) != writers*linesPerWriter { + t.Fatalf("line count = %d, want %d", len(lines), writers*linesPerWriter) + } +} + +func TestProcessRegistrySharesOneWriterAndTracksSessions(t *testing.T) { + root := t.TempDir() + firstRuntime, err := NewRuntime(Options{Directory: root, TranscriptSinks: SinkFile}) + if err != nil { + t.Fatal(err) + } + secondRuntime, err := NewRuntime(Options{Directory: root, TranscriptSinks: SinkFile}) + if err != nil { + t.Fatal(err) + } + path, _ := InstancePath(root, "runner-shared", "guest") + first, err := firstRuntime.OpenTranscript(TranscriptMetadata{SessionID: "first", Category: CategoryInstances, Instance: "runner-shared", Component: "guest"}, path) + if err != nil { + t.Fatal(err) + } + second, err := secondRuntime.OpenTranscript(TranscriptMetadata{SessionID: "second", Category: CategoryInstances, Instance: "runner-shared", Component: "guest"}, path) + if err != nil { + t.Fatal(err) + } + state := readActiveStateForTest(t, root, path) + if len(state.Sessions) != 2 { + t.Fatalf("active sessions = %#v", state.Sessions) + } + _, _ = first.Stdout.Write([]byte("first\n")) + _, _ = second.Stdout.Write([]byte("second\n")) + if err := first.Close(); err != nil { + t.Fatal(err) + } + state = readActiveStateForTest(t, root, path) + if len(state.Sessions) != 1 || state.Sessions[0].SessionID != "second" { + t.Fatalf("remaining active sessions = %#v", state.Sessions) + } + _, _ = second.Stdout.Write([]byte("after-first-close\n")) + if err := second.Close(); err != nil { + t.Fatal(err) + } + _ = firstRuntime.Close() + _ = secondRuntime.Close() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if got := string(data); got != "first\nsecond\nafter-first-close\n" { + t.Fatalf("shared transcript = %q", got) + } + metadataPath := filepath.Join(root, controlDirectoryName, "active", pathHash(mustCanonicalPath(t, path))+".json") + if _, err := os.Stat(metadataPath); !os.IsNotExist(err) { + t.Fatalf("active metadata remains after last close: %v", err) + } +} + +func TestRotationCompressesAndHonorsBackupLimit(t *testing.T) { + root := t.TempDir() + runtime, err := NewRuntime(Options{Directory: root, TranscriptSinks: SinkFile, Rotation: Rotation{MaxSizeMiB: 1, MaxBackups: 2, Compress: true}}) + if err != nil { + t.Fatal(err) + } + path, _ := InstancePath(root, "runner-rotate", "wsl") + transcript, err := runtime.OpenTranscript(TranscriptMetadata{Category: CategoryInstances, Instance: "runner-rotate", Component: "provider"}, path) + if err != nil { + t.Fatal(err) + } + defer runtime.Close() + defer transcript.Close() + chunk := bytes.Repeat([]byte("x"), 700*1024) + for index := 0; index < 4; index++ { + var writeErr error + for attempt := 0; attempt < 5; attempt++ { + if _, writeErr = transcript.File.Write(chunk); writeErr == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + if writeErr != nil { + t.Fatal(writeErr) + } + } + deadline := time.Now().Add(5 * time.Second) + var backups []string + for time.Now().Before(deadline) { + backups = backups[:0] + entries, _ := os.ReadDir(filepath.Dir(path)) + uncompressedBackup := false + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".gz") { + backups = append(backups, filepath.Join(filepath.Dir(path), entry.Name())) + } else if entry.Name() != filepath.Base(path) { + uncompressedBackup = true + } + } + if len(backups) == 2 && !uncompressedBackup { + break + } + time.Sleep(20 * time.Millisecond) + } + if len(backups) != 2 { + t.Fatalf("compressed backups = %v, want exactly two", backups) + } + file, err := os.Open(backups[0]) + if err != nil { + t.Fatal(err) + } + reader, err := gzip.NewReader(file) + if err != nil { + t.Fatal(err) + } + decompressed, err := io.ReadAll(reader) + _ = reader.Close() + _ = file.Close() + if err != nil || len(decompressed) != len(chunk) { + t.Fatalf("decompressed backup bytes = %d, err=%v", len(decompressed), err) + } +} + +func TestRuntimePathLockExcludesAnotherProcess(t *testing.T) { + if os.Getenv("EPAR_LOGGING_LOCK_HELPER") == "1" { + runtime, err := NewRuntime(Options{Directory: os.Getenv("EPAR_LOGGING_LOCK_ROOT"), ManagerSinks: SinkFile}) + if err != nil { + os.Exit(23) + } + _ = runtime.Close() + os.Exit(0) + } + root := t.TempDir() + runtime, err := NewRuntime(Options{Directory: root, ManagerSinks: SinkFile}) + if err != nil { + t.Fatal(err) + } + defer runtime.Close() + command := exec.Command(os.Args[0], "-test.run=^TestRuntimePathLockExcludesAnotherProcess$") + command.Env = append(os.Environ(), "EPAR_LOGGING_LOCK_HELPER=1", "EPAR_LOGGING_LOCK_ROOT="+root) + err = command.Run() + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 23 { + t.Fatalf("child result = %v, want active-path exclusion", err) + } +} + +func decodeEnvelopeLines(t *testing.T, data []byte) []transcriptEnvelope { + t.Helper() + var envelopes []transcriptEnvelope + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + var envelope transcriptEnvelope + if err := json.Unmarshal(scanner.Bytes(), &envelope); err != nil { + t.Fatalf("decode envelope: %v", err) + } + envelopes = append(envelopes, envelope) + } + return envelopes +} + +func readActiveStateForTest(t *testing.T, root, path string) activeState { + t.Helper() + metadataPath := filepath.Join(root, controlDirectoryName, "active", pathHash(mustCanonicalPath(t, path))+".json") + data, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatal(err) + } + var state activeState + if err := json.Unmarshal(data, &state); err != nil { + t.Fatal(err) + } + return state +} + +func mustCanonicalPath(t *testing.T, path string) string { + t.Helper() + canonical, err := canonicalPath(path) + if err != nil { + t.Fatal(err) + } + return canonical +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { + return 0, errors.New("injected sink failure") +} diff --git a/internal/logging/transcript.go b/internal/logging/transcript.go new file mode 100644 index 0000000..24c07f4 --- /dev/null +++ b/internal/logging/transcript.go @@ -0,0 +1,225 @@ +package logging + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + "strings" + "sync" + "time" +) + +// Transcript exposes independent command stdout and stderr writers while +// preserving one raw, coalesced on-disk stream. +type Transcript struct { + Stdout io.Writer + Stderr io.Writer + // File writes only to the raw rotating file and never emits a console + // envelope. It is io.Discard when the file sink is disabled. + File io.Writer + + options Options + metadata TranscriptMetadata + file *rotationHandle + mu sync.Mutex + buffers map[string][]byte + closed bool + closeErr error + onClose func(*Transcript) +} + +type transcriptStream struct { + transcript *Transcript + name string +} + +type transcriptEnvelope struct { + Timestamp string `json:"timestamp"` + Instance string `json:"instance,omitempty"` + Component string `json:"component,omitempty"` + Stream string `json:"stream"` + Message string `json:"message"` + SessionID string `json:"sessionId,omitempty"` + Category Category `json:"category,omitempty"` + Provider string `json:"provider,omitempty"` + Attributes map[string]string `json:"attributes,omitempty"` +} + +func newTranscript(options Options, root, path string, metadata TranscriptMetadata) (*Transcript, error, error) { + if metadata.SessionID == "" { + sessionID, err := randomToken() + if err != nil { + return nil, nil, fmt.Errorf("create transcript session token: %w", err) + } + metadata.SessionID = sessionID + } + transcript := &Transcript{options: options, metadata: cloneMetadata(metadata), buffers: map[string][]byte{"stdout": nil, "stderr": nil}, File: io.Discard} + if options.TranscriptSinks.Has(SinkFile) { + file, err := openRotating(root, path, options.Rotation, metadata, options.Now()) + if err != nil { + err = fmt.Errorf("open transcript: %w", err) + if !options.TranscriptSinks.Has(SinkConsole) { + return nil, nil, err + } + transcript.options.TranscriptSinks &^= SinkFile + transcript.Stdout = &transcriptStream{transcript: transcript, name: "stdout"} + transcript.Stderr = &transcriptStream{transcript: transcript, name: "stderr"} + return transcript, err, nil + } + transcript.file = file + transcript.File = file + } + transcript.Stdout = &transcriptStream{transcript: transcript, name: "stdout"} + transcript.Stderr = &transcriptStream{transcript: transcript, name: "stderr"} + return transcript, nil, nil +} + +func (stream *transcriptStream) Write(data []byte) (int, error) { + if stream == nil || stream.transcript == nil { + return 0, errors.New("nil transcript stream") + } + return stream.transcript.write(stream.name, data) +} + +func (transcript *Transcript) write(stream string, data []byte) (int, error) { + transcript.mu.Lock() + defer transcript.mu.Unlock() + if transcript.closed { + return 0, errors.New("write to closed transcript") + } + written := len(data) + var writeErrors []error + if transcript.file != nil { + var err error + written, err = transcript.file.Write(data) + if err != nil { + writeErrors = append(writeErrors, err) + } + if written != len(data) { + writeErrors = append(writeErrors, io.ErrShortWrite) + } + } + if !transcript.options.TranscriptSinks.Has(SinkConsole) { + return written, errors.Join(writeErrors...) + } + transcript.buffers[stream] = append(transcript.buffers[stream], data...) + for { + newline := bytes.IndexByte(transcript.buffers[stream], '\n') + if newline < 0 { + break + } + line := append([]byte(nil), transcript.buffers[stream][:newline]...) + transcript.buffers[stream] = transcript.buffers[stream][newline+1:] + line = bytes.TrimSuffix(line, []byte{'\r'}) + if err := transcript.writeEnvelope(stream, string(line)); err != nil { + writeErrors = append(writeErrors, err) + } + } + if len(writeErrors) > 0 { + return written, errors.Join(writeErrors...) + } + return len(data), nil +} + +func (transcript *Transcript) writeEnvelope(stream, message string) error { + envelope := transcriptEnvelope{ + Timestamp: transcript.options.Now().UTC().Format(time.RFC3339Nano), + Instance: transcript.metadata.Instance, + Component: transcript.metadata.Component, + Stream: stream, + Message: message, + SessionID: transcript.metadata.SessionID, + Category: transcript.metadata.Category, + Provider: transcript.metadata.Provider, + Attributes: transcript.metadata.Attributes, + } + var data []byte + var err error + if transcript.options.TranscriptConsoleFormat == FormatJSON { + data, err = json.Marshal(envelope) + if err != nil { + return fmt.Errorf("encode transcript console envelope: %w", err) + } + data = append(data, '\n') + } else { + data = formatTextEnvelope(envelope, transcript.options.TranscriptConsoleTextFormat) + } + target := transcript.options.Stdout + if stream == "stderr" { + target = transcript.options.Stderr + } + _, err = target.Write(data) + return err +} + +func formatTextEnvelope(envelope transcriptEnvelope, template string) []byte { + keys := make([]string, 0, len(envelope.Attributes)) + for key := range envelope.Attributes { + keys = append(keys, key) + } + sort.Strings(keys) + attributes := make([]string, 0, len(keys)) + for _, key := range keys { + attributes = append(attributes, key+"="+quoteHumanString(envelope.Attributes[key])) + } + renderedAttributes := strings.Join(attributes, " ") + if renderedAttributes != "" { + renderedAttributes = " " + renderedAttributes + } + line := strings.NewReplacer( + "{time}", envelope.Timestamp, + "{instance}", singleLine(envelope.Instance), + "{component}", singleLine(envelope.Component), + "{stream}", singleLine(envelope.Stream), + "{message}", singleLine(envelope.Message), + "{session}", singleLine(envelope.SessionID), + "{category}", singleLine(string(envelope.Category)), + "{provider}", singleLine(envelope.Provider), + "{attributes}", renderedAttributes, + ).Replace(template) + return []byte(line + "\n") +} + +// Close emits unterminated console fragments, closes the raw file, and +// releases active-path state. It is idempotent. +func (transcript *Transcript) Close() error { + if transcript == nil { + return nil + } + transcript.mu.Lock() + if transcript.closed { + err := transcript.closeErr + transcript.mu.Unlock() + return err + } + transcript.closed = true + var closeErrors []error + if transcript.options.TranscriptSinks.Has(SinkConsole) { + for _, stream := range []string{"stdout", "stderr"} { + if len(transcript.buffers[stream]) == 0 { + continue + } + line := bytes.TrimSuffix(transcript.buffers[stream], []byte{'\r'}) + if err := transcript.writeEnvelope(stream, string(line)); err != nil { + closeErrors = append(closeErrors, err) + } + transcript.buffers[stream] = nil + } + } + if transcript.file != nil { + if err := transcript.file.Close(); err != nil { + closeErrors = append(closeErrors, err) + } + } + transcript.closeErr = errors.Join(closeErrors...) + onClose := transcript.onClose + err := transcript.closeErr + transcript.mu.Unlock() + if onClose != nil { + onClose(transcript) + } + return err +} diff --git a/internal/logging/types.go b/internal/logging/types.go new file mode 100644 index 0000000..32b0073 --- /dev/null +++ b/internal/logging/types.go @@ -0,0 +1,222 @@ +// Package logging provides the manager logger, command transcript writers, and +// conservative log-retention primitives used by ephemeral-action-runner. +package logging + +import ( + "errors" + "fmt" + "io" + "log/slog" + "os" + "strings" + "time" +) + +const ( + // ManagerFilename is the active manager log in the logging root. + ManagerFilename = "epar.log" + // LastErrorFilename is the stable latest-error report in the logging root. + LastErrorFilename = "epar-last-error.log" + defaultMaxSizeMiB = 100 + defaultMaxBackups = 3 +) + +// Format selects a structured or human-readable line representation. +type Format string + +const ( + FormatText Format = "text" + FormatJSON Format = "json" + DefaultManagerConsoleTextFormat = "{time} [{level}] {message}" + DefaultTranscriptConsoleTextFormat = "{time} {stream} {instance} {component} {message}{attributes}" +) + +// Sinks is a bitset describing where a log stream is written. +type Sinks uint8 + +const ( + SinkNone Sinks = 0 + SinkConsole Sinks = 1 << iota + SinkFile + SinkBoth = SinkConsole | SinkFile +) + +// Has reports whether every requested sink is enabled. +func (sinks Sinks) Has(sink Sinks) bool { return sinks&sink == sink } + +// Rotation configures Lumberjack v2. Retention age and aggregate-size policy +// are deliberately separate from per-file rotation. +type Rotation struct { + MaxSizeMiB int + MaxBackups int + Compress bool +} + +func (rotation Rotation) withDefaults() Rotation { + if rotation.MaxSizeMiB == 0 { + rotation.MaxSizeMiB = defaultMaxSizeMiB + if rotation.MaxBackups == 0 { + rotation.MaxBackups = defaultMaxBackups + } + } + return rotation +} + +func (rotation Rotation) validate() error { + if rotation.MaxSizeMiB < 0 { + return errors.New("rotation max size must not be negative") + } + if rotation.MaxBackups < 0 { + return errors.New("rotation max backups must not be negative") + } + return nil +} + +// Options configures a reusable logging runtime. A nil output writer uses the +// corresponding process stream. Level defaults to INFO. +type Options struct { + Directory string + ManagerSinks Sinks + ManagerConsoleFormat Format + ManagerConsoleTextFormat string + ManagerFileFormat Format + TranscriptSinks Sinks + TranscriptConsoleFormat Format + TranscriptConsoleTextFormat string + Rotation Rotation + Level slog.Leveler + Stdout io.Writer + Stderr io.Writer + Now func() time.Time +} + +func (options Options) normalized() (Options, error) { + if options.Directory == "" { + return Options{}, errors.New("logging directory is empty") + } + if options.ManagerSinks&^SinkBoth != 0 || options.TranscriptSinks&^SinkBoth != 0 { + return Options{}, errors.New("logging sinks contain unsupported bits") + } + if options.ManagerConsoleFormat == "" { + options.ManagerConsoleFormat = FormatText + } + if options.ManagerFileFormat == "" { + options.ManagerFileFormat = FormatJSON + } + if options.TranscriptConsoleFormat == "" { + options.TranscriptConsoleFormat = FormatText + } + for label, format := range map[string]Format{ + "manager console": options.ManagerConsoleFormat, + "manager file": options.ManagerFileFormat, + "transcript console": options.TranscriptConsoleFormat, + } { + if format != FormatText && format != FormatJSON { + return Options{}, fmt.Errorf("unsupported %s format %q", label, format) + } + } + if err := validateTextTemplate("manager console", options.ManagerConsoleTextFormat, options.ManagerConsoleFormat, []string{"time", "level", "message", "attributes"}); err != nil { + return Options{}, err + } + if err := validateTextTemplate("transcript console", options.TranscriptConsoleTextFormat, options.TranscriptConsoleFormat, []string{"time", "instance", "component", "stream", "message", "session", "category", "provider", "attributes"}); err != nil { + return Options{}, err + } + if options.ManagerConsoleTextFormat == "" { + options.ManagerConsoleTextFormat = DefaultManagerConsoleTextFormat + } + if options.TranscriptConsoleTextFormat == "" { + options.TranscriptConsoleTextFormat = DefaultTranscriptConsoleTextFormat + } + if err := options.Rotation.validate(); err != nil { + return Options{}, err + } + options.Rotation = options.Rotation.withDefaults() + if options.Level == nil { + options.Level = slog.LevelInfo + } + if options.Stdout == nil { + options.Stdout = os.Stdout + } + if options.Stderr == nil { + options.Stderr = os.Stderr + } + if options.Now == nil { + options.Now = time.Now + } + return options, nil +} + +func validateTextTemplate(label, template string, format Format, allowed []string) error { + if template == "" { + return nil + } + if format != FormatText { + return fmt.Errorf("%s text format is supported only with text output", label) + } + if strings.ContainsAny(template, "\r\n") { + return fmt.Errorf("%s text format must be a single line", label) + } + allowedSet := make(map[string]struct{}, len(allowed)) + for _, placeholder := range allowed { + allowedSet[placeholder] = struct{}{} + } + foundMessage := false + remaining := template + for { + open := strings.IndexByte(remaining, '{') + if open < 0 { + if strings.ContainsRune(remaining, '}') { + return fmt.Errorf("%s text format contains an unmatched closing brace", label) + } + break + } + if strings.ContainsRune(remaining[:open], '}') { + return fmt.Errorf("%s text format contains an unmatched closing brace", label) + } + closeOffset := strings.IndexByte(remaining[open+1:], '}') + if closeOffset < 0 { + return fmt.Errorf("%s text format contains an unmatched opening brace", label) + } + placeholder := remaining[open+1 : open+1+closeOffset] + if _, ok := allowedSet[placeholder]; !ok { + return fmt.Errorf("%s text format contains unsupported placeholder {%s}", label, placeholder) + } + foundMessage = foundMessage || placeholder == "message" + remaining = remaining[open+closeOffset+2:] + } + if !foundMessage { + return fmt.Errorf("%s text format must contain {message}", label) + } + return nil +} + +// Category is a shallow retention and path namespace. +type Category string + +const ( + CategoryManager Category = "manager" + CategoryErrors Category = "errors" + CategoryInstances Category = "instances" + CategoryBuilds Category = "builds" + CategoryBenchmarks Category = "benchmarks" +) + +func (category Category) validTranscriptCategory() bool { + switch category { + case CategoryErrors, CategoryInstances, CategoryBuilds, CategoryBenchmarks: + return true + default: + return false + } +} + +// TranscriptMetadata supplies stable context for console envelopes and active +// session records. Attributes may contain additional low-cardinality context. +type TranscriptMetadata struct { + SessionID string `json:"sessionId,omitempty"` + Category Category `json:"category"` + Instance string `json:"instance,omitempty"` + Component string `json:"component,omitempty"` + Provider string `json:"provider,omitempty"` + Attributes map[string]string `json:"attributes,omitempty"` +} diff --git a/internal/pool/docker_pull.go b/internal/pool/docker_pull.go index 3b3ad71..dbf327e 100644 --- a/internal/pool/docker_pull.go +++ b/internal/pool/docker_pull.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "os" - "path/filepath" "runtime" "strings" "time" @@ -74,7 +73,7 @@ func (m *Manager) pullDockerSource(ctx context.Context, opts dockerSourcePullOpt if err != nil { return fmt.Errorf("Docker Engine pull %s: %w", opts.Image, err) } - if err := renderDockerPullProgress(ctx, response, opts.LogPath); err != nil { + if err := m.renderDockerPullProgress(ctx, response, opts.LogPath); err != nil { return fmt.Errorf("Docker Engine pull %s: %w", opts.Image, err) } m.writeDockerPullNotice(opts.LogPath, "Docker source pull complete: "+opts.Image) @@ -82,14 +81,14 @@ func (m *Manager) pullDockerSource(ctx context.Context, opts dockerSourcePullOpt } func (m *Manager) pullDockerSourceWithCLI(ctx context.Context, opts dockerSourcePullOptions, apiErr error) error { - fmt.Printf("warning: %v; falling back to docker pull CLI\n", apiErr) + m.warnf("warning: %v; falling back to docker pull CLI\n", apiErr) m.writeDockerPullNotice(opts.LogPath, "warning: "+sanitizeTimingError(apiErr)+"; falling back to docker pull CLI") args := []string{"pull"} if opts.Platform != "" { args = append(args, "--platform", opts.Platform) } args = append(args, opts.Image) - return runHostLoggedCommand(ctx, opts.LogPath, "docker", args...) + return m.runHostLogged(ctx, opts.LogPath, "docker", args...) } func (m *Manager) resolveDockerPullPlatform(ctx context.Context, cli *client.Client, configured string) (ocispec.Platform, error) { @@ -210,15 +209,16 @@ func dockerImageReferenceAndAuth(image string) (name.Reference, authn.Authentica return ref, authenticator, nil } -func renderDockerPullProgress(ctx context.Context, response client.ImagePullResponse, logPath string) error { - logFile, err := openDockerPullLog(logPath) +func (m *Manager) renderDockerPullProgress(ctx context.Context, response client.ImagePullResponse, logPath string) error { + transcript, err := m.transcript(logPath, "", "docker-pull") if err != nil { return err } - if logFile != nil { - defer logFile.Close() + interactive := term.IsTerminal(int(os.Stdout.Fd())) && containsString(m.Config.Logging.TranscriptSinks, "console") && m.Config.Logging.TranscriptConsoleFormat == "text" + eventWriter := transcript.Stdout + if interactive { + eventWriter = transcript.File } - interactive := term.IsTerminal(int(os.Stdout.Fd())) layers := map[string]dockerLayerProgress{} lastRender := time.Time{} rendered := false @@ -226,7 +226,7 @@ func renderDockerPullProgress(ctx context.Context, response client.ImagePullResp if streamErr != nil { return streamErr } - writeDockerPullEvent(logFile, message.ID, message.Status, message.Progress, message.Stream, message.Error) + writeDockerPullEvent(eventWriter, message.ID, message.Status, message.Progress, message.Stream, message.Error) if message.Error != nil { return message.Error } @@ -247,13 +247,21 @@ func renderDockerPullProgress(ctx context.Context, response client.ImagePullResp layers[message.ID] = layer } if time.Since(lastRender) >= dockerPullProgressInterval { - writeDockerPullSummary(os.Stdout, interactive, layers) + if interactive { + writeDockerPullSummary(os.Stdout, true, layers) + } else { + writeDockerPullSummary(transcript.Stdout, false, layers) + } lastRender = time.Now() rendered = true } } if rendered { - writeDockerPullSummary(os.Stdout, interactive, layers) + if interactive { + writeDockerPullSummary(os.Stdout, true, layers) + } else { + writeDockerPullSummary(transcript.Stdout, false, layers) + } if interactive { fmt.Fprintln(os.Stdout) } @@ -261,26 +269,22 @@ func renderDockerPullProgress(ctx context.Context, response client.ImagePullResp return nil } -func openDockerPullLog(path string) (*os.File, error) { - if path == "" { - return nil, nil - } - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return nil, err - } - return os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) -} - func (m *Manager) writeDockerPullNotice(logPath, message string) { - fmt.Println(message) - logFile, err := openDockerPullLog(logPath) + transcript, err := m.transcript(logPath, "", "docker-pull") if err != nil { + m.logger().Warn("docker pull transcript unavailable", "operation", "docker-pull", "logPath", logPath, "error", err) return } - if logFile != nil { - defer logFile.Close() - fmt.Fprintf(logFile, "%s %s\n", time.Now().UTC().Format(time.RFC3339Nano), message) + _, _ = fmt.Fprintf(transcript.Stdout, "%s\n", message) +} + +func containsString(values []string, wanted string) bool { + for _, value := range values { + if value == wanted { + return true + } } + return false } func writeDockerPullEvent(logFile io.Writer, id, status string, progress *jsonstream.Progress, stream string, pullErr error) { diff --git a/internal/pool/host_trust.go b/internal/pool/host_trust.go index 3d2e8c2..3dff0b9 100644 --- a/internal/pool/host_trust.go +++ b/internal/pool/host_trust.go @@ -209,7 +209,7 @@ func hostTrustMarkerJSON(snapshot hosttrust.Snapshot) ([]byte, error) { } func (m *Manager) readInstanceHostTrustMarker(ctx context.Context, instanceName string) (hostTrustMarker, error) { - result, err := m.Provider.Exec(ctx, instanceName, []string{"cat", hostTrustMarkerGuest}, provider.ExecOptions{}) + result, err := m.execGuest(ctx, instanceName, []string{"cat", hostTrustMarkerGuest}, provider.ExecOptions{}) if err != nil { return hostTrustMarker{}, fmt.Errorf("read image host trust marker: %w", err) } @@ -295,7 +295,7 @@ func (m *Manager) issueHostTrustLease(ctx context.Context, instanceName string, if err != nil { return err } - if _, err := m.Provider.Exec(ctx, instanceName, provider.ShellCommand("if command -v sudo >/dev/null 2>&1; then sudo install -d -m 0755 /run/epar; else install -d -m 0755 /run/epar; fi"), provider.ExecOptions{}); err != nil { + if _, err := m.execGuest(ctx, instanceName, provider.ShellCommand("if command -v sudo >/dev/null 2>&1; then sudo install -d -m 0755 /run/epar; else install -d -m 0755 /run/epar; fi"), provider.ExecOptions{}); err != nil { return err } return provider.CopyTextAtomic(ctx, m.Provider, instanceName, hostTrustLeaseGuest, "0644", string(content)+"\n") @@ -312,12 +312,12 @@ func (m *Manager) reconcileHostTrustRunners(ctx context.Context, active map[stri // safe for an already-running job (its hook already ran) and closes // the assignment window even when GitHub status is unavailable. if err := m.issueHostTrustLease(ctx, name, current); err != nil { - fmt.Printf("[%s] old-generation revocation warning: %v\n", name, err) + m.warnf("[%s] old-generation revocation warning: %v\n", name, err) } } runner, found, err := m.GitHub.RunnerByName(ctx, name) if err != nil { - fmt.Printf("[%s] host trust reconciliation warning; lease not refreshed: %v\n", name, err) + m.warnf("[%s] host trust reconciliation warning; lease not refreshed: %v\n", name, err) continue } if instance.HostTrustGeneration == current.Generation { @@ -325,17 +325,17 @@ func (m *Manager) reconcileHostTrustRunners(ctx context.Context, active map[stri continue } if err := m.issueHostTrustLease(ctx, name, current); err != nil { - fmt.Printf("[%s] host trust lease refresh warning: %v\n", name, err) + m.warnf("[%s] host trust lease refresh warning: %v\n", name, err) } continue } if found && runner.Busy { - fmt.Printf("[%s] draining busy runner on old host trust generation %s\n", name, instance.HostTrustGeneration) + m.infof("[%s] draining busy runner on old host trust generation %s\n", name, instance.HostTrustGeneration) continue } reason := fmt.Sprintf("host trust generation changed from %s to %s", instance.HostTrustGeneration, current.Generation) if err := m.retireInstance(context.Background(), instance, reason); err != nil { - fmt.Printf("[%s] old-generation retirement warning: %v\n", name, err) + m.warnf("[%s] old-generation retirement warning: %v\n", name, err) continue } delete(active, name) @@ -375,7 +375,7 @@ func (m *Manager) startHostTrustLeaseKeeper(parent context.Context) (func(Provis nextCollection = now.Add(m.hostTrustCollectionInterval()) if err != nil { current = hosttrust.Snapshot{} - fmt.Printf("host trust initial lease refresh warning: %v\n", err) + m.warnf("host trust initial lease refresh warning: %v\n", err) continue } current = snapshot @@ -383,7 +383,7 @@ func (m *Manager) startHostTrustLeaseKeeper(parent context.Context) (func(Provis for name, instance := range active { if instance.HostTrustGeneration != current.Generation { if err := m.issueHostTrustLease(ctx, name, current); err != nil { - fmt.Printf("[%s] host trust initial stale-generation revocation warning: %v\n", name, err) + m.warnf("[%s] host trust initial stale-generation revocation warning: %v\n", name, err) } continue } @@ -392,7 +392,7 @@ func (m *Manager) startHostTrustLeaseKeeper(parent context.Context) (func(Provis continue } if err := m.issueHostTrustLease(ctx, name, current); err != nil { - fmt.Printf("[%s] host trust initial lease refresh warning: %v\n", name, err) + m.warnf("[%s] host trust initial lease refresh warning: %v\n", name, err) } } } diff --git a/internal/pool/host_trust_test.go b/internal/pool/host_trust_test.go index c3b2fbc..809b6da 100644 --- a/internal/pool/host_trust_test.go +++ b/internal/pool/host_trust_test.go @@ -3,6 +3,7 @@ package pool import ( "context" "encoding/json" + "io" "os" "path/filepath" "strings" @@ -224,8 +225,8 @@ func TestHostTrustReconciliationRevokesAndRetiresIdleOldGeneration(t *testing.T) github := &fakeGitHub{runner: gh.Runner{Name: "runner-1", ID: 42, Status: "online", Busy: false}, found: true} manager := Manager{ Config: config.Config{ - Image: config.ImageConfig{HostTrustMode: config.HostTrustModeOverlay, HostTrustScopes: []string{"system"}}, - Pool: config.PoolConfig{LogDir: t.TempDir()}, + Image: config.ImageConfig{HostTrustMode: config.HostTrustModeOverlay, HostTrustScopes: []string{"system"}}, + Logging: config.LoggingConfig{Directory: t.TempDir()}, }, Provider: provider, GitHub: github, @@ -320,7 +321,7 @@ func TestHostTrustImageBuildRetriesChangedGenerationBeforePublishing(t *testing. }, Provider: config.ProviderConfig{Type: "docker-dind"}, Runner: config.RunnerConfig{Ephemeral: true}, - Pool: config.PoolConfig{LogDir: "work/logs"}, + Logging: config.LoggingConfig{Directory: "work/logs"}, }, ProjectRoot: root, } @@ -348,7 +349,7 @@ func TestHostTrustImageBuildRetriesChangedGenerationBeforePublishing(t *testing. }) builds := 0 tagged := false - runHostLoggedCommand = func(_ context.Context, _ string, name string, args ...string) error { + runHostLoggedCommand = func(_ context.Context, _ string, _, _ io.Writer, name string, args ...string) error { if name == "docker" && len(args) > 0 && args[0] == "build" { builds++ } diff --git a/internal/pool/image.go b/internal/pool/image.go index e540f72..4c49f8e 100644 --- a/internal/pool/image.go +++ b/internal/pool/image.go @@ -42,19 +42,21 @@ var ( func (m *Manager) UpdateUpstream(ctx context.Context) error { dir := config.ProjectPath(m.ProjectRoot, m.Config.Image.UpstreamDir) - fmt.Printf("updating runner-images checkout at %s\n", dir) + logPath := m.buildLogPath("runner-images.source.log") + defer m.releaseTranscript(logPath) + m.infof("updating runner-images checkout at %s\n", dir) if _, err := os.Stat(filepath.Join(dir, ".git")); err == nil { - if err := runHost(ctx, "git", "-C", dir, "fetch", "--depth", "1", "origin", "main"); err != nil { + if err := m.runHostLogged(ctx, logPath, "git", "-C", dir, "fetch", "--depth", "1", "origin", "main"); err != nil { return err } - if err := runHost(ctx, "git", "-C", dir, "checkout", "FETCH_HEAD"); err != nil { + if err := m.runHostLogged(ctx, logPath, "git", "-C", dir, "checkout", "FETCH_HEAD"); err != nil { return err } } else { if err := os.MkdirAll(filepath.Dir(dir), 0755); err != nil { return err } - if err := runHost(ctx, "git", "clone", "--depth", "1", "https://github.com/actions/runner-images.git", dir); err != nil { + if err := m.runHostLogged(ctx, logPath, "git", "clone", "--depth", "1", "https://github.com/actions/runner-images.git", dir); err != nil { return err } } @@ -70,8 +72,8 @@ func (m *Manager) UpdateUpstream(ctx context.Context) error { if err := os.WriteFile(lockPath, []byte(commit+"\n"), 0644); err != nil { return err } - fmt.Printf("runner-images pinned at %s\n", commit) - fmt.Printf("lock file written to %s\n", lockPath) + m.infof("runner-images pinned at %s\n", commit) + m.infof("lock file written to %s\n", lockPath) return nil } @@ -107,7 +109,8 @@ func (m *Manager) buildDockerDindImage(ctx context.Context, opts ImageBuildOptio } func (m *Manager) buildDockerDindImageUntimed(ctx context.Context, opts ImageBuildOptions, upstreamDir string) error { - buildLogPath := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), imageLogStem(m.Config.Image.OutputImage)+".docker-build.log") + buildLogPath := m.buildLogPath(imageLogStem(m.Config.Image.OutputImage) + ".docker-build.log") + defer m.releaseTranscript(buildLogPath) if err := resetLogs(buildLogPath); err != nil { return err } @@ -149,7 +152,7 @@ func (m *Manager) buildDockerDindImageUntimed(ctx context.Context, opts ImageBui return err } if current.Generation != snapshot.Generation { - fmt.Printf("host trust changed during image build (%s -> %s); discarding attempt %d/%d\n", snapshot.Generation, current.Generation, attempt, attempts) + m.infof("host trust changed during image build (%s -> %s); discarding attempt %d/%d\n", snapshot.Generation, current.Generation, attempt, attempts) _ = runHostQuiet(context.Background(), "docker", "image", "rm", "-f", targetImage) continue } @@ -158,7 +161,7 @@ func (m *Manager) buildDockerDindImageUntimed(ctx context.Context, opts ImageBui return err } _ = runHostQuiet(context.Background(), "docker", "image", "rm", "-f", targetImage) - fmt.Printf("image build complete: %s is available in `docker image ls`\n", m.Config.Image.OutputImage) + m.infof("image build complete: %s is available in `docker image ls`\n", m.Config.Image.OutputImage) return nil } return fmt.Errorf("host trust changed during all %d image build attempts; retry after the host trust store stabilizes", attempts) @@ -177,8 +180,8 @@ func (m *Manager) buildDockerDindImageAttempt(ctx context.Context, upstreamDir, if err := m.prepareDockerDindBuildContextWithHostTrust(buildCtx, upstreamDir, manifestContent, snapshot); err != nil { return err } - fmt.Printf("building Docker-DinD image %s from %s\n", targetImage, m.Config.Image.SourceImage) - fmt.Printf("log: %s\n", buildLogPath) + m.infof("building Docker-DinD image %s from %s\n", targetImage, m.Config.Image.SourceImage) + m.infof("log: %s\n", buildLogPath) args := []string{"build", "-t", targetImage} if m.Config.Provider.Platform != "" { args = append(args, "--platform", m.Config.Provider.Platform) @@ -190,15 +193,15 @@ func (m *Manager) buildDockerDindImageAttempt(ctx context.Context, upstreamDir, buildCtx, ) if m.DryRun { - fmt.Printf("[dry-run] docker %s\n", strings.Join(args, " ")) - fmt.Printf("image build dry run complete: %s\n", targetImage) + m.infof("[dry-run] docker %s\n", strings.Join(args, " ")) + m.infof("image build dry run complete: %s\n", targetImage) return nil } - if err := runHostLoggedCommand(ctx, buildLogPath, "docker", args...); err != nil { + if err := m.runHostLogged(ctx, buildLogPath, "docker", args...); err != nil { return err } if !m.hostTrustEnabled() { - fmt.Printf("image build complete: %s is available in `docker image ls`\n", targetImage) + m.infof("image build complete: %s is available in `docker image ls`\n", targetImage) } return nil } @@ -219,32 +222,38 @@ func (m *Manager) buildTartImage(ctx context.Context, opts ImageBuildOptions, up } opts.Manifest = &manifest } - buildLogPath := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), imageLogStem(m.Config.Image.OutputImage)+".build.log") - guestLogPath := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), imageLogStem(m.Config.Image.OutputImage)+".guest.log") + buildLogPath := m.buildLogPath(imageLogStem(m.Config.Image.OutputImage) + ".build.log") + guestLogPath := m.buildLogPath(imageLogStem(m.Config.Image.OutputImage) + ".guest.log") + defer m.releaseTranscript(buildLogPath) + defer m.releaseTranscript(guestLogPath) if err := resetLogs(buildLogPath, guestLogPath); err != nil { return err } - fmt.Printf("building Tart image %s from %s\n", m.Config.Image.OutputImage, m.Config.Image.SourceImage) - fmt.Printf("logs: %s, %s\n", buildLogPath, guestLogPath) + m.infof("building Tart image %s from %s\n", m.Config.Image.OutputImage, m.Config.Image.SourceImage) + m.infof("logs: %s, %s\n", buildLogPath, guestLogPath) if opts.Replace { - fmt.Printf("replacing existing Tart image %s if present\n", m.Config.Image.OutputImage) + m.infof("replacing existing Tart image %s if present\n", m.Config.Image.OutputImage) _ = m.Provider.Stop(ctx, m.Config.Image.OutputImage) _ = m.Provider.Delete(ctx, m.Config.Image.OutputImage) } - fmt.Printf("cloning source image\n") + m.infof("cloning source image\n") if err := m.Provider.Clone(ctx, m.Config.Image.SourceImage, m.Config.Image.OutputImage); err != nil { return err } - fmt.Printf("starting image with Tart network mode %q\n", m.Config.Provider.Network) - if _, err := m.Provider.Start(ctx, m.Config.Image.OutputImage, m.startOptions(buildLogPath)); err != nil { + m.infof("starting image with Tart network mode %q\n", m.Config.Provider.Network) + startOptions, err := m.startOptions(buildLogPath, m.Config.Image.OutputImage) + if err != nil { + return err + } + if _, err := m.Provider.Start(ctx, m.Config.Image.OutputImage, startOptions); err != nil { return err } ip, err := m.Provider.IP(ctx, m.Config.Image.OutputImage, m.Config.Timeouts.BootSeconds) if err != nil { return err } - fmt.Printf("guest reachable at %s\n", ip) - fmt.Printf("copying guest scripts\n") + m.infof("guest reachable at %s\n", ip) + m.infof("copying guest scripts\n") if err := m.installGuestScripts(ctx, m.Config.Image.OutputImage); err != nil { return err } @@ -256,19 +265,19 @@ func (m *Manager) buildTartImage(ctx context.Context, opts ImageBuildOptions, up } switch m.runnerImagesCopyMode() { case runnerImagesCopySubset: - fmt.Printf("copying runner-images script subset\n") + m.infof("copying runner-images script subset\n") if err := m.copyRunnerImagesSubset(ctx, m.Config.Image.OutputImage, upstreamDir); err != nil { return err } case runnerImagesCopyNone: - fmt.Printf("skipping runner-images script subset; no selected install script requires it\n") + m.infof("skipping runner-images script subset; no selected install script requires it\n") } - fmt.Printf("installing base runner runtime\n") - if _, err := m.execGuest(ctx, m.Config.Image.OutputImage, []string{"sudo", "bash", "/opt/epar/install-base.sh", "/opt/epar/upstream/runner-images"}, provider.ExecOptions{}); err != nil { + m.infof("installing base runner runtime\n") + if _, err := m.execBuildGuest(ctx, m.Config.Image.OutputImage, []string{"sudo", "bash", "/opt/epar/install-base.sh", "/opt/epar/upstream/runner-images"}, provider.ExecOptions{}); err != nil { return err } - fmt.Printf("installing GitHub Actions runner\n") - if _, err := m.execGuest(ctx, m.Config.Image.OutputImage, []string{"sudo", "bash", "/opt/epar/install-runner.sh", m.Config.Image.RunnerVersion}, provider.ExecOptions{}); err != nil { + m.infof("installing GitHub Actions runner\n") + if _, err := m.execBuildGuest(ctx, m.Config.Image.OutputImage, []string{"sudo", "bash", "/opt/epar/install-runner.sh", m.Config.Image.RunnerVersion}, provider.ExecOptions{}); err != nil { return err } if err := m.installRosettaSupport(ctx, m.Config.Image.OutputImage); err != nil { @@ -277,19 +286,19 @@ func (m *Manager) buildTartImage(ctx context.Context, opts ImageBuildOptions, up if err := m.installCustomInstallScripts(ctx, m.Config.Image.OutputImage); err != nil { return err } - fmt.Printf("validating runner runtime inside the instance\n") + m.infof("validating runner runtime inside the instance\n") if err := m.validateRuntime(ctx, m.Config.Image.OutputImage); err != nil { return err } - fmt.Printf("finalizing image for clean Tart clones\n") - if _, err := m.execGuest(ctx, m.Config.Image.OutputImage, []string{"sudo", "bash", "/opt/epar/finalize-image.sh"}, provider.ExecOptions{}); err != nil { + m.infof("finalizing image for clean Tart clones\n") + if _, err := m.execBuildGuest(ctx, m.Config.Image.OutputImage, []string{"sudo", "bash", "/opt/epar/finalize-image.sh"}, provider.ExecOptions{}); err != nil { return err } - fmt.Printf("stopping image\n") + m.infof("stopping image\n") if err := m.Provider.Stop(ctx, m.Config.Image.OutputImage); err != nil { return err } - fmt.Printf("image build complete: %s is available in `tart list`\n", m.Config.Image.OutputImage) + m.infof("image build complete: %s is available in `tart list`\n", m.Config.Image.OutputImage) return nil } @@ -310,8 +319,10 @@ func (m *Manager) buildWSLImageUntimed(ctx context.Context, opts ImageBuildOptio sourceType = config.ImageSourceRootFSTar } buildName := RunnerName(m.Config.Pool.NamePrefix+"-image", 1, time.Now()) - buildLogPath := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), imageLogStem(m.Config.Image.OutputImage)+".wsl-build.log") - guestLogPath := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), buildName+".guest.log") + buildLogPath := m.buildLogPath(imageLogStem(m.Config.Image.OutputImage) + ".wsl-build.log") + guestLogPath := m.buildLogPath(buildName + ".guest.log") + defer m.releaseTranscript(buildLogPath) + defer m.releaseTranscript(guestLogPath) if err := resetLogs(buildLogPath, guestLogPath); err != nil { return err } @@ -360,44 +371,48 @@ func (m *Manager) buildWSLImageUntimed(ctx context.Context, opts ImageBuildOptio return fmt.Errorf("unsupported WSL image.sourceType %q", sourceType) } - fmt.Printf("building WSL image %s from %s using temporary distro %s\n", outputPath, sourcePath, buildName) - fmt.Printf("logs: %s, %s\n", buildLogPath, guestLogPath) + m.infof("building WSL image %s from %s using temporary distro %s\n", outputPath, sourcePath, buildName) + m.infof("logs: %s, %s\n", buildLogPath, guestLogPath) defer func() { cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() _ = m.Provider.Stop(cleanupCtx, buildName) _ = m.Provider.Delete(cleanupCtx, buildName) }() - fmt.Printf("importing source rootfs\n") + m.infof("importing source rootfs\n") if err := m.Provider.Clone(ctx, sourceForClone, buildName); err != nil { return err } if sourceType == config.ImageSourceDockerImage { - fmt.Printf("preparing Docker image rootfs for WSL systemd\n") + m.infof("preparing Docker image rootfs for WSL systemd\n") if err := m.prepareWSLDockerSourceGuest(ctx, buildName); err != nil { return err } } - fmt.Printf("enabling WSL systemd\n") + m.infof("enabling WSL systemd\n") if err := m.enableWSLSystemd(ctx, buildName); err != nil { return err } - fmt.Printf("restarting temporary distro for systemd\n") + m.infof("restarting temporary distro for systemd\n") if err := m.Provider.Stop(ctx, buildName); err != nil { return err } - if _, err := m.Provider.Start(ctx, buildName, m.startOptions(buildLogPath)); err != nil { + startOptions, err := m.startOptions(buildLogPath, buildName) + if err != nil { + return err + } + if _, err := m.Provider.Start(ctx, buildName, startOptions); err != nil { return err } ip, err := m.Provider.IP(ctx, buildName, m.Config.Timeouts.BootSeconds) if err != nil { return err } - fmt.Printf("guest reachable at %s\n", ip) + m.infof("guest reachable at %s\n", ip) if err := m.waitForSystemd(ctx, buildName); err != nil { return err } - fmt.Printf("copying guest scripts\n") + m.infof("copying guest scripts\n") if err := m.installGuestScripts(ctx, buildName); err != nil { return err } @@ -408,26 +423,26 @@ func (m *Manager) buildWSLImageUntimed(ctx context.Context, opts ImageBuildOptio return err } if sourceEnv != "" { - fmt.Printf("installing source image environment metadata\n") + m.infof("installing source image environment metadata\n") if err := m.installSourceImageEnv(ctx, buildName, sourceEnv); err != nil { return err } } switch m.runnerImagesCopyMode() { case runnerImagesCopySubset: - fmt.Printf("copying runner-images script subset\n") + m.infof("copying runner-images script subset\n") if err := m.copyRunnerImagesSubset(ctx, buildName, upstreamDir); err != nil { return err } case runnerImagesCopyNone: - fmt.Printf("skipping runner-images script subset; no selected install script requires it\n") + m.infof("skipping runner-images script subset; no selected install script requires it\n") } - fmt.Printf("installing base runner runtime\n") - if _, err := m.execGuest(ctx, buildName, []string{"sudo", "bash", "/opt/epar/install-base.sh", "/opt/epar/upstream/runner-images"}, provider.ExecOptions{}); err != nil { + m.infof("installing base runner runtime\n") + if _, err := m.execBuildGuest(ctx, buildName, []string{"sudo", "bash", "/opt/epar/install-base.sh", "/opt/epar/upstream/runner-images"}, provider.ExecOptions{}); err != nil { return err } - fmt.Printf("installing GitHub Actions runner\n") - if _, err := m.execGuest(ctx, buildName, []string{"sudo", "bash", "/opt/epar/install-runner.sh", m.Config.Image.RunnerVersion}, provider.ExecOptions{}); err != nil { + m.infof("installing GitHub Actions runner\n") + if _, err := m.execBuildGuest(ctx, buildName, []string{"sudo", "bash", "/opt/epar/install-runner.sh", m.Config.Image.RunnerVersion}, provider.ExecOptions{}); err != nil { return err } if err := m.installWSLDockerEngine(ctx, buildName); err != nil { @@ -436,18 +451,18 @@ func (m *Manager) buildWSLImageUntimed(ctx context.Context, opts ImageBuildOptio if err := m.installCustomInstallScripts(ctx, buildName); err != nil { return err } - fmt.Printf("validating runner runtime inside the distro\n") + m.infof("validating runner runtime inside the distro\n") if err := m.validateRuntime(ctx, buildName); err != nil { return err } - fmt.Printf("finalizing image for clean WSL imports\n") - if _, err := m.execGuest(ctx, buildName, []string{"sudo", "bash", "/opt/epar/finalize-image.sh"}, provider.ExecOptions{}); err != nil { + m.infof("finalizing image for clean WSL imports\n") + if _, err := m.execBuildGuest(ctx, buildName, []string{"sudo", "bash", "/opt/epar/finalize-image.sh"}, provider.ExecOptions{}); err != nil { return err } if err := m.Provider.Stop(ctx, buildName); err != nil { return err } - fmt.Printf("exporting reusable WSL image to %s\n", outputPath) + m.infof("exporting reusable WSL image to %s\n", outputPath) if err := exporter.Export(ctx, buildName, m.Config.Image.OutputImage); err != nil { return err } @@ -456,7 +471,7 @@ func (m *Manager) buildWSLImageUntimed(ctx context.Context, opts ImageBuildOptio return err } } - fmt.Printf("image build complete: %s is available for WSL imports\n", outputPath) + m.infof("image build complete: %s is available for WSL imports\n", outputPath) return nil } @@ -483,21 +498,21 @@ func (m *Manager) prepareWSLDockerSourceRootfs(ctx context.Context, outputPath, exportArgs := []string{"export", "-o", tmpPath, containerName} if m.DryRun { - fmt.Printf("[dry-run] docker %s\n", strings.Join(pullArgs, " ")) - fmt.Printf("[dry-run] docker %s\n", strings.Join(createArgs, " ")) - fmt.Printf("[dry-run] docker container inspect --format {{json .Config.Env}} %s\n", containerName) - fmt.Printf("[dry-run] docker %s\n", strings.Join(exportArgs, " ")) - fmt.Printf("[dry-run] docker rm -f %s\n", containerName) + m.infof("[dry-run] docker %s\n", strings.Join(pullArgs, " ")) + m.infof("[dry-run] docker %s\n", strings.Join(createArgs, " ")) + m.infof("[dry-run] docker container inspect --format {{json .Config.Env}} %s\n", containerName) + m.infof("[dry-run] docker %s\n", strings.Join(exportArgs, " ")) + m.infof("[dry-run] docker rm -f %s\n", containerName) return rootfsPath, "", nil } if info, err := os.Stat(rootfsPath); err == nil && info.Size() > 0 && sourceCacheMatches(sourceCachePath, sourceCache) { envContent, err := os.ReadFile(envCachePath) if err != nil { - fmt.Printf("using cached WSL source rootfs at %s; source image env cache missing, refreshing metadata from Docker image\n", rootfsPath) + m.infof("using cached WSL source rootfs at %s; source image env cache missing, refreshing metadata from Docker image\n", rootfsPath) refreshedEnv, refreshErr := m.dockerImageEnvContent(ctx, image) if refreshErr != nil { - fmt.Printf("warning: could not refresh WSL source image env metadata: %v\n", refreshErr) + m.warnf("warning: could not refresh WSL source image env metadata: %v\n", refreshErr) return rootfsPath, "", nil } if writeErr := os.WriteFile(envCachePath, []byte(refreshedEnv), 0644); writeErr != nil { @@ -505,12 +520,12 @@ func (m *Manager) prepareWSLDockerSourceRootfs(ctx context.Context, outputPath, } return rootfsPath, refreshedEnv, nil } - fmt.Printf("using cached WSL source rootfs at %s\n", rootfsPath) + m.infof("using cached WSL source rootfs at %s\n", rootfsPath) return rootfsPath, string(envContent), nil } else if err != nil && !os.IsNotExist(err) { return "", "", err } else if err == nil { - fmt.Printf("cached WSL source rootfs is missing source metadata or no longer matches; reconverting %s\n", image) + m.infof("cached WSL source rootfs is missing source metadata or no longer matches; reconverting %s\n", image) if err := os.Remove(rootfsPath); err != nil && !os.IsNotExist(err) { return "", "", err } @@ -528,7 +543,7 @@ func (m *Manager) prepareWSLDockerSourceRootfs(ctx context.Context, outputPath, if err := os.Remove(tmpPath); err != nil && !os.IsNotExist(err) { return "", "", err } - fmt.Printf("preparing WSL source rootfs from Docker image %s\n", image) + m.infof("preparing WSL source rootfs from Docker image %s\n", image) if err := pullDockerSourceCommand(m, ctx, dockerSourcePullOptions{ Image: image, Platform: platform, @@ -536,7 +551,7 @@ func (m *Manager) prepareWSLDockerSourceRootfs(ctx context.Context, outputPath, }); err != nil { return "", "", fmt.Errorf("wsl image.sourceType=docker-image requires Docker Desktop, Docker Engine, or another reachable Docker daemon; alternatively set image.sourceType=rootfs-tar and provide a prepared rootfs tar: %w", err) } - if err := runHostLoggedCommand(ctx, buildLogPath, "docker", createArgs...); err != nil { + if err := m.runHostLogged(ctx, buildLogPath, "docker", createArgs...); err != nil { return "", "", err } defer func() { @@ -553,7 +568,7 @@ func (m *Manager) prepareWSLDockerSourceRootfs(ctx context.Context, outputPath, return "", "", fmt.Errorf("parse Docker source image environment: %w", err) } envContent := sourceImageEnvContent(sourceEnv) - if err := runHostLoggedCommand(ctx, buildLogPath, "docker", exportArgs...); err != nil { + if err := m.runHostLogged(ctx, buildLogPath, "docker", exportArgs...); err != nil { return "", "", err } if err := os.Remove(rootfsPath); err != nil && !os.IsNotExist(err) { @@ -568,7 +583,7 @@ func (m *Manager) prepareWSLDockerSourceRootfs(ctx context.Context, outputPath, if err := writeSourceCacheManifest(sourceCachePath, sourceCache); err != nil { return "", "", err } - fmt.Printf("WSL source rootfs exported to %s\n", rootfsPath) + m.infof("WSL source rootfs exported to %s\n", rootfsPath) return rootfsPath, envContent, nil } @@ -640,7 +655,7 @@ for unit in \ ln -sf /dev/null "/etc/systemd/system/${unit}" done ` - _, err := m.Provider.Exec(ctx, vmName, []string{"bash", "-c", script}, provider.ExecOptions{}) + _, err := m.execBuildGuest(ctx, vmName, []string{"bash", "-c", script}, provider.ExecOptions{LogPath: m.buildLogPath(vmName + ".guest.log")}) return err } @@ -648,8 +663,8 @@ func (m *Manager) installWSLDockerEngine(ctx context.Context, vmName string) err if m.Config.Provider.Type != "wsl" || m.Config.Image.SourceType != config.ImageSourceDockerImage { return nil } - fmt.Printf("validating Docker Engine from WSL Docker source image\n") - _, err := m.execGuest(ctx, vmName, []string{"sudo", "-E", "bash", "/opt/epar/install-docker-engine.sh", "/opt/epar/upstream/runner-images"}, provider.ExecOptions{ + m.infof("validating Docker Engine from WSL Docker source image\n") + _, err := m.execBuildGuest(ctx, vmName, []string{"sudo", "-E", "bash", "/opt/epar/install-docker-engine.sh", "/opt/epar/upstream/runner-images"}, provider.ExecOptions{ Env: map[string]string{"EPAR_REQUIRE_BASE_DOCKER_ENGINE": "true"}, }) return err @@ -701,10 +716,16 @@ func (m *Manager) RefreshScripts(ctx context.Context) error { } func (m *Manager) refreshTartScripts(ctx context.Context) error { - logPath := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), imageLogStem(m.Config.Image.OutputImage)+".refresh.log") - fmt.Printf("refreshing guest scripts in Tart image %s\n", m.Config.Image.OutputImage) - fmt.Printf("log: %s\n", logPath) - if _, err := m.Provider.Start(ctx, m.Config.Image.OutputImage, m.startOptions(logPath)); err != nil { + logPath := m.buildLogPath(imageLogStem(m.Config.Image.OutputImage) + ".refresh.log") + defer m.releaseTranscript(logPath) + defer m.releaseTranscript(m.buildLogPath(imageLogStem(m.Config.Image.OutputImage) + ".guest.log")) + m.infof("refreshing guest scripts in Tart image %s\n", m.Config.Image.OutputImage) + m.infof("log: %s\n", logPath) + startOptions, err := m.startOptions(logPath, m.Config.Image.OutputImage) + if err != nil { + return err + } + if _, err := m.Provider.Start(ctx, m.Config.Image.OutputImage, startOptions); err != nil { return err } shouldStop := true @@ -721,17 +742,17 @@ func (m *Manager) refreshTartScripts(ctx context.Context) error { if err := m.installGuestScripts(ctx, m.Config.Image.OutputImage); err != nil { return err } - if _, err := m.execGuest(ctx, m.Config.Image.OutputImage, []string{"sudo", "bash", "/opt/epar/finalize-image.sh"}, provider.ExecOptions{}); err != nil { + if _, err := m.execBuildGuest(ctx, m.Config.Image.OutputImage, []string{"sudo", "bash", "/opt/epar/finalize-image.sh"}, provider.ExecOptions{}); err != nil { return err } - if _, err := m.Provider.Exec(ctx, m.Config.Image.OutputImage, provider.ShellCommand("sync"), provider.ExecOptions{LogPath: logPath}); err != nil { + if _, err := m.execBuildGuest(ctx, m.Config.Image.OutputImage, provider.ShellCommand("sync"), provider.ExecOptions{LogPath: logPath}); err != nil { return err } shouldStop = false if err := m.Provider.Stop(ctx, m.Config.Image.OutputImage); err != nil { return err } - fmt.Printf("script refresh complete: %s is available in `tart list`\n", m.Config.Image.OutputImage) + m.infof("script refresh complete: %s is available in `tart list`\n", m.Config.Image.OutputImage) return nil } @@ -747,9 +768,11 @@ func (m *Manager) refreshWSLScripts(ctx context.Context) error { } } name := RunnerName(m.Config.Pool.NamePrefix+"-refresh", 1, time.Now()) - logPath := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), imageLogStem(m.Config.Image.OutputImage)+".wsl-refresh.log") - fmt.Printf("refreshing guest scripts in WSL image %s using temporary distro %s\n", imagePath, name) - fmt.Printf("log: %s\n", logPath) + logPath := m.buildLogPath(imageLogStem(m.Config.Image.OutputImage) + ".wsl-refresh.log") + defer m.releaseTranscript(logPath) + defer m.releaseTranscript(m.buildLogPath(name + ".guest.log")) + m.infof("refreshing guest scripts in WSL image %s using temporary distro %s\n", imagePath, name) + m.infof("log: %s\n", logPath) defer func() { cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() @@ -759,7 +782,11 @@ func (m *Manager) refreshWSLScripts(ctx context.Context) error { if err := m.Provider.Clone(ctx, m.Config.Image.OutputImage, name); err != nil { return err } - if _, err := m.Provider.Start(ctx, name, m.startOptions(logPath)); err != nil { + startOptions, err := m.startOptions(logPath, name) + if err != nil { + return err + } + if _, err := m.Provider.Start(ctx, name, startOptions); err != nil { return err } if _, err := m.Provider.IP(ctx, name, m.Config.Timeouts.BootSeconds); err != nil { @@ -771,7 +798,7 @@ func (m *Manager) refreshWSLScripts(ctx context.Context) error { if err := m.installGuestScripts(ctx, name); err != nil { return err } - if _, err := m.execGuest(ctx, name, []string{"sudo", "bash", "/opt/epar/finalize-image.sh"}, provider.ExecOptions{}); err != nil { + if _, err := m.execBuildGuest(ctx, name, []string{"sudo", "bash", "/opt/epar/finalize-image.sh"}, provider.ExecOptions{}); err != nil { return err } if err := m.Provider.Stop(ctx, name); err != nil { @@ -780,7 +807,7 @@ func (m *Manager) refreshWSLScripts(ctx context.Context) error { if err := exporter.Export(ctx, name, m.Config.Image.OutputImage); err != nil { return err } - fmt.Printf("script refresh complete: %s is available for WSL imports\n", imagePath) + m.infof("script refresh complete: %s is available for WSL imports\n", imagePath) return nil } @@ -790,7 +817,7 @@ func (m *Manager) installGuestScripts(ctx context.Context, vmName string) error if err != nil { return err } - if _, err := m.Provider.Exec(ctx, vmName, provider.ShellCommand("if command -v sudo >/dev/null 2>&1; then sudo mkdir -p /opt/epar; else mkdir -p /opt/epar; fi"), provider.ExecOptions{}); err != nil { + if _, err := m.execBuildGuest(ctx, vmName, provider.ShellCommand("if command -v sudo >/dev/null 2>&1; then sudo mkdir -p /opt/epar; else mkdir -p /opt/epar; fi"), provider.ExecOptions{LogPath: m.buildLogPath(vmName + ".guest.log")}); err != nil { return err } for _, entry := range entries { @@ -809,20 +836,33 @@ func (m *Manager) installGuestScripts(ctx context.Context, vmName string) error return nil } -func (m *Manager) startOptions(logPath string) provider.StartOptions { +func (m *Manager) startOptions(logPath, instance string) (provider.StartOptions, error) { + transcript, err := m.transcript(logPath, instance, transcriptComponent(logPath)) + if err != nil { + return provider.StartOptions{}, err + } return provider.StartOptions{ Network: m.Config.Provider.Network, RosettaTag: m.Config.Provider.RosettaTag, LogPath: logPath, + Stdout: transcript.Stdout, + Stderr: transcript.Stderr, + }, nil +} + +func (m *Manager) execBuildGuest(ctx context.Context, name string, command []string, opts provider.ExecOptions) (provider.ExecResult, error) { + if opts.LogPath == "" { + opts.LogPath = m.buildLogPath(imageLogStem(name) + ".guest.log") } + return m.execGuest(ctx, name, command, opts) } func (m *Manager) installRosettaSupport(ctx context.Context, vmName string) error { if m.Config.Provider.Type != "tart" || strings.TrimSpace(m.Config.Provider.RosettaTag) == "" { return nil } - fmt.Printf("installing Tart Rosetta amd64 support with tag %q\n", m.Config.Provider.RosettaTag) - _, err := m.execGuest(ctx, vmName, []string{"sudo", "-E", "bash", "/opt/epar/install-rosetta.sh"}, provider.ExecOptions{ + m.infof("installing Tart Rosetta amd64 support with tag %q\n", m.Config.Provider.RosettaTag) + _, err := m.execBuildGuest(ctx, vmName, []string{"sudo", "-E", "bash", "/opt/epar/install-rosetta.sh"}, provider.ExecOptions{ Env: map[string]string{"EPAR_ROSETTA_TAG": m.Config.Provider.RosettaTag}, }) return err @@ -844,12 +884,12 @@ func (m *Manager) copyRunnerImagesSubset(ctx context.Context, vmName, upstreamDi }, } for _, root := range roots { - if _, err := m.Provider.Exec(ctx, vmName, provider.ShellCommand(mkdirGuestCommand(root.guest)), provider.ExecOptions{}); err != nil { + if _, err := m.execBuildGuest(ctx, vmName, provider.ShellCommand(mkdirGuestCommand(root.guest)), provider.ExecOptions{LogPath: m.buildLogPath(vmName + ".guest.log")}); err != nil { return err } if _, err := os.Stat(root.host); err != nil { if m.DryRun && os.IsNotExist(err) { - fmt.Printf("[dry-run] skipping missing runner-images path %s\n", root.host) + m.infof("[dry-run] skipping missing runner-images path %s\n", root.host) continue } return err @@ -873,7 +913,7 @@ func (m *Manager) copyRunnerImagesSubset(ctx context.Context, vmName, upstreamDi if err != nil { return err } - if _, err := m.Provider.Exec(ctx, vmName, provider.ShellCommand(mkdirGuestCommand(filepath.ToSlash(filepath.Dir(guestPath)))), provider.ExecOptions{}); err != nil { + if _, err := m.execBuildGuest(ctx, vmName, provider.ShellCommand(mkdirGuestCommand(filepath.ToSlash(filepath.Dir(guestPath)))), provider.ExecOptions{LogPath: m.buildLogPath(vmName + ".guest.log")}); err != nil { return err } return provider.CopyText(ctx, m.Provider, vmName, guestPath, "0644", guestText(content)) @@ -882,7 +922,7 @@ func (m *Manager) copyRunnerImagesSubset(ctx context.Context, vmName, upstreamDi } } buildGuestDir := "/opt/epar/upstream/runner-images/images/ubuntu/scripts/build" - if _, err := m.Provider.Exec(ctx, vmName, provider.ShellCommand(mkdirGuestCommand(buildGuestDir)), provider.ExecOptions{}); err != nil { + if _, err := m.execBuildGuest(ctx, vmName, provider.ShellCommand(mkdirGuestCommand(buildGuestDir)), provider.ExecOptions{LogPath: m.buildLogPath(vmName + ".guest.log")}); err != nil { return err } for _, name := range m.runnerImageBuildScripts() { @@ -890,7 +930,7 @@ func (m *Manager) copyRunnerImagesSubset(ctx context.Context, vmName, upstreamDi content, err := os.ReadFile(hostPath) if err != nil { if m.DryRun && os.IsNotExist(err) { - fmt.Printf("[dry-run] skipping missing runner-images file %s\n", hostPath) + m.infof("[dry-run] skipping missing runner-images file %s\n", hostPath) continue } return err @@ -949,7 +989,7 @@ func (m *Manager) prepareDockerDindBuildContextWithHostTrust(buildCtx, upstreamD } switch m.runnerImagesCopyMode() { case runnerImagesCopySubset: - fmt.Printf("preparing Docker-DinD build context with runner-images script subset\n") + m.infof("preparing Docker-DinD build context with runner-images script subset\n") if err := copyRunnerImagesSubsetToDir(upstreamDir, upstreamDest, m.runnerImageBuildScripts()); err != nil { return err } @@ -957,7 +997,7 @@ func (m *Manager) prepareDockerDindBuildContextWithHostTrust(buildCtx, upstreamD return err } case runnerImagesCopyNone: - fmt.Printf("preparing Docker-DinD build context without runner-images resources\n") + m.infof("preparing Docker-DinD build context without runner-images resources\n") } customDir := filepath.Join(buildCtx, "custom-install") if err := os.MkdirAll(customDir, 0755); err != nil { @@ -1043,8 +1083,8 @@ func (m *Manager) installCustomInstallScripts(ctx context.Context, vmName string if len(scripts) == 0 { return nil } - fmt.Printf("running %d image install script(s)\n", len(scripts)) - if _, err := m.execGuest(ctx, vmName, provider.ShellCommand("if command -v sudo >/dev/null 2>&1; then sudo mkdir -p /opt/epar/custom-install; else mkdir -p /opt/epar/custom-install; fi"), provider.ExecOptions{}); err != nil { + m.infof("running %d image install script(s)\n", len(scripts)) + if _, err := m.execBuildGuest(ctx, vmName, provider.ShellCommand("if command -v sudo >/dev/null 2>&1; then sudo mkdir -p /opt/epar/custom-install; else mkdir -p /opt/epar/custom-install; fi"), provider.ExecOptions{}); err != nil { return err } for i, script := range scripts { @@ -1060,8 +1100,8 @@ func (m *Manager) installCustomInstallScripts(ctx context.Context, vmName string if err := provider.CopyText(ctx, m.Provider, vmName, guestPath, "0755", guestText(content)); err != nil { return err } - fmt.Printf("running image install script %d/%d: %s\n", i+1, len(scripts), script) - if _, err := m.execGuest(ctx, vmName, []string{"sudo", "bash", guestPath}, provider.ExecOptions{}); err != nil { + m.infof("running image install script %d/%d: %s\n", i+1, len(scripts), script) + if _, err := m.execBuildGuest(ctx, vmName, []string{"sudo", "bash", guestPath}, provider.ExecOptions{}); err != nil { return err } } @@ -1121,7 +1161,7 @@ func guestScriptName(name string) string { func (m *Manager) enableWSLSystemd(ctx context.Context, name string) error { content := "[boot]\nsystemd=true\n\n[interop]\nappendWindowsPath=false\n\n[user]\ndefault=root\n" - _, err := m.Provider.Exec(ctx, name, provider.ShellCommand("mkdir -p /etc && cat >/etc/wsl.conf"), provider.ExecOptions{Stdin: content}) + _, err := m.execBuildGuest(ctx, name, provider.ShellCommand("mkdir -p /etc && cat >/etc/wsl.conf"), provider.ExecOptions{Stdin: content, LogPath: m.buildLogPath(name + ".guest.log")}) return err } @@ -1133,11 +1173,11 @@ func (m *Manager) waitForSystemd(ctx context.Context, name string) error { deadline := time.Now().Add(time.Duration(waitSeconds) * time.Second) var lastErr error for { - result, err := m.Provider.Exec(ctx, name, provider.ShellCommand(`test "$(ps -p 1 -o comm=)" = systemd && state="$(systemctl is-system-running 2>/dev/null || true)" && case "$state" in running|degraded) echo "$state"; exit 0 ;; *) echo "$state"; exit 1 ;; esac`), provider.ExecOptions{ - LogPath: filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), name+".guest.log"), + result, err := m.execBuildGuest(ctx, name, provider.ShellCommand(`test "$(ps -p 1 -o comm=)" = systemd && state="$(systemctl is-system-running 2>/dev/null || true)" && case "$state" in running|degraded) echo "$state"; exit 0 ;; *) echo "$state"; exit 1 ;; esac`), provider.ExecOptions{ + LogPath: m.buildLogPath(name + ".guest.log"), }) if err == nil { - fmt.Printf("systemd is %s\n", strings.TrimSpace(result.Stdout)) + m.infof("systemd is %s\n", strings.TrimSpace(result.Stdout)) return nil } lastErr = err @@ -1168,10 +1208,7 @@ func guestText(content []byte) string { } func runHost(ctx context.Context, name string, args ...string) error { - cmd := exec.CommandContext(ctx, name, args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() + return exec.CommandContext(ctx, name, args...).Run() } func runHostOutput(ctx context.Context, name string, args ...string) (string, error) { @@ -1188,22 +1225,10 @@ func runHostQuiet(ctx context.Context, name string, args ...string) error { return cmd.Run() } -func runHostLogged(ctx context.Context, logPath, name string, args ...string) error { - if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { - return err - } - logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return err - } - defer logFile.Close() +func runHostLogged(ctx context.Context, _ string, stdout, stderr io.Writer, name string, args ...string) error { cmd := exec.CommandContext(ctx, name, args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if logPath != "" { - cmd.Stdout = io.MultiWriter(os.Stdout, logFile) - cmd.Stderr = io.MultiWriter(os.Stderr, logFile) - } + cmd.Stdout = stdout + cmd.Stderr = stderr if err := cmd.Run(); err != nil { return fmt.Errorf("%s %s failed: %w", name, strings.Join(args, " "), err) } diff --git a/internal/pool/image_manifest.go b/internal/pool/image_manifest.go index 283e1dd..d37dc3f 100644 --- a/internal/pool/image_manifest.go +++ b/internal/pool/image_manifest.go @@ -67,7 +67,7 @@ func (m *Manager) EnsureImage(ctx context.Context) error { return err } if m.DryRun { - fmt.Printf("[dry-run] would ensure image %s has manifest %s\n", m.Config.Image.OutputImage, hash) + m.infof("[dry-run] would ensure image %s has manifest %s\n", m.Config.Image.OutputImage, hash) return m.BuildImage(ctx, ImageBuildOptions{Replace: true, Manifest: &manifest}) } state, err := m.currentImageState(ctx, hash) @@ -76,12 +76,12 @@ func (m *Manager) EnsureImage(ctx context.Context) error { } switch state { case imageStateCurrent: - fmt.Printf("image is current: %s\n", m.Config.Image.OutputImage) + m.infof("image is current: %s\n", m.Config.Image.OutputImage) return nil case imageStateMissing: - fmt.Printf("image is missing; building %s\n", m.Config.Image.OutputImage) + m.infof("image is missing; building %s\n", m.Config.Image.OutputImage) case imageStateOutdated: - fmt.Printf("image is outdated or not aligned with config; rebuilding %s\n", m.Config.Image.OutputImage) + m.infof("image is outdated or not aligned with config; rebuilding %s\n", m.Config.Image.OutputImage) } return m.BuildImage(ctx, ImageBuildOptions{Replace: true, Manifest: &manifest}) } @@ -270,8 +270,9 @@ func (m *Manager) refreshDockerSourceDigestUntimed(ctx context.Context) (string, return "", fmt.Errorf("image.sourceImage is required when image.sourceType=docker-image") } platform := strings.TrimSpace(m.Config.Image.SourcePlatform) - logPath := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), imageLogStem(m.Config.Image.OutputImage)+".source.log") - fmt.Printf("refreshing Docker source image %s\n", image) + logPath := m.buildLogPath(imageLogStem(m.Config.Image.OutputImage) + ".source.log") + defer m.releaseTranscript(logPath) + m.infof("refreshing Docker source image %s\n", image) if err := pullDockerSourceCommand(m, ctx, dockerSourcePullOptions{ Image: image, Platform: platform, diff --git a/internal/pool/image_manifest_test.go b/internal/pool/image_manifest_test.go index a68c4ef..ddd03a6 100644 --- a/internal/pool/image_manifest_test.go +++ b/internal/pool/image_manifest_test.go @@ -3,6 +3,7 @@ package pool import ( "context" "errors" + "io" "os" "path/filepath" "strings" @@ -214,7 +215,7 @@ func TestPrepareWSLDockerSourceRootfsInvalidatesMismatchedCache(t *testing.T) { } var calls []string - runHostLoggedCommand = func(_ context.Context, _ string, name string, args ...string) error { + runHostLoggedCommand = func(_ context.Context, _ string, _, _ io.Writer, name string, args ...string) error { calls = append(calls, name+" "+strings.Join(args, " ")) if len(args) > 0 && args[0] == "export" { for i := 0; i+1 < len(args); i++ { diff --git a/internal/pool/image_test.go b/internal/pool/image_test.go index 53a78aa..b5f057f 100644 --- a/internal/pool/image_test.go +++ b/internal/pool/image_test.go @@ -113,7 +113,7 @@ func TestDockerDindBuildUsesLegacyBuilderCompatibleArgs(t *testing.T) { OutputImage: "epar-docker-dind-catthehacker-ubuntu", RunnerVersion: "latest", }, - Pool: config.PoolConfig{LogDir: "logs"}, + Logging: config.LoggingConfig{Directory: "logs"}, Provider: config.ProviderConfig{Type: "docker-dind", Platform: "linux/amd64"}, }, ProjectRoot: root, @@ -160,7 +160,7 @@ func TestInstallCustomScriptsRunsInOrder(t *testing.T) { Image: config.ImageConfig{ CustomInstallScripts: []string{".local/one.sh", ".local/two.sh"}, }, - Pool: config.PoolConfig{LogDir: "logs"}, + Logging: config.LoggingConfig{Directory: "logs"}, }, Provider: provider, ProjectRoot: root, @@ -269,7 +269,7 @@ func TestConfigureDockerRegistryMirrors(t *testing.T) { Docker: config.DockerConfig{ RegistryMirrors: []string{"http://host.docker.internal:5000", "https://mirror.example.test"}, }, - Pool: config.PoolConfig{LogDir: "logs"}, + Logging: config.LoggingConfig{Directory: "logs"}, }, Provider: provider, ProjectRoot: root, @@ -302,7 +302,7 @@ func TestPrepareWSLDockerSourceRootfsExportsContainerFilesystem(t *testing.T) { }() var calls []string - runHostLoggedCommand = func(_ context.Context, _ string, name string, args ...string) error { + runHostLoggedCommand = func(_ context.Context, _ string, _, _ io.Writer, name string, args ...string) error { calls = append(calls, name+" "+strings.Join(args, " ")) if len(args) > 0 && args[0] == "export" { for i := 0; i+1 < len(args); i++ { @@ -395,7 +395,7 @@ func TestPrepareWSLDockerSourceRootfsUsesCachedRootfs(t *testing.T) { t.Fatal("docker pull should not run when cached rootfs and env metadata exist") return nil } - runHostLoggedCommand = func(context.Context, string, string, ...string) error { + runHostLoggedCommand = func(context.Context, string, io.Writer, io.Writer, string, ...string) error { t.Fatal("docker command should not run when cached rootfs and env metadata exist") return nil } diff --git a/internal/pool/log_paths.go b/internal/pool/log_paths.go new file mode 100644 index 0000000..fb56053 --- /dev/null +++ b/internal/pool/log_paths.go @@ -0,0 +1,19 @@ +package pool + +import ( + "path/filepath" + + "github.com/solutionforest/ephemeral-action-runner/internal/config" +) + +func (m *Manager) logRoot() string { + return config.ProjectPath(m.ProjectRoot, m.Config.Logging.Directory) +} + +func (m *Manager) instanceLogPath(name, suffix string) string { + return filepath.Join(m.logRoot(), "instances", name+suffix) +} + +func (m *Manager) buildLogPath(name string) string { + return filepath.Join(m.logRoot(), "builds", name) +} diff --git a/internal/pool/logging.go b/internal/pool/logging.go new file mode 100644 index 0000000..b4501cc --- /dev/null +++ b/internal/pool/logging.go @@ -0,0 +1,162 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "github.com/solutionforest/ephemeral-action-runner/internal/logging" +) + +func (m *Manager) logger() *slog.Logger { + if m != nil && m.Logging != nil { + return m.Logging.Logger() + } + return slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ReplaceAttr: func(_ []string, attribute slog.Attr) slog.Attr { + if attribute.Key == slog.TimeKey || attribute.Key == slog.LevelKey { + return slog.Attr{} + } + return attribute + }})) +} + +func (m *Manager) infof(format string, args ...any) { + m.logger().Info(fmt.Sprintf(strings.TrimSuffix(format, "\n"), args...)) +} + +func (m *Manager) warnf(format string, args ...any) { + m.logger().Warn(fmt.Sprintf(strings.TrimSuffix(format, "\n"), args...)) +} + +func (m *Manager) Close() error { + if m == nil || m.Logging == nil { + return nil + } + return m.Logging.Close() +} + +func (m *Manager) transcript(path, instance, component string) (*logging.Transcript, error) { + if m.Logging == nil { + return &logging.Transcript{Stdout: io.Discard, Stderr: io.Discard}, nil + } + canonical, err := filepath.Abs(path) + if err != nil { + return nil, err + } + m.transcriptMu.Lock() + defer m.transcriptMu.Unlock() + if transcript := m.transcripts[canonical]; transcript != nil { + return transcript, nil + } + category := logging.CategoryInstances + clean := filepath.Clean(canonical) + switch filepath.Base(filepath.Dir(clean)) { + case "builds": + category = logging.CategoryBuilds + case "benchmarks": + category = logging.CategoryBenchmarks + case "errors": + category = logging.CategoryErrors + case "instances": + category = logging.CategoryInstances + default: + return nil, fmt.Errorf("cannot classify transcript path %s", path) + } + transcript, err := m.Logging.OpenTranscript(logging.TranscriptMetadata{ + Category: category, + Instance: instance, + Component: component, + Provider: m.Config.Provider.Type, + }, canonical) + if err != nil { + return nil, err + } + if m.transcripts == nil { + m.transcripts = make(map[string]*logging.Transcript) + } + m.transcripts[canonical] = transcript + return transcript, nil +} + +func (m *Manager) releaseTranscript(path string) error { + if m == nil || m.Logging == nil || path == "" { + return nil + } + canonical, err := filepath.Abs(path) + if err != nil { + return err + } + m.transcriptMu.Lock() + transcript := m.transcripts[canonical] + delete(m.transcripts, canonical) + m.transcriptMu.Unlock() + if transcript == nil { + return nil + } + return transcript.Close() +} + +func (m *Manager) releaseInstanceTranscripts(vm ProvisionedInstance) error { + return errors.Join(m.releaseTranscript(vm.LogPath), m.releaseTranscript(vm.GuestLogPath)) +} + +func transcriptComponent(path string) string { + name := strings.ToLower(filepath.Base(path)) + switch { + case strings.Contains(name, "guest"): + return "guest" + case strings.Contains(name, "build"): + return "build" + case strings.Contains(name, "source"): + return "source" + default: + return "provider" + } +} + +func (m *Manager) runHostLogged(ctx context.Context, logPath, name string, args ...string) error { + transcript, err := m.transcript(logPath, "", transcriptComponent(logPath)) + if err != nil { + return err + } + return runHostLoggedCommand(ctx, logPath, transcript.Stdout, transcript.Stderr, name, args...) +} + +func (m *Manager) retentionPolicy() logging.RetentionPolicy { + days := func(value int) time.Duration { return time.Duration(value) * 24 * time.Hour } + return logging.RetentionPolicy{ + MaxTotalBytes: int64(m.Config.Logging.RetentionMaxTotalMiB) * 1024 * 1024, + ManagerMaxAge: days(m.Config.Logging.ManagerMaxAgeDays), + InstanceMaxAge: days(m.Config.Logging.InstanceMaxAgeDays), + BuildMaxAge: days(m.Config.Logging.BuildMaxAgeDays), + ErrorMaxAge: days(m.Config.Logging.ErrorMaxAgeDays), + BenchmarkMaxAge: days(m.Config.Logging.BenchmarkMaxAgeDays), + } +} + +func (m *Manager) PruneLogs(dryRun bool) (logging.RetentionReport, error) { + if m.Logging == nil { + return logging.RetentionReport{}, nil + } + return m.Logging.PruneRetention(m.retentionPolicy(), dryRun) +} + +func (m *Manager) pruneLogsBestEffort() { + report, err := m.PruneLogs(false) + if err != nil { + m.logger().Warn("log retention failed", "operation", "logs-prune", "error", err) + return + } + for _, warning := range report.Warnings { + m.logger().Warn("log retention skipped candidate", "operation", "logs-prune", "warning", warning) + } + if report.Deleted > 0 { + m.logger().Info("log retention completed", "operation", "logs-prune", "deleted", report.Deleted, "reclaimedBytes", report.ReclaimedBytes) + } +} diff --git a/internal/pool/manager.go b/internal/pool/manager.go index 1cdfed0..a0ee718 100644 --- a/internal/pool/manager.go +++ b/internal/pool/manager.go @@ -14,6 +14,7 @@ import ( "github.com/solutionforest/ephemeral-action-runner/internal/config" gh "github.com/solutionforest/ephemeral-action-runner/internal/github" "github.com/solutionforest/ephemeral-action-runner/internal/hosttrust" + "github.com/solutionforest/ephemeral-action-runner/internal/logging" "github.com/solutionforest/ephemeral-action-runner/internal/provider" ) @@ -24,7 +25,10 @@ type Manager struct { ProjectRoot string ConfigPath string DryRun bool + Logging *logging.Runtime startupTiming *startupTiming + transcriptMu sync.Mutex + transcripts map[string]*logging.Transcript hostTrustResolver func(context.Context) (hosttrust.Snapshot, error) hostTrustImageEnsurer func(context.Context) error @@ -85,12 +89,11 @@ func (m *Manager) Verify(ctx context.Context, opts VerifyOptions) error { } opts.Instances = m.requestedInstances(opts.Instances) names := RunnerNames(m.Config.Pool.NamePrefix, opts.Instances, time.Now()) - fmt.Printf("verifying %d instance(s): %s\n", opts.Instances, strings.Join(names, ", ")) - fmt.Printf("source image: %s\n", m.Config.Provider.SourceImage) + m.logger().Info("verifying instances", "provider", m.Config.Provider.Type, "operation", "verify", "instances", opts.Instances, "instanceNames", strings.Join(names, ", "), "sourceImage", m.Config.Provider.SourceImage) if opts.RegisterOnly { - fmt.Printf("registration: GitHub ephemeral runners for %s\n", m.Config.GitHub.Organization) + m.infof("registration: GitHub ephemeral runners for %s\n", m.Config.GitHub.Organization) } else { - fmt.Printf("registration: skipped\n") + m.infof("registration: skipped\n") } var ( mu sync.Mutex @@ -119,26 +122,26 @@ func (m *Manager) Verify(ctx context.Context, opts VerifyOptions) error { wg.Wait() stopLeaseKeeper() if opts.Cleanup { - fmt.Printf("cleanup: removing instances and GitHub runners with prefix %q\n", m.Config.Pool.NamePrefix) + m.infof("cleanup: removing instances and GitHub runners with prefix %q\n", m.Config.Pool.NamePrefix) if err := m.cleanupWithFreshContext(); err != nil && errOnce == nil { errOnce = err } } if errOnce == nil { - fmt.Printf("verify complete: %d instance(s) validated", len(created)) + m.infof("verify complete: %d instance(s) validated", len(created)) if opts.RegisterOnly { - fmt.Printf(" and registered online/idle") + m.infof(" and registered online/idle") } if opts.Cleanup { - fmt.Printf("; cleanup complete") + m.infof("; cleanup complete") } - fmt.Printf("\n") + m.infof("\n") for _, vm := range created { - fmt.Printf(" %s ip=%s providerLog=%s guestLog=%s", vm.Name, emptyDash(vm.IP), vm.LogPath, vm.GuestLogPath) + m.infof(" %s ip=%s providerLog=%s guestLog=%s", vm.Name, emptyDash(vm.IP), vm.LogPath, vm.GuestLogPath) if vm.RunnerID != 0 { - fmt.Printf(" runnerID=%d", vm.RunnerID) + m.infof(" runnerID=%d", vm.RunnerID) } - fmt.Printf("\n") + m.infof("\n") } } return errOnce @@ -181,15 +184,27 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { if vm.HostTrustGeneration != "" { poolTrustGeneration = vm.HostTrustGeneration } - fmt.Printf("%s online at %s providerLog=%s guestLog=%s\n", vm.Name, vm.IP, vm.LogPath, vm.GuestLogPath) + m.infof("%s online at %s providerLog=%s guestLog=%s\n", vm.Name, vm.IP, vm.LogPath, vm.GuestLogPath) } stopLeaseKeeper() if !opts.Register || (!opts.ReplaceCompleted && !m.hostTrustEnabled()) { - fmt.Println("pool is running; press Ctrl-C to stop") - <-ctx.Done() - return cleanup() + m.infof("pool is running; press Ctrl-C to stop") + if !m.Config.Logging.RetentionEnabled { + <-ctx.Done() + return cleanup() + } + retentionTicker := time.NewTicker(time.Duration(m.Config.Logging.RetentionIntervalMinutes) * time.Minute) + defer retentionTicker.Stop() + for { + select { + case <-ctx.Done(): + return cleanup() + case <-retentionTicker.C: + m.pruneLogsBestEffort() + } + } } - fmt.Printf("pool supervisor is running; monitoring every %s; press Ctrl-C to stop\n", opts.MonitorInterval) + m.infof("pool supervisor is running; monitoring every %s; press Ctrl-C to stop\n", opts.MonitorInterval) tickInterval := opts.MonitorInterval if m.hostTrustEnabled() && tickInterval > hostTrustRefreshInterval { tickInterval = hostTrustRefreshInterval @@ -197,6 +212,7 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { ticker := time.NewTicker(tickInterval) defer ticker.Stop() nextLivenessCheck := time.Now().Add(opts.MonitorInterval) + nextRetention := time.Now().Add(time.Duration(m.Config.Logging.RetentionIntervalMinutes) * time.Minute) nextHostTrustCollection := time.Time{} var currentHostTrust hosttrust.Snapshot for { @@ -204,6 +220,10 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { case <-ctx.Done(): return cleanup() case <-ticker.C: + if m.Config.Logging.RetentionEnabled && !time.Now().Before(nextRetention) { + m.pruneLogsBestEffort() + nextRetention = time.Now().Add(time.Duration(m.Config.Logging.RetentionIntervalMinutes) * time.Minute) + } trustRetired := 0 trustCapacityReady := true if m.hostTrustEnabled() { @@ -213,7 +233,7 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { nextHostTrustCollection = now.Add(m.hostTrustCollectionInterval()) if err != nil { currentHostTrust = hosttrust.Snapshot{} - fmt.Printf("host trust refresh warning; existing leases will expire closed: %v\n", err) + m.warnf("host trust refresh warning; existing leases will expire closed: %v\n", err) } else { ready := true if poolTrustGeneration != current.Generation { @@ -222,18 +242,18 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { // receive a mismatching lease so no subsequent job can start. currentHostTrust = current trustRetired += m.reconcileHostTrustRunners(ctx, active, current) - fmt.Printf("host trust generation changed (%s -> %s); building replacement image\n", emptyDash(poolTrustGeneration), current.Generation) + m.infof("host trust generation changed (%s -> %s); building replacement image\n", emptyDash(poolTrustGeneration), current.Generation) ready = false for attempt := 1; attempt <= 3; attempt++ { generationBeforeEnsure := current.Generation if err := m.ensureHostTrustImage(ctx); err != nil { - fmt.Printf("host trust replacement image warning: %v\n", err) + m.warnf("host trust replacement image warning: %v\n", err) nextHostTrustCollection = time.Now() break } current, err = m.resolveHostTrust(ctx) if err != nil { - fmt.Printf("host trust post-build refresh warning: %v\n", err) + m.warnf("host trust post-build refresh warning: %v\n", err) nextHostTrustCollection = time.Now() break } @@ -243,9 +263,9 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { break } if attempt < 3 { - fmt.Printf("host trust changed again during replacement image publication (%s -> %s); retrying %d/3\n", generationBeforeEnsure, current.Generation, attempt+1) + m.infof("host trust changed again during replacement image publication (%s -> %s); retrying %d/3\n", generationBeforeEnsure, current.Generation, attempt+1) } else { - fmt.Printf("host trust did not stabilize across 3 replacement image attempts (%s -> %s)\n", generationBeforeEnsure, current.Generation) + m.warnf("host trust did not stabilize across 3 replacement image attempts (%s -> %s)\n", generationBeforeEnsure, current.Generation) } } trustCapacityReady = ready @@ -264,15 +284,15 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { for name, vm := range active { alive, reason, err := m.runnerAlive(ctx, vm) if err != nil { - fmt.Printf("[%s] liveness check warning: %v\n", name, err) + m.logger().Warn("liveness check failed", "provider", m.Config.Provider.Type, "instance", name, "operation", "liveness-check", "error", err) continue } if alive { continue } - fmt.Printf("[%s] runner is finished or unhealthy: %s\n", name, reason) + m.infof("[%s] runner is finished or unhealthy: %s\n", name, reason) if err := m.retireInstance(context.Background(), vm, reason); err != nil { - fmt.Printf("[%s] retirement warning: %v\n", name, err) + m.warnf("[%s] retirement warning: %v\n", name, err) continue } delete(active, name) @@ -298,10 +318,10 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { } name := RunnerName(m.Config.Pool.NamePrefix, sequence, time.Now()) sequence++ - fmt.Printf("[%s] creating replacement runner\n", name) + m.infof("[%s] creating replacement runner\n", name) vm, err := m.provisionOne(ctx, name, opts.Register, true) if err != nil { - fmt.Printf("[%s] replacement failed: %v\n", name, err) + m.warnf("[%s] replacement failed: %v\n", name, err) break } active[vm.Name] = vm @@ -309,7 +329,7 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { if vm.HostTrustGeneration != "" { poolTrustGeneration = vm.HostTrustGeneration } - fmt.Printf("%s online at %s providerLog=%s guestLog=%s\n", vm.Name, vm.IP, vm.LogPath, vm.GuestLogPath) + m.infof("%s online at %s providerLog=%s guestLog=%s\n", vm.Name, vm.IP, vm.LogPath, vm.GuestLogPath) } } } @@ -365,13 +385,24 @@ func (m *Manager) Cleanup(ctx context.Context) error { if !HasPrefix(vm.Name, m.Config.Pool.NamePrefix) { continue } - fmt.Printf("cleanup: deleting instance %s\n", vm.Name) + m.infof("cleanup: deleting instance %s\n", vm.Name) stopCtx, stopCancel := context.WithTimeout(ctx, 60*time.Second) _ = m.Provider.Stop(stopCtx, vm.Name) stopCancel() deleteCtx, deleteCancel := context.WithTimeout(ctx, 60*time.Second) - if err := m.Provider.Delete(deleteCtx, vm.Name); err != nil && firstErr == nil { - firstErr = err + deleteErr := m.Provider.Delete(deleteCtx, vm.Name) + if deleteErr != nil && firstErr == nil { + firstErr = deleteErr + } + if deleteErr == nil { + paths := ProvisionedInstance{ + Name: vm.Name, + LogPath: m.instanceLogPath(vm.Name, "."+m.Config.Provider.Type+".log"), + GuestLogPath: m.instanceLogPath(vm.Name, ".guest.log"), + } + if releaseErr := m.releaseInstanceTranscripts(paths); releaseErr != nil { + m.logger().Warn("instance transcript close failed after cleanup", "provider", m.Config.Provider.Type, "instance", vm.Name, "operation", "cleanup", "error", releaseErr) + } } deleteCancel() } @@ -383,7 +414,7 @@ func (m *Manager) Cleanup(ctx context.Context) error { firstErr = err } for _, runner := range deleted { - fmt.Printf("cleanup: deleted GitHub runner %s id=%d\n", runner.Name, runner.ID) + m.infof("cleanup: deleted GitHub runner %s id=%d\n", runner.Name, runner.ID) } } return firstErr @@ -433,7 +464,7 @@ func (m *Manager) provisionOne(ctx context.Context, name string, register, allow if attempt == attempts { break } - fmt.Printf("[%s] host trust changed before runner publication; rebuilding image (attempt %d/%d)\n", name, attempt+1, attempts) + m.infof("[%s] host trust changed before runner publication; rebuilding image (attempt %d/%d)\n", name, attempt+1, attempts) if err := m.ensureHostTrustImage(ctx); err != nil { return vm, fmt.Errorf("rebuild image after host trust changed during provisioning: %w", err) } @@ -442,9 +473,8 @@ func (m *Manager) provisionOne(ctx context.Context, name string, register, allow } func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register, allowBusy bool) (ProvisionedInstance, error) { - logPath := config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir) - logPath = filepath.Join(logPath, name+"."+m.Config.Provider.Type+".log") - guestLogPath := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), name+".guest.log") + logPath := m.instanceLogPath(name, "."+m.Config.Provider.Type+".log") + guestLogPath := m.instanceLogPath(name, ".guest.log") vm := ProvisionedInstance{Name: name, LogPath: logPath, GuestLogPath: guestLogPath} var trustSnapshot hosttrust.Snapshot if m.hostTrustEnabled() { @@ -458,15 +488,19 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { return vm, err } - fmt.Printf("[%s] cloning %s\n", name, m.Config.Provider.SourceImage) + m.logger().Info("cloning instance", "provider", m.Config.Provider.Type, "instance", name, "operation", "clone", "sourceImage", m.Config.Provider.SourceImage, "logPath", logPath) if err := m.timeFirstInstanceStage(name, "instance_container_create", func() error { return m.Provider.Clone(ctx, m.Config.Provider.SourceImage, name) }); err != nil { return vm, err } - fmt.Printf("[%s] starting instance\n", name) + m.logger().Info("starting instance", "provider", m.Config.Provider.Type, "instance", name, "operation", "start", "logPath", logPath) if err := m.timeFirstInstanceStage(name, m.startupInstanceStartStage(), func() error { - _, err := m.Provider.Start(ctx, name, m.startOptions(logPath)) + startOptions, startOptionsErr := m.startOptions(logPath, name) + if startOptionsErr != nil { + return startOptionsErr + } + _, err := m.Provider.Start(ctx, name, startOptions) return err }); err != nil { return vm, err @@ -476,8 +510,8 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register return vm, err } vm.IP = ip - fmt.Printf("[%s] reachable at %s\n", name, ip) - fmt.Printf("[%s] validating runner runtime\n", name) + m.logger().Info("instance reachable", "provider", m.Config.Provider.Type, "instance", name, "operation", "wait-reachable", "address", ip) + m.logger().Info("validating runner runtime", "provider", m.Config.Provider.Type, "instance", name, "operation", "validate-runtime", "stage", "start") if err := m.timeFirstInstanceStage(name, "runtime_validation", func() error { if err := m.configureDockerRegistryMirrors(ctx, name); err != nil { return err @@ -486,7 +520,7 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register }); err != nil { return vm, err } - fmt.Printf("[%s] runtime validation passed\n", name) + m.infof("[%s] runtime validation passed\n", name) if m.hostTrustEnabled() { marker, err := m.readInstanceHostTrustMarker(ctx, name) if err != nil { @@ -511,7 +545,7 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register } if m.GitHub == nil { if m.DryRun { - fmt.Printf("[dry-run] would register GitHub runner %s with labels %s\n", name, strings.Join(m.Config.Runner.Labels, ",")) + m.infof("[dry-run] would register GitHub runner %s with labels %s\n", name, strings.Join(m.Config.Runner.Labels, ",")) return vm, nil } return vm, fmt.Errorf("github client is required for registration") @@ -522,7 +556,7 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register readiness = "online/idle" ) if err := m.timeFirstInstanceStage(name, "github_registration_and_online_idle", func() error { - fmt.Printf("[%s] requesting GitHub registration token\n", name) + m.infof("[%s] requesting GitHub registration token\n", name) var err error token, err = m.GitHub.RegistrationToken(ctx) if err != nil { @@ -540,14 +574,14 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register if _, err := m.execGuest(ctx, name, []string{"sudo", "-E", "bash", "/opt/epar/configure-runner.sh"}, provider.ExecOptions{Env: env}); err != nil { return err } - fmt.Printf("[%s] starting runner service\n", name) + m.infof("[%s] starting runner service\n", name) if _, err := m.execGuest(ctx, name, []string{"sudo", "bash", "/opt/epar/run-runner.sh"}, provider.ExecOptions{}); err != nil { return err } if allowBusy { readiness = "online" } - fmt.Printf("[%s] waiting for GitHub %s\n", name, readiness) + m.infof("[%s] waiting for GitHub %s\n", name, readiness) runner, err = m.waitRunnerReadyAndHealthy(ctx, vm, time.Duration(m.Config.Timeouts.GitHubOnlineSeconds)*time.Second, allowBusy) return err }); err != nil { @@ -555,7 +589,7 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register return vm, err } vm.RunnerID = runner.ID - fmt.Printf("[%s] GitHub runner %s id=%d busy=%t\n", name, readiness, runner.ID, runner.Busy) + m.infof("[%s] GitHub runner %s id=%d busy=%t\n", name, readiness, runner.ID, runner.Busy) m.finishFirstRunnerReady(name) } else { m.finishFirstRunnerReady(name) @@ -614,7 +648,7 @@ func (m *Manager) waitRunnerReadyAndHealthy(ctx context.Context, vm ProvisionedI } if current.Generation != vm.HostTrustGeneration { if revokeErr := m.issueHostTrustLease(waitCtx, vm.Name, current); revokeErr != nil { - fmt.Printf("[%s] host trust readiness revocation warning: %v\n", vm.Name, revokeErr) + m.warnf("[%s] host trust readiness revocation warning: %v\n", vm.Name, revokeErr) } cancel() return gh.Runner{}, fmt.Errorf("host trust changed while runner %s was registering (%s -> %s)", vm.Name, vm.HostTrustGeneration, current.Generation) @@ -638,7 +672,7 @@ func (m *Manager) waitRunnerReadyAndHealthy(ctx context.Context, vm ProvisionedI consecutiveProbeFailures++ lastProbeErr = err if consecutiveProbeFailures < runnerReadinessProbeFailureLimit { - fmt.Printf("[%s] runner readiness process check failed (%d/%d): %v\n", vm.Name, consecutiveProbeFailures, runnerReadinessProbeFailureLimit, err) + m.warnf("[%s] runner readiness process check failed (%d/%d): %v\n", vm.Name, consecutiveProbeFailures, runnerReadinessProbeFailureLimit, err) continue } cancel() @@ -657,14 +691,14 @@ func (m *Manager) waitRunnerReadyAndHealthy(ctx context.Context, vm ProvisionedI func (m *Manager) captureRunnerReadinessDiagnostics(name, guestLogPath string) { diagnosticCtx, cancel := context.WithTimeout(context.Background(), runnerReadinessDiagnosticsTimeout) defer cancel() - _, err := m.Provider.Exec( + _, err := m.execGuest( diagnosticCtx, name, []string{"sudo", "bash", "/opt/epar/collect-runner-diagnostics.sh"}, provider.ExecOptions{LogPath: guestLogPath}, ) if err != nil { - fmt.Printf("[%s] runner readiness diagnostic collection warning: %v\n", name, err) + m.warnf("[%s] runner readiness diagnostic collection warning: %v\n", name, err) } } @@ -700,7 +734,7 @@ func (m *Manager) runnerProcessAlive(ctx context.Context, vm ProvisionedInstance func (m *Manager) checkRunnerProcess(ctx context.Context, name string) error { checkCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - _, err := m.Provider.Exec(checkCtx, name, provider.ShellCommand("if test -x /opt/epar/check-runner.sh; then sudo bash /opt/epar/check-runner.sh; else systemctl is-active --quiet actions-runner.service; fi"), provider.ExecOptions{}) + _, err := m.execGuest(checkCtx, name, provider.ShellCommand("if test -x /opt/epar/check-runner.sh; then sudo bash /opt/epar/check-runner.sh; else systemctl is-active --quiet actions-runner.service; fi"), provider.ExecOptions{}) return err } @@ -710,7 +744,7 @@ func isTransientGitHubLivenessError(err error) bool { } func (m *Manager) retireInstance(ctx context.Context, vm ProvisionedInstance, reason string) error { - fmt.Printf("[%s] retiring instance: %s\n", vm.Name, reason) + m.infof("[%s] retiring instance: %s\n", vm.Name, reason) var firstErr error if m.GitHub != nil && vm.RunnerID != 0 { deleteCtx, cancel := context.WithTimeout(ctx, 60*time.Second) @@ -724,8 +758,14 @@ func (m *Manager) retireInstance(ctx context.Context, vm ProvisionedInstance, re _ = m.Provider.Stop(stopCtx, vm.Name) stopCancel() deleteCtx, deleteCancel := context.WithTimeout(ctx, 60*time.Second) - if err := m.Provider.Delete(deleteCtx, vm.Name); err != nil && firstErr == nil { - firstErr = err + deleteErr := m.Provider.Delete(deleteCtx, vm.Name) + if deleteErr != nil && firstErr == nil { + firstErr = deleteErr + } + if deleteErr == nil { + if releaseErr := m.releaseInstanceTranscripts(vm); releaseErr != nil { + m.logger().Warn("instance transcript close failed after retirement", "provider", m.Config.Provider.Type, "instance", vm.Name, "operation", "retire", "error", releaseErr) + } } deleteCancel() return firstErr @@ -751,8 +791,8 @@ func (m *Manager) validateRuntimeWithRetry(ctx context.Context, name, guestLogPa if attempt == attempts { break } - fmt.Printf("[%s] runtime validation attempt %d/%d failed: %v\n", name, attempt, attempts, err) - fmt.Printf("[%s] retrying runtime validation in %s; guest log: %s\n", name, runtimeValidationRetryDelay, guestLogPath) + m.warnf("[%s] runtime validation attempt %d/%d failed: %v\n", name, attempt, attempts, err) + m.infof("[%s] retrying runtime validation in %s; guest log: %s\n", name, runtimeValidationRetryDelay, guestLogPath) select { case <-ctx.Done(): return ctx.Err() @@ -766,7 +806,7 @@ func (m *Manager) configureDockerRegistryMirrors(ctx context.Context, name strin if len(m.Config.Docker.RegistryMirrors) == 0 { return nil } - fmt.Printf("[%s] configuring Docker registry mirror(s): %s\n", name, strings.Join(m.Config.Docker.RegistryMirrors, ", ")) + m.infof("[%s] configuring Docker registry mirror(s): %s\n", name, strings.Join(m.Config.Docker.RegistryMirrors, ", ")) hostPath := filepath.Join(m.ProjectRoot, "scripts", "guest", "ubuntu", "configure-docker-daemon.sh") content, err := os.ReadFile(hostPath) if err != nil { @@ -789,8 +829,14 @@ func (m *Manager) execGuest(ctx context.Context, name string, cmd []string, opts timeout = 15 * time.Minute } if opts.LogPath == "" { - opts.LogPath = filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), name+".guest.log") + opts.LogPath = m.instanceLogPath(name, ".guest.log") + } + transcript, err := m.transcript(opts.LogPath, name, transcriptComponent(opts.LogPath)) + if err != nil { + return provider.ExecResult{}, err } + opts.Stdout = transcript.Stdout + opts.Stderr = transcript.Stderr cctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() return m.Provider.Exec(cctx, name, cmd, opts) diff --git a/internal/pool/manager_test.go b/internal/pool/manager_test.go index 828e031..0e8a308 100644 --- a/internal/pool/manager_test.go +++ b/internal/pool/manager_test.go @@ -2,8 +2,11 @@ package pool import ( "context" + "encoding/json" "errors" "fmt" + "os" + "path/filepath" "strings" "sync" "sync/atomic" @@ -13,6 +16,7 @@ import ( "github.com/solutionforest/ephemeral-action-runner/internal/config" gh "github.com/solutionforest/ephemeral-action-runner/internal/github" "github.com/solutionforest/ephemeral-action-runner/internal/hosttrust" + "github.com/solutionforest/ephemeral-action-runner/internal/logging" "github.com/solutionforest/ephemeral-action-runner/internal/provider" ) @@ -36,6 +40,121 @@ func TestRunnerAliveKeepsBusyGitHubRunnerWithoutServiceCheck(t *testing.T) { } } +func TestRetiredInstanceTranscriptsBecomeRetentionEligibleWhileLiveInstanceStaysProtected(t *testing.T) { + root := t.TempDir() + runtime, err := logging.NewRuntime(logging.Options{Directory: root, TranscriptSinks: logging.SinkFile}) + if err != nil { + t.Fatal(err) + } + defer runtime.Close() + manager := Manager{ + Config: config.Config{ + Logging: config.LoggingConfig{Directory: root}, + Provider: config.ProviderConfig{Type: "docker-dind"}, + }, + ProjectRoot: root, + Logging: runtime, + } + retired := ProvisionedInstance{ + Name: "retired-runner", + LogPath: filepath.Join(root, "instances", "retired-runner.docker-dind.log"), + GuestLogPath: filepath.Join(root, "instances", "retired-runner.guest.log"), + } + livePath := filepath.Join(root, "instances", "live-runner.guest.log") + for _, item := range []struct { + path string + instance string + component string + }{ + {retired.LogPath, retired.Name, "provider"}, + {retired.GuestLogPath, retired.Name, "guest"}, + {livePath, "live-runner", "guest"}, + } { + transcript, err := manager.transcript(item.path, item.instance, item.component) + if err != nil { + t.Fatal(err) + } + if _, err := transcript.Stdout.Write([]byte("old\n")); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-48 * time.Hour) + if err := os.Chtimes(item.path, old, old); err != nil { + t.Fatal(err) + } + } + if err := manager.releaseInstanceTranscripts(retired); err != nil { + t.Fatal(err) + } + report, err := logging.PruneRetention(root, logging.RetentionPolicy{InstanceMaxAge: time.Hour}, false) + if err != nil { + t.Fatal(err) + } + if report.Deleted != 2 { + t.Fatalf("deleted = %d, report = %#v", report.Deleted, report) + } + for _, path := range []string{retired.LogPath, retired.GuestLogPath} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("retired transcript %s was not deleted: %v", path, err) + } + } + if _, err := os.Stat(livePath); err != nil { + t.Fatalf("live transcript was not protected: %v", err) + } +} + +func TestRetirementSuccessIsNotReversedByTranscriptCloseFailure(t *testing.T) { + root := t.TempDir() + runtime, err := logging.NewRuntime(logging.Options{Directory: root, TranscriptSinks: logging.SinkFile}) + if err != nil { + t.Fatal(err) + } + defer runtime.Close() + provider := &fakeProvider{} + manager := Manager{ + Config: config.Config{ + Logging: config.LoggingConfig{Directory: root}, + Provider: config.ProviderConfig{Type: "docker-dind"}, + }, + Provider: provider, + ProjectRoot: root, + Logging: runtime, + } + vm := ProvisionedInstance{Name: "retired-runner", LogPath: filepath.Join(root, "instances", "retired-runner.docker-dind.log")} + transcript, err := manager.transcript(vm.LogPath, vm.Name, "provider") + if err != nil { + t.Fatal(err) + } + if _, err := transcript.Stdout.Write([]byte("line\n")); err != nil { + t.Fatal(err) + } + metadataFiles, err := filepath.Glob(filepath.Join(root, ".epar-control", "active", "*.json")) + if err != nil || len(metadataFiles) != 1 { + t.Fatalf("active metadata = %v, err = %v", metadataFiles, err) + } + data, err := os.ReadFile(metadataFiles[0]) + if err != nil { + t.Fatal(err) + } + var state map[string]any + if err := json.Unmarshal(data, &state); err != nil { + t.Fatal(err) + } + state["ownerToken"] = "changed-owner" + data, err = json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(metadataFiles[0], data, 0o600); err != nil { + t.Fatal(err) + } + if err := manager.retireInstance(context.Background(), vm, "test"); err != nil { + t.Fatalf("retireInstance returned transcript close failure after successful provider deletion: %v", err) + } + if got := atomic.LoadInt32(&provider.deleteCalls); got != 1 { + t.Fatalf("provider delete calls = %d, want 1", got) + } +} + func TestRunnerAliveRetiresIdleRunnerWhenServiceIsInactive(t *testing.T) { provider := &fakeProvider{execErr: errors.New("inactive")} github := &fakeGitHub{ @@ -125,7 +244,8 @@ func TestRunPoolDoesNotReplaceWhenRetirementIsDeferred(t *testing.T) { manager := Manager{ Config: config.Config{ Provider: config.ProviderConfig{SourceImage: "image"}, - Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test", LogDir: t.TempDir()}, + Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test"}, + Logging: config.LoggingConfig{Directory: t.TempDir()}, Runner: config.RunnerConfig{Labels: []string{"self-hosted"}}, }, Provider: provider, @@ -157,7 +277,8 @@ func TestRunPoolReplacesCompletedRunnerAfterBusyProvisioning(t *testing.T) { manager := Manager{ Config: config.Config{ Provider: config.ProviderConfig{SourceImage: "image"}, - Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test", LogDir: t.TempDir()}, + Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test"}, + Logging: config.LoggingConfig{Directory: t.TempDir()}, Runner: config.RunnerConfig{Labels: []string{"self-hosted"}, Ephemeral: true}, }, Provider: provider, @@ -200,7 +321,8 @@ func TestRunPoolAddsCurrentTrustCapacityWhileOldGenerationDrains(t *testing.T) { manager := Manager{ Config: config.Config{ Provider: config.ProviderConfig{SourceImage: "image", Type: "docker-dind"}, - Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test", LogDir: t.TempDir()}, + Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test"}, + Logging: config.LoggingConfig{Directory: t.TempDir()}, Runner: config.RunnerConfig{Labels: []string{"self-hosted"}, Ephemeral: true}, Image: config.ImageConfig{ HostTrustMode: config.HostTrustModeOverlay, HostTrustScopes: []string{"system"}, @@ -267,7 +389,8 @@ func TestVerifyUsesIdleReadiness(t *testing.T) { manager := Manager{ Config: config.Config{ Provider: config.ProviderConfig{SourceImage: "image"}, - Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test", LogDir: t.TempDir()}, + Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test"}, + Logging: config.LoggingConfig{Directory: t.TempDir()}, Runner: config.RunnerConfig{Labels: []string{"self-hosted"}, Ephemeral: true}, }, Provider: provider, @@ -291,7 +414,8 @@ func TestRunPoolUsesConfiguredInstancesWhenNoOverride(t *testing.T) { manager := Manager{ Config: config.Config{ Provider: config.ProviderConfig{SourceImage: "image"}, - Pool: config.PoolConfig{Instances: 2, NamePrefix: "epar-test", LogDir: t.TempDir()}, + Pool: config.PoolConfig{Instances: 2, NamePrefix: "epar-test"}, + Logging: config.LoggingConfig{Directory: t.TempDir()}, }, Provider: provider, ProjectRoot: t.TempDir(), @@ -324,7 +448,8 @@ func TestProvisionOneRetriesTransientRuntimeValidationFailure(t *testing.T) { manager := Manager{ Config: config.Config{ Provider: config.ProviderConfig{SourceImage: "image", Type: "docker-dind"}, - Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test", LogDir: t.TempDir()}, + Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test"}, + Logging: config.LoggingConfig{Directory: t.TempDir()}, Timeouts: config.TimeoutConfig{CommandSeconds: 5}, }, Provider: provider, @@ -350,7 +475,8 @@ func TestVerifyCleanupUsesFreshContextAfterCancellation(t *testing.T) { manager := Manager{ Config: config.Config{ Provider: config.ProviderConfig{SourceImage: "image", Type: "docker-dind"}, - Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test", LogDir: t.TempDir()}, + Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test"}, + Logging: config.LoggingConfig{Directory: t.TempDir()}, Timeouts: config.TimeoutConfig{CommandSeconds: 5}, }, Provider: provider, @@ -380,7 +506,8 @@ func TestRunPoolCleanupUsesFreshContextAfterCancellation(t *testing.T) { manager := Manager{ Config: config.Config{ Provider: config.ProviderConfig{SourceImage: "image", Type: "docker-dind"}, - Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test", LogDir: t.TempDir()}, + Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test"}, + Logging: config.LoggingConfig{Directory: t.TempDir()}, Timeouts: config.TimeoutConfig{CommandSeconds: 5}, }, Provider: provider, @@ -418,8 +545,8 @@ func TestProvisionOnePassesRunnerRegistrationControlsWithoutPrivateKey(t *testin Pool: config.PoolConfig{ Instances: 1, NamePrefix: "epar-test", - LogDir: t.TempDir(), }, + Logging: config.LoggingConfig{Directory: t.TempDir()}, Runner: config.RunnerConfig{ Labels: []string{"epar-core-test"}, Ephemeral: true, @@ -609,7 +736,8 @@ func newRegisteredTestManager(t *testing.T, provider provider.Provider, github G return Manager{ Config: config.Config{ Provider: config.ProviderConfig{SourceImage: "image", Type: "docker-dind"}, - Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test", LogDir: t.TempDir()}, + Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test"}, + Logging: config.LoggingConfig{Directory: t.TempDir()}, Runner: config.RunnerConfig{Labels: []string{"self-hosted"}, Ephemeral: true}, Timeouts: config.TimeoutConfig{CommandSeconds: 5, GitHubOnlineSeconds: 5}, }, diff --git a/internal/pool/startup_timing.go b/internal/pool/startup_timing.go index 2ddcff2..3042992 100644 --- a/internal/pool/startup_timing.go +++ b/internal/pool/startup_timing.go @@ -2,15 +2,13 @@ package pool import ( "encoding/json" - "fmt" + "log/slog" "os" "path/filepath" "regexp" "strings" "sync" "time" - - "github.com/solutionforest/ephemeral-action-runner/internal/config" ) var timingSecretPattern = regexp.MustCompile(`(?i)((?:runner_)?token|password|secret|private[_-]?key)=\S+`) @@ -39,6 +37,7 @@ type startupTiming struct { stages []startupTimingStage firstInstance string closed bool + logger *slog.Logger } // StartStartupTiming records the initial Docker-DinD or WSL start path. @@ -47,7 +46,7 @@ func (m *Manager) StartStartupTiming() (string, error) { if !supportsStartupTiming(provider) { return "", nil } - dir := filepath.Join(config.ProjectPath(m.ProjectRoot, m.Config.Pool.LogDir), "benchmarks") + dir := filepath.Join(m.logRoot(), "benchmarks") if err := os.MkdirAll(dir, 0755); err != nil { return "", err } @@ -63,6 +62,7 @@ func (m *Manager) StartStartupTiming() (string, error) { path: path, provider: provider, startedAt: startedAt, + logger: m.logger(), } m.startupTiming = timing timing.eventLocked("total_startup", "started", 0, nil) @@ -152,11 +152,9 @@ func (t *startupTiming) finish(err error) { } else { t.stages = append(t.stages, startupTimingStage{name: "total_startup", elapsed: elapsed}) t.eventLocked("total_startup", "completed", elapsed, nil) - fmt.Printf("%s startup timing (first runner online/idle):\n", startupTimingLabel(t.provider)) for _, stage := range t.stages { - fmt.Printf(" %s: %s\n", stage.name, stage.elapsed.Round(time.Millisecond)) + t.logger.Info("startup timing", "provider", t.provider, "operation", "startup", "stage", stage.name, "duration", stage.elapsed.Round(time.Millisecond), "logPath", t.path) } - fmt.Printf(" timing log: %s\n", t.path) } t.closed = true _ = t.file.Close() diff --git a/internal/pool/trusted_ca.go b/internal/pool/trusted_ca.go index 4767a58..dde68f6 100644 --- a/internal/pool/trusted_ca.go +++ b/internal/pool/trusted_ca.go @@ -129,7 +129,7 @@ func (m *Manager) installTrustedCACertificates(ctx context.Context, vmName strin if len(certificates) == 0 { return nil } - if _, err := m.Provider.Exec(ctx, vmName, provider.ShellCommand("if command -v sudo >/dev/null 2>&1; then sudo mkdir -p "+shellQuote(trustedCAGuestDir)+"; else mkdir -p "+shellQuote(trustedCAGuestDir)+"; fi"), provider.ExecOptions{}); err != nil { + if _, err := m.execGuest(ctx, vmName, provider.ShellCommand("if command -v sudo >/dev/null 2>&1; then sudo mkdir -p "+shellQuote(trustedCAGuestDir)+"; else mkdir -p "+shellQuote(trustedCAGuestDir)+"; fi"), provider.ExecOptions{}); err != nil { return err } for _, certificate := range certificates { diff --git a/internal/provider/dockerdind/docker_dind.go b/internal/provider/dockerdind/docker_dind.go index df04549..d672666 100644 --- a/internal/provider/dockerdind/docker_dind.go +++ b/internal/provider/dockerdind/docker_dind.go @@ -5,9 +5,7 @@ import ( "context" "fmt" "io" - "os" "os/exec" - "path/filepath" "strings" "time" @@ -28,7 +26,7 @@ type Provider struct { runCommand runCommandFunc } -type runCommandFunc func(ctx context.Context, stdin io.Reader, logPath string, args ...string) (provider.ExecResult, error) +type runCommandFunc func(ctx context.Context, stdin io.Reader, logPath string, stdout, stderr io.Writer, args ...string) (provider.ExecResult, error) func New(binary, platform string, dryRun bool) *Provider { return NewWithOptions(binary, platform, false, nil, dryRun) @@ -54,7 +52,7 @@ func (p *Provider) Start(ctx context.Context, name string, opts provider.StartOp if _, err := p.run(ctx, nil, "start", name); err != nil { return nil, err } - if err := p.waitDocker(ctx, name, opts.LogPath); err != nil { + if err := p.waitDocker(ctx, name, opts); err != nil { return nil, err } return &provider.RunningProcess{Name: name, PID: 0, LogPath: opts.LogPath}, nil @@ -74,7 +72,7 @@ func (p *Provider) Exec(ctx context.Context, name string, command []string, opts if opts.Stdin != "" { stdin = strings.NewReader(opts.Stdin) } - return p.runWithLog(ctx, stdin, opts.LogPath, args...) + return p.runWithLog(ctx, stdin, opts.LogPath, opts.Stdout, opts.Stderr, args...) } func (p *Provider) IP(ctx context.Context, name string, waitSeconds int) (string, error) { @@ -172,11 +170,15 @@ func (p *Provider) createArgs(source, name string) []string { return args } -func (p *Provider) waitDocker(ctx context.Context, name, logPath string) error { +func (p *Provider) waitDocker(ctx context.Context, name string, opts provider.StartOptions) error { deadline := time.Now().Add(120 * time.Second) var lastErr error for { - _, err := p.Exec(ctx, name, []string{"docker", "info"}, provider.ExecOptions{LogPath: logPath}) + _, err := p.Exec(ctx, name, []string{"docker", "info"}, provider.ExecOptions{ + LogPath: opts.LogPath, + Stdout: opts.Stdout, + Stderr: opts.Stderr, + }) if err == nil { return nil } @@ -193,12 +195,12 @@ func (p *Provider) waitDocker(ctx context.Context, name, logPath string) error { } func (p *Provider) run(ctx context.Context, stdin io.Reader, args ...string) (provider.ExecResult, error) { - return p.runWithLog(ctx, stdin, "", args...) + return p.runWithLog(ctx, stdin, "", nil, nil, args...) } -func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath string, args ...string) (provider.ExecResult, error) { +func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath string, stdoutSink, stderrSink io.Writer, args ...string) (provider.ExecResult, error) { if p.runCommand != nil { - return p.runCommand(ctx, stdin, logPath, args...) + return p.runCommand(ctx, stdin, logPath, stdoutSink, stderrSink, args...) } if p.DryRun { fmt.Printf("[dry-run] %s %s\n", p.Binary, strings.Join(args, " ")) @@ -209,25 +211,8 @@ func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath stri cmd.Stdin = stdin } var stdout, stderr bytes.Buffer - var logFile *os.File - if logPath != "" { - if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { - return provider.ExecResult{}, err - } - f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return provider.ExecResult{}, err - } - defer f.Close() - logFile = f - } - if logFile != nil { - cmd.Stdout = io.MultiWriter(&stdout, logFile) - cmd.Stderr = io.MultiWriter(&stderr, logFile) - } else { - cmd.Stdout = &stdout - cmd.Stderr = &stderr - } + cmd.Stdout = captureWriter(&stdout, stdoutSink) + cmd.Stderr = captureWriter(&stderr, stderrSink) err := cmd.Run() result := provider.ExecResult{Stdout: stdout.String(), Stderr: stderr.String()} if err != nil { @@ -236,6 +221,13 @@ func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath stri return result, nil } +func captureWriter(capture io.Writer, sink io.Writer) io.Writer { + if sink == nil { + return capture + } + return io.MultiWriter(capture, sink) +} + func isMissingContainer(text string) bool { text = strings.ToLower(text) return strings.Contains(text, "no such container") || strings.Contains(text, "is not a container") diff --git a/internal/provider/dockerdind/docker_dind_test.go b/internal/provider/dockerdind/docker_dind_test.go index b7215e7..eb916bb 100644 --- a/internal/provider/dockerdind/docker_dind_test.go +++ b/internal/provider/dockerdind/docker_dind_test.go @@ -1,6 +1,7 @@ package dockerdind import ( + "bytes" "context" "errors" "io" @@ -11,6 +12,30 @@ import ( "github.com/solutionforest/ephemeral-action-runner/internal/provider" ) +func TestExecPassesInjectedTranscriptWriters(t *testing.T) { + p := New("docker", "", false) + var stdout, stderr bytes.Buffer + p.runCommand = func(_ context.Context, _ io.Reader, logPath string, gotStdout, gotStderr io.Writer, _ ...string) (provider.ExecResult, error) { + if logPath != "display.log" { + t.Fatalf("logPath = %q", logPath) + } + _, _ = gotStdout.Write([]byte("out")) + _, _ = gotStderr.Write([]byte("err")) + return provider.ExecResult{}, nil + } + _, err := p.Exec(context.Background(), "runner", []string{"true"}, provider.ExecOptions{ + LogPath: "display.log", + Stdout: &stdout, + Stderr: &stderr, + }) + if err != nil { + t.Fatal(err) + } + if stdout.String() != "out" || stderr.String() != "err" { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + func TestCreateArgsUsePrivilegedWithoutHostSocketOrPorts(t *testing.T) { p := NewWithOptions("docker", "linux/arm64", true, map[string]string{ "NO_PROXY": "localhost,127.0.0.1", @@ -49,7 +74,7 @@ func TestExecArgsPreserveEnvAndStdin(t *testing.T) { p := New("docker", "", false) var gotArgs []string var gotStdin bool - p.runCommand = func(_ context.Context, stdin io.Reader, _ string, args ...string) (provider.ExecResult, error) { + p.runCommand = func(_ context.Context, stdin io.Reader, _ string, _, _ io.Writer, args ...string) (provider.ExecResult, error) { gotArgs = append([]string(nil), args...) gotStdin = stdin != nil return provider.ExecResult{}, nil @@ -72,7 +97,7 @@ func TestExecArgsPreserveEnvAndStdin(t *testing.T) { func TestListParsesDockerPSOutput(t *testing.T) { p := New("docker", "", false) - p.runCommand = func(_ context.Context, _ io.Reader, _ string, args ...string) (provider.ExecResult, error) { + p.runCommand = func(_ context.Context, _ io.Reader, _ string, _, _ io.Writer, args ...string) (provider.ExecResult, error) { if strings.Join(args, " ") != "ps -a --filter label=epar.provider=docker-dind --format {{.Names}}\t{{.Image}}\t{{.Status}}" { t.Fatalf("unexpected list args: %#v", args) } @@ -109,7 +134,7 @@ func TestStopAndDeleteIgnoreMissingContainer(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { p := New("docker", "", false) - p.runCommand = func(_ context.Context, _ io.Reader, _ string, _ ...string) (provider.ExecResult, error) { + p.runCommand = func(_ context.Context, _ io.Reader, _ string, _, _ io.Writer, _ ...string) (provider.ExecResult, error) { return provider.ExecResult{Stderr: "Error response from daemon: No such container: epar-core-1"}, errors.New("exit status 1") } if err := test.call(p); err != nil { diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 4f0dbf6..6607fda 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -3,6 +3,7 @@ package provider import ( "context" "fmt" + "io" "strings" ) @@ -16,6 +17,8 @@ type StartOptions struct { Network string RosettaTag string LogPath string + Stdout io.Writer + Stderr io.Writer } type RunningProcess struct { @@ -28,6 +31,8 @@ type ExecOptions struct { Stdin string Env map[string]string LogPath string + Stdout io.Writer + Stderr io.Writer } type ExecResult struct { diff --git a/internal/provider/tart/tart.go b/internal/provider/tart/tart.go index a679cf7..edc529b 100644 --- a/internal/provider/tart/tart.go +++ b/internal/provider/tart/tart.go @@ -5,9 +5,7 @@ import ( "context" "fmt" "io" - "os" "os/exec" - "path/filepath" "strings" "github.com/solutionforest/ephemeral-action-runner/internal/provider" @@ -49,36 +47,14 @@ func (p *Provider) Start(ctx context.Context, name string, opts provider.StartOp fmt.Printf("[dry-run] %s %s\n", p.Binary, strings.Join(args, " ")) return &provider.RunningProcess{Name: name, PID: 0, LogPath: opts.LogPath}, nil } - if opts.LogPath != "" { - if err := os.MkdirAll(filepath.Dir(opts.LogPath), 0755); err != nil { - return nil, err - } - } cmd := exec.CommandContext(ctx, p.Binary, args...) - var logFile *os.File - if opts.LogPath != "" { - f, err := os.OpenFile(opts.LogPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return nil, err - } - logFile = f - cmd.Stdout = f - cmd.Stderr = f - } else { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } + cmd.Stdout = writerOrDiscard(opts.Stdout) + cmd.Stderr = writerOrDiscard(opts.Stderr) if err := cmd.Start(); err != nil { - if logFile != nil { - _ = logFile.Close() - } return nil, err } go func() { _ = cmd.Wait() - if logFile != nil { - _ = logFile.Close() - } }() return &provider.RunningProcess{Name: name, PID: cmd.Process.Pid, LogPath: opts.LogPath}, nil } @@ -90,7 +66,7 @@ func (p *Provider) Exec(ctx context.Context, name string, command []string, opts } full = append(full, name) full = append(full, provider.EnvCommand(opts.Env, command)...) - return p.runWithLog(ctx, strings.NewReader(opts.Stdin), opts.LogPath, full...) + return p.runWithLog(ctx, strings.NewReader(opts.Stdin), opts.Stdout, opts.Stderr, full...) } func (p *Provider) IP(ctx context.Context, name string, waitSeconds int) (string, error) { @@ -135,10 +111,10 @@ func (p *Provider) List(ctx context.Context) ([]provider.Instance, error) { } func (p *Provider) run(ctx context.Context, stdin io.Reader, args ...string) (provider.ExecResult, error) { - return p.runWithLog(ctx, stdin, "", args...) + return p.runWithLog(ctx, stdin, nil, nil, args...) } -func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath string, args ...string) (provider.ExecResult, error) { +func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, stdoutSink, stderrSink io.Writer, args ...string) (provider.ExecResult, error) { if p.DryRun { fmt.Printf("[dry-run] %s %s\n", p.Binary, strings.Join(args, " ")) return provider.ExecResult{}, nil @@ -148,25 +124,8 @@ func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath stri cmd.Stdin = stdin } var stdout, stderr bytes.Buffer - var logFile *os.File - if logPath != "" { - if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { - return provider.ExecResult{}, err - } - f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return provider.ExecResult{}, err - } - defer f.Close() - logFile = f - } - if logFile != nil { - cmd.Stdout = io.MultiWriter(&stdout, logFile) - cmd.Stderr = io.MultiWriter(&stderr, logFile) - } else { - cmd.Stdout = &stdout - cmd.Stderr = &stderr - } + cmd.Stdout = captureWriter(&stdout, stdoutSink) + cmd.Stderr = captureWriter(&stderr, stderrSink) err := cmd.Run() result := provider.ExecResult{Stdout: stdout.String(), Stderr: stderr.String()} if err != nil { @@ -174,3 +133,17 @@ func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath stri } return result, nil } + +func captureWriter(capture io.Writer, sink io.Writer) io.Writer { + if sink == nil { + return capture + } + return io.MultiWriter(capture, sink) +} + +func writerOrDiscard(writer io.Writer) io.Writer { + if writer == nil { + return io.Discard + } + return writer +} diff --git a/internal/provider/tart/tart_test.go b/internal/provider/tart/tart_test.go index 9fcd46e..3601b8f 100644 --- a/internal/provider/tart/tart_test.go +++ b/internal/provider/tart/tart_test.go @@ -22,6 +22,17 @@ func TestListParsesTartOutputShape(t *testing.T) { } } +func TestCaptureWriterCopiesToCaptureAndInjectedSink(t *testing.T) { + var capture, sink bytes.Buffer + writer := captureWriter(&capture, &sink) + if _, err := writer.Write([]byte("transcript")); err != nil { + t.Fatal(err) + } + if capture.String() != "transcript" || sink.String() != "transcript" { + t.Fatalf("capture=%q sink=%q", capture.String(), sink.String()) + } +} + func TestStartDryRunIncludesRosettaBeforeName(t *testing.T) { p := New("tart", true) out := captureStdout(t, func() { diff --git a/internal/provider/wsl/wsl.go b/internal/provider/wsl/wsl.go index 9438aca..27a940d 100644 --- a/internal/provider/wsl/wsl.go +++ b/internal/provider/wsl/wsl.go @@ -23,7 +23,7 @@ type Provider struct { runCommand runCommandFunc } -type runCommandFunc func(ctx context.Context, stdin io.Reader, logPath string, args ...string) (provider.ExecResult, error) +type runCommandFunc func(ctx context.Context, stdin io.Reader, logPath string, stdout, stderr io.Writer, args ...string) (provider.ExecResult, error) func New(binary, installRoot, projectRoot string, dryRun bool) *Provider { if binary == "" { @@ -70,11 +70,15 @@ func (p *Provider) Start(ctx context.Context, name string, opts provider.StartOp default: return nil, fmt.Errorf("unsupported wsl network mode %q", opts.Network) } - keeper, err := p.startKeepAlive(name, opts.LogPath) + keeper, err := p.startKeepAlive(name, opts.Stdout, opts.Stderr) if err != nil { return nil, err } - _, err = p.Exec(ctx, name, []string{"/bin/sh", "-c", "true"}, provider.ExecOptions{LogPath: opts.LogPath}) + _, err = p.Exec(ctx, name, []string{"/bin/sh", "-c", "true"}, provider.ExecOptions{ + LogPath: opts.LogPath, + Stdout: opts.Stdout, + Stderr: opts.Stderr, + }) if err != nil { if keeper != nil && keeper.Process != nil { _ = keeper.Process.Kill() @@ -93,7 +97,7 @@ func (p *Provider) Exec(ctx context.Context, name string, command []string, opts if opts.Stdin != "" { stdin = strings.NewReader(opts.Stdin) } - return p.runWithLog(ctx, stdin, opts.LogPath, p.execArgs(name, command, opts.Env)...) + return p.runWithLog(ctx, stdin, opts.LogPath, opts.Stdout, opts.Stderr, p.execArgs(name, command, opts.Env)...) } func (p *Provider) IP(ctx context.Context, name string, waitSeconds int) (string, error) { @@ -220,39 +224,22 @@ func (p *Provider) execArgs(name string, command []string, env map[string]string return append(args, provider.EnvCommand(env, command)...) } -func (p *Provider) startKeepAlive(name, logPath string) (*exec.Cmd, error) { +func (p *Provider) startKeepAlive(name string, stdoutSink, stderrSink io.Writer) (*exec.Cmd, error) { args := p.keepAliveArgs(name) if p.DryRun { fmt.Printf("[dry-run] %s %s\n", p.Binary, strings.Join(args, " ")) return nil, nil } cmd := exec.Command(p.Binary, args...) - var logFile *os.File - if logPath != "" { - if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { - return nil, err - } - f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return nil, err - } - logFile = f - cmd.Stdout = f - cmd.Stderr = f - } + cmd.Stdout = writerOrDiscard(stdoutSink) + cmd.Stderr = writerOrDiscard(stderrSink) if err := cmd.Start(); err != nil { - if logFile != nil { - _ = logFile.Close() - } return nil, fmt.Errorf("%s %s failed: %w", p.Binary, strings.Join(args, " "), err) } go func() { err := cmd.Wait() - if logFile != nil { - if err != nil { - _, _ = fmt.Fprintf(logFile, "\n[wsl keepalive exited: %v]\n", err) - } - _ = logFile.Close() + if err != nil && stderrSink != nil { + _, _ = fmt.Fprintf(stderrSink, "\n[wsl keepalive exited: %v]\n", err) } }() return cmd, nil @@ -282,12 +269,12 @@ func (p *Provider) instanceDir(name string) (string, error) { } func (p *Provider) run(ctx context.Context, stdin io.Reader, args ...string) (provider.ExecResult, error) { - return p.runWithLog(ctx, stdin, "", args...) + return p.runWithLog(ctx, stdin, "", nil, nil, args...) } -func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath string, args ...string) (provider.ExecResult, error) { +func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath string, stdoutSink, stderrSink io.Writer, args ...string) (provider.ExecResult, error) { if p.runCommand != nil { - return p.runCommand(ctx, stdin, logPath, args...) + return p.runCommand(ctx, stdin, logPath, stdoutSink, stderrSink, args...) } if p.DryRun { fmt.Printf("[dry-run] %s %s\n", p.Binary, strings.Join(args, " ")) @@ -298,25 +285,8 @@ func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath stri cmd.Stdin = stdin } var stdout, stderr bytes.Buffer - var logFile *os.File - if logPath != "" { - if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { - return provider.ExecResult{}, err - } - f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return provider.ExecResult{}, err - } - defer f.Close() - logFile = f - } - if logFile != nil { - cmd.Stdout = io.MultiWriter(&stdout, logFile) - cmd.Stderr = io.MultiWriter(&stderr, logFile) - } else { - cmd.Stdout = &stdout - cmd.Stderr = &stderr - } + cmd.Stdout = captureWriter(&stdout, stdoutSink) + cmd.Stderr = captureWriter(&stderr, stderrSink) err := cmd.Run() result := provider.ExecResult{ Stdout: cleanWSLOutput(stdout.Bytes()), @@ -328,6 +298,20 @@ func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath stri return result, nil } +func captureWriter(capture io.Writer, sink io.Writer) io.Writer { + if sink == nil { + return capture + } + return io.MultiWriter(capture, sink) +} + +func writerOrDiscard(writer io.Writer) io.Writer { + if writer == nil { + return io.Discard + } + return writer +} + func parseList(text string) []provider.Instance { var out []provider.Instance for _, line := range strings.Split(cleanWSLOutput([]byte(text)), "\n") { diff --git a/internal/provider/wsl/wsl_test.go b/internal/provider/wsl/wsl_test.go index 4aa8b39..71bbb11 100644 --- a/internal/provider/wsl/wsl_test.go +++ b/internal/provider/wsl/wsl_test.go @@ -1,6 +1,7 @@ package wsl import ( + "bytes" "context" "errors" "io" @@ -13,6 +14,30 @@ import ( "github.com/solutionforest/ephemeral-action-runner/internal/provider" ) +func TestExecPassesInjectedTranscriptWriters(t *testing.T) { + p := New("wsl.exe", t.TempDir(), t.TempDir(), false) + var stdout, stderr bytes.Buffer + p.runCommand = func(_ context.Context, _ io.Reader, logPath string, gotStdout, gotStderr io.Writer, _ ...string) (provider.ExecResult, error) { + if logPath != "display.log" { + t.Fatalf("logPath = %q", logPath) + } + _, _ = gotStdout.Write([]byte("out")) + _, _ = gotStderr.Write([]byte("err")) + return provider.ExecResult{}, nil + } + _, err := p.Exec(context.Background(), "runner", []string{"true"}, provider.ExecOptions{ + LogPath: "display.log", + Stdout: &stdout, + Stderr: &stderr, + }) + if err != nil { + t.Fatal(err) + } + if stdout.String() != "out" || stderr.String() != "err" { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + func TestCommandConstruction(t *testing.T) { p := New("wsl.exe", filepath.Join("C:", "ephemeral"), `D:\repo`, true) installDir, err := p.instanceDir("epar-test-1") @@ -112,7 +137,7 @@ func TestDeleteIgnoresUnregisterFailureOnlyWhenDistroIsAbsent(t *testing.T) { if err := os.MkdirAll(instanceDir, 0755); err != nil { t.Fatal(err) } - p.runCommand = func(_ context.Context, _ io.Reader, _ string, args ...string) (provider.ExecResult, error) { + p.runCommand = func(_ context.Context, _ io.Reader, _ string, _, _ io.Writer, args ...string) (provider.ExecResult, error) { switch strings.Join(args, " ") { case "--unregister epar-wsl-1": return provider.ExecResult{}, errors.New("wsl.exe --unregister epar-wsl-1 failed: exit status 0xffffffff:") @@ -134,7 +159,7 @@ func TestDeleteIgnoresUnregisterFailureOnlyWhenDistroIsAbsent(t *testing.T) { func TestDeleteReturnsUnregisterFailureWhenDistroStillExists(t *testing.T) { p := New("wsl.exe", t.TempDir(), t.TempDir(), false) - p.runCommand = func(_ context.Context, _ io.Reader, _ string, args ...string) (provider.ExecResult, error) { + p.runCommand = func(_ context.Context, _ io.Reader, _ string, _, _ io.Writer, args ...string) (provider.ExecResult, error) { switch strings.Join(args, " ") { case "--unregister epar-wsl-1": return provider.ExecResult{}, errors.New("wsl.exe --unregister epar-wsl-1 failed: exit status 0xffffffff:") diff --git a/scripts/ci/core-runner-controller.sh b/scripts/ci/core-runner-controller.sh index cb8447b..5aae410 100644 --- a/scripts/ci/core-runner-controller.sh +++ b/scripts/ci/core-runner-controller.sh @@ -260,7 +260,25 @@ cat >>"${config_path}" < Date: Thu, 16 Jul 2026 01:48:58 +0800 Subject: [PATCH 3/9] Normalize Windows console line endings --- internal/logging/console_writer.go | 70 +++++++++++++++++ internal/logging/console_writer_test.go | 99 +++++++++++++++++++++++++ internal/logging/types.go | 2 + internal/pool/logging.go | 2 +- 4 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 internal/logging/console_writer.go create mode 100644 internal/logging/console_writer_test.go diff --git a/internal/logging/console_writer.go b/internal/logging/console_writer.go new file mode 100644 index 0000000..7588074 --- /dev/null +++ b/internal/logging/console_writer.go @@ -0,0 +1,70 @@ +package logging + +import ( + "bytes" + "io" + "runtime" + "sync" + + "golang.org/x/term" +) + +// ConsoleWriter preserves native Windows line endings when output is written +// directly to a terminal. Redirected output and non-Windows platforms are left +// unchanged. +func ConsoleWriter(writer io.Writer) io.Writer { + if runtime.GOOS != "windows" { + return writer + } + file, ok := writer.(interface{ Fd() uintptr }) + if !ok || !term.IsTerminal(int(file.Fd())) { + return writer + } + return &windowsConsoleWriter{writer: writer} +} + +type windowsConsoleWriter struct { + writer io.Writer + mu sync.Mutex + previousCR bool +} + +func (writer *windowsConsoleWriter) Write(data []byte) (int, error) { + writer.mu.Lock() + defer writer.mu.Unlock() + + normalized := windowsConsoleLineEndingsAfter(data, writer.previousCR) + written, err := writer.writer.Write(normalized) + if written == len(normalized) && len(data) > 0 { + writer.previousCR = data[len(data)-1] == '\r' + } + if err != nil { + if written == len(normalized) { + return len(data), err + } + return 0, err + } + if written != len(normalized) { + return 0, io.ErrShortWrite + } + return len(data), nil +} + +func windowsConsoleLineEndings(data []byte) []byte { + return windowsConsoleLineEndingsAfter(data, false) +} + +func windowsConsoleLineEndingsAfter(data []byte, previousCR bool) []byte { + if !bytes.Contains(data, []byte{'\n'}) { + return data + } + normalized := make([]byte, 0, len(data)+bytes.Count(data, []byte{'\n'})) + for index, value := range data { + precededByCR := index > 0 && data[index-1] == '\r' || index == 0 && previousCR + if value == '\n' && !precededByCR { + normalized = append(normalized, '\r') + } + normalized = append(normalized, value) + } + return normalized +} diff --git a/internal/logging/console_writer_test.go b/internal/logging/console_writer_test.go new file mode 100644 index 0000000..3ccb9c1 --- /dev/null +++ b/internal/logging/console_writer_test.go @@ -0,0 +1,99 @@ +package logging + +import ( + "bytes" + "errors" + "testing" +) + +type fullWriteErrorOnceWriter struct { + bytes.Buffer + err error +} + +func (writer *fullWriteErrorOnceWriter) Write(data []byte) (int, error) { + written, _ := writer.Buffer.Write(data) + err := writer.err + writer.err = nil + return written, err +} + +func TestWindowsConsoleLineEndings(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {name: "lone LF", in: "one\ntwo\n", want: "one\r\ntwo\r\n"}, + {name: "existing CRLF", in: "one\r\ntwo\r\n", want: "one\r\ntwo\r\n"}, + {name: "mixed", in: "one\r\ntwo\nthree", want: "one\r\ntwo\r\nthree"}, + {name: "no newline", in: "one", want: "one"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := string(windowsConsoleLineEndings([]byte(tt.in))); got != tt.want { + t.Fatalf("windowsConsoleLineEndings(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +func TestWindowsConsoleWriterReportsOriginalByteCount(t *testing.T) { + var output bytes.Buffer + writer := &windowsConsoleWriter{writer: &output} + input := []byte("one\ntwo\n") + + written, err := writer.Write(input) + if err != nil { + t.Fatal(err) + } + if written != len(input) { + t.Fatalf("Write() bytes = %d, want %d", written, len(input)) + } + if got, want := output.String(), "one\r\ntwo\r\n"; got != want { + t.Fatalf("Write() output = %q, want %q", got, want) + } +} + +func TestWindowsConsoleWriterPreservesSplitCRLF(t *testing.T) { + var output bytes.Buffer + writer := &windowsConsoleWriter{writer: &output} + + for _, input := range []string{"one\r", "\ntwo\n"} { + if _, err := writer.Write([]byte(input)); err != nil { + t.Fatal(err) + } + } + if got, want := output.String(), "one\r\ntwo\r\n"; got != want { + t.Fatalf("split Write() output = %q, want %q", got, want) + } +} + +func TestWindowsConsoleWriterReportsFullWriteWithError(t *testing.T) { + sentinel := errors.New("injected write error") + output := &fullWriteErrorOnceWriter{err: sentinel} + writer := &windowsConsoleWriter{writer: output} + + first := []byte("one\r") + written, err := writer.Write(first) + if !errors.Is(err, sentinel) { + t.Fatalf("Write() error = %v, want %v", err, sentinel) + } + if written != len(first) { + t.Fatalf("Write() bytes = %d, want %d", written, len(first)) + } + if _, err := writer.Write([]byte("\ntwo\n")); err != nil { + t.Fatal(err) + } + if got, want := output.String(), "one\r\ntwo\r\n"; got != want { + t.Fatalf("Write() output = %q, want %q", got, want) + } +} + +func TestConsoleWriterLeavesNonFileWriterUnchanged(t *testing.T) { + var output bytes.Buffer + if got := ConsoleWriter(&output); got != &output { + t.Fatalf("ConsoleWriter() wrapped a non-terminal writer: %T", got) + } +} diff --git a/internal/logging/types.go b/internal/logging/types.go index 32b0073..9972d8f 100644 --- a/internal/logging/types.go +++ b/internal/logging/types.go @@ -140,6 +140,8 @@ func (options Options) normalized() (Options, error) { if options.Stderr == nil { options.Stderr = os.Stderr } + options.Stdout = ConsoleWriter(options.Stdout) + options.Stderr = ConsoleWriter(options.Stderr) if options.Now == nil { options.Now = time.Now } diff --git a/internal/pool/logging.go b/internal/pool/logging.go index b4501cc..f898f85 100644 --- a/internal/pool/logging.go +++ b/internal/pool/logging.go @@ -18,7 +18,7 @@ func (m *Manager) logger() *slog.Logger { if m != nil && m.Logging != nil { return m.Logging.Logger() } - return slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ReplaceAttr: func(_ []string, attribute slog.Attr) slog.Attr { + return slog.New(slog.NewTextHandler(logging.ConsoleWriter(os.Stdout), &slog.HandlerOptions{ReplaceAttr: func(_ []string, attribute slog.Attr) slog.Attr { if attribute.Key == slog.TimeKey || attribute.Key == slog.LevelKey { return slog.Attr{} } From 8046202dcb82c46ee5540bbbca8cf0ec86408716 Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 16 Jul 2026 10:17:45 +0800 Subject: [PATCH 4/9] Feature/fix ci failure (#7) Stabilize cross-platform logging tests --- cmd/ephemeral-action-runner/logging_test.go | 8 +++++-- internal/logging/paths.go | 25 +++++++++++++++++---- internal/logging/runtime_test.go | 6 +++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/cmd/ephemeral-action-runner/logging_test.go b/cmd/ephemeral-action-runner/logging_test.go index 0885958..46cae38 100644 --- a/cmd/ephemeral-action-runner/logging_test.go +++ b/cmd/ephemeral-action-runner/logging_test.go @@ -43,8 +43,12 @@ func TestLogsPathListAndPrune(t *testing.T) { if err != nil { t.Fatal(err) } - if !strings.Contains(strings.ToLower(listOutput), strings.ToLower(oldPath)) { - t.Fatalf("logs list output missing %s:\n%s", oldPath, listOutput) + canonicalOldPath, err := filepath.EvalSymlinks(oldPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.ToLower(listOutput), strings.ToLower(canonicalOldPath)) { + t.Fatalf("logs list output missing %s:\n%s", canonicalOldPath, listOutput) } if _, err := captureStdout(t, func() error { diff --git a/internal/logging/paths.go b/internal/logging/paths.go index 668a97a..c2f1607 100644 --- a/internal/logging/paths.go +++ b/internal/logging/paths.go @@ -106,11 +106,28 @@ func canonicalPath(path string) (string, error) { return "", fmt.Errorf("resolve absolute path: %w", err) } absolute = filepath.Clean(absolute) - parent, err := filepath.EvalSymlinks(filepath.Dir(absolute)) - if err == nil { - absolute = filepath.Join(parent, filepath.Base(absolute)) + + // Transcript paths are commonly validated before their category directory or + // log file exists. Resolve the nearest existing ancestor instead of only the + // direct parent so root aliases (such as /var on macOS or Windows short + // paths) cannot leave root and child paths in different representations. + var tail []string + for candidate := absolute; ; { + resolved, resolveErr := filepath.EvalSymlinks(candidate) + if resolveErr == nil { + for index := len(tail) - 1; index >= 0; index-- { + resolved = filepath.Join(resolved, tail[index]) + } + return canonicalCase(resolved), nil + } + + parent := filepath.Dir(candidate) + if parent == candidate { + return canonicalCase(absolute), nil + } + tail = append(tail, filepath.Base(candidate)) + candidate = parent } - return canonicalCase(absolute), nil } func pathWithin(root, path string) bool { diff --git a/internal/logging/runtime_test.go b/internal/logging/runtime_test.go index 7e0f8fc..c6d22f6 100644 --- a/internal/logging/runtime_test.go +++ b/internal/logging/runtime_test.go @@ -414,6 +414,12 @@ func TestRotationCompressesAndHonorsBackupLimit(t *testing.T) { if writeErr != nil { t.Fatal(writeErr) } + // Lumberjack backup names have millisecond precision. Keep forced test + // rotations distinct so this test verifies compression and backup limits + // rather than colliding timestamped filenames on faster hosts. + if index < 3 { + time.Sleep(10 * time.Millisecond) + } } deadline := time.Now().Add(5 * time.Second) var backups []string From 1dc158638106eae68cc46bfe2056307fd040d35c Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 20 Jul 2026 14:44:00 +0800 Subject: [PATCH 5/9] Harden runner replacement during GitHub outages Enforce a strict physical pool cap with lifecycle-aware reconciliation, rollback, quarantine, controller locking, and exponential dependency backoff. Pass registration tokens through stdin, sanitize provider diagnostics, and write private error reports atomically. --- cmd/ephemeral-action-runner/init.go | 8 + cmd/ephemeral-action-runner/init_test.go | 15 + cmd/ephemeral-action-runner/logging_test.go | 62 ++ cmd/ephemeral-action-runner/main.go | 30 +- cmd/ephemeral-action-runner/start.go | 16 + configs/docker-dind.act.example.yml | 4 + configs/docker-dind.core.example.yml | 4 + configs/docker-dind.example.yml | 4 + configs/docker-dind.web-e2e.example.yml | 4 + configs/tart.example.yml | 4 + configs/tart.web-e2e.example.yml | 4 + configs/wsl.example.yml | 4 + configs/wsl.lean.example.yml | 4 + configs/wsl.web-e2e.example.yml | 4 + docs/configuration.md | 18 +- docs/design.md | 6 + docs/operations.md | 12 + docs/usage.md | 4 +- internal/config/config.go | 53 +- internal/config/config_test.go | 113 ++++ internal/github/client.go | 20 + internal/github/client_test.go | 20 + internal/logging/private_file.go | 42 ++ internal/logging/replace_windows.go | 29 +- internal/pool/controller_lock.go | 47 ++ internal/pool/controller_lock_test.go | 97 +++ internal/pool/host_trust.go | 2 + internal/pool/manager.go | 595 +++++++++++++++++- internal/pool/manager_test.go | 564 ++++++++++++++++- internal/pool/runner_script_test.go | 58 +- internal/pool/startup_timing.go | 10 +- internal/pool/startup_timing_test.go | 18 + internal/provider/dockerdind/docker_dind.go | 14 +- .../provider/dockerdind/docker_dind_test.go | 37 ++ internal/provider/provider.go | 11 +- internal/provider/redaction.go | 160 +++++ internal/provider/redaction_test.go | 74 +++ internal/provider/tart/tart.go | 24 +- internal/provider/tart/tart_test.go | 38 ++ internal/provider/wsl/wsl.go | 14 +- internal/provider/wsl/wsl_test.go | 37 ++ scripts/guest/ubuntu/configure-runner.sh | 9 +- 42 files changed, 2204 insertions(+), 89 deletions(-) create mode 100644 internal/logging/private_file.go create mode 100644 internal/pool/controller_lock.go create mode 100644 internal/pool/controller_lock_test.go create mode 100644 internal/pool/startup_timing_test.go create mode 100644 internal/provider/redaction.go create mode 100644 internal/provider/redaction_test.go diff --git a/cmd/ephemeral-action-runner/init.go b/cmd/ephemeral-action-runner/init.go index 4302ef3..3847769 100644 --- a/cmd/ephemeral-action-runner/init.go +++ b/cmd/ephemeral-action-runner/init.go @@ -481,6 +481,10 @@ image: pool: instances: 1 namePrefix: %s + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 logging: directory: work/logs managerSinks: [console] @@ -545,6 +549,10 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: %s + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 logging: directory: work/logs managerSinks: [console] diff --git a/cmd/ephemeral-action-runner/init_test.go b/cmd/ephemeral-action-runner/init_test.go index b5bd13f..0621b2a 100644 --- a/cmd/ephemeral-action-runner/init_test.go +++ b/cmd/ephemeral-action-runner/init_test.go @@ -64,6 +64,18 @@ func TestInitCreatesDefaultDockerDindConfig(t *testing.T) { if got, want := cfg.Pool.NamePrefix, "build-box-01-a4f9c2"; got != want { t.Fatalf("pool.namePrefix = %q, want %q", got, want) } + if got, want := cfg.Pool.ReplacementRetryInitialSeconds, 15; got != want { + t.Fatalf("pool.replacementRetryInitialSeconds = %d, want %d", got, want) + } + if got, want := cfg.Pool.ReplacementRetryMaxSeconds, 1800; got != want { + t.Fatalf("pool.replacementRetryMaxSeconds = %d, want %d", got, want) + } + if got, want := cfg.Pool.ReplacementRetryMultiplier, 2.0; got != want { + t.Fatalf("pool.replacementRetryMultiplier = %v, want %v", got, want) + } + if got, want := cfg.Pool.ReplacementRetryJitterPercent, 20; got != want { + t.Fatalf("pool.replacementRetryJitterPercent = %d, want %d", got, want) + } if got, want := cfg.Logging.Directory, "work/logs"; got != want { t.Fatalf("logging.directory = %q, want %q", got, want) } @@ -74,6 +86,9 @@ func TestInitCreatesDefaultDockerDindConfig(t *testing.T) { if !strings.Contains(string(configText), "logging:\n directory: work/logs\n managerSinks: [console]\n") { t.Fatalf("generated config did not include logging schema:\n%s", configText) } + if !strings.Contains(string(configText), "replacementRetryInitialSeconds: 15\n replacementRetryMaxSeconds: 1800\n replacementRetryMultiplier: 2\n replacementRetryJitterPercent: 20\n") { + t.Fatalf("generated config did not include replacement retry settings:\n%s", configText) + } if got := strings.Join(cfg.Runner.Labels, ","); !strings.Contains(got, "epar-docker-dind-catthehacker-ubuntu") { t.Fatalf("runner labels = %q", got) } diff --git a/cmd/ephemeral-action-runner/logging_test.go b/cmd/ephemeral-action-runner/logging_test.go index 46cae38..2d89e0c 100644 --- a/cmd/ephemeral-action-runner/logging_test.go +++ b/cmd/ephemeral-action-runner/logging_test.go @@ -3,8 +3,10 @@ package main import ( "errors" "io" + "io/fs" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -44,6 +46,9 @@ func TestLogsPathListAndPrune(t *testing.T) { t.Fatal(err) } canonicalOldPath, err := filepath.EvalSymlinks(oldPath) + if runtime.GOOS == "windows" && errors.Is(err, fs.ErrPermission) { + canonicalOldPath, err = filepath.Abs(oldPath) + } if err != nil { t.Fatal(err) } @@ -112,6 +117,63 @@ func TestWriteLastErrorReportUsesConfiguredDirectoryAndFallback(t *testing.T) { } } +func TestWriteLastErrorReportRedactsSecretsAndRestrictsExistingFile(t *testing.T) { + root := t.TempDir() + configPath := writeLoggingCommandConfig(t, root, "custom-logs") + logRoot := filepath.Join(root, "custom-logs") + if err := os.MkdirAll(logRoot, 0o755); err != nil { + t.Fatal(err) + } + lastPath := filepath.Join(logRoot, "epar-last-error.log") + if err := os.WriteFile(lastPath, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + const sentinel = "registration-token-sentinel" + path := writeLastErrorReport( + []string{"start", "--project-root", root, "--config", configPath}, + errors.New("docker exec -e RUNNER_TOKEN="+sentinel+" failed"), + ) + if path != lastPath { + t.Fatalf("report path = %q, want %q", path, lastPath) + } + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(content), sentinel) { + t.Fatalf("last error report exposed sentinel token: %s", content) + } + if runtime.GOOS != "windows" { + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("last error report mode = %#o, want 0600", got) + } + } + archives, err := filepath.Glob(filepath.Join(logRoot, "errors", "epar-*-error.log")) + if err != nil || len(archives) != 1 { + t.Fatalf("error archives = %v, err = %v", archives, err) + } + archiveContent, err := os.ReadFile(archives[0]) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(archiveContent), sentinel) { + t.Fatalf("archived error report exposed sentinel token: %s", archiveContent) + } + if runtime.GOOS != "windows" { + info, err := os.Stat(archives[0]) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("archived error report mode = %#o, want 0600", got) + } + } +} + func writeLoggingCommandConfig(t *testing.T, root, directory string) string { t.Helper() path := filepath.Join(root, "config.yml") diff --git a/cmd/ephemeral-action-runner/main.go b/cmd/ephemeral-action-runner/main.go index e03a181..7c5c680 100644 --- a/cmd/ephemeral-action-runner/main.go +++ b/cmd/ephemeral-action-runner/main.go @@ -52,21 +52,21 @@ func writeLastErrorReport(args []string, runErr error) string { return "" } now := time.Now().UTC() - content := fmt.Sprintf(`EPAR failed + content := provider.RedactText(fmt.Sprintf(`EPAR failed time: %s workingDirectory: %s command: %q %s error: %v -`, now.Format(time.RFC3339), cwd, os.Args, versionString(), runErr) +`, now.Format(time.RFC3339), cwd, os.Args, versionString(), runErr)) lastPath := logging.LastErrorPath(logDir) - if err := os.WriteFile(lastPath, []byte(content), 0644); err != nil { + if err := logging.WritePrivateFileAtomic(lastPath, []byte(content)); err != nil { return "" } stampedPath := logging.ErrorPath(logDir, now) - _ = os.WriteFile(stampedPath, []byte(content), 0644) + _ = logging.WritePrivateFileAtomic(stampedPath, []byte(content)) return lastPath } @@ -232,6 +232,11 @@ func runImage(args []string) error { return err } defer m.Close() + controllerLock, err := m.AcquirePoolControllerLock() + if err != nil { + return err + } + defer controllerLock.Close() return m.UpdateUpstream(context.Background()) case "build": fs := flag.NewFlagSet("image build", flag.ExitOnError) @@ -248,13 +253,17 @@ func runImage(args []string) error { } defer m.Close() ctx := interruptContext() - controllerLock, err := m.AcquireHostTrustControllerLock() + poolControllerLock, err := m.AcquirePoolControllerLock() if err != nil { return err } - defer m.Close() - if controllerLock != nil { - defer controllerLock.Close() + defer poolControllerLock.Close() + hostTrustControllerLock, err := m.AcquireHostTrustControllerLock() + if err != nil { + return err + } + if hostTrustControllerLock != nil { + defer hostTrustControllerLock.Close() } if *update { if err := m.UpdateUpstream(ctx); err != nil { @@ -273,6 +282,11 @@ func runImage(args []string) error { return err } defer m.Close() + controllerLock, err := m.AcquirePoolControllerLock() + if err != nil { + return err + } + defer controllerLock.Close() return m.RefreshScripts(interruptContext()) default: return fmt.Errorf("unknown image subcommand %q", args[0]) diff --git a/cmd/ephemeral-action-runner/start.go b/cmd/ephemeral-action-runner/start.go index fba9142..a22efe8 100644 --- a/cmd/ephemeral-action-runner/start.go +++ b/cmd/ephemeral-action-runner/start.go @@ -23,6 +23,10 @@ type hostTrustLockingStarterManager interface { AcquireHostTrustControllerLock() (io.Closer, error) } +type poolLockingStarterManager interface { + AcquirePoolControllerLock() (io.Closer, error) +} + type startupTimingStarterManager interface { StartStartupTiming() (string, error) FinishStartupTiming(error) @@ -127,6 +131,17 @@ func runStartWithOptions(opts startOptions) (err error) { timingManager.FinishStartupTiming(err) }() } + poolLockHeld := false + if lockingManager, ok := manager.(poolLockingStarterManager); ok { + controllerLock, err := lockingManager.AcquirePoolControllerLock() + if err != nil { + return err + } + if controllerLock != nil { + defer controllerLock.Close() + poolLockHeld = true + } + } hostTrustLockHeld := false if lockingManager, ok := manager.(hostTrustLockingStarterManager); ok { controllerLock, err := lockingManager.AcquireHostTrustControllerLock() @@ -153,6 +168,7 @@ func runStartWithOptions(opts startOptions) (err error) { KeepOnExit: opts.KeepOnExit, ReplaceCompleted: opts.ReplaceCompleted, MonitorInterval: opts.MonitorInterval, + PoolLockHeld: poolLockHeld, HostTrustLockHeld: hostTrustLockHeld, }) return err diff --git a/configs/docker-dind.act.example.yml b/configs/docker-dind.act.example.yml index 9c51164..3afb483 100644 --- a/configs/docker-dind.act.example.yml +++ b/configs/docker-dind.act.example.yml @@ -22,6 +22,10 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 logging: directory: work/logs managerSinks: [console] diff --git a/configs/docker-dind.core.example.yml b/configs/docker-dind.core.example.yml index 2ec64d6..f4fdfbc 100644 --- a/configs/docker-dind.core.example.yml +++ b/configs/docker-dind.core.example.yml @@ -22,6 +22,10 @@ image: pool: instances: 1 namePrefix: epar-ci-core + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 logging: directory: work/logs managerSinks: [console] diff --git a/configs/docker-dind.example.yml b/configs/docker-dind.example.yml index fea1ad1..30c6655 100644 --- a/configs/docker-dind.example.yml +++ b/configs/docker-dind.example.yml @@ -22,6 +22,10 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 logging: directory: work/logs managerSinks: [console] diff --git a/configs/docker-dind.web-e2e.example.yml b/configs/docker-dind.web-e2e.example.yml index abbd6ec..70da56e 100644 --- a/configs/docker-dind.web-e2e.example.yml +++ b/configs/docker-dind.web-e2e.example.yml @@ -23,6 +23,10 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 logging: directory: work/logs managerSinks: [console] diff --git a/configs/tart.example.yml b/configs/tart.example.yml index d182ec4..885278d 100644 --- a/configs/tart.example.yml +++ b/configs/tart.example.yml @@ -18,6 +18,10 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 logging: directory: work/logs managerSinks: [console] diff --git a/configs/tart.web-e2e.example.yml b/configs/tart.web-e2e.example.yml index 22bd36e..ad7fce8 100644 --- a/configs/tart.web-e2e.example.yml +++ b/configs/tart.web-e2e.example.yml @@ -18,6 +18,10 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 logging: directory: work/logs managerSinks: [console] diff --git a/configs/wsl.example.yml b/configs/wsl.example.yml index c39457e..2caf870 100644 --- a/configs/wsl.example.yml +++ b/configs/wsl.example.yml @@ -20,6 +20,10 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 logging: directory: work/logs managerSinks: [console] diff --git a/configs/wsl.lean.example.yml b/configs/wsl.lean.example.yml index 612105c..6fdd411 100644 --- a/configs/wsl.lean.example.yml +++ b/configs/wsl.lean.example.yml @@ -19,6 +19,10 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 logging: directory: work/logs managerSinks: [console] diff --git a/configs/wsl.web-e2e.example.yml b/configs/wsl.web-e2e.example.yml index e0576c5..b454272 100644 --- a/configs/wsl.web-e2e.example.yml +++ b/configs/wsl.web-e2e.example.yml @@ -19,6 +19,10 @@ pool: instances: 1 # Must be unique for this machine/config within the GitHub organization. namePrefix: CHANGE-ME-unique-machine-prefix + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 logging: directory: work/logs managerSinks: [console] diff --git a/docs/configuration.md b/docs/configuration.md index 148f91d..07aa156 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,7 +20,7 @@ EPAR looks for config in this order: | `github` | GitHub App ID, organization, private key path, and optional GitHub API/web URLs. | | `provider` | How EPAR creates disposable runners: `docker-dind`, `wsl`, or `tart`. | | `image` | Source image/rootfs, output image, runner version, and optional install scripts. | -| `pool` | Runner count and instance name prefix. | +| `pool` | Runner count, instance name prefix, and replacement retry policy. | | `logging` | Manager and transcript sinks, formats, rotation, retention, and log directory. | | `runner` | GitHub Actions labels, runner group, default-label policy, and whether to add the host-machine label. | | `docker` | Optional Docker registry mirrors and Docker-DinD daemon proxy settings. | @@ -44,6 +44,22 @@ pool: `pool.namePrefix` is both the prefix for generated GitHub runner names and the cleanup boundary for GitHub runner records. It must be 2-40 characters and should leave room for EPAR's generated `-YYYYMMDD-HHMMSS-###` suffix. Do not reuse the same prefix on different machines or for separate EPAR supervisors in the same organization. If two machines share a prefix, one machine's cleanup can delete the other machine's GitHub runner record, causing the other supervisor to report that the runner record is gone and replace a healthy runner. +Configure replacement retry behavior after a transient GitHub or network outage: + +```yaml +pool: + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 +``` + +These values default to `15`, `1800`, `2`, and `20`, so existing configurations remain valid without changes. `replacementRetryInitialSeconds` must be positive, `replacementRetryMaxSeconds` must be at least the initial delay, `replacementRetryMultiplier` must be at least `1`, and `replacementRetryJitterPercent` must be from `0` through `100`. + +The supervisor backs off only replacement allocation after transient network errors and GitHub HTTP `429` or `5xx` responses. The nominal delay doubles from 15 seconds to a 30-minute cap with the configured jitter; a longer GitHub `Retry-After` response takes precedence. Authentication and deterministic configuration failures remain fail-fast after safe rollback. Initial `pool up` startup also remains fail-fast rather than entering an unattended retry loop. + +`pool.instances` is an absolute local physical-instance cap, not only an online-runner target. Provisioning, ready, draining, quarantined, and cleanup-pending instances all count toward it. Host-trust generation rotation does not receive surge capacity: an old busy runner keeps its slot until it exits or is safely removed. + Add or change workflow labels: ```yaml diff --git a/docs/design.md b/docs/design.md index e5aab2a..5126491 100644 --- a/docs/design.md +++ b/docs/design.md @@ -59,6 +59,12 @@ sequenceDiagram `pool up --instances 2` keeps two runners available in the foreground. Replacement names use `pool.namePrefix` plus a timestamp and sequence suffix, for example `epar-20260703-010530-003`. +The supervisor tracks every prefix-owned local resource as provisioning, ready, draining, quarantined, or cleanup-pending. All five states count against `pool.instances`, which is an absolute physical-instance cap. A cleanup failure, unknown remote state, or busy runner therefore blocks allocation rather than allowing a capacity surge; host-trust rotation follows the same rule. + +Reconciliation runs at startup and before each replacement allocation. It adopts healthy exact-name local/GitHub pairs, deletes proven stopped or unregistered local resources, and deletes exact stale GitHub records when GitHub is reachable. A pre-listener provisioning failure is locally rolled back immediately and queued for later exact-name GitHub reconciliation because registration may have partially succeeded. After the listener starts, uncertain GitHub readiness becomes quarantine rather than deletion until the remote state can be resolved. + +Supervised replacement treats network errors and GitHub HTTP `429` or `5xx` responses as transient: allocation pauses behind the configured exponential retry policy while monitoring and cleanup continue. A fully online replacement or successful adoption resets the retry delay. Initial pool startup remains fail-fast after safe rollback, and authentication or deterministic configuration failures are terminal rather than retryable. + ## Liveness Model The foreground supervisor checks each instance every 15 seconds by default. A runner is considered healthy when: diff --git a/docs/operations.md b/docs/operations.md index e998250..132d9fd 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -22,6 +22,16 @@ When `docker.registryMirrors` is configured, EPAR writes `/etc/docker/daemon.jso `pool up` cleans up prefixed instances and GitHub runner records when it exits. Use `--keep-on-exit` only when intentionally debugging a live instance after the supervisor stops. While the supervisor is not running, EPAR cannot retire or replace completed runners. +## Capacity, Reconciliation, and Outage Recovery + +`pool.instances` is a strict cap on prefix-owned local resources. EPAR counts provisioning, ready, draining, quarantined, and cleanup-pending instances, so a GitHub outage or a failed cleanup cannot create a replacement storm. Host-trust rotation has no temporary surge allowance; old busy-generation instances retain their slots until they finish or can be safely retired. + +Only one controller may mutate a given canonical config, provider, and `pool.namePrefix` at a time. If another EPAR controller holds that lock, stop or reconfigure the other controller instead of forcing concurrent starts; use a distinct unique prefix for intentionally independent pools. + +At startup and before allocating a replacement, EPAR reconciles the provider inventory with exact-name GitHub runner records. Healthy pairs are adopted, stopped or proven unregistered local resources are removed, and exact stale GitHub records are deleted when GitHub is reachable. When GitHub is unavailable, an ambiguous local instance is quarantined and continues to consume capacity instead of being deleted or replaced. If old resources already exceed the configured cap, the supervisor creates nothing until safe cleanup or normal draining restores capacity. + +For transient network failures and GitHub `429` or `5xx` responses during supervised replacement, EPAR pauses allocation and retries with exponential backoff. The default nominal delays are 15, 30, 60, 120, 240, 480, 960, and 1800 seconds, each with ±20% jitter; a longer `Retry-After` response wins. Monitoring and housekeeping continue during that pause, and a successful adoption or fully online replacement resets the delay. Startup remains fail-fast after compensating rollback, and invalid configuration or GitHub authentication failures do not retry indefinitely. + ## Cleanup Safety Cleanup only touches local instances and GitHub runner records matching `pool.namePrefix`: @@ -55,6 +65,8 @@ This section is a compact checklist. For symptom-first diagnostics with host/pro - If repeated jobs still pull slowly after configuring a registry mirror, verify the mirror is reachable from inside the runner instance and that it supports the requested registry, image platform, and authentication model. Docker daemon mirrors primarily target Docker Hub; other registry caches may require workflow image references to use the cache registry URL. - If a mirrored workflow only improves modestly, check where the time is going. Registry mirrors mainly reduce image pull time; container startup, Compose health checks, database initialization, volume sync, browser tests, private image authentication, and CPU-bound or emulated workloads can still dominate the total job time. - If GitHub registration fails, confirm the app has permission to manage organization self-hosted runners and that the private key path is readable by the host user. +- If GitHub returns transient `500`, `502`, `503`, or `429` errors while the supervisor is replacing a runner, leave the supervisor running so it can retain the strict cap, reconcile exact runner names after recovery, and retry on its configured backoff. Do not manually start a second supervisor with the same `pool.namePrefix`. +- If registration fails before the runner listener starts, EPAR rolls back the local candidate immediately and later reconciles a possible exact-name GitHub record. If the listener already started but GitHub readiness is uncertain, EPAR quarantines that candidate; inspect the instance transcript and wait for GitHub recovery before manually deleting it. - If stale runners remain, run `ephemeral-action-runner cleanup`. - If using Tart `softnet`, verify the host has the privileges Tart requires. - If default WSL image build fails before import, confirm Docker Desktop, Docker Engine, or another Docker daemon is reachable so EPAR can export `ghcr.io/catthehacker/ubuntu:full-latest` into a rootfs tar. For lean WSL configs, confirm the clean Ubuntu rootfs was exported from an Ubuntu 24.04 WSL distro. diff --git a/docs/usage.md b/docs/usage.md index 1aadd30..d408748 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -273,7 +273,9 @@ go run ./cmd/ephemeral-action-runner pool up --instances 2 `start` is the recommended public command because it also checks the image before starting runners. `pool up` is the lower-level supervisor command for users who already prepared the image. -`pool up` keeps the requested number of runners online. Each GitHub ephemeral runner exits after one job. EPAR then retires that instance and creates a fresh replacement. +`pool up` keeps the requested number of runners online. Each GitHub ephemeral runner exits after one job. EPAR then retires that instance and creates a fresh replacement. The requested count is a strict physical local-instance cap: provisioning, ready, draining, quarantined, and cleanup-pending resources all consume a slot, including old runners during host-trust rotation. + +When GitHub registration or readiness has a transient network, `429`, or `5xx` failure during supervised replacement, EPAR pauses allocation and retries with the configured exponential backoff while continuing monitoring and cleanup. It does not create extra candidates while remote state is uncertain; see [Configuration](configuration.md#common-edits) and [Operations](operations.md#capacity-reconciliation-and-outage-recovery) for retry settings and recovery steps. Stop the supervisor with Ctrl-C. By default, EPAR cleans up active instances and matching GitHub runner records before it exits. diff --git a/internal/config/config.go b/internal/config/config.go index fdb0256..71c1a1a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,6 +3,7 @@ package config import ( "bufio" "fmt" + "math" "net" "net/url" "os" @@ -59,8 +60,12 @@ const ( ) type PoolConfig struct { - Instances int - NamePrefix string + Instances int + NamePrefix string + ReplacementRetryInitialSeconds int + ReplacementRetryMaxSeconds int + ReplacementRetryMultiplier float64 + ReplacementRetryJitterPercent int } type LoggingConfig struct { @@ -140,8 +145,12 @@ func Default() Config { }, }, Pool: PoolConfig{ - Instances: 1, - NamePrefix: "epar", + Instances: 1, + NamePrefix: "epar", + ReplacementRetryInitialSeconds: 15, + ReplacementRetryMaxSeconds: 1800, + ReplacementRetryMultiplier: 2, + ReplacementRetryJitterPercent: 20, }, Logging: LoggingConfig{ Directory: "work/logs", @@ -345,6 +354,30 @@ func apply(cfg *Config, section, key, value string) error { cfg.Pool.Instances = v case "namePrefix", "vmPrefix": cfg.Pool.NamePrefix = value + case "replacementRetryInitialSeconds": + v, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("invalid pool.replacementRetryInitialSeconds: %w", err) + } + cfg.Pool.ReplacementRetryInitialSeconds = v + case "replacementRetryMaxSeconds": + v, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("invalid pool.replacementRetryMaxSeconds: %w", err) + } + cfg.Pool.ReplacementRetryMaxSeconds = v + case "replacementRetryMultiplier": + v, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid pool.replacementRetryMultiplier: %w", err) + } + cfg.Pool.ReplacementRetryMultiplier = v + case "replacementRetryJitterPercent": + v, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("invalid pool.replacementRetryJitterPercent: %w", err) + } + cfg.Pool.ReplacementRetryJitterPercent = v case "logDir": return fmt.Errorf("pool.logDir is deprecated; use logging.directory") default: @@ -815,6 +848,18 @@ func Validate(cfg Config) error { if cfg.Pool.Instances < 1 { return fmt.Errorf("pool.instances must be 1 or greater") } + if cfg.Pool.ReplacementRetryInitialSeconds <= 0 { + return fmt.Errorf("pool.replacementRetryInitialSeconds must be greater than zero") + } + if cfg.Pool.ReplacementRetryMaxSeconds < cfg.Pool.ReplacementRetryInitialSeconds { + return fmt.Errorf("pool.replacementRetryMaxSeconds must be greater than or equal to pool.replacementRetryInitialSeconds") + } + if math.IsNaN(cfg.Pool.ReplacementRetryMultiplier) || math.IsInf(cfg.Pool.ReplacementRetryMultiplier, 0) || cfg.Pool.ReplacementRetryMultiplier < 1 { + return fmt.Errorf("pool.replacementRetryMultiplier must be 1 or greater") + } + if cfg.Pool.ReplacementRetryJitterPercent < 0 || cfg.Pool.ReplacementRetryJitterPercent > 100 { + return fmt.Errorf("pool.replacementRetryJitterPercent must be between 0 and 100") + } if err := ValidatePrefix(cfg.Pool.NamePrefix); err != nil { return err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 03d67a7..f1c2407 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,6 +1,7 @@ package config import ( + "math" "os" "path/filepath" "slices" @@ -78,6 +79,18 @@ image: if got, want := cfg.Pool.Instances, 3; got != want { t.Fatalf("pool.instances = %d, want %d", got, want) } + if got, want := cfg.Pool.ReplacementRetryInitialSeconds, 15; got != want { + t.Fatalf("pool.replacementRetryInitialSeconds = %d, want %d", got, want) + } + if got, want := cfg.Pool.ReplacementRetryMaxSeconds, 1800; got != want { + t.Fatalf("pool.replacementRetryMaxSeconds = %d, want %d", got, want) + } + if got, want := cfg.Pool.ReplacementRetryMultiplier, 2.0; got != want { + t.Fatalf("pool.replacementRetryMultiplier = %v, want %v", got, want) + } + if got, want := cfg.Pool.ReplacementRetryJitterPercent, 20; got != want { + t.Fatalf("pool.replacementRetryJitterPercent = %d, want %d", got, want) + } if got, want := len(cfg.Image.CustomInstallScripts), 2; got != want { t.Fatalf("custom install scripts = %d, want %d", got, want) } @@ -899,6 +912,106 @@ func TestValidateRejectsInvalidPoolInstances(t *testing.T) { } } +func TestLoadPoolReplacementRetryConfiguration(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yml") + if err := os.WriteFile(path, []byte(` +pool: + replacementRetryInitialSeconds: 12 + replacementRetryMaxSeconds: 720 + replacementRetryMultiplier: 1.5 + replacementRetryJitterPercent: 0 +`), 0644); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if got, want := cfg.Pool.ReplacementRetryInitialSeconds, 12; got != want { + t.Fatalf("pool.replacementRetryInitialSeconds = %d, want %d", got, want) + } + if got, want := cfg.Pool.ReplacementRetryMaxSeconds, 720; got != want { + t.Fatalf("pool.replacementRetryMaxSeconds = %d, want %d", got, want) + } + if got, want := cfg.Pool.ReplacementRetryMultiplier, 1.5; got != want { + t.Fatalf("pool.replacementRetryMultiplier = %v, want %v", got, want) + } + if got, want := cfg.Pool.ReplacementRetryJitterPercent, 0; got != want { + t.Fatalf("pool.replacementRetryJitterPercent = %d, want %d", got, want) + } + if err := Validate(cfg); err != nil { + t.Fatal(err) + } +} + +func TestValidateRejectsInvalidPoolReplacementRetryConfiguration(t *testing.T) { + tests := []struct { + name string + mutate func(*Config) + }{ + { + name: "initial retry delay is not positive", + mutate: func(cfg *Config) { cfg.Pool.ReplacementRetryInitialSeconds = 0 }, + }, + { + name: "maximum retry delay is below initial delay", + mutate: func(cfg *Config) { cfg.Pool.ReplacementRetryMaxSeconds = cfg.Pool.ReplacementRetryInitialSeconds - 1 }, + }, + { + name: "retry multiplier is below one", + mutate: func(cfg *Config) { cfg.Pool.ReplacementRetryMultiplier = 0.5 }, + }, + { + name: "retry multiplier is not finite", + mutate: func(cfg *Config) { cfg.Pool.ReplacementRetryMultiplier = math.NaN() }, + }, + { + name: "retry jitter is below zero", + mutate: func(cfg *Config) { cfg.Pool.ReplacementRetryJitterPercent = -1 }, + }, + { + name: "retry jitter exceeds one hundred percent", + mutate: func(cfg *Config) { cfg.Pool.ReplacementRetryJitterPercent = 101 }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := Default() + test.mutate(&cfg) + if err := Validate(cfg); err == nil { + t.Fatal("Validate accepted invalid replacement retry configuration") + } + }) + } +} + +func TestLoadRejectsInvalidPoolReplacementRetryValues(t *testing.T) { + tests := []struct { + name string + key string + value string + }{ + {name: "initial retry delay", key: "replacementRetryInitialSeconds", value: "invalid"}, + {name: "maximum retry delay", key: "replacementRetryMaxSeconds", value: "invalid"}, + {name: "retry multiplier", key: "replacementRetryMultiplier", value: "invalid"}, + {name: "retry jitter", key: "replacementRetryJitterPercent", value: "invalid"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yml") + content := "pool:\n " + test.key + ": " + test.value + "\n" + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil { + t.Fatal("Load accepted invalid replacement retry value") + } + }) + } +} + func TestValidateRejectsOverlongRunnerLabel(t *testing.T) { cfg := Default() cfg.Runner.IncludeHostLabel = false diff --git a/internal/github/client.go b/internal/github/client.go index c0e1b9a..6ccc2dd 100644 --- a/internal/github/client.go +++ b/internal/github/client.go @@ -236,6 +236,7 @@ func (c *Client) request(ctx context.Context, method, path, auth string, body, o Path: path, StatusCode: resp.StatusCode, Body: strings.TrimSpace(string(respBody)), + RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After"), time.Now()), } } if out == nil || len(respBody) == 0 { @@ -249,8 +250,27 @@ type HTTPError struct { Path string StatusCode int Body string + RetryAfter time.Duration } func (e *HTTPError) Error() string { return fmt.Sprintf("github %s %s returned %d: %s", e.Method, e.Path, e.StatusCode, e.Body) } + +func parseRetryAfter(value string, now time.Time) time.Duration { + value = strings.TrimSpace(value) + if value == "" { + return 0 + } + if seconds, err := time.ParseDuration(value + "s"); err == nil { + if seconds > 0 { + return seconds + } + return 0 + } + when, err := http.ParseTime(value) + if err != nil || !when.After(now) { + return 0 + } + return when.Sub(now) +} diff --git a/internal/github/client_test.go b/internal/github/client_test.go index d380d47..b0c87d6 100644 --- a/internal/github/client_test.go +++ b/internal/github/client_test.go @@ -27,6 +27,26 @@ func TestJWTShape(t *testing.T) { } } +func TestParseRetryAfter(t *testing.T) { + now := time.Date(2026, 7, 20, 10, 0, 0, 0, time.UTC) + for _, test := range []struct { + name string + value string + want time.Duration + }{ + {name: "seconds", value: "120", want: 2 * time.Minute}, + {name: "http date", value: now.Add(3 * time.Minute).Format(http.TimeFormat), want: 3 * time.Minute}, + {name: "past date", value: now.Add(-time.Minute).Format(http.TimeFormat), want: 0}, + {name: "invalid", value: "later", want: 0}, + } { + t.Run(test.name, func(t *testing.T) { + if got := parseRetryAfter(test.value, now); got != test.want { + t.Fatalf("parseRetryAfter(%q) = %s, want %s", test.value, got, test.want) + } + }) + } +} + func TestListRunnersUsesInstallationToken(t *testing.T) { keyPath := writeKey(t) var sawRunnerList bool diff --git a/internal/logging/private_file.go b/internal/logging/private_file.go new file mode 100644 index 0000000..89f65eb --- /dev/null +++ b/internal/logging/private_file.go @@ -0,0 +1,42 @@ +package logging + +import ( + "fmt" + "os" + "path/filepath" +) + +// WritePrivateFileAtomic replaces path with content through a same-directory +// temporary file and enforces owner-only permissions on the result. +func WritePrivateFileAtomic(path string, content []byte) error { + directory := filepath.Dir(path) + temporary, err := os.CreateTemp(directory, "."+filepath.Base(path)+"-*") + if err != nil { + return fmt.Errorf("create private temporary file: %w", err) + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return fmt.Errorf("restrict private temporary file: %w", err) + } + if _, err := temporary.Write(content); err != nil { + _ = temporary.Close() + return fmt.Errorf("write private temporary file: %w", err) + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return fmt.Errorf("sync private temporary file: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close private temporary file: %w", err) + } + if err := replaceFile(temporaryPath, path); err != nil { + return fmt.Errorf("replace private file: %w", err) + } + if err := os.Chmod(path, 0o600); err != nil { + return fmt.Errorf("restrict private file: %w", err) + } + return nil +} diff --git a/internal/logging/replace_windows.go b/internal/logging/replace_windows.go index 503531f..f66344e 100644 --- a/internal/logging/replace_windows.go +++ b/internal/logging/replace_windows.go @@ -3,16 +3,33 @@ package logging import ( - "errors" - "os" + "syscall" + "unsafe" ) +const ( + moveFileReplaceExisting = 0x00000001 + moveFileWriteThrough = 0x00000008 +) + +var moveFileEx = syscall.NewLazyDLL("kernel32.dll").NewProc("MoveFileExW") + func replaceFile(source, destination string) error { - if err := os.Rename(source, destination); err == nil { - return nil + sourcePath, err := syscall.UTF16PtrFromString(source) + if err != nil { + return err } - if err := os.Remove(destination); err != nil && !errors.Is(err, os.ErrNotExist) { + destinationPath, err := syscall.UTF16PtrFromString(destination) + if err != nil { return err } - return os.Rename(source, destination) + result, _, callErr := moveFileEx.Call( + uintptr(unsafe.Pointer(sourcePath)), + uintptr(unsafe.Pointer(destinationPath)), + moveFileReplaceExisting|moveFileWriteThrough, + ) + if result == 0 { + return callErr + } + return nil } diff --git a/internal/pool/controller_lock.go b/internal/pool/controller_lock.go new file mode 100644 index 0000000..c0c259a --- /dev/null +++ b/internal/pool/controller_lock.go @@ -0,0 +1,47 @@ +package pool + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/solutionforest/ephemeral-action-runner/internal/hosttrust" +) + +// AcquirePoolControllerLock excludes another mutating controller for the same +// canonical configuration, provider, and pool prefix on this host. +func (m *Manager) AcquirePoolControllerLock() (io.Closer, error) { + providerType := strings.TrimSpace(strings.ToLower(m.Config.Provider.Type)) + namePrefix := strings.TrimSpace(strings.ToLower(m.Config.Pool.NamePrefix)) + if providerType == "" || namePrefix == "" { + return nil, fmt.Errorf("acquire pool controller lock: provider.type and pool.namePrefix are required") + } + configPath := strings.TrimSpace(m.ConfigPath) + if configPath == "" { + configPath = filepath.Join(m.ProjectRoot, ".local", "config.yml") + } + canonicalConfig, err := filepath.Abs(configPath) + if err != nil { + return nil, fmt.Errorf("acquire pool controller lock: resolve config path: %w", err) + } + if resolved, resolveErr := filepath.EvalSymlinks(canonicalConfig); resolveErr == nil { + canonicalConfig = resolved + } + canonicalConfig = filepath.Clean(canonicalConfig) + if runtime.GOOS == "windows" { + canonicalConfig = strings.ToLower(canonicalConfig) + } + identity := canonicalConfig + "\x00" + providerType + "\x00" + namePrefix + sum := sha256.Sum256([]byte(identity)) + syntheticPath := filepath.Join(os.TempDir(), "ephemeral-action-runner", "pool-controller", hex.EncodeToString(sum[:])+".identity") + lock, err := hosttrust.AcquireConfigLock(syntheticPath) + if err != nil { + return nil, fmt.Errorf("acquire pool controller lock for provider %q prefix %q: %w", providerType, namePrefix, err) + } + return lock, nil +} diff --git a/internal/pool/controller_lock_test.go b/internal/pool/controller_lock_test.go new file mode 100644 index 0000000..93a8fa8 --- /dev/null +++ b/internal/pool/controller_lock_test.go @@ -0,0 +1,97 @@ +package pool + +import ( + "context" + "strings" + "testing" + + "github.com/solutionforest/ephemeral-action-runner/internal/config" +) + +func TestPoolControllerLockConflictsForSameProviderAndPrefix(t *testing.T) { + t.Setenv("LOCALAPPDATA", t.TempDir()) + manager := Manager{ConfigPath: "config.yml", Config: config.Config{Provider: config.ProviderConfig{Type: "docker-dind"}, Pool: config.PoolConfig{NamePrefix: "epar-lock-same"}}} + first, err := manager.AcquirePoolControllerLock() + if err != nil { + t.Fatal(err) + } + defer first.Close() + if second, err := manager.AcquirePoolControllerLock(); err == nil { + _ = second.Close() + t.Fatal("second controller acquired the same provider/prefix lock") + } +} + +func TestVerifyRequiresPoolControllerLock(t *testing.T) { + t.Setenv("LOCALAPPDATA", t.TempDir()) + manager := Manager{ProjectRoot: t.TempDir(), ConfigPath: "verify.yml", Config: config.Config{Provider: config.ProviderConfig{Type: "docker-dind", SourceImage: "image"}, Pool: config.PoolConfig{NamePrefix: "epar-lock-verify"}}, Provider: &fakeProvider{}} + held, err := manager.AcquirePoolControllerLock() + if err != nil { + t.Fatal(err) + } + defer held.Close() + err = manager.Verify(context.Background(), VerifyOptions{Instances: 1}) + if err == nil || !strings.Contains(err.Error(), "lock") { + t.Fatalf("Verify() error = %v, want controller lock conflict", err) + } +} + +func TestCleanupRequiresPoolControllerLock(t *testing.T) { + t.Setenv("LOCALAPPDATA", t.TempDir()) + manager := Manager{ProjectRoot: t.TempDir(), ConfigPath: "cleanup.yml", Config: config.Config{Provider: config.ProviderConfig{Type: "docker-dind"}, Pool: config.PoolConfig{NamePrefix: "epar-lock-cleanup"}}, Provider: &fakeProvider{}} + held, err := manager.AcquirePoolControllerLock() + if err != nil { + t.Fatal(err) + } + defer held.Close() + err = manager.Cleanup(context.Background()) + if err == nil || !strings.Contains(err.Error(), "lock") { + t.Fatalf("Cleanup() error = %v, want controller lock conflict", err) + } +} + +func TestProvisionPoolRequiresPoolControllerLock(t *testing.T) { + t.Setenv("LOCALAPPDATA", t.TempDir()) + manager := Manager{ProjectRoot: t.TempDir(), ConfigPath: "provision.yml", Config: config.Config{Provider: config.ProviderConfig{Type: "docker-dind", SourceImage: "image"}, Pool: config.PoolConfig{NamePrefix: "epar-lock-provision"}}, Provider: &fakeProvider{}} + held, err := manager.AcquirePoolControllerLock() + if err != nil { + t.Fatal(err) + } + defer held.Close() + _, err = manager.ProvisionPool(context.Background(), 1, false) + if err == nil || !strings.Contains(err.Error(), "lock") { + t.Fatalf("ProvisionPool() error = %v, want controller lock conflict", err) + } +} + +func TestPoolControllerLockIsIndependentAcrossPoolIdentity(t *testing.T) { + t.Setenv("LOCALAPPDATA", t.TempDir()) + firstManager := Manager{ConfigPath: "first.yml", Config: config.Config{Provider: config.ProviderConfig{Type: "docker-dind"}, Pool: config.PoolConfig{NamePrefix: "epar-lock-first"}}} + secondManager := Manager{ConfigPath: "second.yml", Config: config.Config{Provider: config.ProviderConfig{Type: "docker-dind"}, Pool: config.PoolConfig{NamePrefix: "epar-lock-second"}}} + first, err := firstManager.AcquirePoolControllerLock() + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := secondManager.AcquirePoolControllerLock() + if err != nil { + t.Fatalf("independent pool identity was blocked: %v", err) + } + defer second.Close() +} + +func TestPoolControllerLockIncludesCanonicalConfigIdentity(t *testing.T) { + t.Setenv("LOCALAPPDATA", t.TempDir()) + firstManager := Manager{ConfigPath: "first.yml", Config: config.Config{Provider: config.ProviderConfig{Type: "docker-dind"}, Pool: config.PoolConfig{NamePrefix: "epar-lock-shared-prefix"}}} + secondManager := Manager{ConfigPath: "second.yml", Config: config.Config{Provider: config.ProviderConfig{Type: "docker-dind"}, Pool: config.PoolConfig{NamePrefix: "epar-lock-shared-prefix"}}} + first, err := firstManager.AcquirePoolControllerLock() + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := secondManager.AcquirePoolControllerLock() + if err != nil { + t.Fatalf("distinct canonical config identity was blocked: %v", err) + } + defer second.Close() +} diff --git a/internal/pool/host_trust.go b/internal/pool/host_trust.go index 3dff0b9..295c46e 100644 --- a/internal/pool/host_trust.go +++ b/internal/pool/host_trust.go @@ -331,6 +331,8 @@ func (m *Manager) reconcileHostTrustRunners(ctx context.Context, active map[stri } if found && runner.Busy { m.infof("[%s] draining busy runner on old host trust generation %s\n", name, instance.HostTrustGeneration) + instance.Phase = LifecycleDraining + active[name] = instance continue } reason := fmt.Sprintf("host trust generation changed from %s to %s", instance.HostTrustGeneration, current.Generation) diff --git a/internal/pool/manager.go b/internal/pool/manager.go index a0ee718..af81fb5 100644 --- a/internal/pool/manager.go +++ b/internal/pool/manager.go @@ -4,9 +4,15 @@ import ( "context" "errors" "fmt" + "math" + "math/rand" + "net" "net/http" "os" "path/filepath" + "regexp" + "sort" + "strconv" "strings" "sync" "time" @@ -33,6 +39,8 @@ type Manager struct { hostTrustResolver func(context.Context) (hosttrust.Snapshot, error) hostTrustImageEnsurer func(context.Context) error hostTrustImageMu sync.Mutex + now func() time.Time + randomFloat64 func() float64 } type GitHubClient interface { @@ -59,8 +67,19 @@ type RunOptions struct { ReplaceCompleted bool MonitorInterval time.Duration HostTrustLockHeld bool + PoolLockHeld bool } +type LifecyclePhase string + +const ( + LifecycleProvisioning LifecyclePhase = "provisioning" + LifecycleReady LifecyclePhase = "ready" + LifecycleDraining LifecyclePhase = "draining" + LifecycleQuarantined LifecyclePhase = "quarantined" + LifecycleCleanupPending LifecyclePhase = "cleanup-pending" +) + type ProvisionedInstance struct { Name string IP string @@ -68,10 +87,12 @@ type ProvisionedInstance struct { GuestLogPath string RunnerID int64 HostTrustGeneration string + Phase LifecyclePhase } var runtimeValidationRetryDelay = 5 * time.Second var runnerReadinessHealthCheckInterval = 2 * time.Second +var dependencyHTTPStatusPattern = regexp.MustCompile(`(?i)(?:status code does not indicate success|http response code)\s*:\s*([0-9]{3})`) const ( cleanupTimeout = 5 * time.Minute @@ -80,6 +101,11 @@ const ( ) func (m *Manager) Verify(ctx context.Context, opts VerifyOptions) error { + poolLock, err := m.AcquirePoolControllerLock() + if err != nil { + return err + } + defer poolLock.Close() controllerLock, err := m.acquireHostTrustControllerLock() if err != nil { return err @@ -148,6 +174,13 @@ func (m *Manager) Verify(ctx context.Context, opts VerifyOptions) error { } func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { + if !opts.PoolLockHeld { + controllerLock, err := m.AcquirePoolControllerLock() + if err != nil { + return err + } + defer controllerLock.Close() + } if !opts.HostTrustLockHeld { controllerLock, err := m.AcquireHostTrustControllerLock() if err != nil { @@ -161,7 +194,20 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { if opts.MonitorInterval <= 0 { opts.MonitorInterval = 15 * time.Second } - active := make(map[string]ProvisionedInstance) + if ctx.Err() != nil { + if opts.KeepOnExit { + return nil + } + return m.cleanupWithFreshContext() + } + active, err := m.reconcilePhysicalPool(ctx, nil, opts.Register) + if err != nil { + return fmt.Errorf("initial pool reconciliation: %w", err) + } + active, err = m.reduceOverCapacity(ctx, active, opts.Instances, opts.Register) + if err != nil { + return fmt.Errorf("initial over-capacity reconciliation: %w", err) + } sequence := 1 poolTrustGeneration := "" cleanup := func() error { @@ -172,14 +218,26 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { } leaseAdd, stopLeaseKeeper := m.startHostTrustLeaseKeeper(ctx) for len(active) < opts.Instances { + active, err = m.reconcilePhysicalPool(ctx, active, opts.Register) + if err != nil { + stopLeaseKeeper() + return fmt.Errorf("initial pool reconciliation before allocation: %w", err) + } + if len(active) >= opts.Instances { + break + } vm, err := m.provisionOne(ctx, RunnerName(m.Config.Pool.NamePrefix, sequence, time.Now()), opts.Register, opts.Register && opts.ReplaceCompleted) sequence++ + if isPhysicalPhase(vm.Phase) { + active[vm.Name] = vm + } if err != nil { stopLeaseKeeper() - _ = cleanup() - return err + if ctx.Err() != nil { + return cleanup() + } + return errors.Join(err, m.cleanupAfterTerminalFailure(active, opts.KeepOnExit)) } - active[vm.Name] = vm leaseAdd(vm) if vm.HostTrustGeneration != "" { poolTrustGeneration = vm.HostTrustGeneration @@ -215,11 +273,14 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { nextRetention := time.Now().Add(time.Duration(m.Config.Logging.RetentionIntervalMinutes) * time.Minute) nextHostTrustCollection := time.Time{} var currentHostTrust hosttrust.Snapshot + retry := replacementRetryState{} for { select { case <-ctx.Done(): return cleanup() case <-ticker.C: + now := m.currentTime() + dependencyCooldown := retry.active(now) if m.Config.Logging.RetentionEnabled && !time.Now().Before(nextRetention) { m.pruneLogsBestEffort() nextRetention = time.Now().Add(time.Duration(m.Config.Logging.RetentionIntervalMinutes) * time.Minute) @@ -241,7 +302,9 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { // Idle runners are removed now; busy runners keep running but // receive a mismatching lease so no subsequent job can start. currentHostTrust = current - trustRetired += m.reconcileHostTrustRunners(ctx, active, current) + if !dependencyCooldown { + trustRetired += m.reconcileHostTrustRunners(ctx, active, current) + } m.infof("host trust generation changed (%s -> %s); building replacement image\n", emptyDash(poolTrustGeneration), current.Generation) ready = false for attempt := 1; attempt <= 3; attempt++ { @@ -275,10 +338,18 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { } } } - if currentHostTrust.Generation != "" { + if currentHostTrust.Generation != "" && !dependencyCooldown { trustRetired += m.reconcileHostTrustRunners(ctx, active, currentHostTrust) } } + if dependencyCooldown { + var localErr error + active, localErr = m.reconcileLocalInventory(active) + if localErr != nil { + m.warnf("local pool housekeeping warning during replacement cooldown: %v\n", localErr) + } + continue + } if opts.ReplaceCompleted && !time.Now().Before(nextLivenessCheck) { nextLivenessCheck = time.Now().Add(opts.MonitorInterval) for name, vm := range active { @@ -298,18 +369,43 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { delete(active, name) } } + var reconcileErr error + beforeReconcile := active + active, reconcileErr = m.reconcilePhysicalPool(ctx, active, opts.Register) + if reconcileErr != nil { + if ctx.Err() != nil { + return cleanup() + } + if !isTransientDependencyError(reconcileErr) { + return errors.Join(fmt.Errorf("pool reconciliation: %w", reconcileErr), m.cleanupAfterTerminalFailure(active, opts.KeepOnExit)) + } + retry.schedule(m, now, reconcileErr) + m.warnf("pool reconciliation deferred during transient dependency failure; retrying after %s: %v\n", retry.remaining(now), reconcileErr) + continue + } + retry.resetAfterAdoption(beforeReconcile, active) + active, reconcileErr = m.reduceOverCapacity(ctx, active, opts.Instances, opts.Register) + if reconcileErr != nil { + if !isTransientDependencyError(reconcileErr) { + return errors.Join(fmt.Errorf("reduce over-capacity pool: %w", reconcileErr), m.cleanupAfterTerminalFailure(active, opts.KeepOnExit)) + } + retry.schedule(m, now, reconcileErr) + continue + } if m.hostTrustEnabled() && currentHostTrust.Generation != "" && currentHostTrust.Generation != poolTrustGeneration { trustCapacityReady = false } replacementCapacity := len(active) needsTrustCapacity := false if m.hostTrustEnabled() && currentHostTrust.Generation != "" { - replacementCapacity = currentHostTrustCapacity(active, currentHostTrust.Generation) - needsTrustCapacity = replacementCapacity < opts.Instances + needsTrustCapacity = currentHostTrustCapacity(active, currentHostTrust.Generation) < opts.Instances } if !trustCapacityReady || (!opts.ReplaceCompleted && trustRetired == 0 && !needsTrustCapacity) { continue } + if retry.active(now) { + continue + } for replacementCapacity < opts.Instances { select { case <-ctx.Done(): @@ -318,13 +414,37 @@ func (m *Manager) RunPool(ctx context.Context, opts RunOptions) error { } name := RunnerName(m.Config.Pool.NamePrefix, sequence, time.Now()) sequence++ + active, err = m.reconcilePhysicalPool(ctx, active, opts.Register) + if err != nil { + if !isTransientDependencyError(err) { + return errors.Join(fmt.Errorf("pool reconciliation before replacement allocation: %w", err), m.cleanupAfterTerminalFailure(active, opts.KeepOnExit)) + } + retry.schedule(m, m.currentTime(), err) + m.warnf("[%s] replacement deferred during transient reconciliation failure: %v\n", name, err) + break + } + replacementCapacity = len(active) + if replacementCapacity >= opts.Instances { + break + } m.infof("[%s] creating replacement runner\n", name) vm, err := m.provisionOne(ctx, name, opts.Register, true) + if isPhysicalPhase(vm.Phase) { + active[vm.Name] = vm + } if err != nil { + if ctx.Err() != nil { + return cleanup() + } m.warnf("[%s] replacement failed: %v\n", name, err) + if !isTransientDependencyError(err) { + return errors.Join(err, m.cleanupAfterTerminalFailure(active, opts.KeepOnExit)) + } + retry.schedule(m, m.currentTime(), err) break } active[vm.Name] = vm + retry.reset() replacementCapacity++ if vm.HostTrustGeneration != "" { poolTrustGeneration = vm.HostTrustGeneration @@ -345,13 +465,401 @@ func currentHostTrustCapacity(active map[string]ProvisionedInstance, generation return capacity } +func isPhysicalPhase(phase LifecyclePhase) bool { + return phase == LifecycleProvisioning || phase == LifecycleReady || phase == LifecycleDraining || phase == LifecycleQuarantined || phase == LifecycleCleanupPending +} + +func adoptedReadyInstance(before, after map[string]ProvisionedInstance) bool { + for name, instance := range after { + previous, found := before[name] + if instance.Phase == LifecycleReady && (!found || previous.Phase != LifecycleReady) { + return true + } + } + return false +} + +func (m *Manager) reconcilePhysicalPool(ctx context.Context, known map[string]ProvisionedInstance, register bool) (map[string]ProvisionedInstance, error) { + reconciled, err := m.reconcileLocalInventoryWithContext(ctx, known) + if err != nil { + return known, err + } + if !register { + for name, vm := range reconciled { + if vm.Phase != LifecycleCleanupPending { + vm.Phase = LifecycleReady + reconciled[name] = vm + } + } + return reconciled, nil + } + if m.GitHub == nil { + return reconciled, fmt.Errorf("github client is required for registered pool reconciliation") + } + remoteByName := make(map[string]gh.Runner) + runners, err := m.GitHub.ListRunners(ctx) + if err != nil { + for name, vm := range reconciled { + if vm.Phase != LifecycleCleanupPending { + vm.Phase = LifecycleQuarantined + reconciled[name] = vm + } + } + return reconciled, err + } + for _, runner := range runners { + if HasPrefix(runner.Name, m.Config.Pool.NamePrefix) { + remoteByName[runner.Name] = runner + } + } + for name, vm := range reconciled { + if vm.Phase == LifecycleCleanupPending { + continue + } + runner, found := remoteByName[name] + if !found && isPhysicalPhase(vm.Phase) { + var lookupErr error + runner, found, lookupErr = m.GitHub.RunnerByName(ctx, name) + if lookupErr != nil { + vm.Phase = LifecycleQuarantined + reconciled[name] = vm + return reconciled, lookupErr + } + } + if !found { + if err := m.deleteLocalInstance(context.Background(), vm); err != nil { + vm.Phase = LifecycleCleanupPending + reconciled[name] = vm + m.warnf("[%s] unregistered-instance cleanup pending: %v\n", name, err) + } else { + delete(reconciled, name) + } + continue + } + delete(remoteByName, name) + vm.RunnerID = runner.ID + if runner.Status == "online" { + vm.Phase = LifecycleReady + reconciled[name] = vm + continue + } + if runner.Busy { + vm.Phase = LifecycleQuarantined + reconciled[name] = vm + continue + } + if vm.Phase == LifecycleQuarantined { + if err := m.retireInstance(context.Background(), vm, "GitHub recovered but quarantined runner remained offline"); err != nil { + vm.Phase = LifecycleCleanupPending + reconciled[name] = vm + m.warnf("[%s] recovered-offline retirement pending: %v\n", name, err) + } else { + delete(reconciled, name) + } + continue + } + alive, _, processErr := m.runnerProcessAlive(ctx, vm) + if processErr == nil && alive { + vm.Phase = LifecycleQuarantined + reconciled[name] = vm + continue + } + if err := m.retireInstance(context.Background(), vm, "reconciliation found offline runner with inactive listener"); err != nil { + vm.Phase = LifecycleCleanupPending + reconciled[name] = vm + m.warnf("[%s] inactive-instance cleanup pending: %v\n", name, err) + } else { + delete(reconciled, name) + } + } + for _, runner := range remoteByName { + if err := m.deleteRemoteRunner(context.Background(), runner); err != nil { + return reconciled, err + } + m.infof("reconciliation: deleted stale GitHub runner %s id=%d\n", runner.Name, runner.ID) + } + return reconciled, nil +} + +func (m *Manager) reduceOverCapacity(ctx context.Context, active map[string]ProvisionedInstance, target int, register bool) (map[string]ProvisionedInstance, error) { + if len(active) <= target || !register || m.GitHub == nil { + return active, nil + } + names := make([]string, 0, len(active)) + for name := range active { + names = append(names, name) + } + sort.Sort(sort.Reverse(sort.StringSlice(names))) + for _, name := range names { + if len(active) <= target { + break + } + vm := active[name] + if vm.Phase != LifecycleReady { + continue + } + runner, found, err := m.GitHub.RunnerByName(ctx, name) + if err != nil { + return active, err + } + if !found || runner.Busy { + continue + } + vm.RunnerID = runner.ID + if err := m.retireInstance(context.Background(), vm, "reconciling legacy physical inventory above pool.instances"); err != nil { + vm.Phase = LifecycleCleanupPending + active[name] = vm + continue + } + delete(active, name) + } + return active, nil +} + +func (m *Manager) reconcileLocalInventory(known map[string]ProvisionedInstance) (map[string]ProvisionedInstance, error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + return m.reconcileLocalInventoryWithContext(ctx, known) +} + +func (m *Manager) reconcileLocalInventoryWithContext(ctx context.Context, known map[string]ProvisionedInstance) (map[string]ProvisionedInstance, error) { + locals, err := m.Provider.List(ctx) + if err != nil { + return known, err + } + reconciled := make(map[string]ProvisionedInstance) + for _, local := range locals { + if !HasPrefix(local.Name, m.Config.Pool.NamePrefix) { + continue + } + vm := m.reconciledInstance(known, local.Name) + if !localInstanceStopped(local.State) { + reconciled[local.Name] = vm + continue + } + if err := m.deleteLocalInstance(context.Background(), vm); err != nil { + vm.Phase = LifecycleCleanupPending + reconciled[local.Name] = vm + m.warnf("[%s] stopped-instance cleanup pending: %v\n", local.Name, err) + } + } + return reconciled, nil +} + +func (m *Manager) reconciledInstance(known map[string]ProvisionedInstance, name string) ProvisionedInstance { + if vm, found := known[name]; found { + return vm + } + return ProvisionedInstance{ + Name: name, + LogPath: m.instanceLogPath(name, "."+m.Config.Provider.Type+".log"), + GuestLogPath: m.instanceLogPath(name, ".guest.log"), + Phase: LifecycleQuarantined, + } +} + +func localInstanceStopped(state string) bool { + switch strings.ToLower(strings.TrimSpace(state)) { + case "stopped", "exited", "dead", "failed": + return true + default: + return false + } +} + +func (m *Manager) deleteLocalInstance(ctx context.Context, vm ProvisionedInstance) error { + cleanupCtx, cancel := context.WithTimeout(ctx, cleanupTimeout) + defer cancel() + stopCtx, stopCancel := context.WithTimeout(cleanupCtx, 60*time.Second) + _ = m.Provider.Stop(stopCtx, vm.Name) + stopCancel() + deleteCtx, deleteCancel := context.WithTimeout(cleanupCtx, 60*time.Second) + err := m.Provider.Delete(deleteCtx, vm.Name) + deleteCancel() + if err != nil { + return err + } + if releaseErr := m.releaseInstanceTranscripts(vm); releaseErr != nil { + m.logger().Warn("instance transcript close failed after local deletion", "provider", m.Config.Provider.Type, "instance", vm.Name, "operation", "reconcile", "error", releaseErr) + } + return nil +} + +func (m *Manager) deleteRemoteRunner(ctx context.Context, runner gh.Runner) error { + deleteCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + return m.GitHub.DeleteRunnerIfExists(deleteCtx, runner.ID) +} + +type replacementRetryState struct { + attempt int + next time.Time +} + +func (s *replacementRetryState) active(now time.Time) bool { + return !s.next.IsZero() && now.Before(s.next) +} + +func (s *replacementRetryState) remaining(now time.Time) time.Duration { + if !s.active(now) { + return 0 + } + return s.next.Sub(now).Round(time.Second) +} + +func (s *replacementRetryState) schedule(m *Manager, now time.Time, err error) { + initial, maximum, multiplier, jitter := m.replacementRetrySettings() + nominal := float64(initial) * math.Pow(multiplier, float64(s.attempt)) + if nominal > float64(maximum) { + nominal = float64(maximum) + } + factor := 1.0 + if jitter > 0 { + factor += ((m.randomValue() * 2) - 1) * jitter + } + delay := time.Duration(nominal * factor) + if delay < time.Second { + delay = time.Second + } + if delay > maximum { + delay = maximum + } + if retryAfter := retryAfterDuration(err); retryAfter > delay { + delay = retryAfter + } + s.attempt++ + s.next = now.Add(delay) +} + +func (s *replacementRetryState) reset() { + s.attempt = 0 + s.next = time.Time{} +} + +func (s *replacementRetryState) resetAfterAdoption(before, after map[string]ProvisionedInstance) { + if adoptedReadyInstance(before, after) { + s.reset() + } +} + +func (m *Manager) replacementRetrySettings() (time.Duration, time.Duration, float64, float64) { + initialSeconds := m.Config.Pool.ReplacementRetryInitialSeconds + if initialSeconds <= 0 { + initialSeconds = 15 + } + maximumSeconds := m.Config.Pool.ReplacementRetryMaxSeconds + if maximumSeconds <= 0 { + maximumSeconds = 1800 + } + multiplier := m.Config.Pool.ReplacementRetryMultiplier + if multiplier < 1 { + multiplier = 2 + } + jitterPercent := m.Config.Pool.ReplacementRetryJitterPercent + if jitterPercent < 0 { + jitterPercent = 0 + } + return time.Duration(initialSeconds) * time.Second, time.Duration(maximumSeconds) * time.Second, multiplier, float64(jitterPercent) / 100 +} + +func (m *Manager) currentTime() time.Time { + if m.now != nil { + return m.now() + } + return time.Now() +} + +func (m *Manager) randomValue() float64 { + if m.randomFloat64 != nil { + return m.randomFloat64() + } + return rand.Float64() +} + +func retryAfterDuration(err error) time.Duration { + var httpErr *gh.HTTPError + if errors.As(err, &httpErr) { + return httpErr.RetryAfter + } + return 0 +} + +func isTransientDependencyError(err error) bool { + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + var httpErr *gh.HTTPError + if errors.As(err, &httpErr) { + return httpErr.StatusCode == http.StatusTooManyRequests || httpErr.StatusCode >= http.StatusInternalServerError + } + var networkErr net.Error + if errors.As(err, &networkErr) { + return true + } + text := strings.ToLower(err.Error()) + for _, match := range dependencyHTTPStatusPattern.FindAllStringSubmatch(text, -1) { + statusCode, parseErr := strconv.Atoi(match[1]) + if parseErr == nil && (statusCode == http.StatusTooManyRequests || statusCode >= http.StatusInternalServerError) { + return true + } + } + for _, marker := range []string{ + "badgateway", "gatewaytimeout", "internalservererror", "serviceunavailable", + "bad gateway", "gateway timeout", "internal server error", "service unavailable", + "connection reset", "connection refused", "temporary failure", + "tls handshake timeout", "i/o timeout", "no such host", + } { + if strings.Contains(text, marker) { + return true + } + } + return false +} + func (m *Manager) cleanupWithFreshContext() error { cleanupCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) defer cancel() - return m.Cleanup(cleanupCtx) + return m.cleanupUnlocked(cleanupCtx) +} + +func (m *Manager) cleanupAfterTerminalFailure(active map[string]ProvisionedInstance, keep bool) error { + if keep { + return nil + } + var firstErr error + for _, vm := range active { + var err error + switch vm.Phase { + case LifecycleReady: + err = m.retireInstance(context.Background(), vm, "controller stopped after terminal pool failure") + case LifecycleCleanupPending, LifecycleProvisioning: + err = m.deleteLocalInstance(context.Background(), vm) + case LifecycleDraining, LifecycleQuarantined: + // These instances may already be running a job. They stay counted and + // are reconciled by the next controller instead of being killed merely + // because GitHub readiness or status became uncertain. + continue + } + if err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr } func (m *Manager) ProvisionPool(ctx context.Context, instances int, register bool) ([]ProvisionedInstance, error) { + poolLock, err := m.AcquirePoolControllerLock() + if err != nil { + return nil, err + } + defer poolLock.Close() + hostTrustLock, err := m.AcquireHostTrustControllerLock() + if err != nil { + return nil, err + } + if hostTrustLock != nil { + defer hostTrustLock.Close() + } instances = m.requestedInstances(instances) names := RunnerNames(m.Config.Pool.NamePrefix, instances, time.Now()) out := make([]ProvisionedInstance, 0, len(names)) @@ -376,6 +884,15 @@ func (m *Manager) requestedInstances(override int) int { } func (m *Manager) Cleanup(ctx context.Context) error { + poolLock, err := m.AcquirePoolControllerLock() + if err != nil { + return err + } + defer poolLock.Close() + return m.cleanupUnlocked(ctx) +} + +func (m *Manager) cleanupUnlocked(ctx context.Context) error { var firstErr error vms, err := m.Provider.List(ctx) if err != nil { @@ -458,7 +975,7 @@ func (m *Manager) provisionOne(ctx context.Context, name string, register, allow return vm, err } lastErr = err - if vm.Name != "" { + if isPhysicalPhase(vm.Phase) { _ = m.retireInstance(context.Background(), vm, "discarding stale host-trust image generation") } if attempt == attempts { @@ -472,10 +989,56 @@ func (m *Manager) provisionOne(ctx context.Context, name string, register, allow return ProvisionedInstance{Name: name}, fmt.Errorf("provision runner after %d host trust image stabilization attempts: %w", attempts, lastErr) } -func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register, allowBusy bool) (ProvisionedInstance, error) { +func (m *Manager) deleteRemoteRunnerByNameBestEffort(name string) { + if m.GitHub == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + runner, found, err := m.GitHub.RunnerByName(ctx, name) + if err != nil { + m.warnf("[%s] deferred exact-name GitHub reconciliation after rollback: %v\n", name, err) + return + } + if !found { + return + } + if err := m.GitHub.DeleteRunnerIfExists(ctx, runner.ID); err != nil { + m.warnf("[%s] deferred exact-name GitHub runner deletion after rollback: %v\n", name, err) + } +} + +func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register, allowBusy bool) (vm ProvisionedInstance, err error) { logPath := m.instanceLogPath(name, "."+m.Config.Provider.Type+".log") guestLogPath := m.instanceLogPath(name, ".guest.log") - vm := ProvisionedInstance{Name: name, LogPath: logPath, GuestLogPath: guestLogPath} + vm = ProvisionedInstance{Name: name, LogPath: logPath, GuestLogPath: guestLogPath, Phase: LifecycleProvisioning} + localMayExist := false + configureAttempted := false + listenerMayBeRunning := false + defer func() { + if err == nil { + vm.Phase = LifecycleReady + return + } + if !localMayExist { + vm.Phase = "" + return + } + if listenerMayBeRunning { + vm.Phase = LifecycleQuarantined + return + } + cleanupErr := m.deleteLocalInstance(context.Background(), vm) + if cleanupErr != nil { + vm.Phase = LifecycleCleanupPending + err = errors.Join(err, fmt.Errorf("rollback local instance %s: %w", name, cleanupErr)) + return + } + vm.Phase = "" + if configureAttempted { + m.deleteRemoteRunnerByNameBestEffort(name) + } + }() var trustSnapshot hosttrust.Snapshot if m.hostTrustEnabled() { var err error @@ -489,6 +1052,7 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register return vm, err } m.logger().Info("cloning instance", "provider", m.Config.Provider.Type, "instance", name, "operation", "clone", "sourceImage", m.Config.Provider.SourceImage, "logPath", logPath) + localMayExist = true if err := m.timeFirstInstanceStage(name, "instance_container_create", func() error { return m.Provider.Clone(ctx, m.Config.Provider.SourceImage, name) }); err != nil { @@ -564,17 +1128,18 @@ func (m *Manager) provisionOneAttempt(ctx context.Context, name string, register } env := map[string]string{ "RUNNER_URL": m.GitHub.OrganizationURL(), - "RUNNER_TOKEN": token.Token, "RUNNER_NAME": name, "RUNNER_LABELS": strings.Join(m.Config.Runner.Labels, ","), "RUNNER_EPHEMERAL": fmt.Sprintf("%t", m.Config.Runner.Ephemeral), "RUNNER_GROUP": m.Config.Runner.Group, "RUNNER_NO_DEFAULT_LABELS": fmt.Sprintf("%t", m.Config.Runner.NoDefaultLabels), } - if _, err := m.execGuest(ctx, name, []string{"sudo", "-E", "bash", "/opt/epar/configure-runner.sh"}, provider.ExecOptions{Env: env}); err != nil { - return err + configureAttempted = true + if _, err := m.execGuest(ctx, name, []string{"sudo", "-E", "bash", "/opt/epar/configure-runner.sh"}, provider.ExecOptions{Env: env, Stdin: token.Token + "\n", SensitiveValues: []string{token.Token}}); err != nil { + return provider.RedactError(err, token.Token) } m.infof("[%s] starting runner service\n", name) + listenerMayBeRunning = true if _, err := m.execGuest(ctx, name, []string{"sudo", "bash", "/opt/epar/run-runner.sh"}, provider.ExecOptions{}); err != nil { return err } diff --git a/internal/pool/manager_test.go b/internal/pool/manager_test.go index 0e8a308..2e0c076 100644 --- a/internal/pool/manager_test.go +++ b/internal/pool/manager_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "net" + "net/http" "os" "path/filepath" "strings" @@ -261,6 +263,7 @@ func TestRunPoolDoesNotReplaceWhenRetirementIsDeferred(t *testing.T) { KeepOnExit: true, ReplaceCompleted: true, MonitorInterval: 5 * time.Millisecond, + PoolLockHeld: true, }); err != nil { t.Fatal(err) } @@ -294,6 +297,7 @@ func TestRunPoolReplacesCompletedRunnerAfterBusyProvisioning(t *testing.T) { KeepOnExit: true, ReplaceCompleted: true, MonitorInterval: 5 * time.Millisecond, + PoolLockHeld: true, }); err != nil { t.Fatal(err) } @@ -366,12 +370,12 @@ func TestRunPoolAddsCurrentTrustCapacityWhileOldGenerationDrains(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond) defer cancel() if err := manager.RunPool(ctx, RunOptions{ - Instances: 1, Register: true, KeepOnExit: true, ReplaceCompleted: false, MonitorInterval: 5 * time.Millisecond, HostTrustLockHeld: true, + Instances: 1, Register: true, KeepOnExit: true, ReplaceCompleted: false, MonitorInterval: 5 * time.Millisecond, HostTrustLockHeld: true, PoolLockHeld: true, }); err != nil { t.Fatal(err) } - if got := atomic.LoadInt32(&fake.cloneCalls); got < 2 { - t.Fatalf("Clone called %d time(s), want G2 replacement while busy G1 drains", got) + if got := atomic.LoadInt32(&fake.cloneCalls); got != 1 { + t.Fatalf("Clone called %d time(s), want strict physical cap while busy G1 drains", got) } if got := atomic.LoadInt32(&fake.deleteCalls); got != 0 { t.Fatalf("busy G1 was deleted %d time(s), want it left draining", got) @@ -382,13 +386,14 @@ func TestRunPoolAddsCurrentTrustCapacityWhileOldGenerationDrains(t *testing.T) { } func TestVerifyUsesIdleReadiness(t *testing.T) { + t.Setenv("LOCALAPPDATA", t.TempDir()) provider := &fakeProvider{ip: "127.0.0.1"} github := &fakeGitHub{ waitRunner: gh.Runner{Name: "epar-test-1", ID: 123, Status: "online"}, } manager := Manager{ Config: config.Config{ - Provider: config.ProviderConfig{SourceImage: "image"}, + Provider: config.ProviderConfig{SourceImage: "image", Type: "docker-dind"}, Pool: config.PoolConfig{Instances: 1, NamePrefix: "epar-test"}, Logging: config.LoggingConfig{Directory: t.TempDir()}, Runner: config.RunnerConfig{Labels: []string{"self-hosted"}, Ephemeral: true}, @@ -428,6 +433,7 @@ func TestRunPoolUsesConfiguredInstancesWhenNoOverride(t *testing.T) { Register: false, KeepOnExit: true, ReplaceCompleted: false, + PoolLockHeld: true, }); err != nil { t.Fatal(err) } @@ -465,6 +471,7 @@ func TestProvisionOneRetriesTransientRuntimeValidationFailure(t *testing.T) { } func TestVerifyCleanupUsesFreshContextAfterCancellation(t *testing.T) { + t.Setenv("LOCALAPPDATA", t.TempDir()) provider := &fakeProvider{ instances: []provider.Instance{ {Name: "epar-test-1"}, @@ -491,11 +498,11 @@ func TestVerifyCleanupUsesFreshContextAfterCancellation(t *testing.T) { if got := atomic.LoadInt32(&provider.canceledListCalls); got != 0 { t.Fatalf("cleanup List received a canceled context %d time(s)", got) } - if got := atomic.LoadInt32(&provider.stopCalls); got != 2 { - t.Fatalf("Stop called %d time(s), want 2 matching prefix-boundary instances", got) + if got := atomic.LoadInt32(&provider.stopCalls); got != 3 { + t.Fatalf("Stop called %d time(s), want 3 matching prefix-boundary instances including the verified candidate", got) } - if got := atomic.LoadInt32(&provider.deleteCalls); got != 2 { - t.Fatalf("Delete called %d time(s), want 2 matching prefix-boundary instances", got) + if got := atomic.LoadInt32(&provider.deleteCalls); got != 3 { + t.Fatalf("Delete called %d time(s), want 3 matching prefix-boundary instances including the verified candidate", got) } } @@ -516,7 +523,7 @@ func TestRunPoolCleanupUsesFreshContextAfterCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - if err := manager.RunPool(ctx, RunOptions{Instances: 1}); err != nil { + if err := manager.RunPool(ctx, RunOptions{Instances: 1, PoolLockHeld: true}); err != nil { t.Fatal(err) } if got := atomic.LoadInt32(&provider.canceledListCalls); got != 0 { @@ -577,12 +584,20 @@ func TestProvisionOnePassesRunnerRegistrationControlsWithoutPrivateKey(t *testin "RUNNER_NO_DEFAULT_LABELS": "true", "RUNNER_LABELS": "epar-core-test", "RUNNER_EPHEMERAL": "true", - "RUNNER_TOKEN": "token", } { if got := env[key]; got != want { t.Errorf("%s = %q, want %q", key, got, want) } } + provider.mu.Lock() + configureOptions := provider.configureOptions + provider.mu.Unlock() + if configureOptions.Stdin != "token\n" { + t.Fatalf("configure stdin = %q, want registration token line", configureOptions.Stdin) + } + if len(configureOptions.SensitiveValues) != 1 || configureOptions.SensitiveValues[0] != "token" { + t.Fatalf("configure sensitive values = %#v, want registration token", configureOptions.SensitiveValues) + } for key, value := range env { if strings.Contains(strings.ToLower(key), "private") || value == "/secret/app.pem" { t.Fatalf("guest registration environment exposes private key through %s", key) @@ -731,6 +746,440 @@ func TestProvisionOneReadinessSucceedsWhileRunnerProcessStaysHealthy(t *testing. } } +func TestProvisioningFailureRollbackBoundary(t *testing.T) { + tests := []struct { + name string + configure func(*fakeProvider, *fakeGitHub) + wantPhase LifecyclePhase + wantLocalDelete bool + }{ + {name: "partial clone", configure: func(p *fakeProvider, _ *fakeGitHub) { p.cloneErr = errors.New("clone failed after partial creation") }, wantLocalDelete: true}, + {name: "start", configure: func(p *fakeProvider, _ *fakeGitHub) { p.startErr = errors.New("start failed") }, wantLocalDelete: true}, + {name: "ip", configure: func(p *fakeProvider, _ *fakeGitHub) { p.ipErr = errors.New("ip failed") }, wantLocalDelete: true}, + {name: "runtime", configure: func(p *fakeProvider, _ *fakeGitHub) { p.execErr = errors.New("runtime validation failed") }, wantLocalDelete: true}, + {name: "token", configure: func(_ *fakeProvider, g *fakeGitHub) { g.registrationErr = errors.New("token failed") }, wantLocalDelete: true}, + {name: "configure runner", configure: func(p *fakeProvider, _ *fakeGitHub) { + p.execFunc = func(_ context.Context, _ string, command []string, _ provider.ExecOptions) (provider.ExecResult, error) { + if strings.Contains(strings.Join(command, " "), "configure-runner.sh") { + return provider.ExecResult{}, errors.New("Http response code: ServiceUnavailable (503)") + } + return provider.ExecResult{}, nil + } + }, wantLocalDelete: true}, + {name: "listener start uncertain", configure: func(p *fakeProvider, _ *fakeGitHub) { + p.execFunc = func(_ context.Context, _ string, command []string, _ provider.ExecOptions) (provider.ExecResult, error) { + if strings.Contains(strings.Join(command, " "), "run-runner.sh") { + return provider.ExecResult{}, errors.New("listener start failed ambiguously") + } + return provider.ExecResult{}, nil + } + }, wantPhase: LifecycleQuarantined}, + {name: "readiness", configure: func(_ *fakeProvider, g *fakeGitHub) { g.waitErr = errors.New("GitHub readiness failed") }, wantPhase: LifecycleQuarantined}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &fakeProvider{ip: "127.0.0.1"} + g := &fakeGitHub{} + tt.configure(p, g) + manager := newRegisteredTestManager(t, p, g) + vm, err := manager.provisionOne(context.Background(), "epar-test-stage", true, false) + if err == nil { + t.Fatal("provisionOne() error = nil") + } + if vm.Phase != tt.wantPhase { + t.Fatalf("phase = %q, want %q", vm.Phase, tt.wantPhase) + } + if tt.wantLocalDelete && atomic.LoadInt32(&p.deleteCalls) == 0 { + t.Fatal("pre-listener failure did not roll back local candidate") + } + if !tt.wantLocalDelete && atomic.LoadInt32(&p.deleteCalls) != 0 { + t.Fatal("post-listener uncertainty deleted local candidate") + } + }) + } +} + +func TestConfigureFailureDeletesExactLocalAndRemoteCandidate(t *testing.T) { + p := &fakeProvider{ip: "127.0.0.1"} + p.execFunc = func(_ context.Context, _ string, command []string, _ provider.ExecOptions) (provider.ExecResult, error) { + if strings.Contains(strings.Join(command, " "), "configure-runner.sh") { + return provider.ExecResult{}, errors.New("Http response code: BadGateway from runner-registration (502)") + } + return provider.ExecResult{}, nil + } + g := &fakeGitHub{runner: gh.Runner{Name: "epar-test-candidate", ID: 456}, found: true} + manager := newRegisteredTestManager(t, p, g) + vm, err := manager.provisionOne(context.Background(), "epar-test-candidate", true, false) + if err == nil { + t.Fatal("provisionOne() error = nil") + } + if vm.Phase != "" { + t.Fatalf("phase = %q, want no surviving physical candidate", vm.Phase) + } + if got := atomic.LoadInt32(&p.deleteCalls); got != 1 { + t.Fatalf("local delete calls = %d, want 1", got) + } + if got := atomic.LoadInt32(&g.deleteCalls); got != 1 { + t.Fatalf("remote delete calls = %d, want 1", got) + } + g.mu.Lock() + defer g.mu.Unlock() + if len(g.deletedIDs) != 1 || g.deletedIDs[0] != 456 { + t.Fatalf("deleted remote IDs = %v, want [456]", g.deletedIDs) + } +} + +func TestConfigureFailureRedactsRegistrationTokenFromErrorAndTiming(t *testing.T) { + const sentinel = "SENTINEL-RUNNER-REGISTRATION-TOKEN" + p := &fakeProvider{ip: "127.0.0.1"} + p.execFunc = func(_ context.Context, _ string, command []string, _ provider.ExecOptions) (provider.ExecResult, error) { + if strings.Contains(strings.Join(command, " "), "configure-runner.sh") { + return provider.ExecResult{}, fmt.Errorf("configure failed with token %s", sentinel) + } + return provider.ExecResult{}, nil + } + g := &fakeGitHub{registrationToken: sentinel} + manager := newRegisteredTestManager(t, p, g) + timingPath, err := manager.StartStartupTiming() + if err != nil { + t.Fatal(err) + } + _, provisionErr := manager.provisionOne(context.Background(), "epar-test-secret", true, false) + if provisionErr == nil { + t.Fatal("provisionOne() error = nil") + } + manager.FinishStartupTiming(provisionErr) + if strings.Contains(provisionErr.Error(), sentinel) { + t.Fatalf("returned error leaked registration token: %v", provisionErr) + } + content, err := os.ReadFile(timingPath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(content), sentinel) { + t.Fatalf("startup timing leaked registration token: %s", content) + } +} + +func TestOneHundredRegistrationOutagesNeverExceedPhysicalCap(t *testing.T) { + p := &fakeProvider{ip: "127.0.0.1"} + p.execFunc = func(_ context.Context, _ string, command []string, _ provider.ExecOptions) (provider.ExecResult, error) { + if strings.Contains(strings.Join(command, " "), "configure-runner.sh") { + return provider.ExecResult{}, errors.New("Http response code: ServiceUnavailable (503)") + } + return provider.ExecResult{}, nil + } + manager := newRegisteredTestManager(t, p, &fakeGitHub{}) + manager.Config.Pool.Instances = 2 + for i := 0; i < 100; i++ { + name := fmt.Sprintf("epar-test-outage-%03d", i) + if _, err := manager.provisionOne(context.Background(), name, true, true); err == nil { + t.Fatalf("cycle %d unexpectedly succeeded", i) + } + } + if got := atomic.LoadInt32(&p.maxInventory); got > 2 { + t.Fatalf("maximum physical inventory = %d, want <= 2", got) + } + instances, err := p.List(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(instances) != 0 { + t.Fatalf("surviving physical inventory = %d, want 0 after local rollback", len(instances)) + } +} + +func TestCleanupFailureRemainsCountedAndBlocksClone(t *testing.T) { + p := &fakeProvider{instances: []provider.Instance{{Name: "epar-test-existing", State: "running"}}, deleteErr: errors.New("delete failed")} + g := &fakeGitHub{} + manager := newRegisteredTestManager(t, p, g) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if err := manager.RunPool(ctx, RunOptions{Instances: 1, Register: true, KeepOnExit: true, ReplaceCompleted: true, MonitorInterval: time.Millisecond, PoolLockHeld: true}); err != nil { + t.Fatal(err) + } + if got := atomic.LoadInt32(&p.cloneCalls); got != 0 { + t.Fatalf("clone calls = %d, want 0 while cleanup-pending resource occupies capacity", got) + } +} + +func TestInitialTerminalFailureCleansReadyInstancesButPreservesQuarantine(t *testing.T) { + t.Run("pre-listener failure cleans earlier ready capacity", func(t *testing.T) { + var configureCalls int32 + p := &fakeProvider{ip: "127.0.0.1"} + p.execFunc = func(_ context.Context, _ string, command []string, _ provider.ExecOptions) (provider.ExecResult, error) { + if strings.Contains(strings.Join(command, " "), "configure-runner.sh") && atomic.AddInt32(&configureCalls, 1) == 2 { + return provider.ExecResult{}, errors.New("Response status code does not indicate success: 401") + } + return provider.ExecResult{}, nil + } + g := &fakeGitHub{runner: gh.Runner{ID: 123, Status: "online"}, found: true, waitRunner: gh.Runner{ID: 123, Status: "online"}} + manager := newRegisteredTestManager(t, p, g) + err := manager.RunPool(context.Background(), RunOptions{Instances: 2, Register: true, ReplaceCompleted: true, PoolLockHeld: true}) + if err == nil { + t.Fatal("RunPool() error = nil") + } + if got := atomic.LoadInt32(&p.deleteCalls); got < 2 { + t.Fatalf("local delete calls = %d, want failed candidate rollback plus ready-instance terminal cleanup", got) + } + }) + + t.Run("post-listener uncertainty survives terminal return", func(t *testing.T) { + p := &fakeProvider{ip: "127.0.0.1"} + g := &fakeGitHub{waitErr: errors.New("Response status code does not indicate success: 401")} + manager := newRegisteredTestManager(t, p, g) + err := manager.RunPool(context.Background(), RunOptions{Instances: 1, Register: true, ReplaceCompleted: true, PoolLockHeld: true}) + if err == nil { + t.Fatal("RunPool() error = nil") + } + if got := atomic.LoadInt32(&p.deleteCalls); got != 0 { + t.Fatalf("local delete calls = %d, want quarantined post-listener candidate preserved", got) + } + }) +} + +func TestShutdownCancellationPreservesPostListenerCandidateWhenKeepOnExit(t *testing.T) { + p := &fakeProvider{ip: "127.0.0.1"} + g := &fakeGitHub{waitFunc: func(ctx context.Context, _ string, _ time.Duration) (gh.Runner, error) { + <-ctx.Done() + return gh.Runner{}, ctx.Err() + }} + manager := newRegisteredTestManager(t, p, g) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if err := manager.RunPool(ctx, RunOptions{Instances: 1, Register: true, KeepOnExit: true, ReplaceCompleted: true, PoolLockHeld: true}); err != nil { + t.Fatalf("RunPool() cancellation error = %v, want clean shutdown", err) + } + if got := atomic.LoadInt32(&p.deleteCalls); got != 0 { + t.Fatalf("local delete calls = %d, want post-listener candidate preserved by keep-on-exit", got) + } + instances, err := p.List(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(instances) != 1 { + t.Fatalf("surviving local instances = %d, want 1 quarantined candidate", len(instances)) + } +} + +func TestStoppedLocalIsCleanedEvenWhenGitHubIsUnavailable(t *testing.T) { + p := &fakeProvider{instances: []provider.Instance{{Name: "epar-test-stopped", State: "stopped"}}} + g := &fakeGitHub{listErr: &gh.HTTPError{StatusCode: 503}} + manager := newRegisteredTestManager(t, p, g) + active, err := manager.reconcilePhysicalPool(context.Background(), nil, true) + if err == nil { + t.Fatal("reconcilePhysicalPool() error = nil, want GitHub outage") + } + if len(active) != 0 { + t.Fatalf("active = %#v, want stopped local removed", active) + } + if got := atomic.LoadInt32(&p.deleteCalls); got != 1 { + t.Fatalf("local delete calls = %d, want 1 during GitHub outage", got) + } +} + +func TestRestartUnknownResourcesCountAndBlockAllocation(t *testing.T) { + p := &fakeProvider{instances: []provider.Instance{{Name: "epar-test-unknown-1", State: "running"}, {Name: "epar-test-unknown-2", State: "running"}}} + g := &fakeGitHub{listErr: &gh.HTTPError{StatusCode: 503}} + manager := newRegisteredTestManager(t, p, g) + active, err := manager.reconcilePhysicalPool(context.Background(), nil, true) + if err == nil { + t.Fatal("reconcilePhysicalPool() error = nil, want GitHub outage") + } + if len(active) != 2 { + t.Fatalf("counted physical resources = %d, want 2", len(active)) + } + for name, vm := range active { + if vm.Phase != LifecycleQuarantined { + t.Fatalf("%s phase = %q, want quarantined", name, vm.Phase) + } + } + if got := atomic.LoadInt32(&p.cloneCalls); got != 0 { + t.Fatalf("clone calls = %d, want 0 while dependency state is ambiguous", got) + } +} + +func TestLegacyOverCapacityInventoryBlocksAllocation(t *testing.T) { + p := &fakeProvider{instances: []provider.Instance{{Name: "epar-test-existing-1", State: "running"}, {Name: "epar-test-existing-2", State: "running"}, {Name: "epar-test-existing-3", State: "running"}}} + manager := newRegisteredTestManager(t, p, nil) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if err := manager.RunPool(ctx, RunOptions{Instances: 2, KeepOnExit: true, PoolLockHeld: true}); err != nil { + t.Fatal(err) + } + if got := atomic.LoadInt32(&p.cloneCalls); got != 0 { + t.Fatalf("clone calls = %d, want 0 while legacy physical inventory exceeds target", got) + } +} + +func TestLegacyIdleOverCapacityIsReducedToTarget(t *testing.T) { + p := &fakeProvider{instances: []provider.Instance{{Name: "epar-test-existing-1", State: "running"}, {Name: "epar-test-existing-2", State: "running"}, {Name: "epar-test-existing-3", State: "running"}}} + g := &fakeGitHub{ + listRunners: []gh.Runner{{Name: "epar-test-existing-1", ID: 1, Status: "online"}, {Name: "epar-test-existing-2", ID: 2, Status: "online"}, {Name: "epar-test-existing-3", ID: 3, Status: "online"}}, + runner: gh.Runner{ID: 9, Status: "online"}, + found: true, + } + manager := newRegisteredTestManager(t, p, g) + active, err := manager.reconcilePhysicalPool(context.Background(), nil, true) + if err != nil { + t.Fatal(err) + } + active, err = manager.reduceOverCapacity(context.Background(), active, 1, true) + if err != nil { + t.Fatal(err) + } + if len(active) != 1 { + t.Fatalf("active count = %d, want target 1", len(active)) + } + if got := atomic.LoadInt32(&p.deleteCalls); got != 2 { + t.Fatalf("local delete calls = %d, want 2 idle excess runners retired", got) + } +} + +func TestQuarantinedRunnersAdoptOrRetireAfterGitHubRecovery(t *testing.T) { + p := &fakeProvider{instances: []provider.Instance{{Name: "epar-test-healthy", State: "running"}, {Name: "epar-test-offline", State: "running"}}} + g := &fakeGitHub{listRunners: []gh.Runner{{Name: "epar-test-healthy", ID: 1, Status: "online"}, {Name: "epar-test-offline", ID: 2, Status: "offline"}}} + manager := newRegisteredTestManager(t, p, g) + known := map[string]ProvisionedInstance{ + "epar-test-healthy": {Name: "epar-test-healthy", Phase: LifecycleQuarantined}, + "epar-test-offline": {Name: "epar-test-offline", Phase: LifecycleQuarantined}, + } + active, err := manager.reconcilePhysicalPool(context.Background(), known, true) + if err != nil { + t.Fatal(err) + } + if got := active["epar-test-healthy"].Phase; got != LifecycleReady { + t.Fatalf("healthy phase = %q, want ready", got) + } + if _, found := active["epar-test-offline"]; found { + t.Fatal("offline quarantined runner survived recovery reconciliation") + } + if got := atomic.LoadInt32(&p.deleteCalls); got != 1 { + t.Fatalf("local delete calls = %d, want offline runner retirement", got) + } +} + +func TestReplacementBackoffSequenceJitterCapRetryAfterAndReset(t *testing.T) { + manager := Manager{Config: config.Config{Pool: config.PoolConfig{ + ReplacementRetryInitialSeconds: 15, + ReplacementRetryMaxSeconds: 1800, + ReplacementRetryMultiplier: 2, + ReplacementRetryJitterPercent: 20, + }}, randomFloat64: func() float64 { return 0.5 }} + now := time.Unix(1_000, 0) + state := replacementRetryState{} + want := []time.Duration{15, 30, 60, 120, 240, 480, 960, 1800, 1800} + for i, seconds := range want { + state.schedule(&manager, now, errors.New("ServiceUnavailable")) + if got := state.next.Sub(now); got != seconds*time.Second { + t.Fatalf("attempt %d delay = %s, want %s", i+1, got, seconds*time.Second) + } + now = state.next + } + state.reset() + if state.attempt != 0 || !state.next.IsZero() { + t.Fatalf("reset state = %#v", state) + } + state = replacementRetryState{attempt: 4, next: now.Add(time.Hour)} + before := map[string]ProvisionedInstance{"runner": {Name: "runner", Phase: LifecycleQuarantined}} + after := map[string]ProvisionedInstance{"runner": {Name: "runner", Phase: LifecycleReady}} + state.resetAfterAdoption(before, after) + if state.attempt != 0 || !state.next.IsZero() { + t.Fatalf("adoption reset state = %#v", state) + } + state.schedule(&manager, now, &gh.HTTPError{StatusCode: 429, RetryAfter: 45 * time.Second}) + if got := state.next.Sub(now); got != 45*time.Second { + t.Fatalf("Retry-After delay = %s, want 45s", got) + } + low := Manager{Config: manager.Config, randomFloat64: func() float64 { return 0 }} + lowState := replacementRetryState{} + lowState.schedule(&low, now, errors.New("ServiceUnavailable")) + if got := lowState.next.Sub(now); got != 12*time.Second { + t.Fatalf("low jitter delay = %s, want 12s", got) + } + high := Manager{Config: manager.Config, randomFloat64: func() float64 { return 1 }} + highState := replacementRetryState{attempt: 20} + highState.schedule(&high, now, errors.New("ServiceUnavailable")) + if got := highState.next.Sub(now); got != 1800*time.Second { + t.Fatalf("jittered cap delay = %s, want 30m", got) + } + minimum := Manager{Config: config.Config{Pool: config.PoolConfig{ + ReplacementRetryInitialSeconds: 1, + ReplacementRetryMaxSeconds: 1, + ReplacementRetryMultiplier: 1, + ReplacementRetryJitterPercent: 100, + }}, randomFloat64: func() float64 { return 0 }} + minimumState := replacementRetryState{} + minimumState.schedule(&minimum, now, errors.New("ServiceUnavailable")) + if got := minimumState.next.Sub(now); got != time.Second { + t.Fatalf("minimum jittered delay = %s, want 1s", got) + } +} + +func TestTransientDependencyClassification(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {name: "network", err: &net.DNSError{IsTimeout: true}, want: true}, + {name: "rate limit", err: &gh.HTTPError{StatusCode: http.StatusTooManyRequests}, want: true}, + {name: "server error", err: &gh.HTTPError{StatusCode: http.StatusBadGateway}, want: true}, + {name: "guest opaque 501", err: errors.New("Response status code does not indicate success: 501 (Not Implemented)"), want: true}, + {name: "guest opaque 599", err: errors.New("Http response code: 599 from runner-registration"), want: true}, + {name: "unauthorized", err: &gh.HTTPError{StatusCode: http.StatusUnauthorized}, want: false}, + {name: "forbidden", err: &gh.HTTPError{StatusCode: http.StatusForbidden}, want: false}, + {name: "guest opaque forbidden", err: errors.New("Response status code does not indicate success: 403 (Forbidden)"), want: false}, + {name: "cancellation", err: context.Canceled, want: false}, + {name: "deterministic configuration", err: errors.New("runner labels are invalid"), want: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := isTransientDependencyError(test.err); got != test.want { + t.Fatalf("isTransientDependencyError(%v) = %t, want %t", test.err, got, test.want) + } + }) + } +} + +func TestReplacementCooldownSkipsGitHubButContinuesLocalHousekeeping(t *testing.T) { + p := &fakeProvider{instances: []provider.Instance{{Name: "epar-test-existing", State: "running"}}} + g := &fakeGitHub{runner: gh.Runner{Name: "epar-test-existing", ID: 1, Status: "online"}, found: true} + g.listFunc = func(context.Context) ([]gh.Runner, error) { + if atomic.LoadInt32(&g.listCalls) == 1 { + return []gh.Runner{{Name: "epar-test-existing", ID: 1, Status: "online"}}, nil + } + p.mu.Lock() + if len(p.instances) > 0 { + p.instances[0].State = "stopped" + } + p.mu.Unlock() + return nil, &gh.HTTPError{StatusCode: 503} + } + manager := newRegisteredTestManager(t, p, g) + manager.Config.Pool.ReplacementRetryInitialSeconds = 15 + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + if err := manager.RunPool(ctx, RunOptions{Instances: 1, Register: true, KeepOnExit: true, ReplaceCompleted: true, MonitorInterval: time.Millisecond, PoolLockHeld: true}); err != nil { + t.Fatal(err) + } + if got := atomic.LoadInt32(&g.listCalls); got != 2 { + t.Fatalf("GitHub ListRunners calls = %d, want initial success plus one failed retry trigger", got) + } + if got := atomic.LoadInt32(&g.runnerByNameCalls); got > 1 { + t.Fatalf("GitHub RunnerByName calls = %d, want no calls during cooldown", got) + } + if got := atomic.LoadInt32(&p.listCalls); got <= 2 { + t.Fatalf("local List calls = %d, want repeated housekeeping during cooldown", got) + } + if got := atomic.LoadInt32(&p.deleteCalls); got != 1 { + t.Fatalf("local delete calls = %d, want stopped resource cleanup during cooldown", got) + } + if got := atomic.LoadInt32(&p.cloneCalls); got != 0 { + t.Fatalf("clone calls = %d, want allocation paused during cooldown", got) + } +} + func newRegisteredTestManager(t *testing.T, provider provider.Provider, github GitHubClient) Manager { t.Helper() return Manager{ @@ -748,31 +1197,45 @@ func newRegisteredTestManager(t *testing.T, provider provider.Provider, github G } type fakeProvider struct { - execErr error - execErrs []error - execFunc func(context.Context, string, []string, provider.ExecOptions) (provider.ExecResult, error) - ip string - mu sync.Mutex - - configureEnv map[string]string - commands []string - execOptions []provider.ExecOptions - instances []provider.Instance + execErr error + execErrs []error + execFunc func(context.Context, string, []string, provider.ExecOptions) (provider.ExecResult, error) + ip string + cloneErr error + startErr error + ipErr error + deleteErr error + listErr error + mu sync.Mutex + + configureEnv map[string]string + configureOptions provider.ExecOptions + commands []string + execOptions []provider.ExecOptions + instances []provider.Instance cloneCalls int32 execCalls int32 stopCalls int32 deleteCalls int32 canceledListCalls int32 + maxInventory int32 + listCalls int32 } -func (p *fakeProvider) Clone(context.Context, string, string) error { +func (p *fakeProvider) Clone(_ context.Context, source, name string) error { atomic.AddInt32(&p.cloneCalls, 1) - return nil + p.mu.Lock() + p.instances = append(p.instances, provider.Instance{Name: name, Source: source, State: "running"}) + if len(p.instances) > int(atomic.LoadInt32(&p.maxInventory)) { + atomic.StoreInt32(&p.maxInventory, int32(len(p.instances))) + } + p.mu.Unlock() + return p.cloneErr } func (p *fakeProvider) Start(context.Context, string, provider.StartOptions) (*provider.RunningProcess, error) { - return &provider.RunningProcess{}, nil + return &provider.RunningProcess{}, p.startErr } func (p *fakeProvider) Exec(ctx context.Context, name string, command []string, opts provider.ExecOptions) (provider.ExecResult, error) { @@ -787,6 +1250,7 @@ func (p *fakeProvider) Exec(ctx context.Context, name string, command []string, for key, value := range opts.Env { p.configureEnv[key] = value } + p.configureOptions = opts p.mu.Unlock() } call := atomic.AddInt32(&p.execCalls, 1) @@ -823,6 +1287,9 @@ func (p *fakeProvider) logPathFor(fragment string) string { } func (p *fakeProvider) IP(context.Context, string, int) (string, error) { + if p.ipErr != nil { + return "", p.ipErr + } if p.ip == "" { return "127.0.0.1", nil } @@ -834,17 +1301,32 @@ func (p *fakeProvider) Stop(context.Context, string) error { return nil } -func (p *fakeProvider) Delete(context.Context, string) error { +func (p *fakeProvider) Delete(_ context.Context, name string) error { atomic.AddInt32(&p.deleteCalls, 1) + if p.deleteErr != nil { + return p.deleteErr + } + p.mu.Lock() + remaining := p.instances[:0] + for _, instance := range p.instances { + if instance.Name != name { + remaining = append(remaining, instance) + } + } + p.instances = remaining + p.mu.Unlock() return nil } func (p *fakeProvider) List(ctx context.Context) ([]provider.Instance, error) { + atomic.AddInt32(&p.listCalls, 1) if ctx.Err() != nil { atomic.AddInt32(&p.canceledListCalls, 1) return nil, ctx.Err() } - return append([]provider.Instance(nil), p.instances...), nil + p.mu.Lock() + defer p.mu.Unlock() + return append([]provider.Instance(nil), p.instances...), p.listErr } type fakeGitHub struct { @@ -857,6 +1339,16 @@ type fakeGitHub struct { deleteErr error waitOnlineCalls int32 waitOnlineIdleCalls int32 + listRunners []gh.Runner + listErr error + listFunc func(context.Context) ([]gh.Runner, error) + registrationErr error + registrationToken string + deleteCalls int32 + deletedIDs []int64 + mu sync.Mutex + listCalls int32 + runnerByNameCalls int32 } func (g *fakeGitHub) OrganizationURL() string { @@ -864,17 +1356,23 @@ func (g *fakeGitHub) OrganizationURL() string { } func (g *fakeGitHub) RegistrationToken(context.Context) (gh.RegistrationToken, error) { - return gh.RegistrationToken{Token: "token"}, nil + token := g.registrationToken + if token == "" { + token = "token" + } + return gh.RegistrationToken{Token: token}, g.registrationErr } -func (g *fakeGitHub) ListRunners(context.Context) ([]gh.Runner, error) { - if !g.found { - return nil, nil +func (g *fakeGitHub) ListRunners(ctx context.Context) ([]gh.Runner, error) { + atomic.AddInt32(&g.listCalls, 1) + if g.listFunc != nil { + return g.listFunc(ctx) } - return []gh.Runner{g.runner}, nil + return append([]gh.Runner(nil), g.listRunners...), g.listErr } func (g *fakeGitHub) RunnerByName(context.Context, string) (gh.Runner, bool, error) { + atomic.AddInt32(&g.runnerByNameCalls, 1) return g.runner, g.found, g.runnerErr } @@ -901,7 +1399,11 @@ func (g *fakeGitHub) waitReady(ctx context.Context, name string, timeout time.Du return g.runner, nil } -func (g *fakeGitHub) DeleteRunnerIfExists(context.Context, int64) error { +func (g *fakeGitHub) DeleteRunnerIfExists(_ context.Context, id int64) error { + atomic.AddInt32(&g.deleteCalls, 1) + g.mu.Lock() + g.deletedIDs = append(g.deletedIDs, id) + g.mu.Unlock() return g.deleteErr } diff --git a/internal/pool/runner_script_test.go b/internal/pool/runner_script_test.go index e07255f..cef619a 100644 --- a/internal/pool/runner_script_test.go +++ b/internal/pool/runner_script_test.go @@ -220,6 +220,62 @@ exit 0 } } +func TestConfigureRunnerReadsTokenFromStdin(t *testing.T) { + const secret = "sentinel-registration-token" + dir := t.TempDir() + argsFile := filepath.Join(dir, "config.args") + for name, content := range map[string]string{ + "sudo": `#!/usr/bin/env bash +if [[ "${1:-}" == "-u" ]]; then + shift 2 +fi +exec "$@" +`, + "config.sh": `#!/usr/bin/env bash +printf '%s\n' "$@" >"${EPAR_CONFIG_ARGS_FILE}" +`, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0755); err != nil { + t.Fatal(err) + } + } + + script := filepath.ToSlash(filepath.Join("..", "..", "scripts", "guest", "ubuntu", "configure-runner.sh")) + cmd := exec.Command(gitBashForRunnerScriptTest(t), script) + cmd.Stdin = strings.NewReader(secret + "\n") + cmd.Env = append(os.Environ(), + "RUNNER_URL=https://github.test/example", + "RUNNER_NAME=epar-test", + "RUNNER_LABELS=self-hosted", + "EPAR_ACTIONS_RUNNER_DIR="+bashPath(dir), + "EPAR_CONFIG_ARGS_FILE="+bashPath(argsFile), + "PATH="+bashPath(dir)+":"+os.Getenv("PATH"), + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("configure-runner stdin token path failed: %v\n%s", err, out) + } + args, err := os.ReadFile(argsFile) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(args), "--token\n"+secret+"\n") { + t.Fatalf("config.sh arguments did not receive stdin token: %q", args) + } + + cmd = exec.Command(gitBashForRunnerScriptTest(t), script) + cmd.Env = append(os.Environ(), + "RUNNER_URL=https://github.test/example", + "RUNNER_NAME=epar-test", + "RUNNER_LABELS=self-hosted", + "EPAR_ACTIONS_RUNNER_DIR="+bashPath(dir), + "PATH="+bashPath(dir)+":"+os.Getenv("PATH"), + ) + out, err := cmd.CombinedOutput() + if err == nil || !strings.Contains(string(out), "must be provided as one nonempty line on stdin") { + t.Fatalf("configure-runner empty stdin error = %v, output = %q", err, out) + } +} + func requireLinuxProc(t *testing.T) { t.Helper() if runtime.GOOS != "linux" { @@ -582,7 +638,7 @@ func TestCollectRunnerDiagnosticsValidatesTailLines(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { command := fmt.Sprintf("EPAR_DIAGNOSTIC_TAIL_LINES=%s exec bash %s", tc.value, script) - cmd := exec.Command("bash", "-c", command) + cmd := exec.Command(gitBashForRunnerScriptTest(t), "-c", command) out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("collect-runner-diagnostics.sh failed: %v\n%s", err, out) diff --git a/internal/pool/startup_timing.go b/internal/pool/startup_timing.go index 3042992..adb065d 100644 --- a/internal/pool/startup_timing.go +++ b/internal/pool/startup_timing.go @@ -5,13 +5,12 @@ import ( "log/slog" "os" "path/filepath" - "regexp" "strings" "sync" "time" -) -var timingSecretPattern = regexp.MustCompile(`(?i)((?:runner_)?token|password|secret|private[_-]?key)=\S+`) + "github.com/solutionforest/ephemeral-action-runner/internal/provider" +) type startupTimingEvent struct { Timestamp string `json:"timestamp"` @@ -190,10 +189,7 @@ func startupTimingLabel(provider string) string { } func sanitizeTimingError(err error) string { - text := strings.Join(strings.Fields(err.Error()), " ") - text = timingSecretPattern.ReplaceAllStringFunc(text, func(match string) string { - return match[:strings.Index(match, "=")+1] + "[REDACTED]" - }) + text := provider.RedactText(strings.Join(strings.Fields(err.Error()), " ")) if len(text) > 500 { return text[:500] + "..." } diff --git a/internal/pool/startup_timing_test.go b/internal/pool/startup_timing_test.go new file mode 100644 index 0000000..10f98bb --- /dev/null +++ b/internal/pool/startup_timing_test.go @@ -0,0 +1,18 @@ +package pool + +import ( + "errors" + "strings" + "testing" +) + +func TestSanitizeTimingErrorRedactsSecretAssignments(t *testing.T) { + const sentinel = "timing-secret-sentinel" + got := sanitizeTimingError(errors.New("configure failed RUNNER_TOKEN=" + sentinel + " PASSWORD=" + sentinel)) + if strings.Contains(got, sentinel) { + t.Fatalf("sanitizeTimingError leaked sentinel: %q", got) + } + if !strings.Contains(got, "RUNNER_TOKEN=[REDACTED]") || !strings.Contains(got, "PASSWORD=[REDACTED]") { + t.Fatalf("sanitizeTimingError did not retain sanitized keys: %q", got) + } +} diff --git a/internal/provider/dockerdind/docker_dind.go b/internal/provider/dockerdind/docker_dind.go index d672666..0beb08b 100644 --- a/internal/provider/dockerdind/docker_dind.go +++ b/internal/provider/dockerdind/docker_dind.go @@ -72,7 +72,7 @@ func (p *Provider) Exec(ctx context.Context, name string, command []string, opts if opts.Stdin != "" { stdin = strings.NewReader(opts.Stdin) } - return p.runWithLog(ctx, stdin, opts.LogPath, opts.Stdout, opts.Stderr, args...) + return p.runWithSensitiveLog(ctx, stdin, opts.LogPath, opts.Stdout, opts.Stderr, opts.SensitiveValues, args...) } func (p *Provider) IP(ctx context.Context, name string, waitSeconds int) (string, error) { @@ -199,11 +199,21 @@ func (p *Provider) run(ctx context.Context, stdin io.Reader, args ...string) (pr } func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath string, stdoutSink, stderrSink io.Writer, args ...string) (provider.ExecResult, error) { + return p.runWithSensitiveLog(ctx, stdin, logPath, stdoutSink, stderrSink, nil, args...) +} + +func (p *Provider) runWithSensitiveLog(ctx context.Context, stdin io.Reader, logPath string, stdoutSink, stderrSink io.Writer, sensitiveValues []string, args ...string) (provider.ExecResult, error) { + bufferedStdout, bufferedStderr, flush := provider.BufferSensitiveSinks(sensitiveValues, stdoutSink, stderrSink) + result, err := p.runWithLogRaw(ctx, stdin, logPath, bufferedStdout, bufferedStderr, sensitiveValues, args...) + return provider.FinishSensitiveExecution(result, err, flush(), sensitiveValues) +} + +func (p *Provider) runWithLogRaw(ctx context.Context, stdin io.Reader, logPath string, stdoutSink, stderrSink io.Writer, sensitiveValues []string, args ...string) (provider.ExecResult, error) { if p.runCommand != nil { return p.runCommand(ctx, stdin, logPath, stdoutSink, stderrSink, args...) } if p.DryRun { - fmt.Printf("[dry-run] %s %s\n", p.Binary, strings.Join(args, " ")) + fmt.Printf("[dry-run] %s %s\n", p.Binary, provider.RedactText(strings.Join(args, " "), sensitiveValues...)) return provider.ExecResult{}, nil } cmd := exec.CommandContext(ctx, p.Binary, args...) diff --git a/internal/provider/dockerdind/docker_dind_test.go b/internal/provider/dockerdind/docker_dind_test.go index eb916bb..306bbf9 100644 --- a/internal/provider/dockerdind/docker_dind_test.go +++ b/internal/provider/dockerdind/docker_dind_test.go @@ -95,6 +95,43 @@ func TestExecArgsPreserveEnvAndStdin(t *testing.T) { } } +func TestExecRedactsSensitiveValuesFromResultErrorAndSinks(t *testing.T) { + const secret = "sentinel-registration-token" + p := New("docker", "", false) + var stdout, stderr bytes.Buffer + p.runCommand = func(_ context.Context, _ io.Reader, _ string, gotStdout, gotStderr io.Writer, args ...string) (provider.ExecResult, error) { + _, _ = gotStdout.Write([]byte("stream " + secret)) + _, _ = gotStderr.Write([]byte("RUNNER_TOKEN=" + secret)) + return provider.ExecResult{Stdout: "result " + secret, Stderr: "SECRET=" + secret}, errors.New("docker " + strings.Join(args, " ") + " failed with " + secret) + } + result, err := p.Exec(context.Background(), "runner", []string{"false"}, provider.ExecOptions{ + Env: map[string]string{"RUNNER_TOKEN": secret}, + SensitiveValues: []string{secret}, + Stdout: &stdout, + Stderr: &stderr, + }) + combined := stdout.String() + stderr.String() + result.Stdout + result.Stderr + err.Error() + if strings.Contains(combined, secret) { + t.Fatalf("sensitive Docker exec leaked secret: %q", combined) + } +} + +func TestExecRedactsSecretAssignmentsWithoutSensitiveValues(t *testing.T) { + const secret = "ordinary-sentinel" + p := New("docker", "", false) + var stdout, stderr bytes.Buffer + p.runCommand = func(_ context.Context, _ io.Reader, _ string, gotStdout, gotStderr io.Writer, _ ...string) (provider.ExecResult, error) { + _, _ = gotStdout.Write([]byte("PASSWORD=" + secret + "\n")) + _, _ = gotStderr.Write([]byte("SECRET=" + secret + "\n")) + return provider.ExecResult{Stdout: "TOKEN=" + secret, Stderr: "PRIVATE_KEY=" + secret}, errors.New("RUNNER_TOKEN=" + secret) + } + result, err := p.Exec(context.Background(), "runner", []string{"false"}, provider.ExecOptions{Stdout: &stdout, Stderr: &stderr}) + combined := stdout.String() + stderr.String() + result.Stdout + result.Stderr + err.Error() + if strings.Contains(combined, secret) { + t.Fatalf("ordinary Docker exec leaked secret assignment: %q", combined) + } +} + func TestListParsesDockerPSOutput(t *testing.T) { p := New("docker", "", false) p.runCommand = func(_ context.Context, _ io.Reader, _ string, _, _ io.Writer, args ...string) (provider.ExecResult, error) { diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 6607fda..71b3949 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -28,11 +28,12 @@ type RunningProcess struct { } type ExecOptions struct { - Stdin string - Env map[string]string - LogPath string - Stdout io.Writer - Stderr io.Writer + Stdin string + Env map[string]string + SensitiveValues []string + LogPath string + Stdout io.Writer + Stderr io.Writer } type ExecResult struct { diff --git a/internal/provider/redaction.go b/internal/provider/redaction.go new file mode 100644 index 0000000..47143b0 --- /dev/null +++ b/internal/provider/redaction.go @@ -0,0 +1,160 @@ +package provider + +import ( + "bytes" + "errors" + "io" + "regexp" + "sort" + "strings" +) + +const redactedValue = "[REDACTED]" + +var sensitiveAssignmentPattern = regexp.MustCompile(`(?i)([[:alnum:]_.-]*(?:token|password|secret|private(?:[_-]?key)?)[[:alnum:]_.-]*=)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^[:space:]]+)`) + +// RedactText removes exact sensitive values and conservative KEY=value secret +// assignments from text before it is returned, logged, or persisted. +func RedactText(text string, sensitiveValues ...string) string { + values := nonemptySensitiveValues(sensitiveValues) + sort.Slice(values, func(i, j int) bool { return len(values[i]) > len(values[j]) }) + for _, value := range values { + text = strings.ReplaceAll(text, value, redactedValue) + } + return sensitiveAssignmentPattern.ReplaceAllString(text, `${1}`+redactedValue) +} + +func HasSensitiveValues(values []string) bool { + return len(nonemptySensitiveValues(values)) > 0 +} + +func RedactExecResult(result ExecResult, sensitiveValues ...string) ExecResult { + result.Stdout = RedactText(result.Stdout, sensitiveValues...) + result.Stderr = RedactText(result.Stderr, sensitiveValues...) + return result +} + +func RedactError(err error, sensitiveValues ...string) error { + if err == nil { + return nil + } + redacted := RedactText(err.Error(), sensitiveValues...) + if redacted == err.Error() { + return err + } + return redactedError{message: redacted, cause: err} +} + +type redactedError struct { + message string + cause error +} + +func (err redactedError) Error() string { return err.message } + +func (err redactedError) Unwrap() error { return err.cause } + +// BufferSensitiveSinks fully buffers executions carrying exact sensitive +// values. Ordinary executions retain line-level streaming while conservative +// KEY=value secret assignments are redacted before reaching either sink. +func BufferSensitiveSinks(sensitiveValues []string, stdout, stderr io.Writer) (io.Writer, io.Writer, func() error) { + if !HasSensitiveValues(sensitiveValues) { + stdoutRedactor := newLineRedactingWriter(stdout) + stderrRedactor := newLineRedactingWriter(stderr) + return stdoutRedactor, stderrRedactor, func() error { + return errors.Join(stdoutRedactor.Flush(), stderrRedactor.Flush()) + } + } + var stdoutBuffer, stderrBuffer bytes.Buffer + flush := func() error { + return errors.Join( + writeRedacted(&stdoutBuffer, stdout, sensitiveValues), + writeRedacted(&stderrBuffer, stderr, sensitiveValues), + ) + } + return &stdoutBuffer, &stderrBuffer, flush +} + +type lineRedactingWriter struct { + destination io.Writer + buffer bytes.Buffer +} + +func newLineRedactingWriter(destination io.Writer) *lineRedactingWriter { + return &lineRedactingWriter{destination: destination} +} + +func (writer *lineRedactingWriter) Write(data []byte) (int, error) { + written := len(data) + if writer.destination == nil { + return written, nil + } + if _, err := writer.buffer.Write(data); err != nil { + return 0, err + } + for { + delimiter := bytes.IndexAny(writer.buffer.Bytes(), "\r\n") + if delimiter < 0 { + break + } + line := append([]byte(nil), writer.buffer.Next(delimiter+1)...) + if err := writer.writeRedacted(string(line)); err != nil { + return written, err + } + } + return written, nil +} + +func (writer *lineRedactingWriter) Flush() error { + if writer.buffer.Len() == 0 { + return nil + } + remaining := writer.buffer.String() + writer.buffer.Reset() + return writer.writeRedacted(remaining) +} + +func (writer *lineRedactingWriter) writeRedacted(text string) error { + if writer.destination == nil { + return nil + } + redacted := RedactText(text) + written, err := io.WriteString(writer.destination, redacted) + if err == nil && written != len(redacted) { + return io.ErrShortWrite + } + return err +} + +func FinishSensitiveExecution(result ExecResult, runErr, sinkErr error, sensitiveValues []string) (ExecResult, error) { + result = RedactExecResult(result, sensitiveValues...) + return result, errors.Join(RedactError(runErr, sensitiveValues...), sinkErr) +} + +func writeRedacted(source *bytes.Buffer, destination io.Writer, sensitiveValues []string) error { + if destination == nil || source.Len() == 0 { + return nil + } + redacted := RedactText(source.String(), sensitiveValues...) + written, err := io.WriteString(destination, redacted) + if err == nil && written != len(redacted) { + return io.ErrShortWrite + } + return err +} + +func nonemptySensitiveValues(values []string) []string { + out := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + if value == "" { + continue + } + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + return out +} diff --git a/internal/provider/redaction_test.go b/internal/provider/redaction_test.go new file mode 100644 index 0000000..8d2bdd0 --- /dev/null +++ b/internal/provider/redaction_test.go @@ -0,0 +1,74 @@ +package provider + +import ( + "bytes" + "errors" + "fmt" + "strings" + "testing" +) + +func TestRedactTextRemovesExactValuesAndSecretAssignments(t *testing.T) { + const secret = "sentinel-registration-token" + input := "docker exec -e RUNNER_TOKEN=" + secret + " PASSWORD=hunter2 normal=value exact=" + secret + got := RedactText(input, secret) + if strings.Contains(got, secret) || strings.Contains(got, "hunter2") { + t.Fatalf("RedactText() leaked a secret: %q", got) + } + for _, want := range []string{"RUNNER_TOKEN=[REDACTED]", "PASSWORD=[REDACTED]", "exact=[REDACTED]"} { + if !strings.Contains(got, want) { + t.Fatalf("RedactText() = %q, want %q", got, want) + } + } +} + +func TestBufferSensitiveSinksRedactsBeforeWriting(t *testing.T) { + const secret = "sentinel-registration-token" + var stdout, stderr bytes.Buffer + bufferedStdout, bufferedStderr, flush := BufferSensitiveSinks([]string{secret}, &stdout, &stderr) + _, _ = bufferedStdout.Write([]byte("out " + secret)) + _, _ = bufferedStderr.Write([]byte("RUNNER_TOKEN=" + secret)) + if stdout.Len() != 0 || stderr.Len() != 0 { + t.Fatal("sensitive output streamed before redaction") + } + if err := flush(); err != nil { + t.Fatal(err) + } + if strings.Contains(stdout.String()+stderr.String(), secret) { + t.Fatalf("sensitive sinks leaked secret: stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestBufferSensitiveSinksRedactsOrdinaryOutputAcrossChunks(t *testing.T) { + var stdout bytes.Buffer + bufferedStdout, _, flush := BufferSensitiveSinks(nil, &stdout, nil) + _, _ = bufferedStdout.Write([]byte("ordinary line\nPASS")) + if got := stdout.String(); got != "ordinary line\n" { + t.Fatalf("ordinary complete-line streaming = %q", got) + } + _, _ = bufferedStdout.Write([]byte("WORD=sentinel\n")) + if err := flush(); err != nil { + t.Fatal(err) + } + if strings.Contains(stdout.String(), "sentinel") { + t.Fatalf("ordinary sink leaked chunked secret assignment: %q", stdout.String()) + } +} + +func TestFinishSensitiveExecutionRedactsResultAndError(t *testing.T) { + const secret = "sentinel-registration-token" + sentinel := errors.New("sentinel cause") + result, err := FinishSensitiveExecution( + ExecResult{Stdout: secret, Stderr: "TOKEN=" + secret}, + fmt.Errorf("command failed with %s: %w", secret, sentinel), + nil, + []string{secret}, + ) + combined := result.Stdout + result.Stderr + err.Error() + if strings.Contains(combined, secret) { + t.Fatalf("sensitive execution outcome leaked secret: %q", combined) + } + if !errors.Is(err, sentinel) { + t.Fatalf("redacted error no longer wraps original cause: %v", err) + } +} diff --git a/internal/provider/tart/tart.go b/internal/provider/tart/tart.go index edc529b..f4fe72f 100644 --- a/internal/provider/tart/tart.go +++ b/internal/provider/tart/tart.go @@ -12,10 +12,13 @@ import ( ) type Provider struct { - Binary string - DryRun bool + Binary string + DryRun bool + runCommand runCommandFunc } +type runCommandFunc func(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, args ...string) (provider.ExecResult, error) + func New(binary string, dryRun bool) *Provider { if binary == "" { binary = "tart" @@ -66,7 +69,7 @@ func (p *Provider) Exec(ctx context.Context, name string, command []string, opts } full = append(full, name) full = append(full, provider.EnvCommand(opts.Env, command)...) - return p.runWithLog(ctx, strings.NewReader(opts.Stdin), opts.Stdout, opts.Stderr, full...) + return p.runWithSensitiveLog(ctx, strings.NewReader(opts.Stdin), opts.Stdout, opts.Stderr, opts.SensitiveValues, full...) } func (p *Provider) IP(ctx context.Context, name string, waitSeconds int) (string, error) { @@ -115,8 +118,21 @@ func (p *Provider) run(ctx context.Context, stdin io.Reader, args ...string) (pr } func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, stdoutSink, stderrSink io.Writer, args ...string) (provider.ExecResult, error) { + return p.runWithSensitiveLog(ctx, stdin, stdoutSink, stderrSink, nil, args...) +} + +func (p *Provider) runWithSensitiveLog(ctx context.Context, stdin io.Reader, stdoutSink, stderrSink io.Writer, sensitiveValues []string, args ...string) (provider.ExecResult, error) { + bufferedStdout, bufferedStderr, flush := provider.BufferSensitiveSinks(sensitiveValues, stdoutSink, stderrSink) + result, err := p.runWithLogRaw(ctx, stdin, bufferedStdout, bufferedStderr, sensitiveValues, args...) + return provider.FinishSensitiveExecution(result, err, flush(), sensitiveValues) +} + +func (p *Provider) runWithLogRaw(ctx context.Context, stdin io.Reader, stdoutSink, stderrSink io.Writer, sensitiveValues []string, args ...string) (provider.ExecResult, error) { + if p.runCommand != nil { + return p.runCommand(ctx, stdin, stdoutSink, stderrSink, args...) + } if p.DryRun { - fmt.Printf("[dry-run] %s %s\n", p.Binary, strings.Join(args, " ")) + fmt.Printf("[dry-run] %s %s\n", p.Binary, provider.RedactText(strings.Join(args, " "), sensitiveValues...)) return provider.ExecResult{}, nil } cmd := exec.CommandContext(ctx, p.Binary, args...) diff --git a/internal/provider/tart/tart_test.go b/internal/provider/tart/tart_test.go index 3601b8f..8dcc363 100644 --- a/internal/provider/tart/tart_test.go +++ b/internal/provider/tart/tart_test.go @@ -3,6 +3,7 @@ package tart import ( "bytes" "context" + "errors" "io" "os" "strings" @@ -33,6 +34,43 @@ func TestCaptureWriterCopiesToCaptureAndInjectedSink(t *testing.T) { } } +func TestExecRedactsSensitiveValuesFromResultErrorAndSinks(t *testing.T) { + const secret = "sentinel-registration-token" + p := New("tart", false) + var stdout, stderr bytes.Buffer + p.runCommand = func(_ context.Context, _ io.Reader, gotStdout, gotStderr io.Writer, args ...string) (provider.ExecResult, error) { + _, _ = gotStdout.Write([]byte("stream " + secret)) + _, _ = gotStderr.Write([]byte("RUNNER_TOKEN=" + secret)) + return provider.ExecResult{Stdout: "result " + secret, Stderr: "SECRET=" + secret}, errors.New("tart " + strings.Join(args, " ") + " failed with " + secret) + } + result, err := p.Exec(context.Background(), "runner", []string{"false"}, provider.ExecOptions{ + Env: map[string]string{"RUNNER_TOKEN": secret}, + SensitiveValues: []string{secret}, + Stdout: &stdout, + Stderr: &stderr, + }) + combined := stdout.String() + stderr.String() + result.Stdout + result.Stderr + err.Error() + if strings.Contains(combined, secret) { + t.Fatalf("sensitive Tart exec leaked secret: %q", combined) + } +} + +func TestExecRedactsSecretAssignmentsWithoutSensitiveValues(t *testing.T) { + const secret = "ordinary-sentinel" + p := New("tart", false) + var stdout, stderr bytes.Buffer + p.runCommand = func(_ context.Context, _ io.Reader, gotStdout, gotStderr io.Writer, _ ...string) (provider.ExecResult, error) { + _, _ = gotStdout.Write([]byte("PASSWORD=" + secret + "\n")) + _, _ = gotStderr.Write([]byte("SECRET=" + secret + "\n")) + return provider.ExecResult{Stdout: "TOKEN=" + secret, Stderr: "PRIVATE_KEY=" + secret}, errors.New("RUNNER_TOKEN=" + secret) + } + result, err := p.Exec(context.Background(), "runner", []string{"false"}, provider.ExecOptions{Stdout: &stdout, Stderr: &stderr}) + combined := stdout.String() + stderr.String() + result.Stdout + result.Stderr + err.Error() + if strings.Contains(combined, secret) { + t.Fatalf("ordinary Tart exec leaked secret assignment: %q", combined) + } +} + func TestStartDryRunIncludesRosettaBeforeName(t *testing.T) { p := New("tart", true) out := captureStdout(t, func() { diff --git a/internal/provider/wsl/wsl.go b/internal/provider/wsl/wsl.go index 27a940d..f4cddd9 100644 --- a/internal/provider/wsl/wsl.go +++ b/internal/provider/wsl/wsl.go @@ -97,7 +97,7 @@ func (p *Provider) Exec(ctx context.Context, name string, command []string, opts if opts.Stdin != "" { stdin = strings.NewReader(opts.Stdin) } - return p.runWithLog(ctx, stdin, opts.LogPath, opts.Stdout, opts.Stderr, p.execArgs(name, command, opts.Env)...) + return p.runWithSensitiveLog(ctx, stdin, opts.LogPath, opts.Stdout, opts.Stderr, opts.SensitiveValues, p.execArgs(name, command, opts.Env)...) } func (p *Provider) IP(ctx context.Context, name string, waitSeconds int) (string, error) { @@ -273,11 +273,21 @@ func (p *Provider) run(ctx context.Context, stdin io.Reader, args ...string) (pr } func (p *Provider) runWithLog(ctx context.Context, stdin io.Reader, logPath string, stdoutSink, stderrSink io.Writer, args ...string) (provider.ExecResult, error) { + return p.runWithSensitiveLog(ctx, stdin, logPath, stdoutSink, stderrSink, nil, args...) +} + +func (p *Provider) runWithSensitiveLog(ctx context.Context, stdin io.Reader, logPath string, stdoutSink, stderrSink io.Writer, sensitiveValues []string, args ...string) (provider.ExecResult, error) { + bufferedStdout, bufferedStderr, flush := provider.BufferSensitiveSinks(sensitiveValues, stdoutSink, stderrSink) + result, err := p.runWithLogRaw(ctx, stdin, logPath, bufferedStdout, bufferedStderr, sensitiveValues, args...) + return provider.FinishSensitiveExecution(result, err, flush(), sensitiveValues) +} + +func (p *Provider) runWithLogRaw(ctx context.Context, stdin io.Reader, logPath string, stdoutSink, stderrSink io.Writer, sensitiveValues []string, args ...string) (provider.ExecResult, error) { if p.runCommand != nil { return p.runCommand(ctx, stdin, logPath, stdoutSink, stderrSink, args...) } if p.DryRun { - fmt.Printf("[dry-run] %s %s\n", p.Binary, strings.Join(args, " ")) + fmt.Printf("[dry-run] %s %s\n", p.Binary, provider.RedactText(strings.Join(args, " "), sensitiveValues...)) return provider.ExecResult{}, nil } cmd := exec.CommandContext(ctx, p.Binary, args...) diff --git a/internal/provider/wsl/wsl_test.go b/internal/provider/wsl/wsl_test.go index 71bbb11..59dc665 100644 --- a/internal/provider/wsl/wsl_test.go +++ b/internal/provider/wsl/wsl_test.go @@ -38,6 +38,43 @@ func TestExecPassesInjectedTranscriptWriters(t *testing.T) { } } +func TestExecRedactsSensitiveValuesFromResultErrorAndSinks(t *testing.T) { + const secret = "sentinel-registration-token" + p := New("wsl.exe", t.TempDir(), t.TempDir(), false) + var stdout, stderr bytes.Buffer + p.runCommand = func(_ context.Context, _ io.Reader, _ string, gotStdout, gotStderr io.Writer, args ...string) (provider.ExecResult, error) { + _, _ = gotStdout.Write([]byte("stream " + secret)) + _, _ = gotStderr.Write([]byte("RUNNER_TOKEN=" + secret)) + return provider.ExecResult{Stdout: "result " + secret, Stderr: "SECRET=" + secret}, errors.New("wsl " + strings.Join(args, " ") + " failed with " + secret) + } + result, err := p.Exec(context.Background(), "runner", []string{"false"}, provider.ExecOptions{ + Env: map[string]string{"RUNNER_TOKEN": secret}, + SensitiveValues: []string{secret}, + Stdout: &stdout, + Stderr: &stderr, + }) + combined := stdout.String() + stderr.String() + result.Stdout + result.Stderr + err.Error() + if strings.Contains(combined, secret) { + t.Fatalf("sensitive WSL exec leaked secret: %q", combined) + } +} + +func TestExecRedactsSecretAssignmentsWithoutSensitiveValues(t *testing.T) { + const secret = "ordinary-sentinel" + p := New("wsl.exe", t.TempDir(), t.TempDir(), false) + var stdout, stderr bytes.Buffer + p.runCommand = func(_ context.Context, _ io.Reader, _ string, gotStdout, gotStderr io.Writer, _ ...string) (provider.ExecResult, error) { + _, _ = gotStdout.Write([]byte("PASSWORD=" + secret + "\n")) + _, _ = gotStderr.Write([]byte("SECRET=" + secret + "\n")) + return provider.ExecResult{Stdout: "TOKEN=" + secret, Stderr: "PRIVATE_KEY=" + secret}, errors.New("RUNNER_TOKEN=" + secret) + } + result, err := p.Exec(context.Background(), "runner", []string{"false"}, provider.ExecOptions{Stdout: &stdout, Stderr: &stderr}) + combined := stdout.String() + stderr.String() + result.Stdout + result.Stderr + err.Error() + if strings.Contains(combined, secret) { + t.Fatalf("ordinary WSL exec leaked secret assignment: %q", combined) + } +} + func TestCommandConstruction(t *testing.T) { p := New("wsl.exe", filepath.Join("C:", "ephemeral"), `D:\repo`, true) installDir, err := p.instanceDir("epar-test-1") diff --git a/scripts/guest/ubuntu/configure-runner.sh b/scripts/guest/ubuntu/configure-runner.sh index e7377c9..6229bdb 100755 --- a/scripts/guest/ubuntu/configure-runner.sh +++ b/scripts/guest/ubuntu/configure-runner.sh @@ -2,14 +2,19 @@ set -euo pipefail : "${RUNNER_URL:?RUNNER_URL is required}" -: "${RUNNER_TOKEN:?RUNNER_TOKEN is required}" : "${RUNNER_NAME:?RUNNER_NAME is required}" : "${RUNNER_LABELS:?RUNNER_LABELS is required}" RUNNER_EPHEMERAL="${RUNNER_EPHEMERAL:-true}" RUNNER_GROUP="${RUNNER_GROUP:-}" RUNNER_NO_DEFAULT_LABELS="${RUNNER_NO_DEFAULT_LABELS:-false}" +EPAR_ACTIONS_RUNNER_DIR="${EPAR_ACTIONS_RUNNER_DIR:-/opt/actions-runner}" -cd /opt/actions-runner +if ! IFS= read -r RUNNER_TOKEN || [[ -z "${RUNNER_TOKEN}" ]]; then + echo "RUNNER_TOKEN must be provided as one nonempty line on stdin" >&2 + exit 1 +fi + +cd "${EPAR_ACTIONS_RUNNER_DIR}" if [[ -f .runner ]]; then sudo -u runner ./config.sh remove --token "${RUNNER_TOKEN}" || true fi From 8cbde7e73a56cbdca4ce745f652b276803a62a53 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 20 Jul 2026 15:48:54 +0800 Subject: [PATCH 6/9] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- internal/logging/replace_windows.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/logging/replace_windows.go b/internal/logging/replace_windows.go index f66344e..2521017 100644 --- a/internal/logging/replace_windows.go +++ b/internal/logging/replace_windows.go @@ -29,6 +29,9 @@ func replaceFile(source, destination string) error { moveFileReplaceExisting|moveFileWriteThrough, ) if result == 0 { + if errno, ok := callErr.(syscall.Errno); ok && errno == 0 { + return syscall.EINVAL + } return callErr } return nil From 4da58b0794134b73ecad10a0bbbdea13e459b663 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 20 Jul 2026 19:16:04 +0800 Subject: [PATCH 7/9] Prepare repository for public release (#8) Prevent the live EPAR canary jobs from running for pull requests originating from forks. Update runner verification documentation for same-repository and fork pull requests. Add contributor guidance, a code of conduct, and issue/PR templates. Expand the security guidance with a private reporting path. --- .github/ISSUE_TEMPLATE/bug_report.yml | 40 +++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.yml | 25 ++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 14 +++++++ .../workflows/core-runner-verification.yml | 4 ++ CODE_OF_CONDUCT.md | 17 ++++++++ CONTRIBUTING.md | 27 +++++++++++++ README.md | 4 +- docs/core-runner-verification.md | 26 ++++-------- docs/security.md | 8 ++++ 9 files changed, 145 insertions(+), 20 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..62f2edf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,40 @@ +name: Bug report +description: Report reproducible behavior that differs from the documentation or expected result. +title: "bug: " +labels: + - bug +body: + - type: markdown + attributes: + value: | + Thank you for reporting a bug. Do not include credentials, private keys, tokens, or other secrets. For a security vulnerability, use the private reporting path in docs/security.md instead. + - type: textarea + id: summary + attributes: + label: What happened? + description: Describe the observed behavior and the expected result. + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Steps to reproduce + description: Include the smallest safe configuration and commands needed to reproduce the issue. + placeholder: | + 1. ... + 2. ... + 3. ... + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment + description: Include EPAR version or commit, provider, host operating system, and Docker or WSL version when relevant. + validations: + required: true + - type: textarea + id: logs + attributes: + label: Sanitized logs + description: Include relevant logs after removing secrets and private infrastructure details. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..8f18a8f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,25 @@ +name: Feature request +description: Suggest an improvement to EPAR. +title: "feature: " +labels: + - enhancement +body: + - type: textarea + id: problem + attributes: + label: What problem does this solve? + description: Describe the workflow or limitation you are trying to address. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed solution + description: Explain the change you would like to see and any alternatives you considered. + validations: + required: true + - type: textarea + id: context + attributes: + label: Additional context + description: Include relevant provider, operating-system, or GitHub Actions details. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..65bc5c6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,14 @@ +## Summary + + + +## Validation + + + +## Checklist + +- [ ] I kept credentials, private keys, tokens, and machine-specific configuration out of this pull request. +- [ ] I added or updated tests where behavior changed. +- [ ] I updated relevant documentation. +- [ ] I read and followed the contributing guide and code of conduct. diff --git a/.github/workflows/core-runner-verification.yml b/.github/workflows/core-runner-verification.yml index c1c5143..6e0f12c 100644 --- a/.github/workflows/core-runner-verification.yml +++ b/.github/workflows/core-runner-verification.yml @@ -9,6 +9,7 @@ on: branches: - develop - main + workflow_dispatch: permissions: contents: read @@ -20,6 +21,7 @@ concurrency: jobs: controller: name: Core runner controller + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest environment: epar-live-ci # Leaves time for a cold image build and bounded cleanup around the @@ -83,6 +85,7 @@ jobs: canary-1: name: Core canary 1 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: group: epar-ci-canary labels: epar-core-${{ github.run_id }}-${{ github.run_attempt }} @@ -128,6 +131,7 @@ jobs: canary-2: name: Core canary 2 needs: canary-1 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: group: epar-ci-canary labels: epar-core-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..73011ab --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,17 @@ +# Code of Conduct + +We are committed to a welcoming, respectful, and constructive community. + +## Expected Behavior + +- Be respectful and professional in issues, pull requests, discussions, and reviews. +- Focus feedback on ideas and technical work rather than people. +- Welcome good-faith questions and contributions from people with different backgrounds and experience levels. + +## Unacceptable Behavior + +Harassment, discrimination, personal attacks, threats, deliberate disruption, and sharing another person's private information without permission are not acceptable. + +## Reporting + +Report conduct concerns privately to a repository maintainer through GitHub. Do not open a public issue for a conduct report. Maintainers will review reports promptly and may remove content, limit participation, or take other action needed to protect the community. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8945162 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,27 @@ +# Contributing to EPAR + +Thanks for taking the time to contribute. + +## Before You Start + +- Use GitHub issues to discuss bugs, documentation gaps, and proposed changes before starting substantial work. +- Do not report security vulnerabilities in public issues. Follow [Security](docs/security.md) instead. +- EPAR runs GitHub Actions jobs on trusted infrastructure. Changes that affect runners, credentials, container privileges, workflow permissions, or cleanup boundaries need clear security reasoning and tests. + +## Development Workflow + +1. Fork the repository and create a focused branch from `develop`. +2. Keep the change small and document any operational or security behavior that it changes. +3. Run the relevant tests locally. The baseline Go test suite is `go test ./...`. +4. Open a pull request targeting `develop` and complete the pull-request template. + +Fork pull requests run the safe hosted verification workflow. The live EPAR canary is reserved for branches in this repository because it uses a protected environment and disposable privileged containers. + +## Pull Request Expectations + +- Explain the problem, the approach, and how you tested it. +- Add or update tests when behavior changes. +- Keep credentials, private keys, tokens, and machine-specific configuration out of commits. +- Update the relevant documentation when a user-visible or operational behavior changes. + +By contributing, you agree to follow the [Code of Conduct](CODE_OF_CONDUCT.md). diff --git a/README.md b/README.md index 6a3779b..a8e6e51 100644 --- a/README.md +++ b/README.md @@ -161,5 +161,7 @@ GitHub also warns against using self-hosted runners with public repositories tha - [Windows Startup](docs/advanced/windows-startup.md): start EPAR after Windows login. - [macOS Startup](docs/advanced/macos-startup.md): start EPAR after macOS login. - [Running EPAR Without Installing Go](docs/advanced/no-go-install.md): run from source with no local Go install. -- [Security](docs/security.md): trust boundaries and secret handling. +- [Security](docs/security.md): trust boundaries, secret handling, and private vulnerability reporting. +- [Contributing](CONTRIBUTING.md): how to propose and validate changes. +- [Code of Conduct](CODE_OF_CONDUCT.md): community expectations and reporting concerns. - [Level 1 Core Runner Verification](docs/core-runner-verification.md): trusted live CI setup, canary behavior, and cleanup. diff --git a/docs/core-runner-verification.md b/docs/core-runner-verification.md index 600bc6a..bdf3c61 100644 --- a/docs/core-runner-verification.md +++ b/docs/core-runner-verification.md @@ -50,9 +50,7 @@ cannot accept them. ### GitHub App and protected environment -The GitHub App must be installed in the target organization and have -organization self-hosted-runner read/write permission. Create a GitHub Actions -environment named `epar-live-ci`, restrict it to trusted branches, and add: +The GitHub App must be installed in the target organization and have organization self-hosted-runner read/write permission. Create a GitHub Actions environment named `epar-live-ci`, choose **Selected branches and tags**, allow `develop`, `main`, and `refs/pull/*/merge`, and add: | Kind | Name | Value | | --- | --- | --- | @@ -65,17 +63,11 @@ not store it in repository variables, workflow YAML, a tracked config file, or an artifact. The workflow materializes it in a mode-restricted temporary file on the controller and removes that file during cleanup. -For the initial feature-branch test, allow -`feature/level-1-core-runner-verification`. Allow `develop` for the ongoing push -trigger. Requiring an environment reviewer is possible, but every matching -push will wait for that approval. +Requiring an environment reviewer is possible, but every matching push and same-repository pull request will wait for that approval. ## Trust Boundaries and Triggers -The controller job is privileged and secret-bearing. It receives the GitHub -App key and a workflow token with Actions write permission, and it can start -privileged containers in its disposable GitHub-hosted VM. It runs only for -trusted repository changes and never for pull-request events. +The controller job is privileged and secret-bearing. It receives the GitHub App key and a workflow token with Actions write permission, and it can start privileged containers in its disposable GitHub-hosted VM. It runs only for trusted repository changes: pushes to `develop` or `main`, and pull requests whose source branch is in this repository. Fork pull requests still trigger the workflow, but all three core-verification jobs skip before they can receive environment secrets or run on the canary group. The two canary jobs do not receive the GitHub App key. They receive only the minimum workflow permissions needed for checkout and artifact operations, and @@ -84,15 +76,11 @@ container. The workflow runs on: -- pushes to `feature/level-1-core-runner-verification` during initial rollout -- pushes to `develop` -- manual `workflow_dispatch` after the workflow is present on the repository's - default branch +- pull requests targeting `develop` or `main` (fork pull requests still trigger this workflow, but core-verification jobs run only when the source branch belongs to this repository) +- pushes to `develop` or `main` +- manual `workflow_dispatch` after the workflow is present on the repository's default branch -It intentionally has no `pull_request` or `pull_request_target` trigger. Do not -add one: code from an untrusted pull request must not reach the controller, -environment secrets, or privileged Docker host. Remove the temporary feature -branch trigger after the initial rollout is complete. +The workflow does not use `pull_request_target`. Keep the same-repository job guard on every core-verification job so code from a forked pull request cannot reach the controller, environment secrets, or privileged Docker host. ## Expected Result and Cleanup diff --git a/docs/security.md b/docs/security.md index 55a9f26..c8c277a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -4,6 +4,14 @@ EPAR is intended for trusted jobs by default. It adds cleanup and isolation arou GitHub's self-hosted runner warning still applies: GitHub recommends using self-hosted runners only with private repositories because public repository forks can run code on the runner machine through pull request workflows. Read the official GitHub guidance before exposing any self-hosted runner to public or untrusted workflows: [Adding self-hosted runners](https://docs.github.com/actions/hosting-your-own-runners/adding-self-hosted-runners). +## Reporting A Vulnerability + +Security fixes are applied to the latest released version and the current default branch. + +Do not disclose suspected vulnerabilities in a public issue, discussion, pull request, or commit. Use this repository's private vulnerability reporting flow: open the **Security** tab, select **Report a vulnerability**, and provide a clear description, affected versions, reproduction steps or a proof of concept, and the potential impact. + +If private reporting is unavailable, contact a repository maintainer privately through GitHub and include the same information. We will acknowledge the report, investigate it, and coordinate a fix and disclosure timeline with you. + ## What EPAR Improves Disposable instances reduce host pollution, stale runner state, and accidental cross-job interference. After a job completes, EPAR retires the instance and creates a replacement. For Docker-DinD, job-created containers, networks, volumes, and inner image cache live inside the runner container's private Docker daemon and are removed with that runner instance. From d5cbd181c1ae7cbdcc517170019d7cd22fd8d128 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 21 Jul 2026 16:36:58 +0800 Subject: [PATCH 8/9] Document and verify cross-architecture Docker support (#10) - document foreign-architecture Docker failure signatures, diagnosis, and the pinned docker/setup-qemu-action remedy - mark Tart container compatibility as experimental and clarify WSL architecture behavior --- .../hosted-runner-docker-architecture.yml | 162 ++++++++++++++++++ README.md | 9 +- SUPPORT.md | 18 ++ cmd/ephemeral-action-runner/init.go | 144 ++++++++++++++-- cmd/ephemeral-action-runner/init_test.go | 86 ++++++++++ cmd/ephemeral-action-runner/start_test.go | 48 ++++++ configs/tart.example.yml | 2 + docs/configuration.md | 2 +- docs/image-build.md | 3 + docs/providers/tart.md | 17 +- docs/providers/wsl.md | 2 + docs/troubleshooting.md | 87 ++++++++++ docs/usage.md | 10 +- start | 2 +- start.ps1 | 4 +- 15 files changed, 561 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/hosted-runner-docker-architecture.yml create mode 100644 SUPPORT.md diff --git a/.github/workflows/hosted-runner-docker-architecture.yml b/.github/workflows/hosted-runner-docker-architecture.yml new file mode 100644 index 0000000..d48bf1c --- /dev/null +++ b/.github/workflows/hosted-runner-docker-architecture.yml @@ -0,0 +1,162 @@ +name: Hosted runner Docker architecture proof + +on: + pull_request: + branches: + - develop + - main + paths: + - .github/workflows/hosted-runner-docker-architecture.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: hosted-docker-architecture-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + capability: + name: Capability (${{ matrix.runner }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + runner: + - ubuntu-latest + - ubuntu-24.04-arm + - windows-latest + - windows-11-arm + - macos-latest + - macos-15-intel + + steps: + - name: Record host and Docker capability + shell: pwsh + run: | + $ErrorActionPreference = 'Continue' + $summary = [System.Collections.Generic.List[string]]::new() + $summary.Add("## $env:RUNNER_OS/$env:RUNNER_ARCH on ``${{ matrix.runner }}``") + $summary.Add('') + $summary.Add("- Runner OS: ``$env:RUNNER_OS``") + $summary.Add("- Runner architecture: ``$env:RUNNER_ARCH``") + + $dockerCommand = Get-Command docker -ErrorAction SilentlyContinue + if (-not $dockerCommand) { + $summary.Add('- Docker CLI: not installed') + $summary.Add('- Linux container test: unavailable') + } else { + $summary.Add("- Docker CLI: ``$($dockerCommand.Source)``") + $versionOutput = (& docker version 2>&1 | Out-String).Trim() + $versionStatus = $LASTEXITCODE + $summary.Add("- Docker daemon reachable: ``$($versionStatus -eq 0)``") + + if ($versionStatus -eq 0) { + $serverOS = (& docker info --format '{{.OSType}}' 2>&1 | Out-String).Trim() + $serverArchitecture = (& docker info --format '{{.Architecture}}' 2>&1 | Out-String).Trim() + $summary.Add("- Docker server: ``$serverOS/$serverArchitecture``") + if ($serverOS -eq 'linux') { + $summary.Add('- Linux container test: available') + } else { + $summary.Add('- Linux container test: unavailable because the reachable daemon is not a Linux daemon') + } + } else { + $summary.Add('- Linux container test: unavailable because no Docker daemon is reachable') + $summary.Add('') + $summary.Add('
docker version diagnostic') + $summary.Add('') + $summary.Add('```text') + $summary.Add($versionOutput) + $summary.Add('```') + $summary.Add('
') + } + } + + $summary | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Encoding utf8 -Append + $summary | ForEach-Object { Write-Host $_ } + + linux-cross-architecture: + name: Linux ${{ matrix.native }} runs ${{ matrix.foreign }} with explicit QEMU + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-latest + native: amd64 + native_platform: linux/amd64 + native_output: x86_64 + foreign: arm64 + foreign_platform: linux/arm64 + foreign_output: aarch64 + - runner: ubuntu-24.04-arm + native: arm64 + native_platform: linux/arm64 + native_output: aarch64 + foreign: amd64 + foreign_platform: linux/amd64 + foreign_output: x86_64 + + steps: + - name: Verify native container execution + shell: bash + run: | + set -euo pipefail + docker info --format 'Docker server: {{.OSType}}/{{.Architecture}}' + native_architecture="$(docker run --rm --platform '${{ matrix.native_platform }}' alpine:3.22 uname -m)" + echo "Native container reported: ${native_architecture}" + [[ "${native_architecture}" == '${{ matrix.native_output }}' ]] + + - name: Observe foreign-architecture behavior before explicit QEMU setup + id: baseline + shell: bash + run: | + set -euo pipefail + set +e + baseline_output="$(docker run --rm --platform '${{ matrix.foreign_platform }}' alpine:3.22 uname -m 2>&1)" + baseline_status=$? + set -e + + printf '%s\n' "${baseline_output}" + { + echo '## ${{ matrix.foreign }} container before explicit QEMU setup' + echo + echo "- Exit status: \`${baseline_status}\`" + echo + echo '```text' + printf '%s\n' "${baseline_output}" + echo '```' + } >>"${GITHUB_STEP_SUMMARY}" + + if [[ ${baseline_status} -eq 0 ]]; then + [[ "${baseline_output}" == *'${{ matrix.foreign_output }}'* ]] + echo 'This hosted image already had a compatible foreign-architecture execution path before the workflow configured QEMU.' >>"${GITHUB_STEP_SUMMARY}" + elif [[ "${baseline_output}" == *"exec format error"* ]]; then + echo 'The baseline failed with the expected missing-emulation error.' >>"${GITHUB_STEP_SUMMARY}" + else + echo 'The baseline failed for a reason other than a recognized architecture mismatch.' >&2 + exit 1 + fi + + - name: Set up foreign-architecture container emulation + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + with: + image: docker.io/tonistiigi/binfmt@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 + platforms: ${{ matrix.foreign }} + + - name: Verify foreign-architecture container execution after QEMU setup + shell: bash + run: | + set -euo pipefail + foreign_architecture="$(docker run --rm --platform '${{ matrix.foreign_platform }}' alpine:3.22 uname -m)" + echo "Emulated container reported: ${foreign_architecture}" + [[ "${foreign_architecture}" == '${{ matrix.foreign_output }}' ]] + { + echo '## ${{ matrix.foreign }} container after explicit QEMU setup' + echo + echo "- Reported architecture: \`${foreign_architecture}\`" + echo '- Result: success' + } >>"${GITHUB_STEP_SUMMARY}" diff --git a/README.md b/README.md index a8e6e51..7dd66e2 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ That's it. #### What Happens -EPAR initializes `.local/config.yml` for you if it does not exist. Docker-DinD is the default. The wizard asks whether new Docker-DinD runners should inherit the host's trusted TLS roots and defaults to yes. On native Windows, it also offers WSL2 when `wsl.exe --status` confirms default version 2; press Enter to keep Docker-DinD. Existing configs do not enable host trust inheritance automatically. You can customize the config afterward; see [Configuration](docs/configuration.md). +EPAR initializes `.local/config.yml` for you if it does not exist. Docker-DinD is the default. The wizard asks whether new Docker-DinD runners should inherit the host's trusted TLS roots and defaults to yes. On native Windows, it also offers WSL2 when `wsl.exe --status` confirms default version 2. On macOS, it offers experimental Tart mode when `tart --version` succeeds. Press Enter to keep Docker-DinD. Existing configs do not enable host trust inheritance automatically. You can customize the config afterward; see [Configuration](docs/configuration.md). Then EPAR checks the configured runner image, builds or replaces it when needed, and starts the configured number of runners. The default config uses `pool.instances: 1`. @@ -117,10 +117,12 @@ Docker-DinD is the default first choice. Other providers are available when they | --- | --- | | Docker-DinD | You have a Docker-compatible daemon on Windows, macOS, or Linux, and want a private Docker daemon per runner. | | WSL2 | You are on Windows and want runners as disposable WSL distros. | -| Tart | You are on Apple Silicon macOS and want Linux VM runners; consider Docker-DinD first for Docker-heavy jobs because virtualization limits can affect compatibility. | +| Tart (experimental) | You are on Apple Silicon macOS and want to experiment with native ARM64 Linux VMs. The default Tart image is a basic Ubuntu OS image and does not include the normal GitHub-hosted runner dependency set. | WSL2 also defaults to Catthehacker's full Ubuntu runner image, but it converts that Docker image into a WSL rootfs during `image build`. +Tart is not a ready-made substitute for GitHub's hosted Ubuntu runners. If you need that environment, build and maintain your own bootable Tart runner image by adapting the scripts from [actions/runner-images](https://github.com/actions/runner-images), then configure EPAR to use it. EPAR does not automate that conversion. + See [Usage](docs/usage.md) for WSL, Tart, source builds, custom configs, and advanced options. ## FAQ @@ -154,10 +156,11 @@ GitHub also warns against using self-hosted runners with public repositories tha - [GitHub App Setup](docs/github-app.md): required GitHub App permissions and fields. - [Docker-DinD Provider](docs/providers/docker-dind.md): default Docker runner mode. - [WSL Provider](docs/providers/wsl.md): Windows WSL2 runners. -- [Tart Provider](docs/providers/tart.md): Apple Silicon Linux VM runners. +- [Tart Provider (experimental)](docs/providers/tart.md): Apple Silicon ARM64 Linux VM runners and Rosetta compatibility limits. - [Image Build](docs/image-build.md): image internals and customization. - [Operations](docs/operations.md): logs, cleanup, and troubleshooting. - [Troubleshooting](docs/troubleshooting.md): symptom-first diagnostics by host and provider. +- [Support](SUPPORT.md): where to start, what diagnostic information to collect, and where to ask for help. - [Windows Startup](docs/advanced/windows-startup.md): start EPAR after Windows login. - [macOS Startup](docs/advanced/macos-startup.md): start EPAR after macOS login. - [Running EPAR Without Installing Go](docs/advanced/no-go-install.md): run from source with no local Go install. diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..72f7890 --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,18 @@ +# Support + +Start with the [troubleshooting guide](docs/troubleshooting.md). It provides symptom-first diagnostics for Docker-DinD, WSL, Tart, host trust, image builds, storage, and cross-architecture containers. + +Before asking for help, search the repository's existing [issues](https://github.com/solutionforest/ephemeral-action-runner/issues) and collect: + +- the EPAR version or commit; +- the host operating system and architecture; +- the selected provider and runner image; +- the relevant configuration with credentials, private keys, tokens, certificate contents, organization names, and other sensitive values removed; +- the smallest reproducible workflow or command; +- the relevant controller, image-build, and guest logs with secrets removed. + +For a reproducible bug or documentation gap, open a [GitHub issue](https://github.com/solutionforest/ephemeral-action-runner/issues/new/choose). For usage questions, include what you expected, what happened, and the diagnostics above so another contributor can reproduce the environment. + +Do not report security vulnerabilities in a public issue. Follow the private reporting instructions in the [security policy](docs/security.md). + +Community support is provided on a best-effort basis; there is no guaranteed response time or service-level agreement. diff --git a/cmd/ephemeral-action-runner/init.go b/cmd/ephemeral-action-runner/init.go index 3847769..f47271f 100644 --- a/cmd/ephemeral-action-runner/init.go +++ b/cmd/ephemeral-action-runner/init.go @@ -37,6 +37,8 @@ var initHostTrustOS = detectedInitHostTrustOS() var initWSLStatus = wslStatus +var initTartVersion = tartVersion + var initResolveHostTrust = hosttrust.Resolve func detectedInitHostTrustOS() string { @@ -105,14 +107,6 @@ func runInitWithOptions(opts initOptions) error { return err } } - if !opts.SkipDockerCheck { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if err := dockerAvailable(ctx); err != nil { - return fmt.Errorf("Docker is required for the default Docker-DinD setup. Install Docker Desktop, Docker Engine, or a compatible Docker host, then rerun %s init. If you want WSL or another custom provider, create .local/config.yml from the examples instead", binaryName) - } - } - fmt.Fprintln(opts.Out, "EPAR first-run setup") fmt.Fprintln(opts.Out, "") fmt.Fprintln(opts.Out, "This creates .local/config.yml for an EPAR runner.") @@ -135,10 +129,23 @@ func runInitWithOptions(opts initOptions) error { } providerType := "docker-dind" if wsl2Available() { - providerType, err = promptProviderType(opts.Out, reader) + providerType, err = promptProviderType(opts.Out, reader, "wsl") if err != nil { return err } + } else if tartAvailable() { + providerType, err = promptProviderType(opts.Out, reader, "tart") + if err != nil { + return err + } + } + if !opts.SkipDockerCheck && providerType != "tart" { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + dockerErr := dockerAvailable(ctx) + cancel() + if dockerErr != nil { + return fmt.Errorf("Docker is required for the selected %s setup. Install Docker Desktop, Docker Engine, or a compatible Docker host, then rerun %s init", providerDisplayName(providerType), binaryName) + } } defaultPrefix, err := generatedPoolNamePrefix() if err != nil { @@ -179,8 +186,11 @@ func runInitWithOptions(opts initOptions) error { } content := defaultDockerDindConfig(appID, organization, privateKeyPath, poolNamePrefix, hostTrustMode, hostTrustScopes) - if providerType == "wsl" { + switch providerType { + case "wsl": content = defaultWSLConfig(appID, organization, privateKeyPath, poolNamePrefix) + case "tart": + content = defaultTartConfig(appID, organization, privateKeyPath, poolNamePrefix) } if err := os.MkdirAll(filepath.Dir(opts.ConfigPath), 0755); err != nil { return err @@ -255,11 +265,12 @@ func promptPoolNamePrefix(out io.Writer, reader *bufio.Reader, defaultValue stri } } -func promptProviderType(out io.Writer, reader *bufio.Reader) (string, error) { +func promptProviderType(out io.Writer, reader *bufio.Reader, alternative string) (string, error) { + alternativeLabel := providerDisplayName(alternative) fmt.Fprintln(out, "") fmt.Fprintln(out, "Runner provider:") fmt.Fprintln(out, " 1. Docker-DinD (default)") - fmt.Fprintln(out, " 2. WSL2") + fmt.Fprintf(out, " 2. %s\n", alternativeLabel) for { value, hitEOF, err := promptDefault(out, reader, "Runner provider", "1") if err != nil { @@ -268,17 +279,33 @@ func promptProviderType(out io.Writer, reader *bufio.Reader) (string, error) { switch strings.ToLower(value) { case "1", "docker", "docker-dind": return "docker-dind", nil - case "2", "wsl", "wsl2": - return "wsl", nil - default: - fmt.Fprintln(out, "Runner provider must be 1 (Docker-DinD) or 2 (WSL2).") - if hitEOF { - return "", fmt.Errorf("invalid runner provider %q", value) + case "2", alternative: + return alternative, nil + case "wsl2": + if alternative == "wsl" { + return alternative, nil } + default: + // Continue below so aliases that belong to an unavailable provider are rejected. + } + fmt.Fprintf(out, "Runner provider must be 1 (Docker-DinD) or 2 (%s).\n", alternativeLabel) + if hitEOF { + return "", fmt.Errorf("invalid runner provider %q", value) } } } +func providerDisplayName(providerType string) string { + switch providerType { + case "wsl": + return "WSL2" + case "tart": + return "Tart (experimental)" + default: + return "Docker-DinD" + } +} + func promptDefault(out io.Writer, reader *bufio.Reader, label string, defaultValue string) (string, bool, error) { fmt.Fprintf(out, "%s (press Enter to use %s): ", label, defaultValue) value, err := reader.ReadString('\n') @@ -439,6 +466,19 @@ func wslStatus(ctx context.Context) ([]byte, error) { return output.Bytes(), nil } +func tartAvailable() bool { + if initGOOS != "darwin" { + return false + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + return initTartVersion(ctx) == nil +} + +func tartVersion(ctx context.Context) error { + return exec.CommandContext(ctx, "tart", "--version").Run() +} + type boundedBuffer struct { bytes.Buffer limit int @@ -595,6 +635,74 @@ timeouts: `, appID, organization, privateKeyPath, poolNamePrefix) } +func defaultTartConfig(appID int64, organization, privateKeyPath string, poolNamePrefix string) string { + return fmt.Sprintf(`# Experimental: this default is a basic Ubuntu ARM64 Tart VM, not a GitHub-hosted runner image. +# It does not include the broad dependency set from https://github.com/actions/runner-images. +github: + appId: %d + organization: %s + privateKeyPath: %s + apiBaseUrl: https://api.github.com + webBaseUrl: https://github.com + +image: + sourceImage: ghcr.io/cirruslabs/ubuntu:latest + outputImage: epar-ubuntu-24-arm64 + upstreamDir: third_party/runner-images + upstreamLock: third_party/runner-images.lock + runnerVersion: latest + customInstallScripts: + # - examples/custom-install/install-extra-apt-tools.sh + +pool: + instances: 1 + # Must be unique for this machine/config within the GitHub organization. + namePrefix: %s + replacementRetryInitialSeconds: 15 + replacementRetryMaxSeconds: 1800 + replacementRetryMultiplier: 2 + replacementRetryJitterPercent: 20 +logging: + directory: work/logs + managerSinks: [console] + managerConsoleFormat: text + managerConsoleTextFormat: "{time} [{level}] {message}" + managerFileFormat: json + transcriptSinks: [file] + transcriptConsoleFormat: text + maxFileSizeMiB: 100 + maxBackups: 3 + compressBackups: true + retentionEnabled: true + retentionMaxTotalMiB: 1024 + managerMaxAgeDays: 14 + instanceMaxAgeDays: 14 + buildMaxAgeDays: 14 + errorMaxAgeDays: 30 + benchmarkMaxAgeDays: 90 + retentionIntervalMinutes: 60 + +runner: + labels: [self-hosted, linux, ARM64, epar-tart-ubuntu-24.04-base] + includeHostLabel: true + ephemeral: true + +provider: + type: tart + sourceImage: epar-ubuntu-24-arm64 + network: default + +docker: + registryMirrors: + # - https://mirror.example.test + +timeouts: + bootSeconds: 180 + githubOnlineSeconds: 180 + commandSeconds: 900 +`, appID, organization, privateKeyPath, poolNamePrefix) +} + var stdinIsInteractive = func() bool { info, err := os.Stdin.Stat() if err != nil { diff --git a/cmd/ephemeral-action-runner/init_test.go b/cmd/ephemeral-action-runner/init_test.go index 0621b2a..641f65d 100644 --- a/cmd/ephemeral-action-runner/init_test.go +++ b/cmd/ephemeral-action-runner/init_test.go @@ -395,6 +395,80 @@ func TestInitWSL2ChoiceDefaultsToDockerDindAndRepromptsInvalidValues(t *testing. } } +func TestInitOffersTartConfigWhenAvailable(t *testing.T) { + stubInitHostAndRandom(t, "Build Box 01", []byte{0xa4, 0xf9, 0xc2}) + stubTartAvailable(t) + oldDockerAvailable := dockerAvailable + dockerAvailable = func(context.Context) error { + t.Fatal("Docker availability should not be checked for Tart") + return nil + } + t.Cleanup(func() { dockerAvailable = oldDockerAvailable }) + + dir := t.TempDir() + path := filepath.Join(dir, ".local", "config.yml") + var out bytes.Buffer + if err := runInitWithOptions(initOptions{ + ProjectRoot: dir, + ConfigPath: path, + SkipHostTrustCheck: true, + In: strings.NewReader("654321\nexample\n.local/github-app.pem\n2\n\n"), + Out: &out, + }); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + want, err := os.ReadFile(filepath.Join("..", "..", "configs", "tart.example.yml")) + if err != nil { + t.Fatal(err) + } + wantText := strings.NewReplacer( + "appId: 123456", "appId: 654321", + "organization: your-org", "organization: example", + "privateKeyPath: ~/.config/ephemeral-action-runner/github-app.pem", "privateKeyPath: .local/github-app.pem", + "namePrefix: CHANGE-ME-unique-machine-prefix", "namePrefix: build-box-01-a4f9c2", + ).Replace(string(want)) + wantText = strings.ReplaceAll(wantText, "\r\n", "\n") + if string(got) != wantText { + t.Fatalf("Tart config did not match configs/tart.example.yml:\nwant:\n%s\ngot:\n%s", wantText, got) + } + if !strings.Contains(out.String(), "2. Tart (experimental)") { + t.Fatalf("init output did not offer Tart:\n%s", out.String()) + } +} + +func TestTartAvailabilityRequiresNativeMacOSAndSuccessfulVersion(t *testing.T) { + oldGOOS := initGOOS + oldTartVersion := initTartVersion + t.Cleanup(func() { + initGOOS = oldGOOS + initTartVersion = oldTartVersion + }) + + initGOOS = "darwin" + initTartVersion = func(context.Context) error { return nil } + if !tartAvailable() { + t.Fatal("tartAvailable() = false when tart --version succeeds on macOS") + } + initTartVersion = func(context.Context) error { return errors.New("tart unavailable") } + if tartAvailable() { + t.Fatal("tartAvailable() = true when tart --version fails") + } + + initGOOS = "linux" + initTartVersion = func(context.Context) error { + t.Fatal("tart --version should not run outside native macOS") + return nil + } + if tartAvailable() { + t.Fatal("tartAvailable() = true outside native macOS") + } +} + func TestWSL2AvailabilityRequiresNativeWindowsSuccessfulVersion2Status(t *testing.T) { stubWSL2Available(t) for _, test := range []struct { @@ -510,3 +584,15 @@ func stubWSL2Available(t *testing.T) { initWSLStatus = oldWSLStatus }) } + +func stubTartAvailable(t *testing.T) { + t.Helper() + oldGOOS := initGOOS + oldTartVersion := initTartVersion + initGOOS = "darwin" + initTartVersion = func(context.Context) error { return nil } + t.Cleanup(func() { + initGOOS = oldGOOS + initTartVersion = oldTartVersion + }) +} diff --git a/cmd/ephemeral-action-runner/start_test.go b/cmd/ephemeral-action-runner/start_test.go index 1ca418c..3bc1d56 100644 --- a/cmd/ephemeral-action-runner/start_test.go +++ b/cmd/ephemeral-action-runner/start_test.go @@ -167,6 +167,54 @@ func TestStartInteractiveMissingConfigCanSelectWSL2(t *testing.T) { } } +func TestStartInteractiveMissingConfigCanSelectTartWithoutDocker(t *testing.T) { + dir := t.TempDir() + stubInitHostAndRandom(t, "Build Box 01", []byte{0xa4, 0xf9, 0xc2}) + stubTartAvailable(t) + oldInteractive := stdinIsInteractive + oldDocker := dockerAvailable + t.Cleanup(func() { + stdinIsInteractive = oldInteractive + dockerAvailable = oldDocker + }) + stdinIsInteractive = func() bool { return true } + dockerAvailable = func(context.Context) error { + t.Fatal("Docker availability should not be checked for Tart") + return nil + } + + fake := &fakeStarterManager{} + var out bytes.Buffer + err := runStartWithOptions(startOptions{ + Context: context.Background(), + ProjectRoot: dir, + In: strings.NewReader("123456\nsolutionforest\n.local/github-app.pem\n2\n\n"), + Out: &out, + ManagerFactory: func(path, _ string, _ bool, _ bool) (starterManager, error) { + if path != filepath.Join(dir, ".local", "config.yml") { + t.Fatalf("config path = %q", path) + } + return fake, nil + }, + }) + if err != nil { + t.Fatal(err) + } + cfg, err := config.Load(filepath.Join(dir, ".local", "config.yml")) + if err != nil { + t.Fatal(err) + } + if got, want := cfg.Provider.Type, "tart"; got != want { + t.Fatalf("provider.type = %q, want %q", got, want) + } + if !strings.Contains(out.String(), "Continuing with") { + t.Fatalf("output missing continuation message:\n%s", out.String()) + } + if fake.ensureCalls != 1 || fake.runCalls != 1 { + t.Fatalf("ensure/run calls = %d/%d, want 1/1", fake.ensureCalls, fake.runCalls) + } +} + func TestStartNonInteractiveMissingConfigFails(t *testing.T) { dir := t.TempDir() oldInteractive := stdinIsInteractive diff --git a/configs/tart.example.yml b/configs/tart.example.yml index 885278d..efe103e 100644 --- a/configs/tart.example.yml +++ b/configs/tart.example.yml @@ -1,3 +1,5 @@ +# Experimental: this default is a basic Ubuntu ARM64 Tart VM, not a GitHub-hosted runner image. +# It does not include the broad dependency set from https://github.com/actions/runner-images. github: appId: 123456 organization: your-org diff --git a/docs/configuration.md b/docs/configuration.md index 07aa156..120b57f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -182,7 +182,7 @@ For `provider.type: docker-dind`, EPAR defaults to Catthehacker's full Ubuntu ru For `provider.type: wsl`, EPAR defaults to Catthehacker's full Ubuntu runner image, converts it into a WSL rootfs, and stores the output under `work/images/`. -For `provider.type: tart`, start from `configs/tart.example.yml` and adjust labels or image scripts as needed. +For the experimental `provider.type: tart`, EPAR defaults to `ghcr.io/cirruslabs/ubuntu:latest`, a basic Ubuntu ARM64 VM image. EPAR installs its runner lifecycle but does not add the broad tool and dependency set found in GitHub's hosted runner images. If you require a GitHub-runner-like environment, build and maintain a bootable Tart image yourself by adapting the scripts in [actions/runner-images](https://github.com/actions/runner-images), then set `image.sourceImage` to that Tart image. Rosetta-based amd64 execution also has compatibility limits and must be validated against the exact workflow. See the provider docs for details: diff --git a/docs/image-build.md b/docs/image-build.md index ca05edc..7c0bedb 100644 --- a/docs/image-build.md +++ b/docs/image-build.md @@ -9,6 +9,9 @@ For Tart, the image build has two image names: These are Tart VM image names. They are stored in Tart's local VM registry and are visible with `tart list`; they are not emitted as repository-local files. +> [!WARNING] +> Tart support is experimental, and its default source is a basic Ubuntu ARM64 OS image rather than a GitHub-hosted runner image. It does not include the usual dependency inventory from [`actions/runner-images`](https://github.com/actions/runner-images). For a GitHub-runner-like Tart environment, adapt those upstream build scripts to produce and maintain your own bootable Tart VM image, then configure EPAR to use it; EPAR does not automate that image conversion. + For WSL, the image build produces a rootfs tar. It can start from either a Docker image or an existing rootfs tar: - `image.sourceType`: `docker-image` or `rootfs-tar`, default `docker-image` for WSL. diff --git a/docs/providers/tart.md b/docs/providers/tart.md index d60e223..30901f1 100644 --- a/docs/providers/tart.md +++ b/docs/providers/tart.md @@ -1,6 +1,9 @@ -# Tart Provider +# Tart Provider (Experimental) -The Tart provider targets Apple Silicon macOS hosts and can run Ubuntu ARM64 or macOS ARM64 VMs. +The Tart provider is experimental. It targets Apple Silicon macOS hosts and currently supports Ubuntu ARM64 guests. Tart itself can run macOS ARM64 VMs, but EPAR's image build, runner service, validation, and cleanup scripts currently depend on Ubuntu, systemd, and Linux process interfaces, so macOS guests are not yet an EPAR provider mode. + +> [!WARNING] +> The default Tart source, `ghcr.io/cirruslabs/ubuntu:latest`, is a basic Ubuntu ARM64 OS image. It does not contain the broad dependency set normally present in GitHub's hosted images from [`actions/runner-images`](https://github.com/actions/runner-images), including many language SDKs, CLIs, browsers, and build tools. Tart uses Apple's Virtualization framework, so an Apple Silicon host runs an ARM64 VM. Rosetta translates supported x86_64 Linux user-space programs inside that ARM64 guest; it does not create an x64 VM, and not every amd64 image or workload is compatible. EPAR currently validates the Ubuntu path: @@ -11,15 +14,17 @@ EPAR currently validates the Ubuntu path: - register an ephemeral GitHub runner from the host - delete the VM after the runner exits -Use `configs/tart.example.yml` for the runner-only image or `configs/tart.web-e2e.example.yml` for the opt-in web/E2E install script. +Use `configs/tart.example.yml` for the basic runner-only Ubuntu image or `configs/tart.web-e2e.example.yml` for the existing opt-in web/E2E and Rosetta experiment. EPAR installs the GitHub Actions runner and its lifecycle scripts, but the default does not install Docker, .NET, PowerShell, Go, browsers, or the rest of GitHub's hosted-runner tool inventory. + +If a workflow needs a GitHub-runner-like environment, build and maintain your own bootable Tart source image. Adapt the Ubuntu build scripts and tool definitions from [`actions/runner-images`](https://github.com/actions/runner-images) to that image, validate the resulting ARM64 tools, push it as a Tart VM image, and set `image.sourceImage` to it. Alternatively, add narrowly scoped `image.customInstallScripts` for only the dependencies your workflows require. EPAR does not convert the Catthehacker Docker image or automatically reproduce the complete GitHub-hosted image for Tart. When Docker/browser support is selected on ARM64, EPAR exposes a Chromium-compatible browser through `epar-browser`, `chromium`, and `chromium-browser`; it is not guaranteed to be Google Chrome. The default network mode is Tart NAT. `softnet` is accepted by the provider, but it can require host-side privileges. -If Docker is installed in the guest, optional `docker.registryMirrors` settings are applied to the guest Docker daemon when each disposable VM starts. Use a mirror URL that is reachable from inside the Tart VM; `host.docker.internal` is Docker-container-specific and may not resolve in Tart guests. See [Docker Registry Mirrors](../advanced/docker-registry-mirrors.md). +If Docker is installed in a custom Tart guest, optional `docker.registryMirrors` settings are applied to the guest Docker daemon when each disposable VM starts. Use a mirror URL that is reachable from inside the Tart VM; `host.docker.internal` is Docker-container-specific and may not resolve in Tart guests. See [Docker Registry Mirrors](../advanced/docker-registry-mirrors.md). -## Rosetta For Linux Amd64 Containers +## Experimental Rosetta Support For Linux Amd64 Containers Tart on Apple Silicon runs ARM64 VMs, but Tart can expose Apple's Linux Rosetta runtime to the guest with `tart run --rosetta `. EPAR supports this as an opt-in Tart-only setting: @@ -43,4 +48,4 @@ Host prerequisites: - Tart version with `--rosetta` support - Apple's Rosetta package installed on the macOS host -This is for Linux amd64 user-space containers. It is not nested virtualization and it does not make the Ubuntu VM an x64 VM. For native amd64 performance and compatibility, use a Windows WSL x64 provider or another x64 Linux host. For Tart Rosetta-capable runners, expose a distinct label such as `epar-tart-rosetta-amd64` so workflows can opt into the behavior explicitly. +This is experimental support for Linux amd64 user-space containers. It is not nested virtualization and it does not make the Ubuntu VM an x64 VM. For native amd64 performance and compatibility, use a Windows WSL x64 provider or another x64 Linux host. For Tart Rosetta-capable runners, expose a distinct label such as `epar-tart-rosetta-amd64` so workflows can opt into the behavior explicitly. diff --git a/docs/providers/wsl.md b/docs/providers/wsl.md index 5dc070c..1a62e9c 100644 --- a/docs/providers/wsl.md +++ b/docs/providers/wsl.md @@ -86,6 +86,8 @@ Runner startup sources `/opt/epar/source-image.env` before launching `/opt/actio WSL x64 is the preferred EPAR target for workflows that pull amd64-only Docker runtime images. +An x64 WSL runner can store an ARM64 image with `docker pull` or `docker load`, but it cannot run that image natively. Running ARM64 containers requires an explicitly configured emulation layer such as QEMU registered through `binfmt_misc`, or a native ARM64 runner. EPAR does not install cross-architecture emulation in WSL by default, so validate both the architecture and execution path before routing ARM64-dependent jobs to an x64 WSL label. + If `docker.registryMirrors` is configured, EPAR applies it to Docker Engine inside each disposable WSL distro before validation. Use a mirror URL reachable from inside WSL, such as an organization DNS name or a host/LAN address. See [Docker Registry Mirrors](../advanced/docker-registry-mirrors.md). ## Caveats diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f5b6f5e..993f380 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -84,6 +84,93 @@ docker run --rm ghcr.io/catthehacker/ubuntu:full-latest df -h / If the container-visible disk is full, adjust or clean the Docker/OrbStack storage from that product's settings. Finder free space by itself may not reflect the Linux VM storage available to containers. +## Docker Container Fails Because Its Architecture Does Not Match The Runner + +### Symptoms + +A Docker or Docker Compose service may fail immediately with one of these messages: + +```text +exec /bin/sh: exec format error +exec user process caused: exec format error +cannot execute binary file: Exec format error +``` + +Docker may also warn that the requested image platform does not match the detected host platform. That warning alone is not a failure: the container can still run when a compatible emulation handler is already registered. + +These related messages point to different problems: + +- `no matching manifest for linux/arm64/v8` or `no matching manifest for linux/amd64` means the image does not publish the requested platform. QEMU cannot supply a missing image manifest; choose an available platform or publish a multi-platform image. +- `qemu-x86_64: Could not open '/lib64/ld-linux-x86-64.so.2'` means translation started but the expected foreign-architecture loader or userspace is unavailable or incompatible. Registration alone may not make that image work. +- Exit code `139` indicates a segmentation fault. Emulation can expose workload-specific incompatibilities, but this code by itself does not prove an architecture mismatch. + +### Confirm The Host And Image Platforms + +Check the runner architecture and the Docker daemon that will execute the container: + +```bash +uname -m +docker info --format '{{.OSType}}/{{.Architecture}}' +``` + +Inspect the locally selected image platform: + +```bash +docker image inspect --format '{{.Os}}/{{.Architecture}}' IMAGE +``` + +Inspect all platforms published by a registry image: + +```bash +docker buildx imagetools inspect IMAGE +``` + +For Docker Compose, also inspect the resolved configuration and look for a service-level `platform:` value: + +```bash +docker compose config +``` + +An x64 Linux Docker daemon normally runs `linux/amd64` images natively, and an ARM64 daemon normally runs `linux/arm64` images natively. Pulling or loading a foreign image does not prove that the daemon can execute it. + +### Match GitHub-Hosted Linux Behavior With Explicit QEMU Setup + +GitHub's Ubuntu runner image installs Docker, but its published installation script and software inventory do not promise pre-registered foreign-architecture emulators. When a trusted Linux job must run foreign-architecture containers, configure the requirement explicitly before the first such container starts: + +```yaml +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Set up ARM64 container emulation + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + with: + image: docker.io/tonistiigi/binfmt@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 + platforms: arm64 + + - name: Verify the foreign container + run: docker run --rm --platform linux/arm64 alpine:3.22 uname -m +``` + +The expected output is `aarch64`. Add only the platforms the workflow needs; emulated compilation and compute-heavy workloads can be substantially slower than native execution. Keep architecture-sensitive jobs on native runners when performance or full compatibility matters. + +`docker/setup-qemu-action` registers user-mode QEMU interpreters through Linux `binfmt_misc`. It helps Linux containers launch foreign-architecture user-space executables; it does not change the runner's CPU architecture, create a foreign-architecture VM, or make arbitrary host executables and libraries compatible. The action uses a privileged helper container, so use it only in trusted workflows and pin reviewed action and helper-image revisions according to your dependency policy. + +Provider notes: + +- Docker-DinD: run the setup action inside the EPAR job before Docker Compose or other foreign-image commands. It configures the disposable runner's Docker execution environment; no EPAR configuration switch is required. +- WSL: run the setup action inside the WSL runner when its Linux Docker daemon must execute a foreign image. An x64 WSL runner does not gain ARM64 container support merely by pulling or loading an ARM64 image. +- Tart: Tart runs an ARM64 VM on Apple Silicon. Its optional Rosetta path is experimental and is not equivalent to QEMU/binfmt compatibility. Prefer Docker-DinD or a native matching architecture when a workload is not compatible. +- GitHub-hosted Windows and macOS: GitHub documents Docker container actions and service containers as Linux-runner features. A Windows or macOS hardware label alone is therefore not a substitute for a Linux Docker daemon with emulation configured. + +Official references: + +- [Docker Setup QEMU action](https://github.com/docker/setup-qemu-action) +- [Docker multi-platform build strategies](https://docs.docker.com/build/building/multi-platform/) +- [GitHub-hosted runner labels and limitations](https://docs.github.com/en/actions/reference/runners/github-hosted-runners) +- [GitHub self-hosted runner container requirements](https://docs.github.com/en/actions/reference/runners/self-hosted-runners#requirements-for-self-hosted-runner-machines) +- [GitHub Ubuntu runner Docker installation](https://github.com/actions/runner-images/blob/main/images/ubuntu/scripts/build/install-docker.sh) + ## Docker Image Build Runs Out Of Space ### Symptom diff --git a/docs/usage.md b/docs/usage.md index d408748..15df358 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -52,7 +52,7 @@ Equivalent without the wrapper: go run ./cmd/ephemeral-action-runner ``` -If no config exists, EPAR starts the initializer, asks for the GitHub App ID, organization, and private key path, then writes `.local/config.yml`. Docker-DinD is the default. For a new Docker-DinD config, the wizard asks whether to inherit the controller host's trusted TLS roots and defaults to yes; existing configs remain disabled unless they explicitly set `image.hostTrustMode: overlay`. On native Windows, when `wsl.exe --status` successfully confirms default version 2, the wizard also offers a WSL2 config; press Enter to retain Docker-DinD. The Docker preflight still applies because the default WSL image uses Docker for its one-time rootfs export. EPAR then checks the configured image, builds or replaces it when the image is missing or no longer matches the config, and starts the configured number of runners. The default config uses `pool.instances: 1`. +If no config exists, EPAR starts the initializer, asks for the GitHub App ID, organization, and private key path, then writes `.local/config.yml`. Docker-DinD is the default. For a new Docker-DinD config, the wizard asks whether to inherit the controller host's trusted TLS roots and defaults to yes; existing configs remain disabled unless they explicitly set `image.hostTrustMode: overlay`. On native Windows, when `wsl.exe --status` successfully confirms default version 2, the wizard also offers a WSL2 config. On macOS, when `tart --version` succeeds, it offers an experimental Tart config. Press Enter to retain Docker-DinD. The Docker preflight applies to Docker-DinD and the default WSL image, which uses Docker for its one-time rootfs export, but not to Tart. EPAR then checks the configured image, builds or replaces it when the image is missing or no longer matches the config, and starts the configured number of runners. The default config uses `pool.instances: 1`. Pass flags through `./start` to choose a config or runner count: @@ -84,7 +84,7 @@ If `--instances` is omitted, `start`, `pool up`, and `pool verify` use `pool.ins ## Configure Only -Use `init` when you only want to create a config without building an image or starting runners. It creates Docker-DinD by default, with the same conditional native-Windows WSL2 choice described above: +Use `init` when you only want to create a config without building an image or starting runners. It creates Docker-DinD by default, with the same conditional native-Windows WSL2 and macOS Tart choices described above: ```bash go run ./cmd/ephemeral-action-runner init @@ -96,11 +96,11 @@ On Windows PowerShell: go run ./cmd/ephemeral-action-runner init ``` -For WSL, Tart, or custom labels, copy one example config into `.local/config.yml`, then edit the GitHub App fields and any labels you want to expose to workflows. +For other WSL or Tart variants, or for custom labels, copy one example config into `.local/config.yml`, then edit the GitHub App fields and any labels you want to expose to workflows. | Host and image | Example config | | --- | --- | -| macOS Tart, runner-only | `configs/tart.example.yml` | +| macOS Tart, experimental basic Ubuntu ARM64 image | `configs/tart.example.yml` | | macOS Tart, web/E2E with Rosetta amd64 Docker support | `configs/tart.web-e2e.example.yml` | | Windows WSL2, default full Catthehacker runner image | `configs/wsl.example.yml` | | Windows WSL2, lean runner-only tar | `configs/wsl.lean.example.yml` | @@ -109,6 +109,8 @@ For WSL, Tart, or custom labels, copy one example config into `.local/config.yml | Docker-DinD, Docker-focused Catthehacker Act image | `configs/docker-dind.act.example.yml` | | Docker-DinD, smaller web/E2E custom image | `configs/docker-dind.web-e2e.example.yml` | +Tart is experimental. Its default image is a basic Ubuntu ARM64 OS image with the EPAR runner lifecycle, not the dependency-rich environment described by [`actions/runner-images`](https://github.com/actions/runner-images). If your workflows depend on that environment, adapt the upstream build scripts to create and maintain your own bootable Tart image and point `image.sourceImage` at it; EPAR does not automatically create one. + macOS: ```bash diff --git a/start b/start index 2c775da..2e8a9eb 100755 --- a/start +++ b/start @@ -5,7 +5,7 @@ set -euo pipefail # # Uses local Go if present. Otherwise runs EPAR from source inside a # containerized Go toolchain via `go run` (no local Go install needed, and -# no binary is built or left on disk). Docker is required either way. +# no binary is built or left on disk). The containerized fallback requires Docker. # See docs/advanced/no-go-install.md. script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" && pwd -P)" diff --git a/start.ps1 b/start.ps1 index ae49026..5dbd9f0 100644 --- a/start.ps1 +++ b/start.ps1 @@ -7,8 +7,8 @@ param( # # Uses local Go if present and actually runnable. Otherwise runs EPAR from # source inside a containerized Go toolchain via `go run` (no local Go -# install needed, and no binary is built or left on disk). Docker is -# required either way. See docs/advanced/no-go-install.md. +# install needed, and no binary is built or left on disk). The containerized +# fallback requires Docker. See docs/advanced/no-go-install.md. $ErrorActionPreference = "Stop" $Root = Split-Path -Parent $MyInvocation.MyCommand.Path From 23bb836229e8217f36c83d68c29c77cd57ae1178 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 21 Jul 2026 17:32:28 +0800 Subject: [PATCH 9/9] Guard Windows console allocation size Count the exact CR insertions before preallocating normalized terminal output, reject unrepresentable sizes without overflowing int arithmetic, and cover the boundary and CRLF cases. --- internal/logging/console_writer.go | 28 ++++++++++++++- internal/logging/console_writer_test.go | 46 +++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/internal/logging/console_writer.go b/internal/logging/console_writer.go index 7588074..145aa8e 100644 --- a/internal/logging/console_writer.go +++ b/internal/logging/console_writer.go @@ -3,6 +3,7 @@ package logging import ( "bytes" "io" + "math" "runtime" "sync" @@ -58,7 +59,14 @@ func windowsConsoleLineEndingsAfter(data []byte, previousCR bool) []byte { if !bytes.Contains(data, []byte{'\n'}) { return data } - normalized := make([]byte, 0, len(data)+bytes.Count(data, []byte{'\n'})) + insertionCount := windowsConsoleLineEndingInsertions(data, previousCR) + capacity, ok := windowsConsoleLineEndingCapacity(len(data), insertionCount) + if !ok { + // The normalized slice cannot be represented by an int. Preserve the + // original output instead of overflowing the allocation size. + return data + } + normalized := make([]byte, 0, capacity) for index, value := range data { precededByCR := index > 0 && data[index-1] == '\r' || index == 0 && previousCR if value == '\n' && !precededByCR { @@ -68,3 +76,21 @@ func windowsConsoleLineEndingsAfter(data []byte, previousCR bool) []byte { } return normalized } + +func windowsConsoleLineEndingCapacity(dataLength, insertionCount int) (int, bool) { + if insertionCount > math.MaxInt-dataLength { + return dataLength, false + } + return dataLength + insertionCount, true +} + +func windowsConsoleLineEndingInsertions(data []byte, previousCR bool) int { + insertionCount := 0 + for index, value := range data { + precededByCR := index > 0 && data[index-1] == '\r' || index == 0 && previousCR + if value == '\n' && !precededByCR { + insertionCount++ + } + } + return insertionCount +} diff --git a/internal/logging/console_writer_test.go b/internal/logging/console_writer_test.go index 3ccb9c1..5fe8805 100644 --- a/internal/logging/console_writer_test.go +++ b/internal/logging/console_writer_test.go @@ -3,6 +3,7 @@ package logging import ( "bytes" "errors" + "math" "testing" ) @@ -39,6 +40,51 @@ func TestWindowsConsoleLineEndings(t *testing.T) { } } +func TestWindowsConsoleLineEndingCapacity(t *testing.T) { + tests := []struct { + name string + dataLength int + insertionCount int + want int + wantOK bool + }{ + {name: "exact capacity", dataLength: 8, insertionCount: 2, want: 10, wantOK: true}, + {name: "maximum capacity", dataLength: math.MaxInt - 1, insertionCount: 1, want: math.MaxInt, wantOK: true}, + {name: "overflow", dataLength: math.MaxInt, insertionCount: 1, want: math.MaxInt, wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := windowsConsoleLineEndingCapacity(tt.dataLength, tt.insertionCount) + if got != tt.want || ok != tt.wantOK { + t.Fatalf("windowsConsoleLineEndingCapacity(%d, %d) = (%d, %t), want (%d, %t)", tt.dataLength, tt.insertionCount, got, ok, tt.want, tt.wantOK) + } + }) + } +} + +func TestWindowsConsoleLineEndingInsertions(t *testing.T) { + tests := []struct { + name string + data string + previousCR bool + want int + }{ + {name: "lone LF", data: "one\ntwo\n", want: 2}, + {name: "existing CRLF", data: "one\r\ntwo\r\n", want: 0}, + {name: "mixed", data: "one\r\ntwo\nthree", want: 1}, + {name: "split CRLF", data: "\ntwo\n", previousCR: true, want: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := windowsConsoleLineEndingInsertions([]byte(tt.data), tt.previousCR); got != tt.want { + t.Fatalf("windowsConsoleLineEndingInsertions(%q, %t) = %d, want %d", tt.data, tt.previousCR, got, tt.want) + } + }) + } +} + func TestWindowsConsoleWriterReportsOriginalByteCount(t *testing.T) { var output bytes.Buffer writer := &windowsConsoleWriter{writer: &output}