feat: ar transport support, lots of cleanups

Author: fastcat

addons/bootstrap/apt/common/google-cloud.go

@@ -17,9 +17,18 @@ const StepNameUpdate = "apt update"
 // WithExtraUpdate adds a secondary `apt update` step with the given name. It
 // will always run after the main `apt update` step. You may pass additional
 // ordering constraints in the options.
+//
+// This is equivalent to calling [bootstrap.WithSteps] with the result of [ExtraUpdateStep].
 func WithExtraUpdate(name string, opts ...bootstrap.StepOpt) bootstrap.Option {
+	return bootstrap.WithSteps(ExtraUpdateStep(name, opts...))
+}
+
+// ExtraUpdateStep creates a secondary `apt update` step with the given name. It
+// will always run after the main `apt update` step. You should pass additional
+// ordering constraints in the options.
+func ExtraUpdateStep(name string, opts ...bootstrap.StepOpt) *bootstrap.Step {
 	opts = append([]bootstrap.StepOpt{bootstrap.AfterSteps(StepNameUpdate)}, opts...)
-	return bootstrap.WithSteps(bootstrap.NewStep(name, DoUpdate, opts...))
+	return bootstrap.NewStep(name, DoUpdate, opts...)
 }
 
 func updateStep() *bootstrap.Step {
@@ -62,9 +71,7 @@ const StepNameInstall = "apt install"
 func installStep() *bootstrap.Step {
 	return bootstrap.NewStep(
 		StepNameInstall,
-		func(ctx *bootstrap.Context) error {
-			return DoInstall(ctx, []string{"--no-install-recommends"}, nil, "")
-		},
+		doInstall,
 		bootstrap.AfterSteps(StepNameUpdate),
 		bootstrap.SimFunc(simInstall),
 	)
@@ -76,13 +83,34 @@ func installStep() *bootstrap.Step {
 //
 // You likely want to pair this with [WithExtraUpdate], one or more steps to
 // add new apt sources that call [ChangedSources] and [AddPackages].
+//
+// This is equivalent to calling [bootstrap.WithSteps] with the result of
+// [ExtraInstallStep].
 func WithExtraInstall(name string, opts ...bootstrap.StepOpt) bootstrap.Option {
-	opts = append([]bootstrap.StepOpt{bootstrap.AfterSteps(StepNameInstall)}, opts...)
-	return bootstrap.WithSteps(bootstrap.NewStep(name, DoUpdate, opts...))
+	return bootstrap.WithSteps(ExtraInstallStep(name, opts...))
+}
+
+// ExtraInstallStep creates a secondary `apt install` step with the given name.
+// It will always run after the main `apt install` step. You should pass
+// additional ordering constraints in the options, e.g. ensuring this runs after
+// a custom update and package selection steps.
+func ExtraInstallStep(name string, opts ...bootstrap.StepOpt) *bootstrap.Step {
+	opts = append(
+		[]bootstrap.StepOpt{
+			bootstrap.AfterSteps(StepNameInstall),
+			bootstrap.SimFunc(simInstall),
+		},
+		opts...,
+	)
+	return bootstrap.NewStep(name, doInstall, opts...)
 }
 
 var pendingPackages = bootstrap.NewKey[map[string]struct{}]("pending-apt-packages")
 
+func doInstall(ctx *bootstrap.Context) error {
+	return DoInstall(ctx, []string{"--no-install-recommends"}, nil, "")
+}
+
 // DoInstall runs `apt install -y ...` with:
 //
 //   - Any extra options you pass. Including `--no-install-recommends` is often
@@ -254,6 +282,21 @@ func AddPackagesStep(
 	)
 }
 
+func AddExtraPackagesStep(
+	stepName string,
+	packages ...string,
+) *bootstrap.Step {
+	mark := func(ctx *bootstrap.Context) error {
+		AddPackages(ctx, packages...)
+		return nil
+	}
+	return bootstrap.NewStep(
+		stepName,
+		mark,
+		bootstrap.SimFunc(mark),
+	)
+}
+
 // WithPackages is an option for [Configure] that will register a step to
 // mark the given package(s) to be installed by the main `apt install` step.
 func WithPackages(

addons/bootstrap/apt/steps-packages.go

@@ -7,8 +7,13 @@ import (
 // SourceInstallStep creates a bootstrap step that installs the given APT
 // source.
 //
-// You should always adjust the returned step with [bootstrap.BeforeSteps],
-// commonly [bootstrap.StepNameAptUpdate].
+// You should always adjust the step with before/after constraints. For example
+// if this is a public source, you would typically use [bootstrap.BeforeSteps],
+// commonly [bootstrap.StepNameAptUpdate] (or call [PublicSourceInstallSteps]).
+// If this is a private source, then you would typically use
+// [bootstrap.AfterSteps] and [bootstrap.BeforeSteps] to ensure it runs after
+// the normal apt setup and before the secondary apt install that uses packages
+// from this private source.
 func SourceInstallStep(
 	installer *SourceInstaller,
 	opts ...bootstrap.StepOpt,

addons/bootstrap/apt/steps-sources.go

@@ -0,0 +1,41 @@
+package bootstrap
+
+import (
+	"os"
+)
+
+var isInContainerKey = NewKey[bool]("is-in-container")
+
+func IsInContainer(ctx *Context) bool {
+	if v, ok := Get(ctx, isInContainerKey); ok {
+		return v
+	}
+	// TODO: better detection than this
+	_, err := os.Stat("/.dockerenv")
+	v := err == nil
+	Save(ctx, isInContainerKey, v)
+	return v
+}
+
+func SkipInContainer() StepOpt {
+	return SkipFunc(func(ctx *Context) (bool, error) {
+		return IsInContainer(ctx), nil
+	})
+}
+
+var hasGUIKey = NewKey[bool]("has-gui")
+
+func HasGUI(ctx *Context) bool {
+	if v, ok := Get(ctx, hasGUIKey); ok {
+		return v
+	}
+	v := os.Getenv("DISPLAY") != "" || os.Getenv("WAYLAND_DISPLAY") != ""
+	Save(ctx, hasGUIKey, v)
+	return v
+}
+
+func SkipIfNoGUI() StepOpt {
+	return SkipFunc(func(ctx *Context) (bool, error) {
+		return !HasGUI(ctx), nil
+	})
+}

addons/bootstrap/env-detect.go

@@ -118,6 +118,7 @@ func (p *Plan) prepare(ctx context.Context) (*Context, error) {
 		for _, s := range p.pending {
 			names = append(names, s.name)
 		}
+		p.debugCircularDeps()
 		return nil, fmt.Errorf("plan has unresolved dependencies blocking %s", strings.Join(names, ", "))
 	}
 	if len(p.ordered) == 0 {
@@ -127,6 +128,62 @@ func (p *Plan) prepare(ctx context.Context) (*Context, error) {
 	return bc, nil
 }
 
+func (p *Plan) debugCircularDeps() {
+	isOrdered := make(map[string]bool, len(p.ordered))
+	for _, s := range p.ordered {
+		isOrdered[s.name] = true
+	}
+	afterByName := make(map[string]map[string]struct{}, len(p.pending))
+	for _, s := range p.pending {
+		m := maps.Clone(s.after)
+		// trim out satisfied deps
+		for n := range m {
+			if _, ok := isOrdered[n]; ok {
+				delete(m, n)
+			}
+		}
+		afterByName[s.name] = m
+	}
+	for _, s := range p.pending {
+		for beforeName := range s.before {
+			if _, ok := isOrdered[beforeName]; !ok {
+				if afterByName[beforeName] == nil {
+					afterByName[beforeName] = make(map[string]struct{})
+				}
+				afterByName[beforeName][s.name] = struct{}{}
+			}
+		}
+	}
+	// find cycles
+	visited := make(map[string]bool, len(afterByName))
+	var visit func(name string, stack []string) bool
+	visit = func(name string, stack []string) bool {
+		if visited[name] {
+			return false
+		}
+		visited[name] = true
+		stack = append(stack, name)
+		for depName := range afterByName[name] {
+			for i, sn := range stack {
+				if sn == depName {
+					fmt.Printf("circular dependency: %s\n", strings.Join(append(stack[i:], depName), " -> "))
+					return true
+				}
+			}
+			if visit(depName, stack) {
+				return true
+			}
+		}
+		return false
+	}
+	for name := range afterByName {
+		if !visited[name] {
+			// don't stop after one cycle, find them all
+			visit(name, nil)
+		}
+	}
+}
+
 func (p *Plan) Sim(ctx context.Context) error {
 	bc, err := p.prepare(ctx)
 	if err != nil {

addons/bootstrap/plan.go

@@ -1,6 +1,10 @@
 package bootstrap
 
-import "fastcat.org/go/gdev/internal"
+import (
+	"fmt"
+
+	"fastcat.org/go/gdev/internal"
+)
 
 type Step struct {
 	name   string
@@ -64,3 +68,35 @@ func AfterSteps(names ...string) StepOpt {
 		}
 	}
 }
+
+func SkipFunc(f func(*Context) (bool, error)) StepOpt {
+	return func(s *Step) {
+		origRun := s.run
+		s.run = func(ctx *Context) error {
+			skip, err := f(ctx)
+			if err != nil {
+				return err
+			}
+			if skip {
+				fmt.Println("Skipping", s.name)
+				return nil
+			}
+			return origRun(ctx)
+		}
+		origSim := s.sim
+		s.sim = func(ctx *Context) error {
+			skip, err := f(ctx)
+			if err != nil {
+				return err
+			}
+			if skip {
+				fmt.Println("Skipping", s.name)
+				return nil
+			}
+			if origSim != nil {
+				return origSim(ctx)
+			}
+			return nil
+		}
+	}
+}

addons/bootstrap/step.go

@@ -17,9 +17,15 @@ var configureBootstrap = sync.OnceFunc(func() {
 			"Select common Docker packages",
 			"docker.io",
 			"docker-buildx",
-			"golang-docker-credential-helpers",
 		),
 		bootstrap.WithSteps(
+			apt.AddPackagesStep(
+				"Select docker credential helper(s)",
+				"golang-docker-credential-helpers",
+			).With(
+				bootstrap.BeforeSteps(apt.StepNameInstall),
+				bootstrap.SkipInContainer(),
+			),
 			apt.AddPackageIfAvailable(
 				"Select docker-cli if needed",
 				// this is only on Ubuntu 25.04+ and Debian 13+

addons/docker/bootstrap.go

@@ -11,7 +11,6 @@ import (
 	"fastcat.org/go/gdev/addons"
 	"fastcat.org/go/gdev/addons/bootstrap"
 	"fastcat.org/go/gdev/addons/bootstrap/apt"
-	apt_common "fastcat.org/go/gdev/addons/bootstrap/apt/common"
 	"fastcat.org/go/gdev/lib/shx"
 )
 
@@ -34,8 +33,9 @@ func init() {
 }
 
 type config struct {
-	skipLogin      bool
-	allowedDomains []string
+	skipLogin        bool
+	includeTransport bool
+	allowedDomains   []string
 }
 
 type option func(*config)
@@ -58,29 +58,49 @@ func initialize() error {
 // WithSkipLogin causes the bootstrap sequence to no-op the login step.
 func WithSkipLogin() option {
 	return func(c *config) {
+		if configuredBootstrap {
+			panic("WithSkipLogin must be called before first Configure()")
+		}
 		c.skipLogin = true
 	}
 }
 
+func WithAptTransport() option {
+	return func(c *config) {
+		if configuredBootstrap {
+			panic("WithAptTransport must be called before first Configure()")
+		}
+		c.includeTransport = true
+	}
+}
+
 func WithAllowedDomains(domains ...string) option {
 	return func(c *config) {
 		c.allowedDomains = domains
 	}
 }
 
-var configureBootstrap = sync.OnceFunc(func() {
-	// TODO: allow customization of the bootstrap steps somehow?
-	// We can't do the bootstrap config in initialize(), so it gets a little tricky
-	bootstrap.Configure(
-		bootstrap.WithSteps(apt.PublicSourceInstallSteps(apt_common.GoogleCloudInstaller())...),
-		apt.WithPackages("Select gcloud packages", "google-cloud-cli"),
-		bootstrap.WithSteps(bootstrap.NewStep(
-			ConfigureStepName,
-			configureGcloud,
-			bootstrap.AfterSteps(apt.StepNameInstall),
-		)),
-	)
-})
+var (
+	configuredBootstrap bool
+	configureBootstrap  = sync.OnceFunc(func() {
+		sources := []*apt.SourceInstaller{CLISourceInstaller()}
+		packages := []string{"google-cloud-cli"}
+		if addon.Config.includeTransport {
+			sources = append(sources, AptTransportSourceInstaller())
+			packages = append(packages, "apt-transport-artifact-registry")
+		}
+		bootstrap.Configure(
+			bootstrap.WithSteps(apt.PublicSourceInstallSteps(sources...)...),
+			apt.WithPackages("Select gcloud packages", packages...),
+			bootstrap.WithSteps(bootstrap.NewStep(
+				ConfigureStepName,
+				configureGcloud,
+				bootstrap.AfterSteps(apt.StepNameInstall),
+			)),
+		)
+		configuredBootstrap = true
+	})
+)
 
 const ConfigureStepName = "Configure gcloud"