diff --git a/.gitignore b/.gitignore index 0feac5ed..b7f0a472 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ dist/ dist-cloudflare/ artifacts/ .crabbox/ +.env.local node_modules/ .wrangler/ *.test diff --git a/README.md b/README.md index eb5d73bc..31fa1fb0 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ Supported providers: - [Cloudflare](docs/providers/cloudflare.md) (`provider: cloudflare`): delegated Cloudflare execution through a Worker and container runner. +- [Freestyle](docs/providers/freestyle.md) (`provider: freestyle`): delegated + Freestyle VM execution through the Freestyle REST API. - [Railway](docs/providers/railway.md) (`provider: railway`): delegated redeploy-and-stream execution against a pre-existing Railway service through the [Railway](https://railway.com) GraphQL API. diff --git a/docs/README.md b/docs/README.md index 227a2b53..bb7e4f88 100644 --- a/docs/README.md +++ b/docs/README.md @@ -87,7 +87,7 @@ Pick whichever matches your intent: - **Start here:** [Getting started](getting-started.md), [How Crabbox Works](how-it-works.md), [Concepts and glossary](concepts.md). - **Get the mental model:** [Vision](vision.md), [Architecture](architecture.md), [Orchestrator](orchestrator.md). - **Use the CLI:** [CLI](cli.md), [Commands](commands/README.md), [Features](features/README.md), [Configuration](features/configuration.md), [Jobs](features/jobs.md), [Actions hydration](features/actions-hydration.md), [Capsules](features/capsules.md), [Checkpoints](features/checkpoints.md), [Browser portal](features/portal.md), [Telemetry](features/telemetry.md). -- **Pick or add a target:** [Provider reference](providers/README.md), [Providers feature overview](features/providers.md), [Provider authoring](features/provider-authoring.md), [Provider backends](provider-backends.md), [AWS](providers/aws.md), [Azure](providers/azure.md), [Google Cloud](providers/gcp.md), [Hetzner](providers/hetzner.md), [Proxmox](providers/proxmox.md), [Parallels](providers/parallels.md), [Local Container](providers/local-container.md), [Static SSH](providers/ssh.md), [Blacksmith Testbox](providers/blacksmith-testbox.md), [Namespace Devbox](providers/namespace-devbox.md), [Semaphore](providers/semaphore.md), [Sprites](providers/sprites.md), [Daytona](providers/daytona.md), [Islo](providers/islo.md), [E2B](providers/e2b.md), [Modal](providers/modal.md), [Tensorlake](providers/tensorlake.md), [Cloudflare](providers/cloudflare.md), [Interactive desktop and VNC](features/interactive-desktop-vnc.md). +- **Pick or add a target:** [Provider reference](providers/README.md), [Providers feature overview](features/providers.md), [Provider authoring](features/provider-authoring.md), [Provider backends](provider-backends.md), [AWS](providers/aws.md), [Azure](providers/azure.md), [Google Cloud](providers/gcp.md), [Hetzner](providers/hetzner.md), [Proxmox](providers/proxmox.md), [Parallels](providers/parallels.md), [Local Container](providers/local-container.md), [Static SSH](providers/ssh.md), [Blacksmith Testbox](providers/blacksmith-testbox.md), [Namespace Devbox](providers/namespace-devbox.md), [Semaphore](providers/semaphore.md), [Sprites](providers/sprites.md), [Daytona](providers/daytona.md), [Islo](providers/islo.md), [E2B](providers/e2b.md), [Modal](providers/modal.md), [Tensorlake](providers/tensorlake.md), [Cloudflare](providers/cloudflare.md), [Freestyle](providers/freestyle.md), [Interactive desktop and VNC](features/interactive-desktop-vnc.md). - **Operate it:** [Operations](operations.md), [Observability](observability.md), [Troubleshooting](troubleshooting.md), [Performance](performance.md). - **Set it up or audit it:** [Infrastructure](infrastructure.md), [Security](security.md), [Source Map](source-map.md). diff --git a/docs/features/providers.md b/docs/features/providers.md index 22ab4bae..edc75e68 100644 --- a/docs/features/providers.md +++ b/docs/features/providers.md @@ -40,6 +40,7 @@ islo Islo sandboxes with delegated command execution e2b E2B sandboxes with delegated command execution modal Modal Sandboxes with delegated command execution tensorlake Tensorlake Firecracker sandboxes with delegated command execution +freestyle Freestyle VMs with delegated command execution ``` ## Provider Pages @@ -63,6 +64,7 @@ tensorlake Tensorlake Firecracker sandboxes with delegated command execution - [E2B](../providers/e2b.md): delegated E2B sandbox execution. - [Modal](../providers/modal.md): delegated Modal Sandbox execution. - [Tensorlake](../providers/tensorlake.md): delegated Tensorlake Firecracker sandbox execution. +- [Freestyle](../providers/freestyle.md): delegated Freestyle VM execution. - [Provider backends](../provider-backends.md): implementation guide for adding a new provider/backend/plugin. ## Hetzner Summary @@ -231,6 +233,11 @@ backend: Crabbox creates Modal Sandboxes through the local Python client, syncs gzipped archive through Sandbox exec, and streams command output from Modal's process API. See [Modal](../providers/modal.md). +Crabbox can use Freestyle VMs with `provider: freestyle`. Freestyle is a delegated +run backend: Crabbox creates VMs through the Freestyle REST API, syncs a gzipped +archive with file-API or exec fallback upload, and runs commands through +Freestyle exec. See [Freestyle](../providers/freestyle.md). + Static SSH targets: ```yaml @@ -280,5 +287,6 @@ Related docs: - [Islo](../providers/islo.md) - [E2B](../providers/e2b.md) - [Modal](../providers/modal.md) +- [Freestyle](../providers/freestyle.md) - [Runner bootstrap](runner-bootstrap.md) - [Cost and usage](cost-usage.md) diff --git a/docs/providers/README.md b/docs/providers/README.md index 54b9ba0a..0eb46f25 100644 --- a/docs/providers/README.md +++ b/docs/providers/README.md @@ -31,6 +31,7 @@ static SSH provider for existing machines. | [Upstash Box](upstash-box.md) | delegated run | Linux | Upstash Box execution through the Box REST API | | [Tensorlake](tensorlake.md) | delegated run | Linux | Tensorlake Firecracker sandbox execution via the `tensorlake` CLI | | [Cloudflare](cloudflare.md) | delegated run | Linux | Cloudflare execution through a Worker and container runner | +| [Freestyle](freestyle.md) | delegated run | Linux | Freestyle VM execution through the Freestyle REST API | | [Railway](railway.md) | delegated run | Linux | redeploy and stream logs for an existing Railway service via the GraphQL API | | [RunPod](runpod.md) | SSH lease | Linux | disposable RunPod pods provisioned via the REST API and accessed over public SSH | | [W&B Sandboxes](wandb.md) | delegated run | Linux | [wandb.ai](https://wandb.ai/) (by CoreWeave) — use an existing `wandb login` credential plus `WANDB_ENTITY_NAME` | diff --git a/docs/providers/freestyle.md b/docs/providers/freestyle.md new file mode 100644 index 00000000..9245099f --- /dev/null +++ b/docs/providers/freestyle.md @@ -0,0 +1,127 @@ +# Freestyle Provider + +Read when: + +- choosing `provider: freestyle`; +- configuring Freestyle VM size or workdir; +- changing `internal/providers/freestyle`. + +Freestyle is a delegated run provider. Crabbox uses the [Freestyle](https://freestyle.sh) +v1 REST API (`https://api.freestyle.sh`) for VM lifecycle, archive sync, and +command execution through pure Go `net/http` calls. Freestyle owns VM state and +exec transport; Crabbox owns local config, repo claims, sync manifests and +guardrails, slugs, timing summaries, and normalized list/status rendering. + +## When To Use + +Use Freestyle when the remote Linux VM should be owned by Freestyle and commands +can run through Freestyle's exec API. Use AWS, Hetzner, Static SSH, or Daytona +when you need Crabbox SSH access. + +## Commands + +```sh +crabbox warmup --provider freestyle --keep +crabbox run --provider freestyle -- pnpm test +crabbox run --provider freestyle --sync-only +crabbox run --provider freestyle --id blue-lobster --shell 'pnpm install && pnpm test' +crabbox status --provider freestyle --id blue-lobster +crabbox stop --provider freestyle blue-lobster +crabbox list --provider freestyle +crabbox doctor --provider freestyle +``` + +## Live Smoke + +Use a live smoke when changing Freestyle lifecycle, sync, or exec code. Keep the +API key in `FREESTYLE_API_KEY`; do not pass it as a command-line argument. + +```sh +export FREESTYLE_API_KEY=... +go build -trimpath -o bin/crabbox ./cmd/crabbox + +bin/crabbox run --provider freestyle --keep 'bash test.sh' +bin/crabbox warmup --provider freestyle --actions-runner # expect exit 2 +bin/crabbox doctor --provider freestyle +``` + +Expected results: + +- `run` prints a Freestyle lease (`fsb_...`), sync summary, command output, and + `exit=0`. +- `warmup --actions-runner` is rejected because Freestyle does not register + GitHub Actions runners. +- `doctor` reports inventory readiness when auth and list calls succeed. + +## Auth + +```sh +export FREESTYLE_API_KEY=... +``` + +`CRABBOX_FREESTYLE_API_KEY` is also accepted and wins over `FREESTYLE_API_KEY`. + +Freestyle API keys must not be passed as CLI flags. Crabbox reads them from +environment variables only. + +`FREESTYLE_API_URL` or `freestyle.apiUrl` can override the default +`https://api.freestyle.sh`. + +## Config + +```yaml +provider: freestyle +target: linux +freestyle: + apiUrl: https://api.freestyle.sh + workdir: crabbox + # Optional. Omit to use Freestyle plan defaults. + vcpus: 2 + memoryMB: 4096 +``` + +Provider flags: + +```text +--freestyle-api-url +--freestyle-workdir +--freestyle-vcpus +--freestyle-memory-mb +``` + +`--freestyle-workdir` / `freestyle.workdir` is interpreted as a relative +directory below `/workspace`. Crabbox rejects absolute paths and `..` escapes +before workspace preparation and sync. + +## Lifecycle + +1. Create or resolve a Crabbox-owned Freestyle VM. +2. Store a local lease ID with the `fsb_` prefix and a friendly slug. +3. Validate the Freestyle workdir, build the Crabbox sync manifest, and upload a + gzipped archive into `/workspace/`. +4. Execute commands through Freestyle's exec API in that workdir via `bash -lc`. +5. Stop deletes the VM and removes the local lease claim. + +## Sync + +Freestyle advertises archive sync (`FeatureArchiveSync`). Crabbox supports +`--sync-only`, `--force-sync-large`, and `--no-sync`. + +Archive upload tries Freestyle's file API first. When that endpoint is +unavailable, Crabbox falls back to chunked base64 upload through exec. Sync still +completes reliably through the fallback path. + +`--checksum` is not supported because Freestyle uses archive sync rather than +Crabbox rsync checksum mode. + +## Limitations + +- `--actions-runner` is not supported for warmup or run. +- `--checksum` is not supported. +- No SSH lease path; this is delegated run only. +- API keys are env-only; there is no `--freestyle-api-key` flag. + +## Doctor + +`crabbox doctor --provider freestyle` checks auth and lists Crabbox-owned VMs in +the Freestyle account. It does not create a VM during the check. diff --git a/internal/cli/config.go b/internal/cli/config.go index 512c6bae..b4ba4705 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -104,6 +104,7 @@ type Config struct { Wandb WandbConfig Islo IsloConfig isloImageExplicit bool + Freestyle FreestyleConfig Tensorlake TensorlakeConfig Modal ModalConfig UpstashBox UpstashBoxConfig @@ -255,6 +256,14 @@ type IsloConfig struct { DiskGB int } +type FreestyleConfig struct { + APIKey string + APIURL string + Workdir string + VCPUs int + MemoryMB int +} + type TensorlakeConfig struct { APIKey string APIURL string @@ -731,6 +740,10 @@ func baseConfig() Config { MemoryMB: 4096, DiskGB: 20, }, + Freestyle: FreestyleConfig{ + APIURL: "https://api.freestyle.sh", + Workdir: "crabbox", + }, Tensorlake: TensorlakeConfig{ APIURL: "https://api.tensorlake.ai", CLIPath: "tensorlake", @@ -828,6 +841,7 @@ type fileConfig struct { Runpod *fileRunpodConfig `yaml:"runpod,omitempty"` Wandb *fileWandbConfig `yaml:"wandb,omitempty"` Islo *fileIsloConfig `yaml:"islo,omitempty"` + Freestyle *fileFreestyleConfig `yaml:"freestyle,omitempty"` Tensorlake *fileTensorlakeConfig `yaml:"tensorlake,omitempty"` Modal *fileModalConfig `yaml:"modal,omitempty"` UpstashBox *fileUpstashBoxConfig `yaml:"upstashBox,omitempty"` @@ -1106,6 +1120,13 @@ type fileIsloConfig struct { DiskGB int `yaml:"diskGB,omitempty"` } +type fileFreestyleConfig struct { + APIURL string `yaml:"apiUrl,omitempty"` + Workdir string `yaml:"workdir,omitempty"` + VCPUs int `yaml:"vcpus,omitempty"` + MemoryMB int `yaml:"memoryMB,omitempty"` +} + type fileTensorlakeConfig struct { APIURL string `yaml:"apiUrl,omitempty"` CLIPath string `yaml:"cliPath,omitempty"` @@ -2055,6 +2076,20 @@ func applyFileConfig(cfg *Config, file fileConfig) { cfg.Islo.DiskGB = file.Islo.DiskGB } } + if file.Freestyle != nil { + if file.Freestyle.APIURL != "" { + cfg.Freestyle.APIURL = file.Freestyle.APIURL + } + if file.Freestyle.Workdir != "" { + cfg.Freestyle.Workdir = file.Freestyle.Workdir + } + if file.Freestyle.VCPUs > 0 { + cfg.Freestyle.VCPUs = file.Freestyle.VCPUs + } + if file.Freestyle.MemoryMB > 0 { + cfg.Freestyle.MemoryMB = file.Freestyle.MemoryMB + } + } if file.Tensorlake != nil { if file.Tensorlake.APIURL != "" { cfg.Tensorlake.APIURL = file.Tensorlake.APIURL @@ -2790,6 +2825,11 @@ func applyEnv(cfg *Config) { cfg.Islo.VCPUs = getenvInt("CRABBOX_ISLO_VCPUS", cfg.Islo.VCPUs) cfg.Islo.MemoryMB = getenvInt("CRABBOX_ISLO_MEMORY_MB", cfg.Islo.MemoryMB) cfg.Islo.DiskGB = getenvInt("CRABBOX_ISLO_DISK_GB", cfg.Islo.DiskGB) + cfg.Freestyle.APIKey = getenv("CRABBOX_FREESTYLE_API_KEY", getenv("FREESTYLE_API_KEY", cfg.Freestyle.APIKey)) + cfg.Freestyle.APIURL = getenv("CRABBOX_FREESTYLE_API_URL", getenv("FREESTYLE_API_URL", cfg.Freestyle.APIURL)) + cfg.Freestyle.Workdir = getenv("CRABBOX_FREESTYLE_WORKDIR", cfg.Freestyle.Workdir) + cfg.Freestyle.VCPUs = getenvInt("CRABBOX_FREESTYLE_VCPUS", cfg.Freestyle.VCPUs) + cfg.Freestyle.MemoryMB = getenvInt("CRABBOX_FREESTYLE_MEMORY_MB", cfg.Freestyle.MemoryMB) cfg.Tensorlake.APIKey = getenv("CRABBOX_TENSORLAKE_API_KEY", getenv("TENSORLAKE_API_KEY", cfg.Tensorlake.APIKey)) cfg.Tensorlake.APIURL = getenv("CRABBOX_TENSORLAKE_API_URL", getenv("TENSORLAKE_API_URL", cfg.Tensorlake.APIURL)) cfg.Tensorlake.CLIPath = getenv("CRABBOX_TENSORLAKE_CLI", cfg.Tensorlake.CLIPath) @@ -2961,7 +3001,7 @@ func serverTypeForConfig(cfg Config) string { if resolved, err := ProviderFor(cfg.Provider); err == nil { cfg.Provider = resolved.Name() } - if isBlacksmithProvider(cfg.Provider) || isStaticProvider(cfg.Provider) || cfg.Provider == "islo" || cfg.Provider == "sprites" || cfg.Provider == "local-container" { + if isBlacksmithProvider(cfg.Provider) || isStaticProvider(cfg.Provider) || cfg.Provider == "islo" || cfg.Provider == "freestyle" || cfg.Provider == "sprites" || cfg.Provider == "local-container" { return "" } if cfg.Provider == "namespace-devbox" || cfg.Provider == "namespace" { @@ -3007,7 +3047,7 @@ func serverTypeForProviderClass(provider, class string) string { if resolved, err := ProviderFor(provider); err == nil { provider = resolved.Name() } - if isBlacksmithProvider(provider) || isStaticProvider(provider) || provider == "islo" || provider == "sprites" || provider == "local-container" { + if isBlacksmithProvider(provider) || isStaticProvider(provider) || provider == "islo" || provider == "freestyle" || provider == "sprites" || provider == "local-container" { return "" } if provider == "namespace-devbox" || provider == "namespace" { diff --git a/internal/cli/config_test.go b/internal/cli/config_test.go index 7eb440f0..c6a3b26e 100644 --- a/internal/cli/config_test.go +++ b/internal/cli/config_test.go @@ -80,6 +80,13 @@ func clearConfigEnv(t *testing.T) { "CRABBOX_ISLO_VCPUS", "CRABBOX_ISLO_MEMORY_MB", "CRABBOX_ISLO_DISK_GB", + "CRABBOX_FREESTYLE_API_KEY", + "FREESTYLE_API_KEY", + "CRABBOX_FREESTYLE_API_URL", + "FREESTYLE_API_URL", + "CRABBOX_FREESTYLE_WORKDIR", + "CRABBOX_FREESTYLE_VCPUS", + "CRABBOX_FREESTYLE_MEMORY_MB", "CRABBOX_TENSORLAKE_API_KEY", "TENSORLAKE_API_KEY", "CRABBOX_TENSORLAKE_API_URL", @@ -467,6 +474,11 @@ islo: vcpus: 4 memoryMB: 8192 diskGB: 40 +freestyle: + apiUrl: https://freestyle.example.test + workdir: team/repo + vcpus: 4 + memoryMB: 8192 tensorlake: apiUrl: https://api.tensorlake.example.test cliPath: /usr/local/bin/tl @@ -624,6 +636,9 @@ ssh: if cfg.Islo.BaseURL != "https://islo.example.test" || cfg.Islo.Image != "docker.io/library/ubuntu:24.04" || cfg.Islo.Workdir != "crabbox" || cfg.Islo.GatewayProfile != "default" || cfg.Islo.SnapshotName != "snap-ready" || cfg.Islo.VCPUs != 4 || cfg.Islo.MemoryMB != 8192 || cfg.Islo.DiskGB != 40 { t.Fatalf("islo config not loaded: %#v", cfg.Islo) } + if cfg.Freestyle.APIURL != "https://freestyle.example.test" || cfg.Freestyle.Workdir != "team/repo" || cfg.Freestyle.VCPUs != 4 || cfg.Freestyle.MemoryMB != 8192 { + t.Fatalf("freestyle config not loaded: %#v", cfg.Freestyle) + } if cfg.Tensorlake.APIURL != "https://api.tensorlake.example.test" || cfg.Tensorlake.CLIPath != "/usr/local/bin/tl" || cfg.Tensorlake.Image != "ubuntu-22.04" || cfg.Tensorlake.Snapshot != "snap-tl" || cfg.Tensorlake.OrganizationID != "org-tl" || cfg.Tensorlake.ProjectID != "proj-tl" || cfg.Tensorlake.Namespace != "ns-tl" || cfg.Tensorlake.Workdir != "/workspace/crabbox-test" || cfg.Tensorlake.CPUs != 4 || cfg.Tensorlake.MemoryMB != 8192 || cfg.Tensorlake.DiskMB != 30000 || cfg.Tensorlake.TimeoutSecs != 1800 || !cfg.Tensorlake.NoInternet { t.Fatalf("tensorlake config not loaded: %#v", cfg.Tensorlake) } @@ -832,6 +847,13 @@ func TestEnvOverridesConfig(t *testing.T) { t.Setenv("CRABBOX_ISLO_VCPUS", "8") t.Setenv("CRABBOX_ISLO_MEMORY_MB", "16384") t.Setenv("CRABBOX_ISLO_DISK_GB", "80") + t.Setenv("FREESTYLE_API_KEY", "freestyle-key-file") + t.Setenv("CRABBOX_FREESTYLE_API_KEY", "freestyle-key-env") + t.Setenv("FREESTYLE_API_URL", "https://freestyle-file.example") + t.Setenv("CRABBOX_FREESTYLE_API_URL", "https://freestyle-env.example") + t.Setenv("CRABBOX_FREESTYLE_WORKDIR", "env/repo") + t.Setenv("CRABBOX_FREESTYLE_VCPUS", "6") + t.Setenv("CRABBOX_FREESTYLE_MEMORY_MB", "12288") t.Setenv("TENSORLAKE_API_KEY", "tl-api-file") t.Setenv("CRABBOX_TENSORLAKE_API_KEY", "tl-api-env") t.Setenv("TENSORLAKE_API_URL", "https://api.tl-file.example") @@ -992,6 +1014,9 @@ func TestEnvOverridesConfig(t *testing.T) { if cfg.Islo.APIKey != "islo-api-env" || cfg.Islo.BaseURL != "https://islo-env.example" || cfg.Islo.Image != "ubuntu:env" || cfg.Islo.Workdir != "env-workdir" || cfg.Islo.GatewayProfile != "env-gateway" || cfg.Islo.SnapshotName != "env-snapshot" || cfg.Islo.VCPUs != 8 || cfg.Islo.MemoryMB != 16384 || cfg.Islo.DiskGB != 80 { t.Fatalf("unexpected islo env: %#v", cfg.Islo) } + if cfg.Freestyle.APIKey != "freestyle-key-env" || cfg.Freestyle.APIURL != "https://freestyle-env.example" || cfg.Freestyle.Workdir != "env/repo" || cfg.Freestyle.VCPUs != 6 || cfg.Freestyle.MemoryMB != 12288 { + t.Fatalf("unexpected freestyle env: %#v", cfg.Freestyle) + } if cfg.Tensorlake.APIKey != "tl-api-env" || cfg.Tensorlake.APIURL != "https://api.tl-env.example" || cfg.Tensorlake.CLIPath != "/opt/tl/bin/tensorlake" || cfg.Tensorlake.Image != "ubuntu:tl-env" || cfg.Tensorlake.Snapshot != "snap-tl-env" || cfg.Tensorlake.OrganizationID != "org-tl-env" || cfg.Tensorlake.ProjectID != "proj-tl-env" || cfg.Tensorlake.Namespace != "ns-tl-env" || cfg.Tensorlake.Workdir != "/workspace/tl-env" || cfg.Tensorlake.CPUs != 2.5 || cfg.Tensorlake.MemoryMB != 4096 || cfg.Tensorlake.DiskMB != 20480 || cfg.Tensorlake.TimeoutSecs != 900 || !cfg.Tensorlake.NoInternet { t.Fatalf("unexpected tensorlake env: %#v", cfg.Tensorlake) } diff --git a/internal/cli/providers_builtin_test.go b/internal/cli/providers_builtin_test.go index 4ee20855..79ded991 100644 --- a/internal/cli/providers_builtin_test.go +++ b/internal/cli/providers_builtin_test.go @@ -17,6 +17,7 @@ func init() { RegisterProvider(testNamespaceProvider{}) RegisterProvider(testDaytonaProvider{}) RegisterProvider(testIsloProvider{}) + RegisterProvider(testFreestyleProvider{}) RegisterProvider(testE2BProvider{}) RegisterProvider(testModalProvider{}) RegisterProvider(testCloudflareProvider{}) @@ -626,6 +627,58 @@ func (p testIsloProvider) Configure(cfg Config, rt Runtime) (Backend, error) { return testDelegatedBackend{spec: p.Spec()}, nil } +type testFreestyleProvider struct{} + +func (testFreestyleProvider) Name() string { return "freestyle" } +func (testFreestyleProvider) Aliases() []string { return nil } +func (testFreestyleProvider) Spec() ProviderSpec { + return ProviderSpec{ + Name: "freestyle", + Kind: ProviderKindDelegatedRun, + Targets: []TargetSpec{{OS: targetLinux}}, + Features: FeatureSet{FeatureArchiveSync}, + Coordinator: CoordinatorNever, + } +} + +type testFreestyleFlagValues struct { + APIURL *string + Workdir *string + VCPUs *int + MemoryMB *int +} + +func (testFreestyleProvider) RegisterFlags(fs *flag.FlagSet, defaults Config) any { + return testFreestyleFlagValues{ + APIURL: fs.String("freestyle-api-url", defaults.Freestyle.APIURL, "Freestyle API URL"), + Workdir: fs.String("freestyle-workdir", defaults.Freestyle.Workdir, "Freestyle sandbox workdir"), + VCPUs: fs.Int("freestyle-vcpus", defaults.Freestyle.VCPUs, "Freestyle sandbox vCPUs"), + MemoryMB: fs.Int("freestyle-memory-mb", defaults.Freestyle.MemoryMB, "Freestyle sandbox memory in MB"), + } +} +func (testFreestyleProvider) ApplyFlags(cfg *Config, fs *flag.FlagSet, values any) error { + v, ok := values.(testFreestyleFlagValues) + if !ok { + return nil + } + if flagWasSet(fs, "freestyle-api-url") { + cfg.Freestyle.APIURL = *v.APIURL + } + if flagWasSet(fs, "freestyle-workdir") { + cfg.Freestyle.Workdir = *v.Workdir + } + if flagWasSet(fs, "freestyle-vcpus") { + cfg.Freestyle.VCPUs = *v.VCPUs + } + if flagWasSet(fs, "freestyle-memory-mb") { + cfg.Freestyle.MemoryMB = *v.MemoryMB + } + return nil +} +func (p testFreestyleProvider) Configure(cfg Config, rt Runtime) (Backend, error) { + return testDelegatedBackend{spec: p.Spec()}, nil +} + type testE2BProvider struct{} func (testE2BProvider) Name() string { return "e2b" } diff --git a/internal/providers/all/all.go b/internal/providers/all/all.go index 53e53c5b..03640eb2 100644 --- a/internal/providers/all/all.go +++ b/internal/providers/all/all.go @@ -8,6 +8,7 @@ import ( _ "github.com/openclaw/crabbox/internal/providers/daytona" _ "github.com/openclaw/crabbox/internal/providers/e2b" _ "github.com/openclaw/crabbox/internal/providers/exedev" + _ "github.com/openclaw/crabbox/internal/providers/freestyle" _ "github.com/openclaw/crabbox/internal/providers/gcp" _ "github.com/openclaw/crabbox/internal/providers/hetzner" _ "github.com/openclaw/crabbox/internal/providers/islo" diff --git a/internal/providers/all/all_test.go b/internal/providers/all/all_test.go index d7fc3ffd..3bf25359 100644 --- a/internal/providers/all/all_test.go +++ b/internal/providers/all/all_test.go @@ -15,6 +15,7 @@ func TestAllBuiltInProvidersExposeDoctor(t *testing.T) { "daytona", "e2b", "exe-dev", + "freestyle", "gcp", "hetzner", "islo", diff --git a/internal/providers/freestyle/backend.go b/internal/providers/freestyle/backend.go new file mode 100644 index 00000000..871d2620 --- /dev/null +++ b/internal/providers/freestyle/backend.go @@ -0,0 +1,528 @@ +package freestyle + +import ( + "context" + "crypto/rand" + "encoding/hex" + "flag" + "fmt" + "sort" + "strings" + "time" + + core "github.com/openclaw/crabbox/internal/cli" +) + +type Config = core.Config +type ProviderSpec = core.ProviderSpec +type Runtime = core.Runtime +type Backend = core.Backend +type FreestyleConfig = core.FreestyleConfig +type SyncConfig = core.SyncConfig +type WarmupRequest = core.WarmupRequest +type RunRequest = core.RunRequest +type RunResult = core.RunResult +type ListRequest = core.ListRequest +type LeaseView = core.LeaseView +type StatusRequest = core.StatusRequest +type StatusView = core.StatusView +type StopRequest = core.StopRequest +type Server = core.Server +type Repo = core.Repo +type ExitError = core.ExitError +type timingReport = core.TimingReport +type timingPhase = core.TimingPhase + +const ( + targetLinux = core.TargetLinux + NetworkPublic = core.NetworkPublic +) + +const ( + freestyleProvider = "freestyle" + freestyleLeasePrefix = "fsb_" + freestyleNamePrefix = "crabbox-" +) + +type freestyleFlagValues struct { + APIURL *string + Workdir *string + VCPUs *int + MemoryMB *int +} + +func RegisterFreestyleProviderFlags(fs *flag.FlagSet, defaults Config) any { + return freestyleFlagValues{ + APIURL: fs.String("freestyle-api-url", defaults.Freestyle.APIURL, "Freestyle API URL"), + Workdir: fs.String("freestyle-workdir", defaults.Freestyle.Workdir, "Freestyle sandbox workdir"), + VCPUs: fs.Int("freestyle-vcpus", defaults.Freestyle.VCPUs, "Freestyle sandbox vCPUs"), + MemoryMB: fs.Int("freestyle-memory-mb", defaults.Freestyle.MemoryMB, "Freestyle sandbox memory in MB"), + } +} + +func ApplyFreestyleProviderFlags(cfg *Config, fs *flag.FlagSet, values any) error { + v, ok := values.(freestyleFlagValues) + if !ok { + return nil + } + if flagWasSet(fs, "freestyle-api-url") { + cfg.Freestyle.APIURL = *v.APIURL + } + if flagWasSet(fs, "freestyle-workdir") { + cfg.Freestyle.Workdir = *v.Workdir + } + if flagWasSet(fs, "freestyle-vcpus") { + cfg.Freestyle.VCPUs = *v.VCPUs + } + if flagWasSet(fs, "freestyle-memory-mb") { + cfg.Freestyle.MemoryMB = *v.MemoryMB + } + return nil +} + +func NewFreestyleBackend(spec ProviderSpec, cfg Config, rt Runtime) Backend { + cfg.Provider = freestyleProvider + return &freestyleBackend{spec: spec, cfg: cfg, rt: rt} +} + +type freestyleBackend struct { + spec ProviderSpec + cfg Config + rt Runtime +} + +func (b *freestyleBackend) Spec() ProviderSpec { return b.spec } + +func (b *freestyleBackend) Warmup(ctx context.Context, req WarmupRequest) error { + if req.ActionsRunner { + return exit(2, "--actions-runner is not supported for provider=%s", freestyleProvider) + } + started := b.now() + client, err := newFreestyleClient(b.cfg, b.rt) + if err != nil { + return err + } + leaseID, name, slug, err := b.createSandbox(ctx, client, req.Repo, req.Reclaim, req.RequestedSlug) + if err != nil { + return err + } + fmt.Fprintf(b.rt.Stdout, "leased %s slug=%s provider=freestyle sandbox=%s\n", leaseID, slug, name) + if !req.Keep { + fmt.Fprintf(b.rt.Stderr, "warning: freestyle warmup keeps the sandbox until explicit stop\n") + } + total := b.now().Sub(started) + fmt.Fprintf(b.rt.Stdout, "warmup complete total=%s\n", total.Round(time.Millisecond)) + if req.TimingJSON { + return writeTimingJSON(b.rt.Stderr, timingReport{ + Provider: freestyleProvider, + LeaseID: leaseID, + Slug: slug, + TotalMs: total.Milliseconds(), + ExitCode: 0, + }) + } + return nil +} + +func (b *freestyleBackend) Run(ctx context.Context, req RunRequest) (RunResult, error) { + if err := rejectFreestyleSyncOptions(req); err != nil { + return RunResult{}, err + } + workspace, err := freestyleWorkspacePath(b.cfg) + if err != nil { + return RunResult{}, err + } + started := b.now() + client, err := newFreestyleClient(b.cfg, b.rt) + if err != nil { + return RunResult{}, err + } + leaseID, name, slug := "", "", "" + acquired := false + if req.ID == "" { + leaseID, name, slug, err = b.createSandbox(ctx, client, req.Repo, req.Reclaim, req.RequestedSlug) + if err != nil { + return RunResult{}, err + } + fmt.Fprintf(b.rt.Stderr, "leased %s slug=%s provider=freestyle sandbox=%s\n", leaseID, slug, name) + acquired = true + } else { + leaseID, name, err = resolveFreestyleLeaseID(ctx, client, req.ID, req.Repo.Root, req.Reclaim) + if err != nil { + return RunResult{}, err + } + slug = newLeaseSlug(leaseID) + } + shouldStop := acquired && !req.Keep + if shouldStop { + defer func() { + if !shouldStop { + return + } + if err := client.DeleteVM(context.Background(), name); err != nil { + fmt.Fprintf(b.rt.Stderr, "warning: freestyle stop failed for %s: %v\n", name, err) + return + } + removeLeaseClaim(leaseID) + }() + } + fmt.Fprintf(b.rt.Stderr, "provider=freestyle lease=%s sandbox=%s\n", leaseID, name) + syncDuration := time.Duration(0) + syncPhases := []timingPhase{{Name: "sync", Skipped: true, Reason: "--no-sync"}} + if !req.NoSync { + var err error + syncPhases, syncDuration, err = b.syncWorkspace(ctx, client, name, req) + if err != nil { + return RunResult{}, err + } + fmt.Fprintf(b.rt.Stderr, "sync complete in %s\n", syncDuration.Round(time.Millisecond)) + } else if err := b.prepareWorkspace(ctx, client, name, workspace, false); err != nil { + return RunResult{}, err + } + if req.SyncOnly { + result := RunResult{ + Total: b.now().Sub(started), + SyncDelegated: true, + } + fmt.Fprintf(b.rt.Stdout, "synced %s\n", workspace) + if req.TimingJSON { + err := writeTimingJSON(b.rt.Stderr, timingReport{ + Provider: freestyleProvider, + LeaseID: leaseID, + Slug: slug, + SyncDelegated: true, + SyncMs: syncDuration.Milliseconds(), + SyncPhases: syncPhases, + SyncSkipped: req.NoSync, + TotalMs: result.Total.Milliseconds(), + ExitCode: 0, + Label: strings.TrimSpace(req.Label), + }) + return result, err + } + return result, nil + } + if len(req.Command) == 0 || (len(req.Command) == 1 && strings.TrimSpace(req.Command[0]) == "") { + return RunResult{}, exit(2, "missing command") + } + commandStart := b.now() + exitCode, runErr := b.exec(ctx, client, name, workspace, req.Command, req.ShellMode, req.Env) + commandDuration := b.now().Sub(commandStart) + result := RunResult{ + ExitCode: exitCode, + Command: commandDuration, + Total: b.now().Sub(started), + SyncDelegated: true, + } + if req.NoSync { + fmt.Fprintf(b.rt.Stderr, "freestyle run summary sync_skipped=true command=%s total=%s exit=%d\n", result.Command.Round(time.Millisecond), result.Total.Round(time.Millisecond), exitCode) + } else { + fmt.Fprintf(b.rt.Stderr, "freestyle run summary sync=%s command=%s total=%s exit=%d\n", syncDuration.Round(time.Millisecond), result.Command.Round(time.Millisecond), result.Total.Round(time.Millisecond), exitCode) + } + if req.TimingJSON { + if err := writeTimingJSON(b.rt.Stderr, timingReport{ + Provider: freestyleProvider, + LeaseID: leaseID, + SyncDelegated: true, + SyncMs: syncDuration.Milliseconds(), + SyncPhases: syncPhases, + SyncSkipped: req.NoSync, + CommandMs: result.Command.Milliseconds(), + TotalMs: result.Total.Milliseconds(), + ExitCode: exitCode, + Label: strings.TrimSpace(req.Label), + }); err != nil { + return result, err + } + } + if runErr != nil { + handleDelegatedRunFailure(b.rt.Stderr, req, freestyleProvider, leaseID, slug, b.cfg.IdleTimeout, b.cfg.TTL, acquired, &shouldStop) + return result, ExitError{Code: 1, Message: fmt.Sprintf("freestyle run failed: %v", runErr)} + } + if exitCode != 0 { + handleDelegatedRunFailure(b.rt.Stderr, req, freestyleProvider, leaseID, slug, b.cfg.IdleTimeout, b.cfg.TTL, acquired, &shouldStop) + return result, ExitError{Code: exitCode, Message: fmt.Sprintf("freestyle run exited %d", exitCode)} + } + return result, nil +} + +func (b *freestyleBackend) List(ctx context.Context, req ListRequest) ([]LeaseView, error) { + _ = req + client, err := newFreestyleClient(b.cfg, b.rt) + if err != nil { + return nil, err + } + vms, err := client.ListVMs(ctx) + if err != nil { + return nil, freestyleError("list vms", err) + } + servers := make([]Server, 0, len(vms)) + for _, vm := range vms { + if !isCrabboxFreestyleSandboxName(vm.Name) { + continue + } + servers = append(servers, freestyleVMToServer(vm)) + } + return servers, nil +} + +func (b *freestyleBackend) Doctor(ctx context.Context, _ core.DoctorRequest) (core.DoctorResult, error) { + servers, err := b.List(ctx, ListRequest{}) + if err != nil { + return core.DoctorResult{}, err + } + return core.InventoryDoctorResult(freestyleProvider, len(servers)), nil +} + +func (b *freestyleBackend) Status(ctx context.Context, req StatusRequest) (statusView, error) { + client, err := newFreestyleClient(b.cfg, b.rt) + if err != nil { + return statusView{}, err + } + leaseID, id, err := resolveFreestyleLeaseID(ctx, client, req.ID, "", false) + if err != nil { + return statusView{}, err + } + deadline := b.now().Add(req.WaitTimeout) + if req.WaitTimeout <= 0 { + deadline = b.now().Add(5 * time.Minute) + } + for { + vm, err := client.GetVM(ctx, id) + if err != nil { + return statusView{}, freestyleError("get vm", err) + } + view := freestyleStatusView(leaseID, vm) + if !req.Wait || view.Ready { + return view, nil + } + if b.now().After(deadline) { + return statusView{}, exit(5, "timed out waiting for vm %s to become ready", id) + } + select { + case <-ctx.Done(): + return statusView{}, ctx.Err() + case <-time.After(2 * time.Second): + } + } +} + +func (b *freestyleBackend) Stop(ctx context.Context, req StopRequest) error { + client, err := newFreestyleClient(b.cfg, b.rt) + if err != nil { + return err + } + leaseID, id, err := resolveFreestyleLeaseID(ctx, client, req.ID, "", false) + if err != nil { + return err + } + if err := client.DeleteVM(ctx, id); err != nil { + return freestyleError("delete vm", err) + } + removeLeaseClaim(leaseID) + fmt.Fprintf(b.rt.Stderr, "released lease=%s sandbox=%s\n", leaseID, id) + return nil +} + +func (b *freestyleBackend) createSandbox(ctx context.Context, client freestyleAPI, repo Repo, reclaim bool, requestedSlug string) (string, string, string, error) { + workdir, err := freestyleRelativeWorkdir(b.cfg) + if err != nil { + return "", "", "", err + } + _ = workdir + name := newFreestyleSandboxName(repo) + create := freestyleCreateVMRequest{ + Name: name, + VcpuCount: b.cfg.Freestyle.VCPUs, + MemSizeMb: b.cfg.Freestyle.MemoryMB, + } + vm, err := client.CreateVM(ctx, create) + if err != nil { + return "", "", "", freestyleError("create vm", err) + } + if vm.ID == "" { + return "", "", "", exit(5, "freestyle create vm returned no id") + } + leaseID := freestyleLeasePrefix + vm.ID + slug, err := allocateClaimLeaseSlug(leaseID, requestedSlug) + if err != nil { + _ = client.DeleteVM(context.Background(), vm.ID) + return "", "", "", err + } + if err := claimLeaseForRepoProvider(leaseID, slug, freestyleProvider, repo.Root, b.cfg.IdleTimeout, reclaim); err != nil { + _ = client.DeleteVM(context.Background(), vm.ID) + return "", "", "", err + } + return leaseID, vm.ID, slug, nil +} + +func (b *freestyleBackend) exec(ctx context.Context, client freestyleAPI, id, workdir string, command []string, shellMode bool, env map[string]string) (int, error) { + execCommand := freestyleExecCommand(command, shellMode) + parts := make([]string, 0, 3) + if workdir != "" { + parts = append(parts, "cd "+shellQuote(workdir)) + } + if envCommand := freestyleEnvExportCommand(env); envCommand != "" { + parts = append(parts, envCommand) + } + parts = append(parts, execCommand) + fullCommand := strings.Join(parts, " && ") + return client.Exec(ctx, id, "bash -lc "+shellQuote(fullCommand), b.rt.Stdout, b.rt.Stderr) +} + +func freestyleEnvExportCommand(env map[string]string) string { + if len(env) == 0 { + return "" + } + keys := make([]string, 0, len(env)) + for name := range env { + if validFreestyleEnvName(name) { + keys = append(keys, name) + } + } + if len(keys) == 0 { + return "" + } + sort.Strings(keys) + var b strings.Builder + b.WriteString("export") + for _, name := range keys { + b.WriteByte(' ') + b.WriteString(name) + b.WriteByte('=') + b.WriteString(shellQuote(env[name])) + } + return b.String() +} + +func validFreestyleEnvName(name string) bool { + if name == "" || strings.Contains(name, "=") { + return false + } + return isShellEnvAssignment(name + "=x") +} + +func freestyleExecCommand(command []string, shellMode bool) string { + if len(command) == 0 { + return "" + } + if shellMode { + return strings.Join(command, " ") + } + if len(command) == 1 && shouldUseShell(command) { + return command[0] + } + if shouldUseShell(command) || leadingEnvAssignment(command) { + return shellScriptFromArgv(command) + } + return strings.Join(shellWords(command), " ") +} + +func resolveFreestyleLeaseID(ctx context.Context, client freestyleAPI, id, repoRoot string, reclaim bool) (string, string, error) { + if id == "" { + return "", "", exit(2, "provider=freestyle requires a Crabbox-created vm name, lease id, or slug") + } + if claim, ok, err := resolveLeaseClaim(id); err != nil { + return "", "", err + } else if ok && claim.Provider == freestyleProvider { + if repoRoot != "" { + if err := claimLeaseForRepoProvider(claim.LeaseID, claim.Slug, freestyleProvider, repoRoot, time.Duration(claim.IdleTimeoutSeconds)*time.Second, reclaim); err != nil { + return "", "", err + } + } + return claim.LeaseID, strings.TrimPrefix(claim.LeaseID, freestyleLeasePrefix), nil + } + if strings.HasPrefix(id, freestyleLeasePrefix) { + vmID := strings.TrimPrefix(id, freestyleLeasePrefix) + if vmID == "" { + return "", "", exit(4, "freestyle vm %q is not claimed by Crabbox", id) + } + vm, err := client.GetVM(ctx, vmID) + if err != nil { + return "", "", freestyleError("get vm", err) + } + if !isCrabboxFreestyleSandboxName(vm.Name) { + return "", "", exit(4, "freestyle vm %q is not claimed by Crabbox", id) + } + return id, blank(vm.ID, vmID), nil + } + return "", "", exit(4, "freestyle vm %q is not claimed by Crabbox; use a Crabbox slug or claimed Freestyle lease id", id) +} + +func freestyleVMToServer(vm freestyleVM) Server { + leaseID := freestyleLeasePrefix + vm.ID + labels := applyFreestyleClaimLabels(leaseID, vm) + return Server{ + Provider: freestyleProvider, + CloudID: vm.ID, + Name: vm.Name, + Status: vm.State, + Labels: labels, + } +} + +func freestyleStatusView(leaseID string, vm freestyleVM) statusView { + return statusView{ + ID: leaseID, + Slug: newLeaseSlug(leaseID), + Provider: freestyleProvider, + TargetOS: targetLinux, + State: vm.State, + ServerID: vm.ID, + ServerType: vm.Name, + Network: NetworkPublic, + Ready: freestyleStatusReady(vm.State), + Labels: map[string]string{ + "provider": freestyleProvider, + "lease": leaseID, + "state": vm.State, + }, + } +} + +func applyFreestyleClaimLabels(leaseID string, vm freestyleVM) map[string]string { + return map[string]string{ + "provider": freestyleProvider, + "lease": leaseID, + "slug": newLeaseSlug(leaseID), + "target": targetLinux, + "state": vm.State, + } +} + +func freestyleStatusReady(status string) bool { + switch strings.ToLower(strings.TrimSpace(status)) { + case "ready", "running", "started", "active": + return true + default: + return false + } +} + +func newFreestyleSandboxName(repo Repo) string { + base := normalizeLeaseSlug(repo.Name) + if base == "" { + base = "crabbox" + } + base = strings.TrimPrefix(base, strings.TrimSuffix(freestyleNamePrefix, "-")+"-") + return freestyleNamePrefix + base + "-" + freestyleRandomSuffix() +} + +func isCrabboxFreestyleSandboxName(name string) bool { + return strings.HasPrefix(normalizeLeaseSlug(name), freestyleNamePrefix) +} + +func freestyleRandomSuffix() string { + var b [3]byte + if _, err := rand.Read(b[:]); err != nil { + return fmt.Sprintf("%x", time.Now().UnixNano())[:6] + } + return hex.EncodeToString(b[:]) +} + +func (b *freestyleBackend) now() time.Time { + if b.rt.Clock != nil { + return b.rt.Clock.Now() + } + return time.Now() +} diff --git a/internal/providers/freestyle/backend_test.go b/internal/providers/freestyle/backend_test.go new file mode 100644 index 00000000..fa1570c6 --- /dev/null +++ b/internal/providers/freestyle/backend_test.go @@ -0,0 +1,571 @@ +package freestyle + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "flag" + "io" + "os" + "os/exec" + "strings" + "testing" +) + +func TestFreestyleExecCommandPreservesShellString(t *testing.T) { + got := freestyleExecCommand([]string{"pnpm install && pnpm test"}, true) + want := "pnpm install && pnpm test" + if got != want { + t.Fatalf("command=%q want %q", got, want) + } +} + +func TestFreestyleExecCommandQuotesImplicitShellArgv(t *testing.T) { + if got := freestyleExecCommand([]string{"go", "test", "./..."}, false); got != "'go' 'test' './...'" { + t.Fatalf("command=%q", got) + } + got := freestyleExecCommand([]string{"FOO=bar", "pnpm", "test"}, false) + if !strings.Contains(got, "FOO=") || !strings.Contains(got, "'pnpm'") { + t.Fatalf("command=%q", got) + } +} + +func TestFreestyleExecCommandPreservesSpacedArguments(t *testing.T) { + got := freestyleExecCommand([]string{"echo", "hello world"}, false) + want := "'echo' 'hello world'" + if got != want { + t.Fatalf("command=%q want %q", got, want) + } +} + +func TestFreestyleExecCommandPreservesSingleShellString(t *testing.T) { + got := freestyleExecCommand([]string{"echo hello from freestyle"}, false) + want := "echo hello from freestyle" + if got != want { + t.Fatalf("command=%q want %q", got, want) + } +} + +func TestFreestyleEnvExportCommandQuotesValuesOnly(t *testing.T) { + got := freestyleEnvExportCommand(map[string]string{ + "GREETING": "hello world", + "Z_TOKEN": "abc'123", + "BAD; id >&2 #": "boom", + }) + want := "export GREETING='hello world' Z_TOKEN='abc'\\''123'" + if got != want { + t.Fatalf("env export=%q want %q", got, want) + } + if strings.Contains(got, "'GREETING'") || strings.Contains(got, "BAD;") { + t.Fatalf("env export contains unsafe name quoting or invalid name: %q", got) + } +} + +func TestFreestyleExecForwardsEnvAfterWorkdir(t *testing.T) { + client := &fakeFreestyleClient{} + backend := &freestyleBackend{rt: Runtime{Stderr: io.Discard}} + code, err := backend.exec(context.Background(), client, "vm123", "/workspace/repo", []string{`echo "$GREETING"`}, false, map[string]string{ + "GREETING": "hello world", + }) + if err != nil { + t.Fatal(err) + } + if code != 0 { + t.Fatalf("exit code=%d", code) + } + if len(client.execCommands) != 1 { + t.Fatalf("exec commands=%#v", client.execCommands) + } + command := client.execCommands[0] + want := `bash -lc 'cd '\''/workspace/repo'\'' && export GREETING='\''hello world'\'' && echo "$GREETING"'` + if command != want { + t.Fatalf("command=%q want %q", command, want) + } + if strings.Contains(command, "'GREETING'=") { + t.Fatalf("command quotes env name: %s", command) + } +} + +func TestFreestyleAPIKeyFlagIsNotRegistered(t *testing.T) { + cfg := Config{} + cfg.Freestyle.APIKey = "secret-key" + fs := flag.NewFlagSet("test", flag.ContinueOnError) + RegisterFreestyleProviderFlags(fs, cfg) + for _, name := range []string{"freestyle-api-key", "freestyle-api-token", "freestyle-key", "freestyle-token"} { + if fs.Lookup(name) != nil { + t.Fatalf("freestyle API key surfaced as a flag --%s", name) + } + } + for _, name := range []string{"freestyle-api-url", "freestyle-workdir", "freestyle-vcpus", "freestyle-memory-mb"} { + if fs.Lookup(name) == nil { + t.Fatalf("%s flag missing", name) + } + } +} + +func TestFreestyleWarmupRejectsActionsRunner(t *testing.T) { + backend := &freestyleBackend{rt: Runtime{Stderr: io.Discard}} + err := backend.Warmup(context.Background(), WarmupRequest{ActionsRunner: true}) + if err == nil || !strings.Contains(err.Error(), "--actions-runner") { + t.Fatalf("Warmup err=%v, want actions-runner rejection", err) + } +} + +func TestFreestyleStatusReady(t *testing.T) { + for _, status := range []string{"ready", "running", "started", "active"} { + if !freestyleStatusReady(status) { + t.Fatalf("expected %q ready", status) + } + } + if freestyleStatusReady("stopped") { + t.Fatal("stopped should not be ready") + } +} + +func TestResolveFreestyleLeaseIDRejectsUnclaimedRawSandbox(t *testing.T) { + client := &fakeFreestyleClient{getVM: freestyleVM{ + ID: "vm123", + Name: "personal-vm", + State: "running", + }} + if _, _, err := resolveFreestyleLeaseID(context.Background(), client, "random-vm-id", "", false); err == nil { + t.Fatal("expected raw non-Crabbox vm to be rejected") + } + if _, _, err := resolveFreestyleLeaseID(context.Background(), client, "fsb_vm123", "", false); err == nil { + t.Fatal("expected unclaimed Freestyle vm to be rejected") + } +} + +func TestResolveFreestyleLeaseIDAcceptsCrabboxSandbox(t *testing.T) { + client := &fakeFreestyleClient{getVM: freestyleVM{ + ID: "vm123", + Name: "crabbox-repo-abc123", + State: "running", + }} + leaseID, name, err := resolveFreestyleLeaseID(context.Background(), client, "fsb_vm123", "", false) + if err != nil { + t.Fatal(err) + } + if leaseID != "fsb_vm123" || name != "vm123" { + t.Fatalf("lease=%q name=%q", leaseID, name) + } +} + +func TestFreestyleWorkspacePathDefaultsUnderWorkspace(t *testing.T) { + cfg := Config{Freestyle: FreestyleConfig{}} + if got, err := freestyleWorkspacePath(cfg); err != nil || got != "/workspace/crabbox" { + t.Fatalf("workspace=%q err=%v", got, err) + } + cfg = Config{Freestyle: FreestyleConfig{Workdir: "repo"}} + if got, err := freestyleWorkspacePath(cfg); err != nil || got != "/workspace/repo" { + t.Fatalf("workspace=%q err=%v", got, err) + } + cfg = Config{Freestyle: FreestyleConfig{Workdir: "team/repo"}} + if got, err := freestyleWorkspacePath(cfg); err != nil || got != "/workspace/team/repo" { + t.Fatalf("workspace=%q err=%v", got, err) + } +} + +func TestFreestyleWorkspacePathRejectsEscapes(t *testing.T) { + for _, workdir := range []string{"/work/repo", "/etc", "../etc", "repo/../../../etc", ".", "./.."} { + t.Run(workdir, func(t *testing.T) { + if got, err := freestyleWorkspacePath(Config{Freestyle: FreestyleConfig{Workdir: workdir}}); err == nil { + t.Fatalf("workspace=%q, want error for workdir %q", got, workdir) + } + }) + } +} + +func TestFreestyleRunRejectsUnsafeWorkdirBeforeProviderClient(t *testing.T) { + backend := &freestyleBackend{ + cfg: Config{Freestyle: FreestyleConfig{Workdir: "../etc"}}, + rt: Runtime{Stderr: io.Discard}, + } + _, err := backend.Run(context.Background(), RunRequest{NoSync: true}) + if err == nil || !strings.Contains(err.Error(), "escapes /workspace") { + t.Fatalf("Run err=%v, want workdir containment error", err) + } +} + +func TestFreestyleRunRejectsMissingCommand(t *testing.T) { + t.Setenv("XDG_STATE_HOME", t.TempDir()) + client := &fakeFreestyleClient{createID: "vm123"} + oldClient := newFreestyleClient + newFreestyleClient = func(cfg Config, rt Runtime) (freestyleAPI, error) { + return client, nil + } + defer func() { newFreestyleClient = oldClient }() + backend := &freestyleBackend{ + cfg: Config{Freestyle: FreestyleConfig{}}, + rt: Runtime{Stderr: io.Discard}, + } + _, err := backend.Run(context.Background(), RunRequest{ + Repo: Repo{Root: t.TempDir(), Name: "repo"}, + NoSync: true, + Command: nil, + }) + if err == nil || !strings.Contains(err.Error(), "missing command") { + t.Fatalf("Run err=%v, want missing command", err) + } +} + +func TestFreestyleRunSyncOnlySkipsUserExec(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + if _, err := exec.LookPath("tar"); err != nil { + t.Skip("tar not available") + } + t.Setenv("XDG_STATE_HOME", t.TempDir()) + root := t.TempDir() + if err := os.WriteFile(root+"/go.mod", []byte("module example.test/repo\n"), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command("git", "init") + cmd.Dir = root + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git init: %v\n%s", err, out) + } + client := &fakeFreestyleClient{createID: "vm-sync"} + oldClient := newFreestyleClient + newFreestyleClient = func(cfg Config, rt Runtime) (freestyleAPI, error) { + return client, nil + } + defer func() { newFreestyleClient = oldClient }() + var stdout bytes.Buffer + backend := &freestyleBackend{ + cfg: Config{Freestyle: FreestyleConfig{}}, + rt: Runtime{Stdout: &stdout, Stderr: io.Discard}, + } + _, err := backend.Run(context.Background(), RunRequest{ + Repo: Repo{Root: root, Name: "repo"}, + SyncOnly: true, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(stdout.String(), "synced /workspace/crabbox") { + t.Fatalf("stdout=%q", stdout.String()) + } + for _, command := range client.execCommands { + if strings.Contains(command, "bash -lc") && !strings.Contains(command, "mkdir") && !strings.Contains(command, "tar") && !strings.Contains(command, "base64") && !strings.Contains(command, "printf") && !strings.Contains(command, "rm -f") { + t.Fatalf("unexpected user exec: %q", command) + } + } +} + +func TestFreestyleRunNoSyncDoesNotDeleteExistingWorkspace(t *testing.T) { + t.Setenv("XDG_STATE_HOME", t.TempDir()) + client := &fakeFreestyleClient{getVM: freestyleVM{ + ID: "vm123", + Name: "crabbox-repo-abc123", + State: "running", + }} + oldClient := newFreestyleClient + newFreestyleClient = func(cfg Config, rt Runtime) (freestyleAPI, error) { + return client, nil + } + defer func() { newFreestyleClient = oldClient }() + backend := &freestyleBackend{ + cfg: Config{ + Freestyle: FreestyleConfig{}, + Sync: SyncConfig{Delete: true}, + }, + rt: Runtime{Stderr: io.Discard}, + } + _, err := backend.Run(context.Background(), RunRequest{ + ID: "fsb_vm123", + Repo: Repo{Root: t.TempDir(), Name: "repo"}, + NoSync: true, + Command: []string{"test", "-f", "kept.txt"}, + }) + if err != nil { + t.Fatalf("Run err=%v", err) + } + if len(client.execCommands) != 2 { + t.Fatalf("exec commands=%#v want prepare and user command", client.execCommands) + } + prepare := client.execCommands[0] + if strings.Contains(prepare, "rm -rf") { + t.Fatalf("--no-sync prepare deleted workspace: %q", prepare) + } + if !strings.Contains(prepare, "mkdir -p") { + t.Fatalf("--no-sync prepare should ensure workspace: %q", prepare) + } +} + +func TestFreestyleCreateSandboxWorksWithoutWorkdir(t *testing.T) { + t.Setenv("XDG_STATE_HOME", t.TempDir()) + client := &fakeFreestyleClient{createID: "vm123"} + backend := &freestyleBackend{ + cfg: Config{Freestyle: FreestyleConfig{}}, + rt: Runtime{Stderr: io.Discard}, + } + leaseID, id, slug, err := backend.createSandbox(context.Background(), client, Repo{Root: t.TempDir(), Name: "repo"}, false, "") + if err != nil { + t.Fatal(err) + } + if leaseID != "fsb_vm123" { + t.Fatalf("leaseID=%q", leaseID) + } + if id != "vm123" { + t.Fatalf("id=%q", id) + } + if slug == "" { + t.Fatal("slug is empty") + } + if client.createReq == nil { + t.Fatal("create request was nil") + } + if client.createReq.VcpuCount != 0 || client.createReq.MemSizeMb != 0 { + t.Fatalf("create sizing=%d/%d want omitted", client.createReq.VcpuCount, client.createReq.MemSizeMb) + } +} + +func TestFreestyleCreateSandboxPassesNameWithoutWorkdir(t *testing.T) { + t.Setenv("XDG_STATE_HOME", t.TempDir()) + client := &fakeFreestyleClient{createID: "vm456"} + backend := &freestyleBackend{ + cfg: Config{Freestyle: FreestyleConfig{VCPUs: 4, MemoryMB: 8192}}, + rt: Runtime{Stderr: io.Discard}, + } + _, _, _, err := backend.createSandbox(context.Background(), client, Repo{Root: t.TempDir(), Name: "repo"}, false, "") + if err != nil { + t.Fatal(err) + } + if client.createReq == nil { + t.Fatal("create request was nil") + } + if !strings.HasPrefix(client.createReq.Name, "crabbox-repo-") { + t.Fatalf("name=%q", client.createReq.Name) + } + if client.createReq.VcpuCount != 4 || client.createReq.MemSizeMb != 8192 { + t.Fatalf("create sizing=%d/%d", client.createReq.VcpuCount, client.createReq.MemSizeMb) + } +} + +func TestFreestyleCreateSandboxStoresClaimForList(t *testing.T) { + t.Setenv("XDG_STATE_HOME", t.TempDir()) + client := &fakeFreestyleClient{createID: "vm789"} + backend := &freestyleBackend{ + cfg: Config{Freestyle: FreestyleConfig{}}, + rt: Runtime{Stderr: io.Discard}, + } + _, _, _, err := backend.createSandbox(context.Background(), client, Repo{Root: t.TempDir(), Name: "repo"}, false, "") + if err != nil { + t.Fatal(err) + } + claim, ok, err := resolveLeaseClaim("fsb_vm789") + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("claim not found for fsb_vm789") + } + if claim.Provider != "freestyle" { + t.Fatalf("claim provider=%q", claim.Provider) + } +} + +func TestFreestyleSyncWorkspaceUploadsRepoArchive(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + if _, err := exec.LookPath("tar"); err != nil { + t.Skip("tar not available") + } + root := t.TempDir() + if err := os.WriteFile(root+"/go.mod", []byte("module example.test/repo\n"), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command("git", "init") + cmd.Dir = root + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git init: %v\n%s", err, out) + } + client := &fakeFreestyleClient{} + backend := &freestyleBackend{ + cfg: Config{Freestyle: FreestyleConfig{Workdir: "repo"}}, + rt: Runtime{Stderr: io.Discard}, + } + _, _, err := backend.syncWorkspace(context.Background(), client, "crabbox-test", RunRequest{ + Repo: Repo{Root: root, Name: "repo"}, + }) + if err != nil { + t.Fatal(err) + } + if client.writeFilePath != "/tmp/crabbox-" { + if !strings.HasPrefix(client.writeFilePath, "/tmp/crabbox-") || !strings.HasSuffix(client.writeFilePath, ".tgz") { + t.Fatalf("write file path=%q", client.writeFilePath) + } + } + if client.writeFileEncoding != "base64" { + t.Fatalf("write file encoding=%q", client.writeFileEncoding) + } + if len(client.prepareCommands) < 1 || !strings.Contains(client.prepareCommands[0], "mkdir") || !strings.Contains(client.prepareCommands[0], "/workspace/repo") { + t.Fatalf("prepare commands=%#v", client.prepareCommands) + } +} + +func TestFreestyleSyncWorkspaceFallsBackToExecUpload(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + if _, err := exec.LookPath("tar"); err != nil { + t.Skip("tar not available") + } + root := t.TempDir() + if err := os.WriteFile(root+"/go.mod", []byte("module example.test/repo\n"), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command("git", "init") + cmd.Dir = root + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git init: %v\n%s", err, out) + } + client := &fakeFreestyleClient{writeFileErr: errors.New("file api upload failed")} + backend := &freestyleBackend{ + cfg: Config{Freestyle: FreestyleConfig{Workdir: "repo"}}, + rt: Runtime{Stderr: io.Discard}, + } + _, _, err := backend.syncWorkspace(context.Background(), client, "crabbox-test", RunRequest{ + Repo: Repo{Root: root, Name: "repo"}, + }) + if err != nil { + t.Fatal(err) + } + if !client.commandContains("base64 -d") || !client.commandContains("tar -xzf") { + t.Fatalf("fallback commands=%#v", client.prepareCommands) + } +} + +func TestFreestyleFallbackExtractCommandCleansUploadsOnFailure(t *testing.T) { + cmd := freestyleFallbackExtractCommand("/tmp/crabbox-test.tgz.b64", "/tmp/crabbox-test.tgz", "/workspace/repo") + for _, want := range []string{ + "base64 -d '/tmp/crabbox-test.tgz.b64' > '/tmp/crabbox-test.tgz'", + "tar -xzf '/tmp/crabbox-test.tgz' -C '/workspace/repo'", + "; status=$?; rm -f '/tmp/crabbox-test.tgz.b64' '/tmp/crabbox-test.tgz'; exit $status", + } { + if !strings.Contains(cmd, want) { + t.Fatalf("command missing %q: %s", want, cmd) + } + } + if strings.Index(cmd, "rm -f '/tmp/crabbox-test.tgz.b64'") < strings.Index(cmd, "tar -xzf") { + t.Fatalf("cleanup should run after extract attempt: %s", cmd) + } +} + +func TestRejectFreestyleSyncOptionsAllowsForceSyncLarge(t *testing.T) { + if err := rejectFreestyleSyncOptions(RunRequest{ForceSyncLarge: true}); err != nil { + t.Fatalf("force sync large should be honored by Freestyle archive sync: %v", err) + } + if err := rejectFreestyleSyncOptions(RunRequest{SyncOnly: true}); err != nil { + t.Fatalf("sync-only should be supported: %v", err) + } + if err := rejectFreestyleSyncOptions(RunRequest{ChecksumSync: true}); err == nil || !strings.Contains(err.Error(), "--checksum") { + t.Fatalf("checksum err=%v", err) + } +} + +func TestNewFreestyleSandboxNameUsesCrabboxPrefix(t *testing.T) { + name := newFreestyleSandboxName(Repo{Name: "repo"}) + if !strings.HasPrefix(name, "crabbox-repo-") { + t.Fatalf("name=%q", name) + } + if !isCrabboxFreestyleSandboxName(name) { + t.Fatalf("expected %q to be recognized as Crabbox-owned", name) + } +} + +type fakeFreestyleClient struct { + createID string + createReq *freestyleCreateVMRequest + getVM freestyleVM + getVMErr error + prepareCommands []string + writeFilePath string + writeFileContent string + writeFileEncoding string + writeFileErr error + execCommands []string +} + +func (f *fakeFreestyleClient) CreateVM(_ context.Context, req freestyleCreateVMRequest) (freestyleVM, error) { + f.createReq = &req + id := f.createID + if id == "" { + id = "vm-test-abcdef" + } + return freestyleVM{ID: id, State: "running"}, nil +} + +func (f *fakeFreestyleClient) GetVM(_ context.Context, id string) (freestyleVM, error) { + if f.getVMErr != nil { + return freestyleVM{}, f.getVMErr + } + if f.getVM.ID != "" || f.getVM.Name != "" || f.getVM.State != "" { + return f.getVM, nil + } + return freestyleVM{ID: id, State: "running"}, nil +} + +func (f *fakeFreestyleClient) ListVMs(_ context.Context) ([]freestyleVM, error) { + return nil, nil +} + +func (f *fakeFreestyleClient) DeleteVM(_ context.Context, _ string) error { + return nil +} + +func (f *fakeFreestyleClient) Exec(_ context.Context, _ string, command string, _, _ io.Writer) (int, error) { + f.execCommands = append(f.execCommands, command) + f.prepareCommands = append(f.prepareCommands, command) + return 0, nil +} + +func (f *fakeFreestyleClient) WriteFile(_ context.Context, _ string, path, content, encoding string) error { + f.writeFilePath = path + f.writeFileContent = content + f.writeFileEncoding = encoding + if f.writeFileErr != nil { + return f.writeFileErr + } + return nil +} + +func (f *fakeFreestyleClient) ReadFile(_ context.Context, _, _ string) (string, error) { + return "", nil +} + +func (f *fakeFreestyleClient) commandContains(value string) bool { + for _, command := range f.prepareCommands { + if strings.Contains(command, value) { + return true + } + } + return false +} + +func tarGzipContains(t *testing.T, data []byte, name string) bool { + t.Helper() + gz, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tr := tar.NewReader(gz) + for { + header, err := tr.Next() + if err == io.EOF { + return false + } + if err != nil { + t.Fatal(err) + } + if header.Name == name { + return true + } + } +} diff --git a/internal/providers/freestyle/client.go b/internal/providers/freestyle/client.go new file mode 100644 index 00000000..e1c1ed02 --- /dev/null +++ b/internal/providers/freestyle/client.go @@ -0,0 +1,260 @@ +package freestyle + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +type freestyleAPI interface { + CreateVM(ctx context.Context, req freestyleCreateVMRequest) (freestyleVM, error) + GetVM(ctx context.Context, id string) (freestyleVM, error) + ListVMs(ctx context.Context) ([]freestyleVM, error) + DeleteVM(ctx context.Context, id string) error + Exec(ctx context.Context, id string, command string, stdout, stderr io.Writer) (int, error) + WriteFile(ctx context.Context, id, path, content, encoding string) error + ReadFile(ctx context.Context, id, path string) (string, error) +} + +type freestyleVM struct { + ID string `json:"id"` + Name string `json:"name"` + State string `json:"state"` +} + +type freestyleListVMsResponse struct { + VMs []freestyleVM `json:"vms"` + TotalCount int `json:"totalCount"` +} + +type freestyleCreateVMResponse struct { + ID string `json:"id"` +} + +type freestyleCreateVMRequest struct { + Name string `json:"name"` + VcpuCount int `json:"vcpuCount,omitempty"` + MemSizeMb int `json:"memSizeMb,omitempty"` +} + +type freestyleExecRequest struct { + Command string `json:"command"` +} + +type freestyleExecResponse struct { + StatusCode int `json:"statusCode"` + Stdout *string `json:"stdout"` + Stderr *string `json:"stderr"` +} + +type freestyleWriteFileRequest struct { + Content string `json:"content"` + Encoding string `json:"encoding"` +} + +type freestyleReadFileResponse struct { + Content string `json:"content"` + Encoding string `json:"encoding"` +} + +type freestyleHTTPClient struct { + apiKey string + apiURL string + httpClient *http.Client +} + +var newFreestyleClient = func(cfg Config, rt Runtime) (freestyleAPI, error) { + apiKey := strings.TrimSpace(cfg.Freestyle.APIKey) + if apiKey == "" { + return nil, exit(2, "provider=freestyle requires FREESTYLE_API_KEY") + } + apiURL := strings.TrimRight(blank(cfg.Freestyle.APIURL, "https://api.freestyle.sh"), "/") + httpClient := rt.HTTP + if httpClient == nil { + httpClient = http.DefaultClient + } + return &freestyleHTTPClient{ + apiKey: apiKey, + apiURL: apiURL, + httpClient: httpClient, + }, nil +} + +func (c *freestyleHTTPClient) do(ctx context.Context, method, urlPath string, body io.Reader) (*http.Response, error) { + u, err := url.Parse(c.apiURL + urlPath) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, method, u.String(), body) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+c.apiKey) + req.Header.Set("Accept", "application/json") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return c.httpClient.Do(req) +} + +func (c *freestyleHTTPClient) CreateVM(ctx context.Context, req freestyleCreateVMRequest) (freestyleVM, error) { + payload, err := json.Marshal(req) + if err != nil { + return freestyleVM{}, err + } + resp, err := c.do(ctx, http.MethodPost, "/v1/vms", bytes.NewReader(payload)) + if err != nil { + return freestyleVM{}, freestyleError("create vm", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return freestyleVM{}, freestyleError("create vm", fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(snippet)))) + } + var parsed freestyleCreateVMResponse + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return freestyleVM{}, freestyleError("create vm", err) + } + return freestyleVM{ID: parsed.ID, State: "running"}, nil +} + +func (c *freestyleHTTPClient) GetVM(ctx context.Context, id string) (freestyleVM, error) { + resp, err := c.do(ctx, http.MethodGet, "/v1/vms/"+url.PathEscape(id), nil) + if err != nil { + return freestyleVM{}, freestyleError("get vm", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return freestyleVM{}, freestyleError("get vm", fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(snippet)))) + } + var vm freestyleVM + if err := json.NewDecoder(resp.Body).Decode(&vm); err != nil { + return freestyleVM{}, freestyleError("get vm", err) + } + return vm, nil +} + +func (c *freestyleHTTPClient) ListVMs(ctx context.Context) ([]freestyleVM, error) { + resp, err := c.do(ctx, http.MethodGet, "/v1/vms", nil) + if err != nil { + return nil, freestyleError("list vms", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, freestyleError("list vms", fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(snippet)))) + } + var parsed freestyleListVMsResponse + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return nil, freestyleError("list vms", err) + } + return parsed.VMs, nil +} + +func (c *freestyleHTTPClient) DeleteVM(ctx context.Context, id string) error { + resp, err := c.do(ctx, http.MethodDelete, "/v1/vms/"+url.PathEscape(id), nil) + if err != nil { + return freestyleError("delete vm", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return freestyleError("delete vm", fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(snippet)))) + } + return nil +} + +func (c *freestyleHTTPClient) Exec(ctx context.Context, id string, command string, stdout, stderr io.Writer) (int, error) { + payload, err := json.Marshal(freestyleExecRequest{Command: command}) + if err != nil { + return 1, err + } + resp, err := c.do(ctx, http.MethodPost, "/v1/vms/"+url.PathEscape(id)+"/exec-await", bytes.NewReader(payload)) + if err != nil { + return 1, freestyleError("exec", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return 1, freestyleError("exec", fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(snippet)))) + } + var parsed freestyleExecResponse + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return 1, freestyleError("exec", err) + } + if parsed.Stdout != nil { + _, _ = stdout.Write([]byte(*parsed.Stdout)) + } + if parsed.Stderr != nil { + _, _ = stderr.Write([]byte(*parsed.Stderr)) + } + return parsed.StatusCode, nil +} + +func (c *freestyleHTTPClient) WriteFile(ctx context.Context, id, path string, content, encoding string) error { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + payload, err := json.Marshal(freestyleWriteFileRequest{Content: content, Encoding: encoding}) + if err != nil { + return err + } + resp, err := c.do(ctx, http.MethodPut, freestyleFileURLPath(id, path), bytes.NewReader(payload)) + if err != nil { + return freestyleError("write file", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return freestyleError("write file", fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(snippet)))) + } + return nil +} + +func (c *freestyleHTTPClient) ReadFile(ctx context.Context, id, path string) (string, error) { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + resp, err := c.do(ctx, http.MethodGet, freestyleFileURLPath(id, path), nil) + if err != nil { + return "", freestyleError("read file", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return "", freestyleError("read file", fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(snippet)))) + } + var parsed freestyleReadFileResponse + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return "", freestyleError("read file", err) + } + if parsed.Encoding == "base64" { + decoded, err := base64.StdEncoding.DecodeString(parsed.Content) + if err != nil { + return "", freestyleError("read file", err) + } + return string(decoded), nil + } + return parsed.Content, nil +} + +func freestyleFileURLPath(id, path string) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return "/v1/vms/" + url.PathEscape(id) + "/files/" + url.PathEscape(path) +} + +func freestyleError(action string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("freestyle %s: %w", action, err) +} diff --git a/internal/providers/freestyle/client_test.go b/internal/providers/freestyle/client_test.go new file mode 100644 index 00000000..da2cbe2a --- /dev/null +++ b/internal/providers/freestyle/client_test.go @@ -0,0 +1,99 @@ +package freestyle + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestFreestyleFileOperationsUseFilesPathPrefix(t *testing.T) { + const ( + writePath = "/v1/vms/vm123/files/%2Ftmp%2Fcrabbox%20upload.tgz" + readPath = "/v1/vms/vm123/files/%2Fworkspace%2Frepo%2Ffile.txt" + ) + var sawWrite, sawRead bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("Authorization"), "Bearer test-key"; got != want { + t.Errorf("authorization header=%q want %q", got, want) + } + switch r.Method { + case http.MethodPut: + sawWrite = true + if got := r.URL.EscapedPath(); got != writePath { + t.Errorf("write path=%q want %q", got, writePath) + } + w.WriteHeader(http.StatusNoContent) + case http.MethodGet: + sawRead = true + if got := r.URL.EscapedPath(); got != readPath { + t.Errorf("read path=%q want %q", got, readPath) + } + if err := json.NewEncoder(w).Encode(freestyleReadFileResponse{Content: "ok"}); err != nil { + t.Errorf("encode response: %v", err) + } + default: + t.Errorf("unexpected method %s", r.Method) + w.WriteHeader(http.StatusMethodNotAllowed) + } + })) + defer server.Close() + + client := &freestyleHTTPClient{ + apiKey: "test-key", + apiURL: server.URL, + httpClient: server.Client(), + } + if err := client.WriteFile(context.Background(), "vm123", "/tmp/crabbox upload.tgz", "payload", "base64"); err != nil { + t.Fatalf("WriteFile err=%v", err) + } + got, err := client.ReadFile(context.Background(), "vm123", "workspace/repo/file.txt") + if err != nil { + t.Fatalf("ReadFile err=%v", err) + } + if got != "ok" { + t.Fatalf("ReadFile content=%q", got) + } + if !sawWrite || !sawRead { + t.Fatalf("sawWrite=%v sawRead=%v", sawWrite, sawRead) + } +} + +func TestFreestyleCreateVMOmitsUnsetSizing(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Method, http.MethodPost; got != want { + t.Errorf("method=%s want %s", got, want) + } + if got, want := r.URL.EscapedPath(), "/v1/vms"; got != want { + t.Errorf("path=%q want %q", got, want) + } + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("decode request: %v", err) + } + if _, ok := body["vcpuCount"]; ok { + t.Fatalf("vcpuCount should be omitted from default create request: %#v", body) + } + if _, ok := body["memSizeMb"]; ok { + t.Fatalf("memSizeMb should be omitted from default create request: %#v", body) + } + if err := json.NewEncoder(w).Encode(freestyleCreateVMResponse{ID: "vm123"}); err != nil { + t.Errorf("encode response: %v", err) + } + })) + defer server.Close() + + client := &freestyleHTTPClient{ + apiKey: "test-key", + apiURL: server.URL, + httpClient: server.Client(), + } + vm, err := client.CreateVM(context.Background(), freestyleCreateVMRequest{Name: "crabbox-test"}) + if err != nil { + t.Fatalf("CreateVM err=%v", err) + } + if vm.ID != "vm123" { + t.Fatalf("vm.ID=%q", vm.ID) + } +} diff --git a/internal/providers/freestyle/core.go b/internal/providers/freestyle/core.go new file mode 100644 index 00000000..04de8f58 --- /dev/null +++ b/internal/providers/freestyle/core.go @@ -0,0 +1,99 @@ +package freestyle + +import ( + "flag" + "io" + "time" + + core "github.com/openclaw/crabbox/internal/cli" +) + +type statusView = core.StatusView + +func exit(code int, format string, args ...any) core.ExitError { + return core.Exit(code, format, args...) +} + +func flagWasSet(fs *flag.FlagSet, name string) bool { + return core.FlagWasSet(fs, name) +} + +func rejectDelegatedSyncOptions(provider string, req RunRequest) error { + return core.RejectDelegatedSyncOptions(provider, req) +} + +func writeTimingJSON(w io.Writer, report timingReport) error { + return core.WriteTimingJSON(w, report) +} + +func printKeepOnFailureDelegatedHint(w io.Writer, provider, leaseID, slug string, idleTimeout, ttl time.Duration) { + core.PrintKeepOnFailureDelegatedHint(w, provider, leaseID, slug, idleTimeout, ttl) +} + +func handleDelegatedRunFailure(w io.Writer, req RunRequest, provider, leaseID, slug string, idleTimeout, ttl time.Duration, acquired bool, shouldStop *bool) { + core.HandleDelegatedRunFailure(w, req, provider, leaseID, slug, idleTimeout, ttl, acquired, shouldStop) +} + +func shouldUseShell(command []string) bool { + return core.ShouldUseShell(command) +} + +func shellScriptFromArgv(command []string) string { + return core.ShellScriptFromArgv(command) +} + +func shellWords(words []string) []string { + return core.ShellWords(words) +} + +func leadingEnvAssignment(command []string) bool { + return core.LeadingEnvAssignment(command) +} + +func isShellEnvAssignment(word string) bool { + return core.IsShellEnvAssignment(word) +} + +func newLeaseSlug(leaseID string) string { + return core.NewLeaseSlug(leaseID) +} + +func normalizeLeaseSlug(value string) string { + return core.NormalizeLeaseSlug(value) +} + +func allocateClaimLeaseSlug(leaseID, requested string) (string, error) { + return core.AllocateClaimLeaseSlug(leaseID, requested) +} + +func blank(value, fallback string) string { + return core.Blank(value, fallback) +} + +func claimLeaseForRepoProvider(leaseID, slug, provider, repoRoot string, idleTimeout time.Duration, reclaim bool) error { + return core.ClaimLeaseForRepoProvider(leaseID, slug, provider, repoRoot, idleTimeout, reclaim) +} + +func resolveLeaseClaim(identifier string) (core.LeaseClaim, bool, error) { + return core.ResolveLeaseClaim(identifier) +} + +func removeLeaseClaim(leaseID string) { + core.RemoveLeaseClaim(leaseID) +} + +func syncExcludes(root string, cfg Config) ([]string, error) { + return core.SyncExcludes(root, cfg) +} + +func syncManifest(root string, excludes []string) (core.SyncManifest, error) { + return core.BuildSyncManifest(root, excludes) +} + +func checkSyncPreflight(manifest core.SyncManifest, cfg Config, force bool, stderr io.Writer) error { + return core.CheckSyncPreflight(manifest, cfg, force, stderr) +} + +func shellQuote(s string) string { + return core.ShellQuote(s) +} diff --git a/internal/providers/freestyle/provider.go b/internal/providers/freestyle/provider.go new file mode 100644 index 00000000..c5b28bb4 --- /dev/null +++ b/internal/providers/freestyle/provider.go @@ -0,0 +1,48 @@ +package freestyle + +import ( + "flag" + + core "github.com/openclaw/crabbox/internal/cli" +) + +func init() { + core.RegisterProvider(Provider{}) +} + +type Provider struct{} + +func (Provider) Name() string { return "freestyle" } +func (Provider) Aliases() []string { + return nil +} +func (Provider) Spec() core.ProviderSpec { + return core.ProviderSpec{ + Name: "freestyle", + Kind: core.ProviderKindDelegatedRun, + Targets: []core.TargetSpec{{OS: core.TargetLinux}}, + Features: core.FeatureSet{core.FeatureArchiveSync}, + Coordinator: core.CoordinatorNever, + } +} +func (Provider) RegisterFlags(fs *flag.FlagSet, defaults core.Config) any { + return RegisterFreestyleProviderFlags(fs, defaults) +} +func (Provider) ApplyFlags(cfg *core.Config, fs *flag.FlagSet, values any) error { + return ApplyFreestyleProviderFlags(cfg, fs, values) +} +func (p Provider) Configure(cfg core.Config, rt core.Runtime) (core.Backend, error) { + return NewFreestyleBackend(p.Spec(), cfg, rt), nil +} + +func (p Provider) ConfigureDoctor(cfg core.Config, rt core.Runtime) (core.DoctorBackend, error) { + backend, err := p.Configure(cfg, rt) + if err != nil { + return nil, err + } + doctor, ok := backend.(core.DoctorBackend) + if !ok { + return nil, core.Exit(2, "freestyle doctor backend unavailable") + } + return doctor, nil +} diff --git a/internal/providers/freestyle/sync.go b/internal/providers/freestyle/sync.go new file mode 100644 index 00000000..88620ba7 --- /dev/null +++ b/internal/providers/freestyle/sync.go @@ -0,0 +1,193 @@ +package freestyle + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "os" + "os/exec" + "path" + "strings" + "time" + + core "github.com/openclaw/crabbox/internal/cli" +) + +type SyncManifest = core.SyncManifest + +func rejectFreestyleSyncOptions(req RunRequest) error { + if req.ChecksumSync { + return exit(2, "%s uses Freestyle archive sync; --checksum is not supported", freestyleProvider) + } + return nil +} + +func (b *freestyleBackend) syncWorkspace(ctx context.Context, client freestyleAPI, name string, req RunRequest) ([]timingPhase, time.Duration, error) { + start := b.now() + excludes, err := syncExcludes(req.Repo.Root, b.cfg) + if err != nil { + return nil, 0, err + } + manifestStarted := b.now() + manifest, err := syncManifest(req.Repo.Root, excludes) + if err != nil { + return nil, 0, exit(6, "build sync file list: %v", err) + } + manifestDuration := b.now().Sub(manifestStarted) + preflightStarted := b.now() + if err := checkSyncPreflight(manifest, b.cfg, req.ForceSyncLarge, b.rt.Stderr); err != nil { + return nil, 0, err + } + preflightDuration := b.now().Sub(preflightStarted) + workspace, err := freestyleWorkspacePath(b.cfg) + if err != nil { + return nil, 0, err + } + prepareStarted := b.now() + if err := b.prepareWorkspace(ctx, client, name, workspace, b.cfg.Sync.Delete); err != nil { + return nil, 0, err + } + prepareDuration := b.now().Sub(prepareStarted) + archiveStarted := b.now() + archive, err := createFreestyleSyncArchive(ctx, req.Repo, manifest, b.rt.Stderr) + if err != nil { + return nil, 0, err + } + defer os.Remove(archive.Name()) + defer archive.Close() + archiveDuration := b.now().Sub(archiveStarted) + uploadStarted := b.now() + if _, err := archive.Seek(0, 0); err != nil { + return nil, 0, fmt.Errorf("freestyle rewind archive: %w", err) + } + archiveData, err := io.ReadAll(archive) + if err != nil { + return nil, 0, fmt.Errorf("freestyle read archive: %w", err) + } + b64Content := base64.StdEncoding.EncodeToString(archiveData) + suffix := freestyleRandomSuffix() + remoteArchive := "/tmp/crabbox-" + suffix + ".tgz" + if err := client.WriteFile(ctx, name, remoteArchive, b64Content, "base64"); err != nil { + fmt.Fprintf(b.rt.Stderr, "warning: freestyle file API upload failed; falling back to exec upload: %v\n", err) + if fallbackErr := b.uploadArchiveViaExec(ctx, client, name, workspace, archiveData); fallbackErr != nil { + return nil, 0, fallbackErr + } + } else { + if err := b.execShell(ctx, client, name, "tar -xzf "+shellQuote(remoteArchive)+" -C "+shellQuote(workspace)+" && rm -f "+shellQuote(remoteArchive)); err != nil { + return nil, 0, err + } + } + uploadDuration := b.now().Sub(uploadStarted) + total := b.now().Sub(start) + return []timingPhase{ + {Name: "manifest", Ms: manifestDuration.Milliseconds()}, + {Name: "preflight", Ms: preflightDuration.Milliseconds()}, + {Name: "prepare", Ms: prepareDuration.Milliseconds()}, + {Name: "archive", Ms: archiveDuration.Milliseconds()}, + {Name: "upload", Ms: uploadDuration.Milliseconds()}, + {Name: "freestyle_sync", Ms: total.Milliseconds()}, + }, total, nil +} + +func (b *freestyleBackend) prepareWorkspace(ctx context.Context, client freestyleAPI, name, workspace string, delete bool) error { + command := "mkdir -p " + shellQuote(workspace) + if delete { + command = "rm -rf " + shellQuote(workspace) + " && " + command + } + return b.execShell(ctx, client, name, command) +} + +func (b *freestyleBackend) uploadArchiveViaExec(ctx context.Context, client freestyleAPI, name, workspace string, archiveData []byte) error { + suffix := freestyleRandomSuffix() + remoteB64 := "/tmp/crabbox-" + suffix + ".tgz.b64" + remoteArchive := "/tmp/crabbox-" + suffix + ".tgz" + if err := b.execShell(ctx, client, name, "rm -f "+shellQuote(remoteB64)+" "+shellQuote(remoteArchive)); err != nil { + return err + } + buf := archiveData + chunkSize := 48 * 1024 + for i := 0; i < len(buf); i += chunkSize { + end := i + chunkSize + if end > len(buf) { + end = len(buf) + } + chunk := base64.StdEncoding.EncodeToString(buf[i:end]) + command := "printf %s " + shellQuote(chunk) + " >> " + shellQuote(remoteB64) + if err := b.execShell(ctx, client, name, command); err != nil { + return err + } + } + return b.execShell(ctx, client, name, freestyleFallbackExtractCommand(remoteB64, remoteArchive, workspace)) +} + +func freestyleFallbackExtractCommand(remoteB64, remoteArchive, workspace string) string { + extract := strings.Join([]string{ + "if base64 -d " + shellQuote(remoteB64) + " > " + shellQuote(remoteArchive) + " 2>/dev/null; then :; else base64 --decode " + shellQuote(remoteB64) + " > " + shellQuote(remoteArchive) + "; fi", + "tar -xzf " + shellQuote(remoteArchive) + " -C " + shellQuote(workspace), + }, " && ") + cleanup := "rm -f " + shellQuote(remoteB64) + " " + shellQuote(remoteArchive) + return extract + "; status=$?; " + cleanup + "; exit $status" +} + +func (b *freestyleBackend) execShell(ctx context.Context, client freestyleAPI, name, command string) error { + code, err := client.Exec(ctx, name, "bash -lc "+shellQuote(command), io.Discard, b.rt.Stderr) + if err != nil { + return fmt.Errorf("freestyle exec %q: %w", command, err) + } + if code != 0 { + return exit(code, "freestyle exec %q exited %d", command, code) + } + return nil +} + +func createFreestyleSyncArchive(ctx context.Context, repo Repo, manifest SyncManifest, stderr io.Writer) (*os.File, error) { + var input bytes.Buffer + input.Write(manifest.NUL()) + archive, err := os.CreateTemp("", "crabbox-freestyle-sync-*.tgz") + if err != nil { + return nil, fmt.Errorf("create sync archive temp file: %w", err) + } + keep := false + defer func() { + if !keep { + name := archive.Name() + _ = archive.Close() + _ = os.Remove(name) + } + }() + cmd := exec.CommandContext(ctx, "tar", "--no-xattrs", "-czf", "-", "-C", repo.Root, "--null", "-T", "-") + cmd.Stdin = &input + cmd.Env = append(os.Environ(), "COPYFILE_DISABLE=1") + cmd.Stdout = archive + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + return nil, exit(6, "create sync archive: %v", err) + } + keep = true + return archive, nil +} + +func freestyleWorkspacePath(cfg Config) (string, error) { + workdir, err := freestyleRelativeWorkdir(cfg) + if err != nil { + return "", err + } + return path.Join("/workspace", workdir), nil +} + +func freestyleRelativeWorkdir(cfg Config) (string, error) { + workdir := strings.TrimSpace(cfg.Freestyle.Workdir) + if workdir == "" { + workdir = "crabbox" + } + if strings.HasPrefix(workdir, "/") { + return "", exit(2, "freestyle workdir %q must be relative under /workspace", workdir) + } + workdir = path.Clean(workdir) + if workdir == "." || workdir == ".." || strings.HasPrefix(workdir, "../") { + return "", exit(2, "freestyle workdir %q escapes /workspace", workdir) + } + return workdir, nil +}