diff --git a/.air.cli.toml b/.air.cli.toml index 893d9f2..33671ae 100644 --- a/.air.cli.toml +++ b/.air.cli.toml @@ -2,7 +2,7 @@ root = "." tmp_dir = "tmp" [build] - cmd = "make build-cli-dev" + cmd = "mage buildCLIDev" bin = "/home/hersi/.nextdeploy/bin/nextdeploy" full_bin = "" include_ext = ["go", "toml"] diff --git a/.air.daemon.toml b/.air.daemon.toml index 313dfe5..49bf2ab 100644 --- a/.air.daemon.toml +++ b/.air.daemon.toml @@ -2,7 +2,7 @@ root = "." tmp_dir = "tmp" [build] - cmd = "make build-daemon-dev" + cmd = "mage buildDaemonDev" bin = "~/.nextdeploy/bin/nextdeployd" full_bin = "~/.nextdeploy/bin/nextdeployd --foreground=true" include_ext = ["go", "toml", "yml"] diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..98dce49 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# NextDeploy pre-commit hook +# +# Blocks commits that contain: +# - Compiled binaries (Mach-O, ELF, PE) +# - Files larger than MAX_FILE_KB +# - Obvious secrets (.env, AWS keys, private keys) +# - Files matching ignored patterns +# +# Install with: mage preCommitInstall +# Bypass (use sparingly, with explicit reason): git commit --no-verify + +set -euo pipefail + +MAX_FILE_KB=512 +RED='\033[0;31m' +YELLOW='\033[0;33m' +GREEN='\033[0;32m' +NC='\033[0m' + +fail=0 + +# Collect staged files (added/copied/modified, not deleted) +staged=$(git diff --cached --name-only --diff-filter=ACM) +if [ -z "$staged" ]; then + exit 0 +fi + +err() { echo -e "${RED}✘ $*${NC}" >&2; fail=1; } +warn() { echo -e "${YELLOW}⚠ $*${NC}" >&2; } +ok() { echo -e "${GREEN}✓ $*${NC}"; } + +# 1. Block compiled binaries by content sniffing +for f in $staged; do + [ -f "$f" ] || continue + mime=$(file --mime-type -b "$f" 2>/dev/null || echo "unknown") + case "$mime" in + application/x-mach-binary|application/x-executable|application/x-sharedlib|application/x-pie-executable|application/x-dosexec) + err "Binary file blocked: $f ($mime)" + ;; + esac +done + +# 2. Block oversized files +for f in $staged; do + [ -f "$f" ] || continue + size_kb=$(du -k "$f" | cut -f1) + if [ "$size_kb" -gt "$MAX_FILE_KB" ]; then + err "File too large: $f (${size_kb}KB > ${MAX_FILE_KB}KB). Use Git LFS or .gitignore." + fi +done + +# 3. Block secrets by filename +for f in $staged; do + case "$f" in + .env|.env.*|*.pem|*.key|.nextdeploy/.env|.secrets|credentials.json|service-account*.json) + # allow examples and testdata + case "$f" in + .env.example|*/testdata/*) continue ;; + esac + err "Secret file blocked: $f" + ;; + esac +done + +# 4. Block secret PATTERNS in staged content (text files only) +secret_pattern='(AKIA[0-9A-Z]{16}|aws_secret_access_key\s*=\s*[A-Za-z0-9/+=]{40}|-----BEGIN [A-Z ]*PRIVATE KEY-----|ghp_[A-Za-z0-9]{36}|xox[baprs]-[A-Za-z0-9-]+)' +for f in $staged; do + [ -f "$f" ] || continue + mime=$(file --mime-type -b "$f" 2>/dev/null || echo "unknown") + case "$mime" in text/*|application/json|application/xml|application/yaml) ;; *) continue ;; esac + if grep -EH "$secret_pattern" "$f" >/dev/null 2>&1; then + err "Possible secret detected in $f" + grep -EHn "$secret_pattern" "$f" | sed 's/^/ /' >&2 || true + fi +done + +# 5. gofmt check on staged Go files +go_files=$(echo "$staged" | grep '\.go$' || true) +if [ -n "$go_files" ]; then + if ! command -v gofmt >/dev/null 2>&1; then + warn "gofmt not found — skipping format check" + else + bad=$(gofmt -s -l $go_files || true) + if [ -n "$bad" ]; then + err "Go files need gofmt:" + echo "$bad" | sed 's/^/ /' >&2 + echo " Fix with: mage fmt" >&2 + fi + fi +fi + +# 6. go vet on changed packages (cheap, catches obvious bugs) +if [ -n "$go_files" ] && command -v go >/dev/null 2>&1; then + pkgs=$(echo "$go_files" | xargs -n1 dirname | sort -u | sed 's|^|./|') + if ! go vet $pkgs >/dev/null 2>&1; then + err "go vet failed on changed packages:" + go vet $pkgs 2>&1 | sed 's/^/ /' >&2 || true + fi +fi + +if [ "$fail" -ne 0 ]; then + echo + echo -e "${RED}Pre-commit hook blocked the commit.${NC}" >&2 + echo " - Fix the issues above, OR" >&2 + echo " - Bypass with --no-verify ONLY if you understand the risk and document why." >&2 + exit 1 +fi + +ok "Pre-commit checks passed" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a69da8..46d23c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,54 +16,106 @@ permissions: env: GO_VERSION: "1.25.x" GOTOOLCHAIN: local + GOLANGCI_LINT_VERSION: "v2.5.0" jobs: - build: - name: Build & Test + modules: + name: Modules runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} cache: true cache-dependency-path: go.sum - - name: Verify modules run: | go mod verify go mod tidy git diff --exit-code go.mod go.sum - - name: Build CLI - run: go build -o /dev/null ./cli + fmt: + name: Format Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + cache-dependency-path: go.sum + - name: Install mage + run: go install github.com/magefile/mage@latest + - name: gofmt + run: mage fmtCheck + build: + name: Build (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + cache-dependency-path: go.sum + - name: Install mage + run: go install github.com/magefile/mage@latest + - name: Build CLI + run: mage buildCLI - name: Build Daemon - run: go build -o /dev/null ./daemon/cmd/nextdeployd + if: runner.os == 'Linux' + run: mage buildDaemon - - name: Test - run: go test -race -timeout 5m ./... + test: + name: Unit Tests + runs-on: ubuntu-latest + needs: [modules] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + cache-dependency-path: go.sum + - name: Install mage + run: go install github.com/magefile/mage@latest + - name: Run unit tests + run: mage testUnit + - name: Coverage summary + run: go tool cover -func=coverage.out | tail -1 + - name: Upload coverage + uses: actions/upload-artifact@v4 + with: + name: coverage + path: coverage.out + retention-days: 7 lint: name: Lint runs-on: ubuntu-latest + needs: [modules] permissions: contents: read pull-requests: read steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} cache: true cache-dependency-path: go.sum - - name: golangci-lint - uses: golangci/golangci-lint-action@v6 + # v7 is required for golangci-lint v2.x (v6 only supports v1.x and + # errors with `invalid version string 'v2.5.0'`). + uses: golangci/golangci-lint-action@v7 with: - version: latest + version: ${{ env.GOLANGCI_LINT_VERSION }} args: --timeout 10m vuln: @@ -71,9 +123,42 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: govulncheck uses: golang/govulncheck-action@v1 with: go-version-input: ${{ env.GO_VERSION }} check-latest: true + + hook-smoke: + name: Pre-commit Hook Smoke Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Validate hook syntax + run: bash -n .githooks/pre-commit + - name: Run hook against HEAD + run: | + # Stage everything from the last commit so the hook has something + # to inspect, then run it. Catches accidental binaries on main. + git -c user.email=ci@example.com -c user.name=ci \ + reset --soft HEAD~1 || true + ./.githooks/pre-commit || (echo "Hook failed on HEAD"; exit 1) + + quality: + name: Code Quality (informational) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + cache-dependency-path: go.sum + - name: Install scc + run: go install github.com/boyter/scc/v3@latest + - name: Lines of code + run: scc --format wide --exclude-dir vendor,test-serverless-app,.next + - name: Benchmarks (best-effort) + run: go test -bench=. -benchmem -run='^$' ./... 2>/dev/null || echo "No benchmarks found" diff --git a/.gitignore b/.gitignore index 7f0a089..0298e9c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,25 +1,74 @@ -/bin/ -/.env -./.keys -.secrets +# ── Build artifacts ────────────────────────────────────────────── +/bin/ /dist/ +/nextdeploy +/nextdeployd +*.exe +*.test +*.out +*.prof +coverage.out +coverage.html -/daemon.md +# ── Secrets and credentials ────────────────────────────────────── +.env +.env.* +!.env.example .secrets +.keys +.nextdeploy/.env +.nextdeploy/secrets/ +*.pem +*.key +!**/testdata/**/*.pem +!**/testdata/**/*.key +credentials.json +service-account*.json -# Documentation -*.md -!README.md -!TECH_DOCS.md +# ── Generated / packaged ───────────────────────────────────────── +*.tar.gz +*.tgz +*.zip +!assets/**/*.zip +node_modules/ +.next/ +out/ -# Root-level binaries -/nextdeploy -/nextdeployd +# ── Tooling / IDE / OS ─────────────────────────────────────────── +.idea/ +.vscode/ +*.swp +*.swo +*~ +.DS_Store +Thumbs.db +.air.toml +.air.*.toml.bak -# Test artifacts -test*.go +# ── Test scratch ───────────────────────────────────────────────── testfile* testdir*/ testsym* testwalk/ test_src_dir/ + +# ── Documentation ──────────────────────────────────────────────── +# Track only the docs we curate. Anything else is scratch/draft. +*.md +!README.md +!TECH_DOCS.md +!CODE_QUALITY.md +!CHANGELOG.md +!CONTRIBUTING.md +!cli/internal/serverless/REVIEW.md +!**/README.md +!CLOUDFLARE_PARITY.md + +# ── Misc ───────────────────────────────────────────────────────── +/daemon.md +vendor/ + +# ── Dev-local ──────────────────────────────────────────────────── +tmp/ +NextDeploy/ +graphify-out/ diff --git a/.golangci.yml b/.golangci.yml index 143e9e6..1a11a76 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,19 +1,94 @@ +version: "2" linters: - # Disable all linters by default and enable only these - disable-all: true + default: none enable: - - gofmt - - goimports + # Correctness - govet - staticcheck - errcheck - - typecheck - unused - ineffassign + - bodyclose + - contextcheck + - noctx + - rowserrcheck + - sqlclosecheck + + # Style - misspell + - whitespace + - unconvert + - predeclared + + # Complexity & Maintainability + - cyclop + - gocognit + - gocyclo + - maintidx + - funlen + + # Security + - gosec + + # Performance + - prealloc + - makezero + + # Best Practices + - godot + - gocritic + - revive + - exhaustive + - errorlint + - nilerr + + settings: + cyclop: + max-complexity: 15 + gocognit: + min-complexity: 20 + gocyclo: + min-complexity: 15 + funlen: + lines: 80 + statements: 50 + maintidx: + under: 20 + gosec: + excludes: + - G304 # file path from variable - acceptable in CLI tools + revive: + rules: + - name: exported + - name: var-naming + - name: error-return + - name: error-strings + - name: increment-decrement + gocritic: + enabled-tags: + - diagnostic + - performance + - style + + exclusions: + rules: + - path: _test\.go + linters: + - errcheck + - funlen + - gocognit + - cyclop + - path: magefile\.go + linters: + - funlen + - gocognit + +formatters: + enable: + - gofmt + - goimports + issues: - exclude-rules: - - path: _test\.go - linters: - - errcheck + max-issues-per-linter: 50 + max-same-issues: 10 diff --git a/.goreleaser.yml b/.goreleaser.yml index b502704..5b4b8af 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -8,6 +8,8 @@ env: before: hooks: - go mod tidy + - env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o assets/lambda/revalidator/bootstrap ./cmd/revalidator + - env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o assets/lambda/imgopt/bootstrap ./cmd/imgopt - go generate ./... builds: @@ -27,6 +29,7 @@ builds: ldflags: - -s -w - -X github.com/Golangcodes/nextdeploy/shared.Version={{.Version}} + - -X github.com/Golangcodes/nextdeploy/shared.Commit={{.ShortCommit}} - -X github.com/Golangcodes/nextdeploy/shared/updater.Version={{.Version}} - -X main.commit={{.Commit}} - -X main.date={{.Date}} @@ -44,6 +47,7 @@ builds: ldflags: - -s -w - -X github.com/Golangcodes/nextdeploy/shared.Version={{.Version}} + - -X github.com/Golangcodes/nextdeploy/shared.Commit={{.ShortCommit}} - -X github.com/Golangcodes/nextdeploy/shared/updater.Version={{.Version}} - -X main.commit={{.Commit}} - -X main.date={{.Date}} diff --git a/CLOUDFLARE_PARITY.md b/CLOUDFLARE_PARITY.md new file mode 100644 index 0000000..608def2 --- /dev/null +++ b/CLOUDFLARE_PARITY.md @@ -0,0 +1,386 @@ +# Cloudflare Parity: What nextdeploy needs to ship Pesastream-shaped apps in one shot + +> Status: scoping doc, not yet implemented. Written 2026-04-18 during Pesastream's OpenNext-on-CF migration. +> Target: `nextdeploy deploy --target cloudflare` completes for a complex Next.js app with DOs, crons, queues, +> Hyperdrive, middleware, and server actions — with no post-deploy manual fixes. + +## 1. What "Pesastream-shaped" means + +This doc uses Pesastream (a real Next.js 16 app) as the reference target, because if nextdeploy can deploy +this, it can deploy anything a Kenyan fintech / SaaS startup is likely to build. The feature surface: + +- **Next.js 16.1.6**, App Router, `output: 'standalone'`, Turbopack, strict TypeScript +- **428 routes** — 209 static + 219 dynamic; 207 of them are `/api/*` (auth, payments, webhooks, crons, admin) +- **`src/middleware.ts`** — subdomain routing, critical path +- **Server Actions** (`"use server"`) used across 54 files +- **6 Durable Object classes** — `ConversationThread`, `CallSession`, `RafikiAgent`, `JibuAgent`, `ZidiAgent`, `AkibaAgent` +- **Cron** — `* * * * *` payment-fanout +- **Hyperdrive** — single binding, pg → `hyperdrive.local:5432` TCP proxy +- **45+ Doppler secrets** deployed via `wrangler secret put` +- **R2** for static assets + Next image originals +- **Streaming SSR** — POS UI depends on progressive render +- **PWA + Service Worker** — `sw.js` must carry `X-Build-ID` header per deploy + +## 2. What nextdeploy can do today (2026-04-18) + +Confirmed by reading `cli/internal/serverless/cloudflare_adapter.go` and `templates/worker_shim.mjs`: + +- Drop an embedded `worker_shim.mjs` into the standalone dir +- Invoke `npx esbuild` with `--bundle --platform=node --format=esm --target=esnext --conditions=worker,node --external:node:* --external:cloudflare:*` +- Bundle the Next standalone server into one ESM worker module +- The shim bridges workerd `fetch` → Next's Node-style request handler (buffered response) +- Static asset short-circuit against `env.ASSETS` (R2 binding) +- `.nextdeploy/metadata.json` emission with: buildId, route classification, static asset inventory, + prerender manifest, detected features + +**What this is good for today:** blog / marketing site / simple SSR app, 1 route handler, no DOs, no crons. + +**What it cannot ship today:** everything on the list below. + +## 3. Gap list — blocking items for Pesastream-shape deploy + +Ranked by "will Pesastream run at all without this" → "cosmetic." + +### Tier A — Pesastream won't function without these + +| # | Gap | Why it blocks | +|---|---|---| +| A1 | **`scheduled()` handler export** | Cron fires the CF runtime's `scheduled()` entry. nextdeploy shim only exports `fetch`. Payment fanout never runs. | +| A2 | **Durable Object class re-exports** | `wrangler.toml` `durable_objects.bindings[].class_name` must resolve to a class exported by the worker module. 6 classes missing → deploy fails at `wrangler deploy` validation. | +| A3 | **`queue()` handler export** | Any CF Queue consumer binding dispatches to `queue()`. Same mechanism as A1. | +| A4 | **workerd esbuild conditions** | Current flag `--conditions=worker,node` misses the `workerd` condition. Causes `pg-cloudflare` and any library with `workerd` conditional exports to resolve to empty stubs at build, producing `TypeError: r2 is not a constructor` at runtime. | +| A5 | **`cloudflare:sockets` external + ESM preservation** | Required by Hyperdrive/pg. If esbuild wraps the dynamic `import('cloudflare:sockets')` in its CJS `__require` shim, runtime dies with `Dynamic require of "cloudflare:sockets" is not supported`. Must (a) mark external, (b) ensure the consuming module is bundled as ESM. | +| A6 | **Hyperdrive binding injection** | Shim must read `env.HYPERDRIVE.connectionString` and make it visible to the pg pool before any handler runs. Pesastream does this with a `globalThis.__PS_DATABASE_URL` override — nextdeploy needs a generic equivalent (config-declared binding → `process.env` + `globalThis` injection at isolate init). | +| A7 | **Middleware invocation** | nextdeploy metadata reports `middleware: null` even when `src/middleware.ts` exists. Detector must find it, and the shim must invoke Next's middleware proxy before the route handler — otherwise subdomain routing (`{subdomain}.pesastream.com` → `/store/{subdomain}/*`) is dead. | +| A8 | **Streaming SSR** | Shim's `res.end()` path accumulates into `chunks[]` and resolves a single `Response(body)`. POS UI degrades visibly. Replace with `TransformStream` writer; pipe as chunks arrive. | + +### Tier B — production-grade but not first-deploy blockers + +| # | Gap | Why it matters | +|---|---|---| +| B1 | **Server Actions detection + routing** | `detected_features.HasServerActions = false` today. Pesastream has 54 files with `"use server"`. Action requests go through Next's RSC handler — should be invoked, but the flow isn't tested without detection/audit. | +| B2 | **R2 asset upload with per-type Cache-Control** | Currently expects pre-uploaded assets. Deploy should upload `.next/static/*` (immutable) + `public/*` (short TTL) in a single pass. Metadata has absolute paths + sizes already; just needs a driver. | +| B3 | **Secret sync from Doppler / .env / KV** | Pesastream deploys 45+ secrets. Current pipeline uses `wrangler secret put` 45 times serially and hits rate limit (error 10013) at request ~13. Needs batched `wrangler secret bulk` or equivalent. | +| B4 | **API route inventory** | Metadata has `api_routes: null`. 207 API routes should be emitted, both for audit and for smoke-testing. | +| B5 | **Build-ID injection** | `BUILD_ID` must land in both (a) `env` for the worker, (b) SW headers (`X-Build-ID`), (c) asset URLs. nextdeploy has the `buildId` in metadata — needs wiring into deploy. | +| B6 | **WebSocket upgrade** | Shim explicitly states unsupported. Pesastream doesn't use WS today (uses Ably over HTTPS), but OpenClaw agents may. Implement via workerd `WebSocketPair`. | +| B7 | **Image optimization** | `/_next/image` currently returns 501. Options: (a) bounce to Cloudflare Images, (b) embed WASM resizer. Pick one and document. | +| B8 | **Incremental cache / tag cache** | Pesastream doesn't use ISR → not strictly blocking. But any Next app reaching for `revalidate` / `unstable_cache` breaks. Define a pluggable interface (R2-backed or DO-backed); ship a no-op default. | + +### Tier C — polish + +- C1: route preloading behavior (cold-start optimization) +- C2: skew protection (multi-version coexistence during rollouts) +- C3: sourcemap upload (observability) +- C4: custom error pages beyond Next's built-in +- C5: `robots.txt` + `sitemap.xml` dynamic generation passthrough + +## 4. Metadata.json extensions required + +Currently missing fields that must be emitted for the deploy pipeline to be driveable: + +```jsonc +{ + // Existing: + "nextbuildmetadata": { "...": "..." }, + "static_assets": { "...": "..." }, + "route_info": { "static_routes": [...], "dynamic_routes": [...] }, + "detected_features": { "...": "..." }, + + // NEW — required: + "middleware": { + "path": "src/middleware.ts", + "matcher": ["/((?!api|_next|...).*)"], + "runtime": "nodejs" + }, + "server_actions": { + "files": ["src/actions/invoice.ts", "..."], // found via `"use server"` scan + "count": 54 + }, + "api_routes": [ + { "path": "/api/webhooks/paystack", "file": "src/app/(api)/api/webhooks/paystack/route.ts", + "methods": ["POST"], "runtime": "nodejs", "dynamic": "force-dynamic" }, + "..." + ], + "cloudflare": { + "durable_objects": [ + { "class_name": "ConversationThread", "source_file": "src/worker/do/conversation-thread.ts" }, + "..." + ], + "queues": { "producers": [], "consumers": [] }, + "crons": [{ "schedule": "* * * * *", "path": "/api/cron/payment-fanout" }], + "bindings": { + "hyperdrive": [{ "binding": "HYPERDRIVE", "id": "eea12bef..." }], + "r2": [{ "binding": "ASSETS", "bucket": "pesastream-assets" }], + "kv": [], "d1": [], "vectorize": [], "ai": [] + }, + "compatibility_flags": ["nodejs_compat", "global_fetch_strictly_public"], + "compatibility_date": "2025-09-15" + }, + "secrets": { + "required": ["DATABASE_URL", "PAYSTACK_SECRET_KEY", "..."], // scanned from process.env usage + "source": "doppler" // detected from scripts/.env/CLI + } +} +``` + +Detectors to add: + +- `src/middleware.ts` + `middleware.ts` at root +- Regex scan for `^"use server"$` directive at top of `.ts`/`.tsx` files +- Parse `src/app/**/route.{ts,js,tsx,jsx}` for HTTP method exports + `dynamic`/`runtime` consts +- Read `wrangler.toml` if present → DO classes, queue bindings, crons, bindings, compat flags +- AST-scan `src/**/*.{ts,tsx}` for `new DurableObject(...)` subclasses → DO class list +- Env var usage scan for required secret names + +## 5. Worker shim extensions required + +Current shim exports only `{ fetch }`. Target shape: + +```js +// generated worker.mjs (not hand-written) +import { RafikiAgent, JibuAgent, /* ... */ } from "./_user_do_classes.mjs"; // re-exported +export { RafikiAgent, JibuAgent, ConversationThread, CallSession, ZidiAgent, AkibaAgent }; + +export default { + async fetch(request, env, ctx) { + applyBindingOverrides(env); // Hyperdrive → globalThis + process.env + await runMiddlewareIfPresent(...); // if metadata.middleware, invoke Next's middleware proxy + return runNextHandler(request, env, ctx); // existing path, but returning streaming Response + }, + async scheduled(event, env, ctx) { + applyBindingOverrides(env); + const path = resolveCronPath(event.cron); // from metadata.cloudflare.crons + return invokeRouteAsRequest("GET", path, env, ctx); + }, + async queue(batch, env, ctx) { + applyBindingOverrides(env); + const handlerPath = resolveQueueHandler(batch.queue); + return invokeQueueHandler(handlerPath, batch, env, ctx); + }, +}; +``` + +New helpers needed: + +- `applyBindingOverrides(env)` — per metadata.cloudflare.bindings, push into `globalThis.__PS_*` + `process.env.*` +- `runMiddlewareIfPresent(req, env, ctx)` — only if metadata.middleware is present +- `invokeRouteAsRequest(method, path, env, ctx)` — synthetic Request for cron +- `invokeQueueHandler(path, batch, env, ctx)` — dispatch queue message to user code +- Streaming response wiring — use `TransformStream`, return `Response` with readable half immediately + +DO class sourcing is the tricky part: the shim needs to import the user's DO classes, but they live in `src/worker/do/*.ts`. Options: +- User declares them in `nextdeploy.config.ts`, the deploy step generates `_user_do_classes.mjs` that imports and re-exports them, esbuild bundles it in +- Or AST-scan + auto-generate the same file + +The config-declared approach is safer (explicit + type-checkable). AST scan is nice as a "did you forget one?" warning. + +## 6. Deploy pipeline additions required + +Order of operations for a one-shot `nextdeploy deploy --target cloudflare`: + +``` +1. Discovery + ├─ build metadata.json (enhanced, per §4) + └─ validate: all DO classes found, all bindings declared, compat flags set + +2. Build + ├─ next build (standalone) + └─ esbuild pass with --conditions=workerd,import,default + - externals: node:*, cloudflare:*, user-declared + - alias map for known quirky packages (pg-cloudflare ESM swap, Ably lazy, etc.) + - single-pass, sourcemap on + +3. Asset upload (R2) + ├─ batch upload .next/static/* (immutable cache) + ├─ batch upload public/* (short TTL) + └─ dedupe by sha256 against remote manifest + +4. Secret sync (if declared) + └─ wrangler secret bulk from .env / Doppler / user-provided + +5. Deploy (wrangler) + ├─ render wrangler.toml from metadata.cloudflare.* if not already present + ├─ wrangler deploy worker.mjs + └─ poll for deploy ID + +6. Post-deploy smoke + ├─ curl every route_info.static_routes entry → expect 200/308 + ├─ curl sample dynamic_routes → expect non-500 + ├─ curl /api/diag/db if present → expect ok:true + └─ fail + auto-rollback if any check fails + +7. Emit deploy record + └─ .nextdeploy/deploys/.json: built_id, route results, size deltas +``` + +Each step should be independently runnable (`nextdeploy build`, `nextdeploy upload`, `nextdeploy smoke`) so debugging doesn't require re-running the world. + +## 7. Suggested phasing + +Assumes one focused developer (you). + +**Phase 1 — "Pesastream deploys without crashing" (target: 1 week)** + +- Fix A4 (workerd conditions) + A5 (cloudflare:sockets ESM preservation) — unblocks any pg user, ~2 hours +- A1/A2/A3 — DO + cron + queue exports via user-declared config in `nextdeploy.config.ts`, ~2 days +- A6 — generic binding → `globalThis`/`process.env` override, ~0.5 day +- A7 — middleware detector + invocation, ~1 day +- A8 — streaming response, ~0.5 day +- B2 — R2 asset upload with per-type Cache-Control, ~1 day +- B3 — secret sync with `wrangler secret bulk`, ~0.5 day +- Validation gate: Pesastream deploys via nextdeploy → `/api/diag/db` returns `{ok: true}`, subdomain routing works, cron fires every minute + +**Phase 2 — production-grade (target: 1 week)** + +- B1, B4, B5, B6 — server actions audit, API route inventory, build-id wiring, WebSocket support +- Post-deploy smoke test runner +- Auto-rollback on smoke failure +- Sourcemap upload + +**Phase 3 — framework parity (target: 1 week)** + +- B7 — image optimization (Cloudflare Images route recommended) +- B8 — incremental cache / tag cache pluggable +- Skew protection, route preloading +- `nextdeploy.config.ts` typed config + validation + +## 8. Non-goals — things nextdeploy should NOT match from OpenNext + +Deliberate simplifications worth keeping: + +- **No runtime bundle patching.** OpenNext applies 11+ esbuild plugins rewriting source mid-bundle. nextdeploy's single-pass + user-declared config model is the point. Patches go into the *pre-esbuild* source tree (e.g. `patch-pg-cloudflare.mjs`-style install-time rewrites) where they're debuggable. +- **No implicit cache behaviour.** If the user doesn't declare an incremental cache, nextdeploy deploys without one. OpenNext silently provisions DO-backed caches that cost money whether you asked or not. +- **No AWS fallback path.** OpenNext is aws-shaped first, CF as overlay. nextdeploy-cloudflare should be CF-native, no AWS shims. +- **No hidden magic.** Everything that ends up in the deploy (bindings, crons, DO classes, secrets) must be either declared in `nextdeploy.config.ts` or emitted into `metadata.json` where the user can inspect it. No reading env vars and "doing the right thing." + +## 9. Test matrix to call nextdeploy CF-ready + +A minimal Pesastream-shaped fixture app that the CI must deploy green: + +- [ ] App Router + 1 static page + 1 dynamic `[slug]` page + 1 API route + 1 server action + middleware +- [ ] 1 Durable Object class with SQLite storage +- [ ] 1 cron at `*/5 * * * *` hitting an internal API route +- [ ] 1 Queue consumer +- [ ] Hyperdrive binding pointing to a real Postgres (Neon dev project) +- [ ] R2 binding serving `/public/*` +- [ ] 3 secrets via `wrangler secret bulk` +- [ ] `nodejs_compat` + `global_fetch_strictly_public` compat flags +- [ ] One `"use server"` action that writes through Hyperdrive to Postgres +- [ ] Smoke test: every route returns expected status, cron fires within 10 min of deploy, DO method round-trips + +When this fixture deploys and stays green for 24h: nextdeploy-cloudflare is ready for Pesastream. + +--- + +**Bottom line:** OpenNext is ~3 years of Vercel-adjacent plumbing ported to CF. nextdeploy's architecture (single-pass esbuild, explicit shim, metadata-as-contract, Go-owned pipeline) is the correct long-term shape — but hitting parity for a real app like Pesastream is ~2–3 weeks of focused work, not a weekend. Do not cut over any production traffic until the Phase 1 + Phase 2 gates are green. + +--- + +## 10. Init, config, and build-flag concerns (added 2026-04-18) + +Surfaced while adding Cloudflare to the `init` flow (`cli/internal/initialcommand/init.go`, +`shared/config/template.go`) and wiring Turbopack/webpack builder detection +(`shared/nextcore/build_flags.go`). None of these block today's commits but each will bite +before the Phase 1 gate closes. + +### 10.1 `init` template is a dead end for Pesastream-shaped apps + +The new `cloudflareTemplate` in `shared/config/template.go` emits: + +- `CloudProvider.name: cloudflare` + `account_id` +- `serverless.provider: cloudflare` +- `serverless.cloudflare: { compatibility_date, compatibility_flags: [nodejs_compat_v2] }` +- commented stubs for `custom_domains`, `triggers.crons`, `bindings.r2` + +Missing — every item below is called out as required in §3/§4/§5 above, but `init` gives the +user zero scaffolding for any of it: + +- **Durable Object class declarations** (A2) — no prompt, no `durable_objects:` stub in yaml. + Worse: the yaml schema has `serverless.cloudflare.bindings.durable_objects`, but §5 proposes a + separate `nextdeploy.config.ts` as the canonical place to declare DO *source files* for the + shim's `_user_do_classes.mjs` generation. We have not picked one. Pick before users start + writing either form. +- **`scheduled()` / `queue()` handler wiring** (A1, A3) — yaml has a commented `triggers.crons` + stub but no syntax for pointing a cron at a route handler. The Pesastream reference uses + `/api/cron/payment-fanout` — nextdeploy has no way to express "cron `* * * * *` invokes this + path" in the current yaml. +- **Hyperdrive binding injection** (A6) — `bindings.hyperdrive` slot exists, but nothing in the + init flow documents that the shim is supposed to read `env.HYPERDRIVE.connectionString` and + push it onto `process.env` / `globalThis` before the first request. Users will declare the + binding, deploy, and fail at runtime with confusing `pg` connection errors. +- **Middleware** (A7) — no yaml block; detection is meant to be automatic but §4 says the + detector currently misses `src/middleware.ts`. Subdomain routing silently dies. + +**Concern:** the template compiles via `config.Load` and passes every static check, so users who +pick "Cloudflare (Workers + R2)" in `init` get a green yaml that is feature-incomplete for any +non-trivial app. We should either (a) keep the template minimal and put a `# See docs before +deploying a DO/queue/cron app` banner at the top, or (b) expand it with commented sections for +each of the items above so users see the shape of what's missing. + +### 10.2 Config contract: two provider fields for one decision + +`init` writes both `CloudProvider.name: cloudflare` AND `serverless.provider: cloudflare`. +Nothing in `shared/config` validates they agree. A user editing between providers can end up +with `CloudProvider.name: aws` + `serverless.provider: cloudflare` and either (a) silently pick +up AWS credentials for a CF deploy, or (b) fail opaquely at credential resolution. + +**Fix options:** + +- Make `serverless.provider` the single source of truth for dispatch; treat `CloudProvider.name` + as a hint for credential file lookup only, OR +- Add a `config.Validate()` pass that errors loudly if the two disagree. + +Second option is cheaper and preserves existing yaml. + +### 10.3 Turbopack / webpack builder injection not verified on the CF path + +`MaybeInjectWebpackFlag` runs unconditionally in `GenerateMetadata` for every target type, +appending `-- --webpack` to the `next build` command when Next ≥ 16 + `webpack:` key present + +no `turbopack:` key. + +Unknowns on the Cloudflare path: + +- `next build --webpack` produces the same `.next/standalone` layout as `next build + --turbopack`. *Assumed*, not verified. The CF adapter (`cloudflare_adapter.go`) consumes that + dir — if layouts diverge we'll bundle a broken worker silently. +- For a project that migrates to Turbopack (drops `webpack:`), the CF esbuild bundler still + needs A4's `--conditions=workerd` fix. The flag injector has no opinion on this; the CF + adapter has to own the workerd condition regardless. +- `build_flags.go` has zero unit tests. `scriptEndsWithNextBuild` uses prefix-match heuristics + that will silently fail on scripts like `next build && next-sitemap`, `NODE_OPTIONS='...' next + build`, or monorepo forms like `turbo run build --filter=web`. Users will see the old + Turbopack error with no indication that nextdeploy tried and gave up. + +**Minimum ask:** unit tests for `scriptEndsWithNextBuild` covering at least: + +- `next build` (trivial) +- `prisma generate && next build` +- `next build && next-sitemap` (should warn, not inject) +- `NODE_OPTIONS='--max-old-space-size=4096' next build` +- `turbo run build --filter=web` (monorepo — can't reason about, should warn) +- `bun run build` (already the wrapper — detection should look at the resolved script body) + +### 10.4 `nextdeploy build` is target-agnostic where it shouldn't be + +`GenerateMetadata` runs the same `next build` regardless of `cfg.Serverless.Provider`. The CF +path will need target-specific build-time decisions: + +- Force `output: 'standalone'` (CF adapter assumes it) +- Set `serverActions.bodySizeLimit` ≤ CF's 100MB request cap (hard fail if higher) +- Error if `experimental.turbo` is set but the CF worker shim isn't turbopack-compatible + +The build-flags layer should take `cfg.Serverless.Provider` as input and branch. Right now it +only branches on Next version. Same applies to metadata emission: §4's CF-specific metadata +extensions should only be computed when deploying to CF, to keep the AWS pipeline unchanged. + +### 10.5 End-to-end smoke: `init → build → deploy` has never run for Cloudflare + +Memory note `cloudflare_state.md` says the IaC + adapter + shim compile but have never been +exercised against a real CF account. With §10.1's init now pointing users at that path, the +risk shifts: a user can pick "Cloudflare" in `init`, hit `nextdeploy build` (works — it's just +`next build`), then `nextdeploy deploy` and land in the bug-iceberg §3 enumerates. + +**Ask:** gate the `init` Cloudflare option behind an explicit "experimental — see +CLOUDFLARE_PARITY.md" prompt until Phase 1 closes, OR run a minimum fixture deploy in CI (the §9 +test matrix) before advertising CF-as-a-target via `init`. diff --git a/CODE_QUALITY.md b/CODE_QUALITY.md new file mode 100644 index 0000000..e4945d1 --- /dev/null +++ b/CODE_QUALITY.md @@ -0,0 +1,88 @@ +# Code Quality Plan — Go + +> **Enforcement model:** these rules apply to **new code and code we touch**. We do not stop work to retrofit the whole tree. Every PR leaves the area it touched cleaner than it found it (boy-scout rule). Track regressions in PR review. + +## Size targets (soft caps) + +| Unit | Target | Hard ceiling | Notes | +|---|---|---|---| +| File | ≤ 300 LOC | 500 LOC | Split by responsibility, not by line count. AWS SDK call sites can push toward the ceiling — that's OK if the file is one cohesive job. | +| Function | ≤ 20 LOC | 40 LOC | The 20-line target is aspirational. SDK input-struct literals don't count against the budget but should be extracted to a builder if reused. | +| Struct | ≤ 20 fields | 30 fields | If you cross 20, ask: is this two structs glued together? | +| Interface | ≤ 5 methods | 8 methods | Bigger interfaces = weaker abstractions. Prefer many small interfaces (`io.Reader`-style). | + +When you touch a file >300 LOC: extract at least one cohesive unit out. Don't try to fix the whole file. + +## Decoupling rules + +1. **No SDK clients constructed in business logic.** `lambda.NewFromConfig(...)` belongs in a constructor or a `clients` struct on the provider — never inside a `Deploy*` or `Reconcile*` method. New code that does this gets bounced. +2. **Define small interfaces at the consumer side.** If `DeployCompute` needs only `GetFunction` and `UpdateFunctionCode`, declare a 2-method interface in the same file. Mock that. Don't depend on the full `*lambda.Client`. +3. **Orchestration layers don't import provider internals.** `deploy.go` orchestrates via the `Provider` interface only. If it needs an AWS-specific concept, the interface is wrong. +4. **One package = one responsibility.** When `serverless/` becomes painful (it already is), split into subpackages: `serverless/aws/lambda`, `serverless/aws/cloudfront`, etc. Don't pre-split — split when adding the next feature in that area. + +## Error handling + +1. **Errors are values — handle them, don't just return them.** Wrap with `fmt.Errorf("doing X for %s: %w", name, err)`. Future-you will thank you. +2. **Never match errors by `strings.Contains(err.Error(), ...)`** — use `errors.As` against typed SDK errors. The only exceptions: AWS errors that aren't exposed as types (document why with a comment). +3. **No silent `_ = doThing()`** — if you don't care about the error, write a one-line comment explaining why ("non-fatal: cleanup, see X"). +4. **Distinguish fatal vs degraded.** A failed CloudWatch alarm is degraded (warn + continue). A failed Lambda update is fatal (return). Be explicit, not accidental. +5. **No `panic` outside `main` and tests.** Use `log.Fatal` in `main`, return errors elsewhere. + +## Naming & structure + +1. Package names: short, lowercase, no underscores, no `util`/`common`/`helpers`. +2. File names mirror the dominant type or responsibility (`lambda_url.go`, not `helpers.go`). +3. Constructors: `NewX` returns `*X` or `(X, error)`. No `MakeX`. +4. Public API needs a doc comment that starts with the identifier name (`golint` rule). +5. Acronyms stay uppercase: `URL`, `ID`, `ARN`, `OAC`, `IAM`. Not `Url`, `Id`. + +## Concurrency + +1. Channels for orchestration, mutexes for protecting state. Don't mix. +2. Always pass `context.Context` as the first parameter. Always honor cancellation. +3. No goroutines without a clear lifecycle — every goroutine needs a way to stop. + +## Testing + +1. New business logic ships with at least one test. Bug fixes ship with a regression test. +2. Tests depend on interfaces, not concrete SDK types. If you can't mock it, the design is wrong. +3. Prefer table-driven tests for anything with branches. +4. Integration tests live behind a build tag (`//go:build integration`). + +## Comments & docs + +1. Comments explain *why*, not *what*. The code shows what. +2. Every exported symbol has a doc comment. Unexported only when non-obvious. +3. `TODO` / `FIXME` must include a name or issue link. `// FIXME` alone is forbidden. +4. No commented-out code in commits. Delete it; git remembers. + +## Forbidden patterns + +- `interface{}` / `any` in public APIs (use generics or a real type). +- `init()` for anything other than registering with a registry. +- Global mutable state. +- `panic` for control flow. +- `reflect` outside encoding/decoding glue. +- Hand-rolled JSON parsing of secrets / config (use `encoding/json` + tagged structs). +- Hardcoded ARNs, account IDs, or region-specific resources without a config override. + +## What "boy-scout" means in practice + +When you open a file to fix bug X: + +1. Fix bug X. +2. If the file is >300 LOC, extract one cohesive function or section into a new file. +3. If a function you touched is >40 LOC, split it. +4. If you spot a `strings.Contains(err.Error(), ...)` near your change, swap it for `errors.As`. +5. Stop. Don't refactor things you didn't touch — that goes in its own PR. + +## Review checklist (paste into PR descriptions) + +- [ ] No new file >300 LOC +- [ ] No new function >40 LOC +- [ ] No new SDK clients constructed inside business logic +- [ ] No new `strings.Contains(err.Error(), ...)` +- [ ] No new silent `_ =` on errors +- [ ] All exported symbols documented +- [ ] Boy-scout: at least one cleanup in a touched file >300 LOC +- [ ] Tests added or updated diff --git a/Makefile b/Makefile deleted file mode 100644 index 4dca1d9..0000000 --- a/Makefile +++ /dev/null @@ -1,220 +0,0 @@ -# NextDeploy Build Makefile -.PHONY: help build build-cli build-daemon build-all clean test lint security-scan cross-build install dev mage-install - -# Build variables - VERSION comes from the current git tag (set automatically by CI or manually) -VERSION ?= $(shell git describe --tags --exact-match 2>/dev/null || git describe --tags 2>/dev/null || echo "dev") -COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") -BUILD_DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) - -# Go build flags -export PATH := $(PATH):$(shell go env GOPATH)/bin -GOFLAGS := -trimpath -LDFLAGS := -s -w \ - -X github.com/Golangcodes/nextdeploy/shared.Version=$(VERSION) \ - -X main.commit=$(COMMIT) \ - -X main.date=$(BUILD_DATE) - -# Directories -BIN_DIR := bin -DIST_DIR := dist - -# Platform targets for CLI (multiplatform) -CLI_PLATFORMS := \ - linux/amd64 \ - linux/arm64 \ - darwin/amd64 \ - darwin/arm64 \ - windows/amd64 - -# Platform targets for Daemon (Linux only) -DAEMON_PLATFORMS := \ - linux/amd64 \ - linux/arm64 - -# Default target -help: ## Display this help message - @echo "NextDeploy Build System" - @echo "=======================" - @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) - -# Clean build artifacts -clean: ## Clean build artifacts - @echo "Cleaning build artifacts..." - @rm -rf $(BIN_DIR)/* $(DIST_DIR)/* - @echo "Clean complete" - -# Install dependencies -deps: ## Install build dependencies - @echo "Installing dependencies..." - @go mod download - @go mod verify - @echo "Dependencies installed" - -# Run tests -test: ## Run tests with coverage - @echo "Running tests..." - @go test -race -coverprofile=coverage.out -covermode=atomic -v ./... - @go tool cover -html=coverage.out -o coverage.html - @echo "Tests complete - coverage report: coverage.html" - -# Run linting -lint: ## Run linting and formatting checks - @echo "Running linting..." - @command -v golangci-lint >/dev/null 2>&1 || { echo "Installing golangci-lint..."; go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; } - @golangci-lint run --timeout=5m - @echo "Checking formatting..." - @if [ "$$(gofmt -s -l . | wc -l)" -gt 0 ]; then echo "Files need formatting:"; gofmt -s -l .; exit 1; fi - @echo "Linting complete" - -# Security scanning -security-scan: ## Run security scans - @echo "Running security scan..." - @command -v gosec >/dev/null 2>&1 || { echo "Installing gosec..."; go install github.com/securecodewarrior/gosec/v2/cmd/gosec@latest; } - @gosec ./... - @command -v govulncheck >/dev/null 2>&1 || { echo "Installing govulncheck..."; go install golang.org/x/vuln/cmd/govulncheck@latest; } - @govulncheck ./... - @echo "Security scan complete" - -# Build single CLI binary (native platform) -build-cli: ## Build CLI binary for current platform - @echo "Building CLI for current platform..." - @mkdir -p $(BIN_DIR) - @CGO_ENABLED=0 go build $(GOFLAGS) -ldflags="$(LDFLAGS)" -o $(BIN_DIR)/nextdeploy ./cli - @echo "CLI built: $(BIN_DIR)/nextdeploy" - -build-cli-dev: ## Build CLI binary directly into ~/.nextdeploy/bin for local development - @echo "Building CLI for local dev environment..." - @mkdir -p $(HOME)/.nextdeploy/bin - @go build $(GOFLAGS) -ldflags="$(LDFLAGS)" -o $(HOME)/.nextdeploy/bin/nextdeploy ./cli - @if ! grep -q '$(HOME)/.nextdeploy/bin' $(HOME)/.bashrc 2>/dev/null; then \ - echo 'export PATH="$$HOME/.nextdeploy/bin:$$PATH"' >> $(HOME)/.bashrc; \ - echo "Added ~/.nextdeploy/bin to your ~/.bashrc. Please run 'source ~/.bashrc' or restart your terminal."; \ - fi - @echo "Dev CLI built: $(HOME)/.nextdeploy/bin/nextdeploy" - -# Build single daemon binary (Linux only) -build-daemon: ## Build daemon binary for current platform (Linux) - @echo "Building daemon for current platform..." - @mkdir -p $(BIN_DIR) - @if [ "$$(go env GOOS)" != "linux" ]; then \ - echo "Daemon only supports Linux - building for linux/amd64"; \ - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GOFLAGS) -ldflags="$(LDFLAGS)" -o $(BIN_DIR)/nextdeployd ./daemon/cmd/nextdeployd; \ - else \ - CGO_ENABLED=0 go build $(GOFLAGS) -ldflags="$(LDFLAGS)" -o $(BIN_DIR)/nextdeployd ./daemon/cmd/nextdeployd; \ - fi - @echo "Daemon built: $(BIN_DIR)/nextdeployd" - -build-daemon-dev: ## Build daemon directly into ~/.nextdeploy/bin for local development - @echo "Building daemon for local dev environment..." - @mkdir -p $(HOME)/.nextdeploy/bin - @go build $(GOFLAGS) -ldflags="$(LDFLAGS)" -o $(HOME)/.nextdeploy/bin/nextdeployd ./daemon/cmd/nextdeployd - @echo "Dev Daemon built: $(HOME)/.nextdeploy/bin/nextdeployd" - -# Build both binaries -build: build-cli build-daemon ## Build both CLI and daemon - -# Cross-compile CLI for all platforms -cross-build-cli: ## Cross-compile CLI for all supported platforms - @echo "Cross-compiling CLI for all platforms..." - @mkdir -p $(DIST_DIR) - @for platform in $(CLI_PLATFORMS); do \ - GOOS=$$(echo $$platform | cut -d/ -f1); \ - GOARCH=$$(echo $$platform | cut -d/ -f2); \ - OUTPUT_NAME="nextdeploy-$$GOOS-$$GOARCH"; \ - if [ "$$GOOS" = "windows" ]; then OUTPUT_NAME="$$OUTPUT_NAME.exe"; fi; \ - echo "Building $$OUTPUT_NAME..."; \ - CGO_ENABLED=0 GOOS=$$GOOS GOARCH=$$GOARCH go build $(GOFLAGS) \ - -ldflags="$(LDFLAGS)" \ - -o $(DIST_DIR)/$$OUTPUT_NAME ./cli; \ - if command -v sha256sum >/dev/null; then \ - cd $(DIST_DIR) && sha256sum $$OUTPUT_NAME > $$OUTPUT_NAME.sha256 && cd ..; \ - fi; \ - done - @echo "CLI cross-compilation complete" - -# Cross-compile daemon for Linux platforms -cross-build-daemon: ## Cross-compile daemon for Linux platforms - @echo "Cross-compiling daemon for Linux platforms..." - @mkdir -p $(DIST_DIR) - @for platform in $(DAEMON_PLATFORMS); do \ - GOOS=$$(echo $$platform | cut -d/ -f1); \ - GOARCH=$$(echo $$platform | cut -d/ -f2); \ - OUTPUT_NAME="nextdeployd-$$GOOS-$$GOARCH"; \ - echo "Building $$OUTPUT_NAME..."; \ - CGO_ENABLED=0 GOOS=$$GOOS GOARCH=$$GOARCH go build $(GOFLAGS) \ - -ldflags="$(LDFLAGS)" \ - -o $(DIST_DIR)/$$OUTPUT_NAME ./daemon/cmd/nextdeployd; \ - if command -v sha256sum >/dev/null; then \ - cd $(DIST_DIR) && sha256sum $$OUTPUT_NAME > $$OUTPUT_NAME.sha256 && cd ..; \ - fi; \ - done - @echo "Daemon cross-compilation complete" - -# Cross-compile everything -cross-build: cross-build-cli cross-build-daemon ## Cross-compile for all supported platforms - -# Build everything (current + cross-platform) -build-all: build cross-build ## Build everything (local + cross-platform) - -# Install binaries to system -install: build ## Install binaries to system PATH - @echo "Installing binaries to system..." - @sudo cp $(BIN_DIR)/nextdeploy /usr/local/bin/ - @sudo cp $(BIN_DIR)/nextdeployd /usr/local/bin/ - @sudo chmod +x /usr/local/bin/nextdeploy /usr/local/bin/nextdeployd - @echo "Binaries installed to /usr/local/bin/" - -# Development workflow -dev-cli: ## Watch CLI code and rebuild binary on changes - @command -v air >/dev/null 2>&1 || { echo "Installing air..."; go install github.com/air-verse/air@latest; } - @air -c .air.cli.toml - -dev-daemon: ## Watch daemon code, rebuild and restart on changes - @command -v air >/dev/null 2>&1 || { echo "Installing air..."; go install github.com/air-verse/air@latest; } - @air -c .air.daemon.toml - -dev-check: deps lint test security-scan ## Run all development checks - -# Release preparation -release-prep: clean dev-check build-all ## Prepare for release - -# Show build info -info: ## Show build information - @echo "Build Information" - @echo "=================" - @echo "Version: $(VERSION)" - @echo "Commit: $(COMMIT)" - @echo "Build Date: $(BUILD_DATE)" - @echo "Builder: $(BUILDER)" - @echo "Go Version: $$(go version)" - @echo "GOOS: $$(go env GOOS)" - @echo "GOARCH: $$(go env GOARCH)" - -# Docker build -docker-build: ## Build Docker image - @echo "Building Docker image..." - @docker build -t nextdeploy:$(VERSION) . - @docker build -t nextdeploy:latest . - @echo "Docker image built" - -# Docker multi-platform build -docker-buildx: ## Build multi-platform Docker image - @echo "Building multi-platform Docker image..." - @docker buildx build --platform linux/amd64,linux/arm64 -t nextdeploy:$(VERSION) -t nextdeploy:latest . - @echo "Multi-platform Docker image built" - -# List all available targets -list: ## List all make targets - @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' - -# Mage bootstrap -mage-install: ## Install mage build tool to /usr/local/bin - @echo "Installing mage..." - @go install github.com/magefile/mage@latest - @sudo cp "$(shell go env GOPATH)/bin/mage" /usr/local/bin/mage - @echo "mage installed: $$(mage --version)" - -# Dev workflow -dev: build-cli ## Build the CLI and run it (alias for quick local iteration) - @echo "Running nextdeploy (dev build)..." - @./$(BIN_DIR)/nextdeploy \ No newline at end of file diff --git a/NEXTCOMPILE_ROADMAP.md b/NEXTCOMPILE_ROADMAP.md new file mode 100644 index 0000000..262b4ab --- /dev/null +++ b/NEXTCOMPILE_ROADMAP.md @@ -0,0 +1,668 @@ +# nextcompile — remaining work + +Pick up here when we resume. This file is the authoritative punch list so +we don't have to re-derive the plan from conversation history. + +The commitment is: **our own runtime, composed from public React APIs. +No wrapping of Next's internal `app-render` module.** What we write is a +thin composition + dispatch layer on top of React, not a rendering engine. + +--- + +## Strategic framing — is this worth developer time? + +**Not yet.** We'd be a worse OpenNext with extra steps today. OpenNext +already renders real App Router apps on Cloudflare. Our runtime doesn't, +and we have zero production dogfooding. If we only ship runtime parity, +no one picks us over OpenNext + wrangler. + +### The one thing that would change that + +**Auto-provisioned infrastructure from code.** Not RSC. Not multi-provider. +Not any single runtime feature. The moment that makes someone post about +this tool: + +> I wrote `process.env.DATABASE_URL` in my Next app. I ran +> `nextdeploy ship`. It noticed the reference, asked me once for the +> value, created the binding, deployed the Worker, and the app worked. +> No wrangler.toml. No AWS console. No terraform. + +Extended cases: + +- `fetch("https://my-kv-host/...")` → "looks like you're hitting a KV; + bind one?" +- `revalidateTag("posts")` → auto-provisions the revalidation queue. +- `` → adds the remote + pattern, enables CF Images binding. +- `import { Ai } from "cloudflare:ai"` → detects, prompts for binding + name. +- `fetch("https://*.r2.cloudflarestorage.com/...")` → suggests R2 + binding and prompts for bucket name. + +### Why this is the killer feature + +- Vercel has zero-config because they built the runtime AND the + platform. You can't get that elsewhere. +- OpenNext + wrangler has config the developer writes by hand in + `wrangler.toml`. +- Pulumi + Terraform has config the developer writes by hand. +- **Nobody reads your code and provisions the infrastructure it + actually needs.** + +The gap isn't in rendering. It's in the human step between "I wrote +the code" and "the infrastructure exists." + +### Concrete work to ship this + +Scope: ~400 Go LOC + a clean interactive prompt UX. Two weeks of +focused work. Ships **before** the RSC runtime phases because it's the +reason someone would pick nextdeploy over existing tools. RSC runtime +becomes a Week 3+ priority once there are users asking for it. + +1. **Finish `deriveBindings`.** Move + `shared/nextcompile/compiler.go:deriveBindingsStub` from "echo every + env ref" to a real extractor that recognizes: + - `process.env.X` references → secret binding hints + - `fetch(...)` URL patterns matching known platform services → + binding suggestions: + - `*.r2.cloudflarestorage.com` → R2 binding + - `api.cloudflare.com/client/v4/accounts/*/d1/*` → D1 binding + - `api.cloudflare.com/client/v4/accounts/*/storage/kv/*` → KV binding + - Well-known imports (`cloudflare:ai`, `@cloudflare/workers-types`, + hyperdrive references) → specific binding suggestions + - De-duplicate against bindings already declared in + `cfg.Serverless.Cloudflare.Bindings` +2. **First-deploy interactive flow.** New module + `cli/cmd/ship_provision.go`. When `CompiledBundle.SuggestedBindings` + has entries not already declared in cfg: + - Print the hint + reason + detected source file path + - Prompt: "Create as ? [Y/n/skip/edit]" + - On Y: collect the value (for secrets) or the identifier (for + KV/R2/D1/etc.) + - Save the accepted decisions back into `nextdeploy.yml` under a + generated `# auto-provisioned:` section so re-runs skip the prompt +3. **Automatic provisioning.** Extend `CloudflareProvider.Plan` + + `Provision` to consume `SuggestedBindings`: + - Plan phase surfaces proposed creations alongside user-declared + resources in the same drift table + - Provision phase creates them via existing helpers + (`cloudflare_resources.go`: `ensureHyperdrive` / `ensureQueue` / + `ensureVectorize` / `ensureAIGateway`) — secrets flow through the + existing `UpdateSecrets` path + - New helpers needed: `ensureKV`, `ensureR2Bucket` (R2 bucket exists + via `ensureR2BucketExists` at `cloudflare.go:666`), `ensureD1` +4. **Idempotent re-runs.** Once a hint is accepted and written into + cfg, subsequent scans skip it. The generated section in + `nextdeploy.yml` is the declarative record of what was provisioned. + Re-running `ship` against an unchanged codebase does nothing new. +5. **Graceful escape hatches.** + - `--no-auto-provision` — fail loudly on any missing binding instead + of prompting. For teams that want manual control. + - `--yes` / `CI=true` — auto-approve safe hints (secrets-only) and + hard-fail on ambiguous ones (which binding matches this fetch?). + Used by CI pipelines. + - `nextdeploy provision` — run just the prompt/create loop without + a deploy. Useful for first-time setup. + +### Why this goes before runtime phases + +Runtime correctness only matters to people who are already using the +tool. The "aha" moment that makes someone install it in the first +place is the auto-provisioning. RSC rendering is the reason they keep +using it. + +**Priority order flips to:** + +1. Ship auto-provisioning against a real trivial Next app (API routes + + one env var + one KV access). Smoke-test the flow end-to-end. +2. Dogfood publicly — get 5–10 devs using it. Collect the real + failures. +3. Then fix what breaks in Phase 1+ of the runtime roadmap based on + real app failures, not hypothetical ones. + +This is the actual hill. Everything in the Phased plan below is secondary. + +### What makes this hard (the honest risks) + +1. **False positives.** Suggesting a binding for every `fetch()` call + produces noise. The extractor must be conservative; wrong suggestions + destroy trust in the auto-provisioning faster than no suggestions. +2. **Ambiguous bindings.** `fetch("https://api.example.com/")` could be + a third-party API or a custom service binding. Default to no hint; + prompt only when patterns match known platform services. +3. **Interactive in CI.** Prompting in CI is a deploy-stopper. Must + detect non-TTY early and either auto-approve safe hints or fail with + a clear message pointing to `--yes`. +4. **Migration from existing wrangler.toml users.** Some devs already + have bindings declared. The first-deploy flow must read existing + declarations and not double-provision. Idempotence is load-bearing. +5. **Secrets UX.** Prompting for a database URL at the terminal is fine + for first-run. For rotation, operators use `nextdeploy secrets set`. + Don't re-prompt on every deploy — detect that a secret *binding* + exists even if the value is only in Secrets Manager / Worker secrets. + +--- + +## Current state + +Built and tested (all green, `go build ./...` + `go vet ./...` clean). + +- **Go side** (`shared/nextcompile/`, ~3200 LOC) + - 13-phase `Compile()` pipeline: DetectVersions → ScanCompiledServer → + DetectServerActions → DeriveBindings (stub) → ElideDeadRoutes (stub) + → ensure OutDir → EmitManifest → EmitDispatchTable → ExtractRuntime + → VendorRSC → EmitActionManifest → EmitWorkerEntry → content hash. + - Reproducible, content-addressable output. Parallel scan via + errgroup. Version-aware runtime variant picker. Honest vendoring + step that copies `react-server-dom-webpack/server.edge` from + node_modules into the bundle. +- **JS runtime** (`shared/nextcompile/runtime_src/`, ~1400 LOC) + - `dispatcher.mjs` — fetch handler with proxy/middleware → static/SSG + → dynamic routes dispatch. + - `context.mjs` — AsyncLocalStorage with async `cookies()`, `headers()`, + `draftMode()`, `after()`. + - `rsc.mjs` — RSC renderer skeleton; vendored Flight encoder loads at + request time or returns 501. + - `actions.mjs` — Server Actions POST dispatch with CSRF, context, + JSON-fallback response. + - `cache.mjs` — `revalidatePath` / `revalidateTag` / `unstable_cache` + with in-memory + KV tiers. + - `image.mjs` — `/_next/image` with remote-pattern validation + + Cloudflare Images binding. + - `serve.mjs`, `route_match.mjs`, `errors.mjs` — supporting modules. + - `next_shims/{cache,headers,server}.mjs` — esbuild-aliased shims so + user imports from `next/cache`, `next/headers`, `next/server` resolve + into our runtime without source modification. +- **Adapter integration** (`cli/internal/serverless/`) + - `cloudflare_adapter.go` — runs `nextcompile.Compile()` then + `esbuild` on the generated entry. Prints a capability report. + - `nextcompile_bridge.go` — `nextcore.NextCorePayload` → + `nextcompile.Payload`. + - `smoke.go` — post-deploy probe with retries, warn-only by default. +- **CLI** + - `nextdeploy ship` wired through the new pipeline via `serverless.Deploy`. + - `nextdeploy explain [--code]` for all 16 commands. + +Capability state of a deployed Worker: + +| Feature | Works | Notes | +|---|---|---| +| API routes (App Router + Pages Router) | yes | inside ALS context | +| Static / `/public` / `/_next/static` | yes | from R2 | +| Dynamic routes with params | yes | specificity-ordered | +| Middleware + `proxy.ts` | partial | passthrough OK; `NextResponse.rewrite/redirect` semantics not honored | +| async `cookies()` / `headers()` / `draftMode()` | yes | real ALS | +| Server Actions POST + invoke + result | yes | CSRF + body parse + context; JSON fallback response (not Flight) | +| RSC pages with no client components | partial | renders, minimal shell, no layouts yet | +| RSC pages with `"use client"` | **no** | Phase 1 target | +| Suspense + streaming | **no** | Phase 3 target | +| `loading.js` / `error.js` / `not-found.js` | **no** | Phase 4 target | +| `revalidatePath` / `revalidateTag` | yes | local + KV tier; no cross-worker queue fan-out | +| `/_next/image` via CF Images | yes | binding optional, passthrough fallback | +| `after()` | yes | via ctx.waitUntil | +| PPR | detected → **501** | Phase 6 target | + +--- + +## Phased plan + +Every phase is independent and improves correctness on its own. Numbers are +solo-engineer estimates; halve with two focused engineers. + +### Phase 1 — Client-reference-manifest threading (3–5 days) + +**Unlocks:** RSC pages with `"use client"` components finally render +without throwing in the Flight encoder. + +**What already exists** +- `shared/nextcompile/scanner.go:attachClientManifests` already populates + `ModuleRef.ClientManifestPath` for each page that has a sibling + `page_client-reference-manifest.json`. +- `shared/nextcompile/dispatch.go:renderEntryFields` already emits + `loadClientManifest: () => import(...)` per entry. +- `shared/nextcompile/runtime_src/rsc.mjs` already has the slot where + `clientManifest.clientModules` becomes `bundlerConfig` in the + `renderToReadableStream` call. + +**What's missing** + +1. **Go-side copy step.** The Next-emitted client manifests live under + `.next/server/app/**/page_client-reference-manifest.json`. They stay + in place as part of the standalone tree esbuild traverses, so the + `loadClientManifest` `import()` finds them — but we should verify this + by adding an integration assertion. If esbuild's bundling doesn't + follow JSON imports with the dynamic-import-attribute form, we need + to either: + - pass `--loader:.json=json` (already set) plus ensure the dynamic + import uses the attribute form (done), OR + - copy the manifests into `_nextdeploy/client-manifests/` and + reference those paths instead. +2. **Bundler-config shape translation.** React's `bundlerConfig` is a + map keyed on `"file:export"` strings with values + `{id, name, chunks, async}`. Next's manifest sometimes wraps this + under `.clientModules` or `.ssrModuleMapping`. The rsc.mjs currently + picks one or the other defensively — needs a real test against a + real Next 14 + 15 manifest pair. +3. **Vendor `react-server-dom-webpack/client`** (not just `/server.edge`). + Clients decode the Flight stream via + `createFromReadableStream(stream, { moduleLoader, ... })`. This is a + different vendored bundle file. Extend `vendor.go` to copy it too. +4. **Emit chunk script tags in the HTML shell.** When a page has client + components, the emitted HTML must include ``); + + return new ReadableStream({ + async start(controller) { + controller.enqueue(shellPrefix); + const reader = flightStream.getReader(); + while (true) { + const { value, done } = await reader.read(); + if (done) break; + controller.enqueue(value); + } + controller.enqueue(shellSuffix); + controller.close(); + }, + }); +} + +function vendorMissingResponse() { + const body = + "nextcompile: React Server Components runtime not vendored.\n\n" + + "This deployment contains App Router pages that use Server Components,\n" + + "but the build did not include the vendored React server bundle at:\n" + + " _nextdeploy/runtime/vendor/react-server-dom-webpack/server.edge.mjs\n\n" + + "Fix: invoke nextcompile's adapter build step with RSC vendoring enabled,\n" + + "or ship the matching react-server-dom-webpack bundle for the detected\n" + + "React version. See runtime/vendor/README.md for the exact steps.\n\n" + + "Load error: " + String(vendoredLoadError); + return new Response(body, { + status: 501, + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "no-store", + }, + }); +} + +function pprNotImplemented() { + const body = + "nextcompile: Partial Prerendering (PPR) is not yet implemented.\n\n" + + "This page is marked PPR in the compiled output, which means Next\n" + + "expects the server to stream a static shell with dynamic holes\n" + + "resolved inline. nextcompile's RSC renderer handles fully-dynamic\n" + + "pages today; PPR support is tracked as a follow-up milestone.\n\n" + + "Workaround: remove `experimental.ppr` or `experimental_ppr` from\n" + + "this route's config and redeploy."; + return new Response(body, { + status: 501, + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "no-store", + }, + }); +} diff --git a/shared/nextcompile/runtime_src/serve.mjs b/shared/nextcompile/runtime_src/serve.mjs new file mode 100644 index 0000000..ff58b5f --- /dev/null +++ b/shared/nextcompile/runtime_src/serve.mjs @@ -0,0 +1,74 @@ +// Static + SSG asset serving from R2. +// +// Conventions upheld by the nextdeploy packager when uploading to R2: +// - /_next/static//... → keyed under "_next/static//..." +// - /public/foo.png → keyed under "foo.png" (Next serves +// /public content at the root) +// - SSG/ISR HTML → keyed under the file path from the +// manifest (e.g. "index.html", +// "blog.html", "news.html") +// +// env.ASSETS is the R2 binding for the public bundle; it's declared +// automatically by cloudflare_bindings.go when compute routes exist. + +/** + * Serve a /_next/static or /public path from R2. Returns null on miss so + * the dispatcher can fall through to a 404 or compute route. + */ +export async function serveStaticFromR2(env, pathname) { + if (!env.ASSETS) return null; + + const key = r2KeyForStatic(pathname); + if (!key) return null; + + const obj = await env.ASSETS.get(key); + if (!obj) return null; + + const headers = new Headers(); + obj.writeHttpMetadata?.(headers); + if (obj.httpEtag) headers.set("etag", obj.httpEtag); + if (!headers.has("cache-control")) { + headers.set( + "cache-control", + pathname.startsWith("/_next/static/") + ? "public, max-age=31536000, immutable" + : "public, max-age=300", + ); + } + return new Response(obj.body, { headers }); +} + +/** + * Serve a pre-rendered HTML file (SSG or ISR) from R2. The dispatcher + * resolves the R2 key from manifest.routes.ssg/isr and passes it here. + */ +export async function serveSSGFromR2(env, htmlKey) { + if (!env.ASSETS) return null; + // Manifest entries are typically "/foo.html" — normalize to the bare key. + const key = htmlKey.replace(/^\/+/, ""); + const obj = await env.ASSETS.get(key); + if (!obj) return null; + + const headers = new Headers(); + obj.writeHttpMetadata?.(headers); + if (!headers.has("content-type")) { + headers.set("content-type", "text/html; charset=utf-8"); + } + if (!headers.has("cache-control")) { + headers.set("cache-control", "public, max-age=0, must-revalidate"); + } + return new Response(obj.body, { headers }); +} + +function r2KeyForStatic(pathname) { + if (pathname.startsWith("/_next/static/")) { + return pathname.slice(1); // drop leading slash + } + if (pathname.startsWith("/public/")) { + return pathname.slice("/public/".length); + } + // Next also serves public/* at root — these only reach here via a + // caller that hinted "this is a public asset". Keep the contract tight: + // return null so we don't accidentally 200 on non-static paths. + return null; +} diff --git a/shared/nextcompile/runtime_test.go b/shared/nextcompile/runtime_test.go new file mode 100644 index 0000000..f1731f5 --- /dev/null +++ b/shared/nextcompile/runtime_test.go @@ -0,0 +1,91 @@ +package nextcompile + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRuntimeSourceFiles_IncludesMVP(t *testing.T) { + files, err := RuntimeSourceFiles() + if err != nil { + t.Fatal(err) + } + // MVP runtime must include these — they're what the dispatcher imports. + required := []string{ + "runtime_src/dispatcher.mjs", + "runtime_src/serve.mjs", + "runtime_src/route_match.mjs", + "runtime_src/errors.mjs", + "runtime_src/context.mjs", + "runtime_src/rsc.mjs", + "runtime_src/actions.mjs", + "runtime_src/cache.mjs", + "runtime_src/image.mjs", + "runtime_src/next_shims/cache.mjs", + "runtime_src/next_shims/headers.mjs", + "runtime_src/next_shims/server.mjs", + } + for _, r := range required { + if !containsPath(files, r) { + t.Errorf("embedded runtime missing %s\nfound: %v", r, files) + } + } +} + +func TestExtractRuntime_WritesFiles(t *testing.T) { + dir := t.TempDir() + written, err := ExtractRuntime(dir) + if err != nil { + t.Fatalf("ExtractRuntime: %v", err) + } + if len(written) == 0 { + t.Fatal("no files written") + } + + // dispatcher.mjs must land at /_nextdeploy/runtime/dispatcher.mjs + // and must parse as ESM (start with comment or import / export). + dispatcher := filepath.Join(dir, "_nextdeploy", "runtime", "dispatcher.mjs") + data, err := os.ReadFile(dispatcher) // #nosec G304 + if err != nil { + t.Fatalf("dispatcher.mjs not written: %v", err) + } + if !strings.Contains(string(data), "export async function dispatch") { + t.Errorf("dispatcher.mjs contents unexpected:\n%.400s", data) + } + + // serve.mjs, route_match.mjs, errors.mjs, context.mjs, rsc.mjs must all be there. + for _, name := range []string{"serve.mjs", "route_match.mjs", "errors.mjs", "context.mjs", "rsc.mjs", "actions.mjs", "cache.mjs", "image.mjs"} { + p := filepath.Join(dir, "_nextdeploy", "runtime", name) + if _, err := os.Stat(p); err != nil { + t.Errorf("missing %s: %v", name, err) + } + } + + // vendor/README.md must be there (subdirectory walk works). + readme := filepath.Join(dir, "_nextdeploy", "runtime", "vendor", "README.md") + if _, err := os.Stat(readme); err != nil { + t.Errorf("vendor/README.md missing: %v", err) + } +} + +func TestExtractRuntime_Idempotent(t *testing.T) { + dir := t.TempDir() + if _, err := ExtractRuntime(dir); err != nil { + t.Fatal(err) + } + // Second run must not error and must produce the same file contents. + if _, err := ExtractRuntime(dir); err != nil { + t.Fatalf("second ExtractRuntime: %v", err) + } +} + +func containsPath(files []string, want string) bool { + for _, f := range files { + if f == want { + return true + } + } + return false +} diff --git a/shared/nextcompile/scanner.go b/shared/nextcompile/scanner.go new file mode 100644 index 0000000..06bc493 --- /dev/null +++ b/shared/nextcompile/scanner.go @@ -0,0 +1,469 @@ +package nextcompile + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "golang.org/x/sync/errgroup" +) + +// scanConcurrency caps parallel file reads. Standalone trees for large apps +// can contain 50k+ files; uncapped errgroup Go routines thrash open-file +// limits on macOS (default 256). 32 is empirically a sweet spot across +// common dev hardware. +const scanConcurrency = 32 + +// Compiled-path prefixes that classify where a module lives in Next's +// standalone output. `app/` for App Router, `pages/` for Pages Router. +const ( + appRouterPrefix = "app/" + pagesRouterPrefix = "pages/" +) + +// ScanCompiledServer walks the server subtree of a Next standalone build +// in parallel and returns a ModuleRef per compiled route/handler/middleware. +// Non-server assets (client chunks, static files) are skipped — those flow +// to the CDN via packaging.S3Assets, not into the Worker bundle. +// +// The caller's Payload supplies route classification; the scanner's job is +// to attach compiled file paths to each classified route and extract the +// static-analysis facts (env refs, fetch targets, RSC markers) that later +// phases consume. +func ScanCompiledServer(ctx context.Context, standaloneDir string, payload Payload) ([]ModuleRef, error) { + serverDir := filepath.Join(standaloneDir, payloadDistDir(payload), "server") + if _, err := os.Stat(serverDir); err != nil { + return nil, fmt.Errorf("server dir not found at %s: %w", serverDir, err) + } + + paths, err := collectCompiledFiles(serverDir) + if err != nil { + return nil, err + } + + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(scanConcurrency) + + // classifyRoot is the parent of `server/` — i.e. the .next directory. + // Paths relative to classifyRoot start with "server/..." which is what + // routePathFromCompiled / kindFromCompiled expect. + classifyRoot := filepath.Dir(serverDir) + + refs := make([]ModuleRef, len(paths)) + for i, p := range paths { + g.Go(func() error { + if err := gctx.Err(); err != nil { + return err + } + ref, err := analyzeModule(standaloneDir, classifyRoot, p) + if err != nil { + return fmt.Errorf("analyze %s: %w", p, err) + } + refs[i] = ref + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + + // Tag each ref against the route classification. This is the bit that + // distinguishes an /api/users handler from a /dashboard page — the + // compiled path alone is ambiguous (both land under server/app/...). + refs = classifyRefs(refs, payload) + + // Second pass: attach client reference manifests + layout chains to + // page refs. Cheap — each is a stat(). Done after classification so we + // only walk up the tree for confirmed page kinds. + refs = attachClientManifests(refs, standaloneDir, classifyRoot) + refs = attachLayoutChains(refs, standaloneDir, classifyRoot) + + sort.Slice(refs, func(i, j int) bool { + return refs[i].RoutePath < refs[j].RoutePath + }) + return refs, nil +} + +// attachClientManifests looks for the Next-emitted +// `page_client-reference-manifest.json` sibling next to each page.js and +// attaches its relative path to the ModuleRef. Next 14+ emits one per RSC +// page; its contents are the bundlerConfig the Flight encoder needs to +// serialize "use client" boundaries. +func attachClientManifests(refs []ModuleRef, standaloneDir, classifyRoot string) []ModuleRef { + for i := range refs { + r := &refs[i] + if r.Kind != RouteKindPage { + continue + } + // Sibling convention: for ...//page.js, look for + // ...//page_client-reference-manifest.json. + abs := filepath.Join(standaloneDir, r.CompiledPath) + dir := filepath.Dir(abs) + base := filepath.Base(abs) + stem := strings.TrimSuffix(base, filepath.Ext(base)) + candidate := filepath.Join(dir, stem+"_client-reference-manifest.json") + if _, err := os.Stat(candidate); err != nil { + continue + } + rel, err := filepath.Rel(standaloneDir, candidate) + if err != nil { + continue + } + r.ClientManifestPath = filepath.ToSlash(rel) + } + return refs +} + +// attachLayoutChains walks the App Router tree for each page ref and +// records every layout.js on the path from root to the page's containing +// directory. classifyRoot lets us compute directory structure without +// re-parsing the standalone tree. +func attachLayoutChains(refs []ModuleRef, standaloneDir, classifyRoot string) []ModuleRef { + // Index layouts by their enclosing directory (relative to classifyRoot). + // A layout at `server/app/dashboard/layout.js` applies to everything + // under `server/app/dashboard/**`. + layoutByDir := map[string]string{} + for _, r := range refs { + if r.Kind != RouteKindLayout { + continue + } + // r.CompiledPath is relative to standaloneDir; we want it relative + // to classifyRoot for comparison with page dirs. + layoutAbs := filepath.Join(standaloneDir, r.CompiledPath) + classifyRel, err := filepath.Rel(classifyRoot, layoutAbs) + if err != nil { + continue + } + dir := filepath.Dir(filepath.ToSlash(classifyRel)) + layoutByDir[dir] = r.CompiledPath + } + + for i := range refs { + r := &refs[i] + if r.Kind != RouteKindPage { + continue + } + pageAbs := filepath.Join(standaloneDir, r.CompiledPath) + pageRel, err := filepath.Rel(classifyRoot, pageAbs) + if err != nil { + continue + } + pageDir := filepath.Dir(filepath.ToSlash(pageRel)) + + // Walk from the page's directory back up to server/app, collecting + // any layout.js sitting in an ancestor directory. Result is ordered + // from page-nearest to root-nearest; we reverse so the runtime can + // apply root-first. + var chain []string + for dir := pageDir; dir != "." && dir != "/"; dir = filepath.Dir(dir) { + if layoutPath, ok := layoutByDir[dir]; ok { + chain = append(chain, layoutPath) + } + } + // Also check the top-level (server/app) layout. + if top, ok := layoutByDir["server/app"]; ok { + chain = append(chain, top) + } else if top, ok := layoutByDir[filepath.Dir(pageDir)]; ok && len(chain) == 0 { + chain = append(chain, top) + } + + // Reverse to root→leaf ordering. + for l, rr := 0, len(chain)-1; l < rr; l, rr = l+1, rr-1 { + chain[l], chain[rr] = chain[rr], chain[l] + } + r.LayoutChain = chain + } + return refs +} + +// payloadDistDir returns the configured dist directory or the Next default. +func payloadDistDir(p Payload) string { + if p.DistDir != "" { + return p.DistDir + } + return ".next" +} + +// collectCompiledFiles walks serverDir and returns every .js / .mjs file +// below it. Ordered deterministically for reproducible builds. +func collectCompiledFiles(serverDir string) ([]string, error) { + var out []string + err := filepath.WalkDir(serverDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + // Skip Next's own internal chunks — they're required by the + // compiled modules but not directly dispatchable. esbuild + // pulls them in transitively when bundling the ModuleRef set. + base := d.Name() + if base == "chunks" || base == "pages-manifest" { + return fs.SkipDir + } + return nil + } + ext := filepath.Ext(path) + if ext == ".js" || ext == ".mjs" { + out = append(out, path) + } + return nil + }) + if err != nil { + return nil, err + } + sort.Strings(out) + return out, nil +} + +// analyzeModule reads a single compiled file and extracts the facts the +// compiler needs for dispatch + binding derivation. It does NOT parse JS — +// Next's compiled output is regular enough that regex grep on the source +// is accurate and orders of magnitude faster than a full AST pass. If +// that assumption ever breaks we'll swap in esbuild's parser here. +// +// standaloneDir is the bundle root (used for CompiledPath display). +// classifyRoot is the directory whose children are `server/`, `static/`, +// etc. — normally the Next dist dir. Paths relative to classifyRoot are +// what routePathFromCompiled / kindFromCompiled are calibrated for. +func analyzeModule(standaloneDir, classifyRoot, absPath string) (ModuleRef, error) { + displayRel, err := filepath.Rel(standaloneDir, absPath) + if err != nil { + return ModuleRef{}, err + } + classifyRel, err := filepath.Rel(classifyRoot, absPath) + if err != nil { + return ModuleRef{}, err + } + info, err := os.Stat(absPath) + if err != nil { + return ModuleRef{}, err + } + + data, err := os.ReadFile(absPath) // #nosec G304 — compiler reads its own output tree + if err != nil { + return ModuleRef{}, err + } + src := string(data) + + classifySlash := filepath.ToSlash(classifyRel) + ref := ModuleRef{ + CompiledPath: filepath.ToSlash(displayRel), + RoutePath: routePathFromCompiled(classifySlash), + Kind: kindFromCompiled(classifySlash), + ByteSize: info.Size(), + EnvRefs: uniqueStrings(envRefPattern.FindAllStringSubmatch(src, -1), 1), + FetchTargets: uniqueStrings(fetchURLPattern.FindAllStringSubmatch(src, -1), 1), + UsesRSC: usesRSCPattern.MatchString(src), + HasActions: hasActionsPattern.MatchString(src), + PPREnabled: pprPattern.MatchString(src), + } + return ref, nil +} + +// ── Regex surface ──────────────────────────────────────────────────────────── +// +// These are tuned against Next 13/14/15 compiled output. Patterns are +// conservative — false negatives (missing a ref) are cheaper than false +// positives (suggesting a binding that doesn't exist). When a new Next +// version lands, the CI fixture matrix (testdata/fixtures/*) will flag +// any pattern that drifts. + +var ( + // process.env.FOO_BAR — minified output preserves the exact form because + // Next explicitly doesn't rename env references (they're static-replaced + // by webpack/turbopack in some paths, left intact in others). + envRefPattern = regexp.MustCompile(`process\.env\.([A-Z][A-Z0-9_]*)`) + + // fetch("https://...") or fetch('https://...') — double OR single quote, + // url-ish body up to the next quote. Template literals are intentionally + // not matched here; they'd produce too many false positives. + fetchURLPattern = regexp.MustCompile(`\bfetch\(\s*['"]([a-z][a-z0-9+.-]*://[^'"]+)['"]`) + + // Flight / RSC markers. Two tells: + // - "use client" pragma preserved in compiled output + // - imports from react-server-dom-* runtime modules + usesRSCPattern = regexp.MustCompile(`"use client"|react-server-dom-`) + + // Server Actions: compiled modules tag their action exports with a + // specific comment marker plus a $$typeof reference. We look for either. + hasActionsPattern = regexp.MustCompile(`"use server"|__next_internal_action_entry_do_not_use__`) + + // PPR opt-in markers. Next 14+ exports `experimental_ppr = true` from + // pages that opt into Partial Prerendering. Compiled output preserves + // the identifier. `__NEXT_PPR_STATIC_` is a second tell observed in + // some Next 15 canary builds where the static shell has already been + // materialized at build time. + pprPattern = regexp.MustCompile(`experimental_ppr\s*=\s*true|__NEXT_PPR_STATIC_`) +) + +// uniqueStrings deduplicates a regex FindAllStringSubmatch result by the +// capture group at `group` (1 = first paren group). Sorted for determinism. +func uniqueStrings(matches [][]string, group int) []string { + if len(matches) == 0 { + return nil + } + seen := make(map[string]struct{}, len(matches)) + for _, m := range matches { + if len(m) <= group { + continue + } + seen[m[group]] = struct{}{} + } + out := make([]string, 0, len(seen)) + for s := range seen { + out = append(out, s) + } + sort.Strings(out) + return out +} + +// ── Path classification ───────────────────────────────────────────────────── + +// routePathFromCompiled translates a compiled path into the URL route. +// Examples: +// +// server/app/dashboard/page.js → /dashboard +// server/app/api/users/route.js → /api/users +// server/pages/index.js → / +// server/pages/blog/[slug].js → /blog/[slug] +// server/middleware.js → /_middleware +// server/proxy.js → /_proxy +// server/app/layout.js → /_root_layout +func routePathFromCompiled(rel string) string { + rel = strings.TrimPrefix(rel, "server/") + + switch { + case rel == "middleware.js" || rel == "middleware.mjs": + return "/_middleware" + case rel == "proxy.js" || rel == "proxy.mjs": + return "/_proxy" + case strings.HasPrefix(rel, appRouterPrefix): + return appRouteFromPath(strings.TrimPrefix(rel, appRouterPrefix)) + case strings.HasPrefix(rel, pagesRouterPrefix): + return pagesRouteFromPath(strings.TrimPrefix(rel, pagesRouterPrefix)) + } + return "" +} + +// appRouteFromPath handles App Router conventions. page.js / route.js / +// layout.js are all nested under directories that name the route segment. +// Route groups like (marketing) are stripped. Intercepting routes +// ((..)) are preserved verbatim — the dispatcher resolves them. +func appRouteFromPath(rel string) string { + rel = strings.TrimSuffix(rel, filepath.Ext(rel)) + base := filepath.Base(rel) + dir := filepath.Dir(rel) + + // Special files that don't contribute their filename to the URL. + switch base { + case "page", "route": + // fall through — directory is the URL + case "layout": + if dir == "." { + return "/_root_layout" + } + return "/" + stripRouteGroups(dir) + "/_layout" + default: + // Other compiled files (e.g. not-found.js, loading.js) — tag by name. + return "/" + stripRouteGroups(dir) + "/_" + base + } + + if dir == "." { + return "/" + } + return "/" + stripRouteGroups(dir) +} + +// pagesRouteFromPath handles the older Pages Router. index.js collapses to /, +// other files become their path. API routes under pages/api/ are flagged by +// kindFromCompiled, not here. +func pagesRouteFromPath(rel string) string { + rel = strings.TrimSuffix(rel, filepath.Ext(rel)) + if rel == "index" { + return "/" + } + if strings.HasSuffix(rel, "/index") { + rel = strings.TrimSuffix(rel, "/index") + } + return "/" + rel +} + +// stripRouteGroups removes Next.js route group segments like (marketing) +// from a path. Route groups are directory-only organization; they never +// appear in URLs. +func stripRouteGroups(p string) string { + parts := strings.Split(p, "/") + out := parts[:0] + for _, part := range parts { + if strings.HasPrefix(part, "(") && strings.HasSuffix(part, ")") { + continue + } + out = append(out, part) + } + return strings.Join(out, "/") +} + +// kindFromCompiled classifies a compiled path without reading the source. +// The source-based checks (UsesRSC, HasActions) refine Kind later in +// classifyRefs — this function just gets us to a reasonable default. +func kindFromCompiled(rel string) RouteKind { + rel = strings.TrimPrefix(rel, "server/") + + switch { + case rel == "middleware.js" || rel == "middleware.mjs": + return RouteKindMiddleware + case rel == "proxy.js" || rel == "proxy.mjs": + return RouteKindProxy + case strings.HasSuffix(rel, "/route.js") || strings.HasSuffix(rel, "/route.mjs"): + return RouteKindAPI + case strings.HasPrefix(rel, "pages/api/"): + return RouteKindAPI + case strings.HasSuffix(rel, "/page.js") || strings.HasSuffix(rel, "/page.mjs"): + return RouteKindPage + case strings.HasSuffix(rel, "/layout.js") || strings.HasSuffix(rel, "/layout.mjs"): + return RouteKindLayout + case strings.HasPrefix(rel, pagesRouterPrefix): + return RouteKindPage + } + return RouteKindUnknown +} + +// classifyRefs refines Kind against the Payload.Routes classification. +// The compiled-path heuristic in kindFromCompiled gets the common cases +// (middleware.js, route.js, page.js); the manifest overrides only for +// edge cases like a custom API route declared outside server/api/. +// +// Intentionally not consulted here: +// - RouteInfo.MiddlewareRoutes — this is "paths middleware APPLIES TO", +// not "modules that ARE middleware". Mixing them reclassifies pages. +// - HasActions — action promotion is confirmed by actions.go against +// Next's server-reference-manifest, not the path shape alone. +func classifyRefs(refs []ModuleRef, payload Payload) []ModuleRef { + apiSet := sliceToSet(payload.Routes.APIRoutes) + + for i := range refs { + r := &refs[i] + // File-name heuristics already identified these special modules; + // don't let a RoutePath coincidence reclassify them. + if r.Kind == RouteKindMiddleware || r.Kind == RouteKindProxy { + continue + } + if _, ok := apiSet[r.RoutePath]; ok { + r.Kind = RouteKindAPI + } + } + return refs +} + +func sliceToSet(ss []string) map[string]struct{} { + out := make(map[string]struct{}, len(ss)) + for _, s := range ss { + out[s] = struct{}{} + } + return out +} diff --git a/shared/nextcompile/scanner_test.go b/shared/nextcompile/scanner_test.go new file mode 100644 index 0000000..35f7b5d --- /dev/null +++ b/shared/nextcompile/scanner_test.go @@ -0,0 +1,192 @@ +package nextcompile + +import ( + "context" + "path/filepath" + "testing" +) + +func TestRoutePathFromCompiled(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"server/app/page.js", "/"}, + {"server/app/dashboard/page.js", "/dashboard"}, + {"server/app/api/users/route.js", "/api/users"}, + {"server/app/blog/[slug]/page.js", "/blog/[slug]"}, + {"server/app/(marketing)/about/page.js", "/about"}, + {"server/app/layout.js", "/_root_layout"}, + {"server/app/dashboard/layout.js", "/dashboard/_layout"}, + {"server/pages/index.js", "/"}, + {"server/pages/blog/[slug].js", "/blog/[slug]"}, + {"server/pages/api/users.js", "/api/users"}, + {"server/middleware.js", "/_middleware"}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + got := routePathFromCompiled(tc.in) + if got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + }) + } +} + +func TestKindFromCompiled(t *testing.T) { + cases := []struct { + in string + want RouteKind + }{ + {"server/app/page.js", RouteKindPage}, + {"server/app/api/users/route.js", RouteKindAPI}, + {"server/pages/api/users.js", RouteKindAPI}, + {"server/pages/index.js", RouteKindPage}, + {"server/app/layout.js", RouteKindLayout}, + {"server/middleware.js", RouteKindMiddleware}, + {"server/proxy.js", RouteKindProxy}, + {"server/proxy.mjs", RouteKindProxy}, + {"server/chunks/some-chunk.js", RouteKindUnknown}, + } + for _, tc := range cases { + got := kindFromCompiled(tc.in) + if got != tc.want { + t.Errorf("%s: got %s, want %s", tc.in, got, tc.want) + } + } +} + +func TestStripRouteGroups(t *testing.T) { + cases := []struct { + in, want string + }{ + {"(marketing)/about", "about"}, + {"(group1)/(group2)/deep", "deep"}, + {"plain/path", "plain/path"}, + {"blog/[slug]", "blog/[slug]"}, + } + for _, tc := range cases { + got := stripRouteGroups(tc.in) + if got != tc.want { + t.Errorf("%q → %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestAnalyzeModule_EnvAndFetch(t *testing.T) { + dir := t.TempDir() + src := ` +export async function GET() { + const key = process.env.ALGOLIA_KEY; + const other = process.env.DATABASE_URL; + const dup = process.env.ALGOLIA_KEY; + const r = await fetch("https://api.example.com/v1/data"); + const r2 = await fetch('https://other.example.com/q'); + return Response.json({key, other}); +} +` + path := filepath.Join(dir, "server", "app", "api", "users", "route.js") + writeFile(t, path, src) + + ref, err := analyzeModule(dir, dir, path) + if err != nil { + t.Fatalf("analyzeModule: %v", err) + } + + if ref.RoutePath != "/api/users" { + t.Errorf("RoutePath: got %q, want /api/users", ref.RoutePath) + } + if ref.Kind != RouteKindAPI { + t.Errorf("Kind: got %s, want api", ref.Kind) + } + if len(ref.EnvRefs) != 2 { + t.Errorf("EnvRefs: got %v, want 2 unique entries", ref.EnvRefs) + } + if !contains(ref.EnvRefs, "ALGOLIA_KEY") || !contains(ref.EnvRefs, "DATABASE_URL") { + t.Errorf("EnvRefs missing expected keys: %v", ref.EnvRefs) + } + if len(ref.FetchTargets) != 2 { + t.Errorf("FetchTargets: got %v, want 2", ref.FetchTargets) + } +} + +func TestAnalyzeModule_RSCAndActions(t *testing.T) { + dir := t.TempDir() + src := ` +"use client"; +import { fn } from "react-server-dom-webpack/client"; +// __next_internal_action_entry_do_not_use__ [["action123","default"]] +"use server"; +export default function Page() { return null } +` + path := filepath.Join(dir, "server", "app", "page.js") + writeFile(t, path, src) + + ref, err := analyzeModule(dir, dir, path) + if err != nil { + t.Fatalf("analyzeModule: %v", err) + } + if !ref.UsesRSC { + t.Error("expected UsesRSC=true") + } + if !ref.HasActions { + t.Error("expected HasActions=true") + } +} + +func TestScanCompiledServer_Minimal(t *testing.T) { + dir := t.TempDir() + + writeFile(t, filepath.Join(dir, ".next", "server", "app", "page.js"), ` +export default function Home() { return null } +`) + writeFile(t, filepath.Join(dir, ".next", "server", "app", "api", "users", "route.js"), ` +export async function GET() { return new Response(process.env.X) } +`) + writeFile(t, filepath.Join(dir, ".next", "server", "middleware.js"), ` +export default function mw() {} +`) + // A chunk that should be skipped. + writeFile(t, filepath.Join(dir, ".next", "server", "chunks", "util.js"), `export const x = 1`) + + payload := Payload{ + DistDir: ".next", + Routes: RouteInfo{ + APIRoutes: []string{"/api/users"}, + }, + } + + refs, err := ScanCompiledServer(context.Background(), dir, payload) + if err != nil { + t.Fatalf("ScanCompiledServer: %v", err) + } + if len(refs) != 3 { + t.Fatalf("got %d refs, want 3: %+v", len(refs), refs) + } + + byPath := map[string]ModuleRef{} + for _, r := range refs { + byPath[r.RoutePath] = r + } + if byPath["/"].Kind != RouteKindPage { + t.Errorf("/ kind: got %s", byPath["/"].Kind) + } + if byPath["/api/users"].Kind != RouteKindAPI { + t.Errorf("/api/users kind: got %s", byPath["/api/users"].Kind) + } + if !contains(byPath["/api/users"].EnvRefs, "X") { + t.Errorf("/api/users env refs: %v", byPath["/api/users"].EnvRefs) + } + if byPath["/_middleware"].Kind != RouteKindMiddleware { + t.Errorf("middleware kind: got %s", byPath["/_middleware"].Kind) + } +} + +func contains(xs []string, want string) bool { + for _, x := range xs { + if x == want { + return true + } + } + return false +} diff --git a/shared/nextcompile/types.go b/shared/nextcompile/types.go new file mode 100644 index 0000000..b8eb254 --- /dev/null +++ b/shared/nextcompile/types.go @@ -0,0 +1,294 @@ +// Package nextcompile transforms a Next.js standalone build into a +// runtime-native Worker/Lambda bundle by analyzing the compiled output +// and emitting a dispatch table + manifest that a minimal JS runtime +// consumes at request time. +// +// This is the build-time half of the nextcore adapter story. The JS +// runtime half lives in runtime_src/ and is embedded as pre-built +// bundles under runtime_assets/. +// +// Pipeline (see compiler.go): +// +// NextCorePayload (from shared/nextcore) + .next/standalone +// │ +// ▼ +// DetectVersions ── pick runtime variant (v13 / v14 / v15) +// │ +// ▼ +// ScanCompiledServer ── walk .next/server/**, build ModuleRef graph +// │ +// ▼ +// DetectServerActions ── parse server-reference-manifest.json +// │ +// ▼ +// DeriveBindings ── static analysis → binding hints +// │ +// ▼ +// ElideDeadRoutes ── drop orphans +// │ +// ▼ +// Emit{Manifest,DispatchTable,ActionManifest} → /_nextdeploy/ +// │ +// ▼ +// ExtractRuntimeForVersion + AssembleBundle → CompiledBundle +package nextcompile + +import "time" + +// Target selects which deploy surface the bundle is compiled for. +// The same scan phase feeds every target; emit phases diverge. +type Target string + +const ( + TargetCloudflareWorker Target = "cloudflare-worker" + TargetAWSLambda Target = "aws-lambda" + TargetVPS Target = "vps" +) + +// RouteKind categorizes a compiled module. Dispatch order in the runtime +// follows this roughly: Middleware → Static/SSG → ISR → SSR/Page → API/Action. +type RouteKind string + +const ( + RouteKindPage RouteKind = "page" + RouteKindLayout RouteKind = "layout" + RouteKindAPI RouteKind = "api" + RouteKindAction RouteKind = "action" + RouteKindMiddleware RouteKind = "middleware" + // RouteKindProxy is Next 15's proxy.ts — a Node-runtime middleware + // complement. Same dispatch position as middleware but uses the Node + // execution model (can call crypto, Node streams, etc.). If both + // proxy and middleware exist, proxy wins (Next's documented order). + RouteKindProxy RouteKind = "proxy" + RouteKindStatic RouteKind = "static" + RouteKindUnknown RouteKind = "unknown" +) + +// NextVersion is the semver-ish breakdown of the detected Next.js version. +// Raw is preserved verbatim (e.g. "15.0.0-canary.42") for diagnostics. +type NextVersion struct { + Major int + Minor int + Patch int + Raw string +} + +// ReactVersion mirrors NextVersion. Tracked separately because the RSC +// runtime bundle is keyed on React version, not Next version — two Next +// minors can ship against the same React minor. +type ReactVersion struct { + Major int + Minor int + Patch int + Raw string +} + +// CompileOpts is the input bag for Compile. StandaloneDir and Payload are +// required; everything else has a defensible zero value. +type CompileOpts struct { + // StandaloneDir is the path to .next/standalone (or its extracted tarball). + StandaloneDir string + + // Payload is the nextcore extraction. Routes, middleware, image config, + // and ISR tag data all flow from here into the emitted manifest. + Payload Payload + + // OutDir is where _nextdeploy/{manifest.json,dispatch.mjs,...} land. + // Defaults to /.nextdeploy-build when empty. + OutDir string + + // Target picks the emit strategy. Defaults to TargetCloudflareWorker. + Target Target + + // Verbose toggles per-step timing logs. + Verbose bool + + // Log sinks diagnostics. Compile tolerates a nil logger (no output). + Log Logger +} + +// Payload is the subset of nextcore.NextCorePayload that the compiler +// actually reads. Declared as a minimal interface-ish struct so the +// compiler package can stay free of the nextcore import cycle risk +// while staying strongly typed at the adapter boundary. +// +// Callers populate this by translating from nextcore.NextCorePayload +// in the adapter (cli/internal/serverless/cloudflare_adapter.go). +type Payload struct { + AppName string + DistDir string + OutputMode string + BasePath string + HasAppRouter bool + Routes RouteInfo + Middleware *MiddlewareConfig + ImageConfig *ImageConfig + I18n *I18nConfig + BuildID string + GitCommit string +} + +// RouteInfo mirrors nextcore.RouteInfo. Duplicated here so the compiler +// never imports nextcore directly (see Payload doc). +type RouteInfo struct { + StaticRoutes []string + DynamicRoutes []string + SSGRoutes map[string]string // route -> HTML path + SSRRoutes []string + ISRRoutes map[string]string + ISRDetail []ISRRoute + APIRoutes []string + FallbackRoutes map[string]string + MiddlewareRoutes []string +} + +// ISRRoute mirrors nextcore.ISRRoute for the same duplication reason. +type ISRRoute struct { + Path string + Tags []string + Revalidate int +} + +// MiddlewareConfig mirrors the subset of nextcore.MiddlewareConfig the +// compiler forwards into the runtime manifest. The full matcher shape is +// opaque here — it gets emitted as JSON into manifest.json verbatim. +type MiddlewareConfig struct { + Path string + Matchers []MiddlewareMatcher + Runtime string +} + +type MiddlewareMatcher struct { + Pathname string + Pattern string +} + +// ImageConfig mirrors the minimal shape the /_next/image runtime handler +// needs: the remote-pattern whitelist. Everything else (device sizes, +// format preference, etc.) is included raw in the emitted manifest. +type ImageConfig struct { + RemotePatterns []ImageRemotePattern + Domains []string + Formats []string + Unoptimized bool +} + +type ImageRemotePattern struct { + Protocol string + Hostname string + Port string + Pathname string +} + +// I18nConfig mirrors nextcore.I18nConfig for locale-aware dispatch. +type I18nConfig struct { + Locales []string + DefaultLocale string + LocaleDetection bool +} + +// ModuleRef is one compiled server module — a page, layout, route handler, +// or middleware — plus the static-analysis facts the compiler derived. +type ModuleRef struct { + // RoutePath is the user-facing URL (e.g. "/api/users" or "/dashboard/[id]"). + RoutePath string + + // Kind is what the dispatcher should do with this module. + Kind RouteKind + + // CompiledPath is relative to StandaloneDir (e.g. "server/app/api/users/route.js"). + CompiledPath string + + // HasActions is true when Next's server-reference-manifest lists an + // action ID rooted in this module. + HasActions bool + + // UsesRSC is true when the compiled source contains Server Component + // markers ("use client" boundaries or Flight-payload imports). + UsesRSC bool + + // ClientManifestPath points at the Next-emitted + // page_client-reference-manifest.json sibling, when present. + // Relative to StandaloneDir. Used by rsc.mjs as Flight bundlerConfig. + ClientManifestPath string + + // LayoutChain is the ordered list of compiled layout.js paths that + // wrap this page, from root to nearest. Empty for non-page kinds. + LayoutChain []string + + // EnvRefs are the unique process.env.X identifiers the compiler + // found via lexical scan. Used by DeriveBindings to suggest secrets. + EnvRefs []string + + // FetchTargets are literal fetch() URL prefixes extracted from the + // module. Used to suggest KV/R2/D1/service bindings. + FetchTargets []string + + // ByteSize is the raw size of the compiled source on disk. + ByteSize int64 + + // PPREnabled is true when Next compiled this page with Partial + // Prerendering. The runtime dispatcher returns a clear 501 for now + // since our renderer doesn't implement the static-shell / dynamic- + // holes protocol yet. + PPREnabled bool +} + +// BindingHint is a compiler-derived suggestion for the deployment config. +// Emitted as warnings rather than errors — the user has final say. +type BindingHint struct { + Kind string // "secret" | "kv" | "r2" | "d1" | "service" | "queue" + Name string // env var name or logical binding name + Reason string // human-readable justification + Sources []string // ModuleRef.CompiledPath list where the hint was derived +} + +// CompileStats is the post-run summary. Logged in full at info, content-hashed +// for reproducible-build verification. +type CompileStats struct { + RouteCount int + ActionCount int + DeadRoutesElided int + BundleBytes int64 + Duration time.Duration + ContentHash string +} + +// CompiledBundle is the output of Compile. Everything in BundleDir is +// ready for the downstream esbuild step; the adapter doesn't need to +// know the subtree layout beyond EntryPath. +type CompiledBundle struct { + BundleDir string + EntryPath string + ManifestPath string + DispatchPath string + ActionManifest string + DetectedVersion NextVersion + DetectedReact ReactVersion + SuggestedBindings []BindingHint + // VendoredRSC is populated when the target requires vendoring and the + // react-server-dom-webpack package was located in node_modules. Nil + // when the app does not use RSC, or when vendoring was not applicable + // for the target. Checked by adapter logs and bundle reports. + VendoredRSC *VendoredPackage + Stats CompileStats +} + +// Logger is the minimal sink the compiler writes to. Matches the subset +// of shared.Logger the package actually uses; kept as an interface so +// tests can pass a no-op sink without pulling in the full shared package. +type Logger interface { + Info(format string, args ...any) + Warn(format string, args ...any) + Debug(format string, args ...any) +} + +// nopLogger is the zero-value sink used when CompileOpts.Log is nil. +// Tests + callers that don't care about output get a coherent Logger +// without having to import shared/. The three methods are intentionally +// empty — this is a discard sink, not a stub. +type nopLogger struct{} + +func (nopLogger) Info(string, ...any) { /* intentional no-op: discard sink */ } +func (nopLogger) Warn(string, ...any) { /* intentional no-op: discard sink */ } +func (nopLogger) Debug(string, ...any) { /* intentional no-op: discard sink */ } diff --git a/shared/nextcompile/vendor.go b/shared/nextcompile/vendor.go new file mode 100644 index 0000000..98ae495 --- /dev/null +++ b/shared/nextcompile/vendor.go @@ -0,0 +1,162 @@ +package nextcompile + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +// ErrRSCPackageNotFound signals that react-server-dom-webpack is not +// present in the standalone tree's node_modules. Callers match on this +// via errors.Is to branch: +// - manifest.Features.RSC == true → surface to user with vendoring steps +// - manifest.Features.RSC == false → silently skip, rsc.mjs will 501 +// anyway if ever reached +var ErrRSCPackageNotFound = errors.New("react-server-dom-webpack not found in node_modules") + +// VendoredPackage records what VendorRSC copied into the bundle. The +// adapter logs this and includes it in CompileStats. +type VendoredPackage struct { + Name string + Version string + SourcePath string + TargetPath string + Bytes int64 + BuildKind string // "production" | "development" | "legacy" +} + +// VendorRSC resolves react-server-dom-webpack from the standalone tree's +// node_modules and copies its server.edge ESM bundle into +// /_nextdeploy/runtime/vendor/react-server-dom-webpack/server.edge.mjs. +// +// Lookup order (first hit wins): +// 1. /node_modules/react-server-dom-webpack +// 2. /../node_modules/react-server-dom-webpack (app root) +// +// Returns ErrRSCPackageNotFound when neither location resolves. The CF +// Workers runtime has no npm at request time, so vendoring is the only +// way Server Components render without OpenNext. +func VendorRSC(standaloneDir, bundleDir string) (*VendoredPackage, error) { + pkgDir, err := locateRSCPackage(standaloneDir) + if err != nil { + return nil, err + } + + meta, err := readRSCPackageMeta(pkgDir) + if err != nil { + return nil, fmt.Errorf("read react-server-dom-webpack metadata: %w", err) + } + + sourcePath, buildKind, err := findRSCServerEdge(pkgDir) + if err != nil { + return nil, err + } + + targetDir := filepath.Join(bundleDir, "_nextdeploy", "runtime", "vendor", "react-server-dom-webpack") + if err := os.MkdirAll(targetDir, 0o750); err != nil { + return nil, fmt.Errorf("mkdir vendor: %w", err) + } + targetPath := filepath.Join(targetDir, "server.edge.mjs") + + n, err := copyFile(sourcePath, targetPath) + if err != nil { + return nil, fmt.Errorf("copy %s → %s: %w", sourcePath, targetPath, err) + } + + return &VendoredPackage{ + Name: meta.Name, + Version: meta.Version, + SourcePath: sourcePath, + TargetPath: targetPath, + Bytes: n, + BuildKind: buildKind, + }, nil +} + +// locateRSCPackage walks upward from standaloneDir looking for a +// node_modules/react-server-dom-webpack. Next's standalone build lands at +// /.next/standalone, so the package may be two levels up in a +// workspace or monorepo. Cap the walk at 5 levels to avoid unbounded +// upward search in pathological filesystems. +// +// Symlink-transparent by design — pnpm's flat .pnpm store resolves +// because os.Stat follows symlinks. +func locateRSCPackage(standaloneDir string) (string, error) { + current := standaloneDir + for i := 0; i < 5; i++ { + candidate := filepath.Join(current, "node_modules", "react-server-dom-webpack") + if _, err := os.Stat(filepath.Join(candidate, "package.json")); err == nil { + return candidate, nil + } + parent := filepath.Dir(current) + if parent == current { + break // hit root + } + current = parent + } + return "", ErrRSCPackageNotFound +} + +type rscPackageMeta struct { + Name string `json:"name"` + Version string `json:"version"` +} + +func readRSCPackageMeta(pkgDir string) (rscPackageMeta, error) { + data, err := os.ReadFile(filepath.Join(pkgDir, "package.json")) // #nosec G304 + if err != nil { + return rscPackageMeta{}, err + } + var m rscPackageMeta + if err := json.Unmarshal(data, &m); err != nil { + return rscPackageMeta{}, err + } + return m, nil +} + +// findRSCServerEdge tries the known on-disk layouts for the package and +// returns the first existing file along with its build flavor. +// +// Ordering rationale: prefer ESM production (smallest, no dev warnings), +// fall back to ESM development, then the legacy flat CJS layout as a +// last resort. React 18 published both ESM and CJS; React 19 is ESM-only +// but the file names are stable. +func findRSCServerEdge(pkgDir string) (string, string, error) { + candidates := []struct { + rel string + buildKind string + }{ + {"esm/react-server-dom-webpack-server.edge.production.js", "production"}, + {"server.edge.production.js", "production"}, + {"esm/react-server-dom-webpack-server.edge.development.js", "development"}, + {"server.edge.development.js", "development"}, + {"server.edge.js", "legacy"}, + } + for _, c := range candidates { + p := filepath.Join(pkgDir, c.rel) + if _, err := os.Stat(p); err == nil { + return p, c.buildKind, nil + } + } + return "", "", fmt.Errorf("no server.edge build found in %s (tried esm/ and legacy layouts)", pkgDir) +} + +// copyFile byte-copies src to dst and returns the number of bytes written. +// Uses 0640 on the target — worker bundles contain code that can reference +// secrets via bindings, so conservative permissions are correct. +func copyFile(src, dst string) (int64, error) { + in, err := os.Open(src) // #nosec G304 + if err != nil { + return 0, err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640) // #nosec G304 + if err != nil { + return 0, err + } + defer out.Close() + return io.Copy(out, in) +} diff --git a/shared/nextcompile/vendor_test.go b/shared/nextcompile/vendor_test.go new file mode 100644 index 0000000..d31490d --- /dev/null +++ b/shared/nextcompile/vendor_test.go @@ -0,0 +1,145 @@ +package nextcompile + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestVendorRSC_ESMProduction(t *testing.T) { + dir := t.TempDir() + bundleDir := filepath.Join(dir, "out") + + pkgDir := filepath.Join(dir, "node_modules", "react-server-dom-webpack") + writeFile(t, filepath.Join(pkgDir, "package.json"), + `{"name":"react-server-dom-webpack","version":"18.3.1"}`) + writeFile(t, filepath.Join(pkgDir, "esm", "react-server-dom-webpack-server.edge.production.js"), + `// react 18.3.1 server.edge production build +export function renderToReadableStream(){}`) + + got, err := VendorRSC(dir, bundleDir) + if err != nil { + t.Fatalf("VendorRSC: %v", err) + } + if got.Version != "18.3.1" { + t.Errorf("version: got %s", got.Version) + } + if got.BuildKind != "production" { + t.Errorf("buildKind: got %s", got.BuildKind) + } + if got.Bytes == 0 { + t.Error("zero bytes written") + } + + // Target file must exist at the expected path. + expPath := filepath.Join(bundleDir, "_nextdeploy", "runtime", "vendor", + "react-server-dom-webpack", "server.edge.mjs") + if got.TargetPath != expPath { + t.Errorf("target path: got %s, want %s", got.TargetPath, expPath) + } + data, err := os.ReadFile(expPath) // #nosec G304 + if err != nil { + t.Fatalf("vendored file unreadable: %v", err) + } + if len(data) == 0 { + t.Error("vendored file is empty") + } +} + +func TestVendorRSC_PrefersProductionOverDev(t *testing.T) { + dir := t.TempDir() + bundleDir := filepath.Join(dir, "out") + + pkgDir := filepath.Join(dir, "node_modules", "react-server-dom-webpack") + writeFile(t, filepath.Join(pkgDir, "package.json"), + `{"name":"react-server-dom-webpack","version":"19.0.0"}`) + // Both flavors present. + writeFile(t, filepath.Join(pkgDir, "esm", "react-server-dom-webpack-server.edge.production.js"), + `// prod`) + writeFile(t, filepath.Join(pkgDir, "esm", "react-server-dom-webpack-server.edge.development.js"), + `// dev`) + + got, err := VendorRSC(dir, bundleDir) + if err != nil { + t.Fatal(err) + } + if got.BuildKind != "production" { + t.Errorf("expected production, got %s", got.BuildKind) + } + + data, _ := os.ReadFile(got.TargetPath) // #nosec G304 + if string(data) != "// prod" { + t.Errorf("wrong build vendored: %q", data) + } +} + +func TestVendorRSC_FallsBackToDevThenLegacy(t *testing.T) { + dir := t.TempDir() + bundleDir := filepath.Join(dir, "out") + + pkgDir := filepath.Join(dir, "node_modules", "react-server-dom-webpack") + writeFile(t, filepath.Join(pkgDir, "package.json"), + `{"name":"react-server-dom-webpack","version":"18.0.0"}`) + // Only the legacy flat file exists. + writeFile(t, filepath.Join(pkgDir, "server.edge.js"), `// legacy`) + + got, err := VendorRSC(dir, bundleDir) + if err != nil { + t.Fatal(err) + } + if got.BuildKind != "legacy" { + t.Errorf("expected legacy, got %s", got.BuildKind) + } +} + +func TestVendorRSC_AppRootFallback(t *testing.T) { + // Standalone tree lacks node_modules but the app root above it has it. + // pnpm-workspace-style layouts hit this path. + root := t.TempDir() + standalone := filepath.Join(root, ".next", "standalone") + if err := os.MkdirAll(standalone, 0o755); err != nil { + t.Fatal(err) + } + + pkgDir := filepath.Join(root, "node_modules", "react-server-dom-webpack") + writeFile(t, filepath.Join(pkgDir, "package.json"), + `{"name":"react-server-dom-webpack","version":"18.3.1"}`) + writeFile(t, filepath.Join(pkgDir, "esm", "react-server-dom-webpack-server.edge.production.js"), + `// root-level`) + + got, err := VendorRSC(standalone, filepath.Join(root, "out")) + if err != nil { + t.Fatalf("expected fallback to succeed: %v", err) + } + data, _ := os.ReadFile(got.TargetPath) // #nosec G304 + if string(data) != "// root-level" { + t.Errorf("unexpected content: %q", data) + } +} + +func TestVendorRSC_NotFound(t *testing.T) { + dir := t.TempDir() + _, err := VendorRSC(dir, filepath.Join(dir, "out")) + if !errors.Is(err, ErrRSCPackageNotFound) { + t.Errorf("expected ErrRSCPackageNotFound, got %v", err) + } +} + +func TestVendorRSC_PackageWithoutServerEdge(t *testing.T) { + // Package is installed but no server.edge file — corrupt or unexpected + // publish. We treat this as non-recoverable and surface the exact dir. + dir := t.TempDir() + pkgDir := filepath.Join(dir, "node_modules", "react-server-dom-webpack") + writeFile(t, filepath.Join(pkgDir, "package.json"), + `{"name":"react-server-dom-webpack","version":"99.0.0"}`) + // Intentionally no server.edge build present. + + _, err := VendorRSC(dir, filepath.Join(dir, "out")) + if err == nil { + t.Fatal("expected error for missing server.edge") + } + if errors.Is(err, ErrRSCPackageNotFound) { + t.Errorf("should NOT be ErrRSCPackageNotFound — package exists, build is missing") + } +} diff --git a/shared/nextcompile/version_detect.go b/shared/nextcompile/version_detect.go new file mode 100644 index 0000000..8632c2d --- /dev/null +++ b/shared/nextcompile/version_detect.go @@ -0,0 +1,195 @@ +package nextcompile + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +// packageJSONFile is the filename npm + pnpm + yarn all use for the +// package manifest. Centralized so the lookup-path list stays literal- +// free. +const packageJSONFile = "package.json" + +// DetectVersions reads Next.js and React versions from the standalone +// build's dependency graph. Lookup order: +// 1. /package.json (standalone builds vendor their own) +// 2. /node_modules/next/package.json +// 3. /../package.json (repo root, last resort) +// +// Returns ErrVersionNotFound when none resolve. Callers should treat +// that as fatal — the runtime bundle selection depends on knowing which +// Next major is in play. +func DetectVersions(standaloneDir string) (NextVersion, ReactVersion, error) { + var zeroNext NextVersion + var zeroReact ReactVersion + + nextRaw, reactRaw, err := readDepVersions(standaloneDir) + if err != nil { + return zeroNext, zeroReact, err + } + + nv, err := parseNextVersion(nextRaw) + if err != nil { + return zeroNext, zeroReact, fmt.Errorf("parse next version %q: %w", nextRaw, err) + } + + rv, err := parseReactVersion(reactRaw) + if err != nil { + // React version is nice-to-have — not every standalone tree resolves + // it cleanly. Fall through with an empty struct and a readable Raw. + rv = ReactVersion{Raw: reactRaw} + } + + return nv, rv, nil +} + +// ErrVersionNotFound is returned when no package.json yields a Next version. +var ErrVersionNotFound = fmt.Errorf("nextcompile: could not locate next.js version in standalone tree") + +// readDepVersions walks the lookup order and returns the first (next,react) +// pair where at least `next` is present. +func readDepVersions(standaloneDir string) (string, string, error) { + candidates := []string{ + filepath.Join(standaloneDir, packageJSONFile), + filepath.Join(standaloneDir, "node_modules", "next", packageJSONFile), + filepath.Join(filepath.Dir(standaloneDir), packageJSONFile), + } + + for _, path := range candidates { + nv, rv, ok := readPackageJSON(path) + if ok { + return nv, rv, nil + } + } + return "", "", ErrVersionNotFound +} + +// readPackageJSON returns (nextVersion, reactVersion, ok). Two shapes are +// handled: a direct package (the `version` field — used by node_modules/next) +// and a consumer package (dependencies block — used by the app's own file). +func readPackageJSON(path string) (string, string, bool) { + data, err := os.ReadFile(path) // #nosec G304 + if err != nil { + return "", "", false + } + + var pkg struct { + Name string `json:"name"` + Version string `json:"version"` + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` + } + if err := json.Unmarshal(data, &pkg); err != nil { + return "", "", false + } + + // Shape 1: this file IS next/package.json. + if pkg.Name == "next" && pkg.Version != "" { + return pkg.Version, lookupDep(pkg.Dependencies, "react"), true + } + + // Shape 2: app/consumer package with a next dependency. + if v := lookupDep(pkg.Dependencies, "next"); v != "" { + return v, lookupDep(pkg.Dependencies, "react"), true + } + if v := lookupDep(pkg.DevDependencies, "next"); v != "" { + return v, lookupDep(pkg.DevDependencies, "react"), true + } + + return "", "", false +} + +func lookupDep(m map[string]string, key string) string { + if m == nil { + return "" + } + return m[key] +} + +// parseNextVersion handles the formats observed in real package.json files: +// - Exact: "14.2.3" +// - Caret: "^14.2.3" +// - Tilde: "~14.2.3" +// - Canary: "15.0.0-canary.42" +// - RC: "14.0.0-rc.1" +// +// Anything else (workspace:*, git refs, latest) returns an error. Callers +// should pin Next to a concrete version in production, so this is intentional. +func parseNextVersion(raw string) (NextVersion, error) { + return parseSemver(raw, "next") +} + +func parseReactVersion(raw string) (ReactVersion, error) { + nv, err := parseSemver(raw, "react") + return ReactVersion{ + Major: nv.Major, + Minor: nv.Minor, + Patch: nv.Patch, + Raw: nv.Raw, + }, err +} + +func parseSemver(raw, label string) (NextVersion, error) { + trimmed := strings.TrimLeft(strings.TrimSpace(raw), "^~>= ") + if trimmed == "" { + return NextVersion{Raw: raw}, fmt.Errorf("empty %s version", label) + } + + // Strip pre-release / build metadata for numeric parsing, but keep Raw intact. + core := trimmed + if idx := strings.IndexAny(core, "-+"); idx >= 0 { + core = core[:idx] + } + + parts := strings.Split(core, ".") + if len(parts) < 2 { + return NextVersion{Raw: raw}, fmt.Errorf("unexpected %s version format: %s", label, raw) + } + + out := NextVersion{Raw: raw} + var err error + if out.Major, err = strconv.Atoi(parts[0]); err != nil { + return NextVersion{Raw: raw}, fmt.Errorf("major in %q: %w", raw, err) + } + if out.Minor, err = strconv.Atoi(parts[1]); err != nil { + return NextVersion{Raw: raw}, fmt.Errorf("minor in %q: %w", raw, err) + } + if len(parts) >= 3 { + // Patch may have extra dot segments in pre-release; take the leading int. + patchDigits := takeLeadingDigits(parts[2]) + if patchDigits != "" { + out.Patch, _ = strconv.Atoi(patchDigits) + } + } + return out, nil +} + +func takeLeadingDigits(s string) string { + for i := 0; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return s[:i] + } + } + return s +} + +// RuntimeVariant picks which embedded runtime bundle matches the detected +// Next version. Only majors are consulted — minors share a runtime within +// a major because our JS runtime targets the stable public surface, not +// the NextServer internals that churn per-minor. +func (v NextVersion) RuntimeVariant() string { + switch { + case v.Major >= 15: + return "v15" + case v.Major == 14: + return "v14" + case v.Major == 13: + return "v13" + default: + return "v14" // conservative default — most common in the wild today + } +} diff --git a/shared/nextcompile/version_detect_test.go b/shared/nextcompile/version_detect_test.go new file mode 100644 index 0000000..14d11fe --- /dev/null +++ b/shared/nextcompile/version_detect_test.go @@ -0,0 +1,139 @@ +package nextcompile + +import ( + "os" + "path/filepath" + "testing" +) + +type parseCase struct { + name string + in string + major int + minor int + patch int + wantErr bool +} + +func TestParseNextVersion(t *testing.T) { + cases := []parseCase{ + {"exact", "14.2.3", 14, 2, 3, false}, + {"caret", "^15.0.1", 15, 0, 1, false}, + {"tilde", "~13.4.19", 13, 4, 19, false}, + {"canary", "15.0.0-canary.42", 15, 0, 0, false}, + {"rc", "14.0.0-rc.1", 14, 0, 0, false}, + {"two-segment", "14.2", 14, 2, 0, false}, + {"workspace", "workspace:*", 0, 0, 0, true}, + {"empty", "", 0, 0, 0, true}, + {"garbage", "latest", 0, 0, 0, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { runParseCase(t, tc) }) + } +} + +func runParseCase(t *testing.T, tc parseCase) { + t.Helper() + v, err := parseNextVersion(tc.in) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error for %q, got %+v", tc.in, v) + } + return + } + if err != nil { + t.Fatalf("unexpected error for %q: %v", tc.in, err) + } + if v.Major != tc.major || v.Minor != tc.minor || v.Patch != tc.patch { + t.Errorf("got %d.%d.%d, want %d.%d.%d", v.Major, v.Minor, v.Patch, tc.major, tc.minor, tc.patch) + } + if v.Raw != tc.in { + t.Errorf("Raw not preserved: got %q, want %q", v.Raw, tc.in) + } +} + +func TestRuntimeVariant(t *testing.T) { + cases := []struct { + major int + want string + }{ + {13, "v13"}, + {14, "v14"}, + {15, "v15"}, + {16, "v15"}, // newer defaults forward + {12, "v14"}, // older defaults to safe middle + } + for _, tc := range cases { + got := NextVersion{Major: tc.major}.RuntimeVariant() + if got != tc.want { + t.Errorf("major=%d: got %q, want %q", tc.major, got, tc.want) + } + } +} + +func TestDetectVersions_AppPackageJSON(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "package.json"), `{ + "name": "my-app", + "dependencies": { + "next": "14.2.3", + "react": "18.3.1" + } + }`) + + nv, rv, err := DetectVersions(dir) + if err != nil { + t.Fatalf("DetectVersions: %v", err) + } + if nv.Major != 14 || nv.Minor != 2 || nv.Patch != 3 { + t.Errorf("next: got %+v, want 14.2.3", nv) + } + if rv.Major != 18 || rv.Minor != 3 { + t.Errorf("react: got %+v, want 18.3.x", rv) + } +} + +func TestDetectVersions_NodeModulesNextPackage(t *testing.T) { + dir := t.TempDir() + nextDir := filepath.Join(dir, "node_modules", "next") + if err := os.MkdirAll(nextDir, 0o755); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(nextDir, "package.json"), `{ + "name": "next", + "version": "15.0.0-canary.42", + "dependencies": {"react": "19.0.0-rc.0"} + }`) + + nv, rv, err := DetectVersions(dir) + if err != nil { + t.Fatalf("DetectVersions: %v", err) + } + if nv.Major != 15 || nv.Raw != "15.0.0-canary.42" { + t.Errorf("next: got %+v, want 15.0.0-canary.42", nv) + } + if rv.Major != 19 { + t.Errorf("react: got %+v, want major=19", rv) + } + if nv.RuntimeVariant() != "v15" { + t.Errorf("runtime variant: got %q, want v15", nv.RuntimeVariant()) + } +} + +func TestDetectVersions_NotFound(t *testing.T) { + dir := t.TempDir() + _, _, err := DetectVersions(dir) + if err == nil { + t.Fatal("expected ErrVersionNotFound, got nil") + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/shared/nextcore/build_flags.go b/shared/nextcore/build_flags.go new file mode 100644 index 0000000..571bb78 --- /dev/null +++ b/shared/nextcore/build_flags.go @@ -0,0 +1,170 @@ +package nextcore + +import ( + "encoding/json" + "os" + "path/filepath" + "strconv" + "strings" +) + +// MajorVersion parses a semver-ish string like "16.1.6", "^16.0.0", "~15.2" and +// returns the major component. Returns 0 if it cannot be parsed (which the +// callers treat as "don't apply Next 16-specific behaviour"). +func MajorVersion(v string) int { + v = strings.TrimSpace(v) + v = strings.TrimLeft(v, "^~>== 16 +} + +// scriptEndsWithNextBuild reports whether the final command in an npm-style +// build script is a `next build` invocation. Used to decide if appending +// `-- --webpack` via the package-manager argv forwarding will land on next. +// +// Recognises: `next build`, `next-build`, `npx next build`, `bunx next build`, +// and yarn's `yarn next build`. Flags after `next build` (like `--profile`) +// are fine — argv forwarding appends to the tail of the script string. +func scriptEndsWithNextBuild(script string) bool { + last := lastCommandInScript(script) + last = strings.TrimSpace(last) + if last == "" { + return false + } + candidates := []string{ + "next build", + "next-build", + "npx next build", + "npx --no-install next build", + "bunx next build", + "yarn next build", + "pnpm exec next build", + } + for _, c := range candidates { + if strings.HasPrefix(last, c) { + return true + } + } + return false +} + +// lastCommandInScript returns the trailing command of a shell-style chain, +// splitting on the highest-precedence separators we care about (&&, ||, ;). +// Pipes (|) are deliberately ignored — piping into `next build` is not a +// thing we want to reason about. +func lastCommandInScript(script string) string { + seps := []string{"&&", "||", ";"} + tail := script + changed := true + for changed { + changed = false + for _, sep := range seps { + if i := strings.LastIndex(tail, sep); i >= 0 { + tail = tail[i+len(sep):] + changed = true + } + } + } + return tail +} + +// scriptAlreadyPicksBuilder returns true if the user's build script already +// explicitly asks for a builder (either --webpack or --turbopack). In that +// case we leave it alone. +func scriptAlreadyPicksBuilder(script string) bool { + return strings.Contains(script, "--webpack") || strings.Contains(script, "--turbopack") +} + +// readBuildScriptBody returns the raw string of the "build" entry under +// "scripts" in package.json. Empty string on any error (callers treat that as +// "can't reason about the script, skip injection"). +func readBuildScriptBody(projectDir string) string { + // #nosec G304 + data, err := os.ReadFile(filepath.Join(projectDir, "package.json")) + if err != nil { + return "" + } + var pkg PackageJSON + if err := json.Unmarshal(data, &pkg); err != nil { + return "" + } + return pkg.Scripts["build"] +} + +// buildFlagLogger is a minimal subset of the project logger surface — kept +// narrow so this file stays testable without pulling the whole logger. +type buildFlagLogger interface { + Info(format string, args ...any) + Warn(format string, args ...any) +} + +// MaybeInjectWebpackFlag conditionally appends `-- --webpack` to buildCmd so +// the underlying `next build` runs under the webpack builder. Returns the +// (possibly unchanged) command. +// +// Decision table (only modifies when all four are true): +// - next.config has a `webpack:` key +// - next.config has no `turbopack:` key +// - Next.js major version >= 16 +// - package.json scripts.build ends with a `next build` invocation and +// does not already pin --webpack / --turbopack +// +// If the first three are true but the script is too exotic to patch (e.g. +// `next build && post-step`), we log a clear warning and leave the command +// untouched so the user sees the real Turbopack error. +func MaybeInjectWebpackFlag( + buildCmd, projectDir string, + cfg *NextConfig, + nextVersion string, + log buildFlagLogger, +) string { + if !needsWebpackFlag(cfg, nextVersion) { + return buildCmd + } + + script := readBuildScriptBody(projectDir) + if script == "" { + if log != nil { + log.Warn("Detected Next >=16 with webpack config but no turbopack key, and could not read scripts.build from package.json — leaving build command unchanged. Add `turbopack: {}` to next.config or run `next build --webpack` manually.") + } + return buildCmd + } + + if scriptAlreadyPicksBuilder(script) { + return buildCmd + } + + if !scriptEndsWithNextBuild(script) { + if log != nil { + log.Warn("Detected Next >=16 with webpack config but no turbopack key. Your `build` script doesn't end with a plain `next build` invocation (got: %q), so NextDeploy cannot safely inject --webpack. The build will likely fail under Turbopack — either add `turbopack: {}` to next.config or pin `--webpack` in your build script.", script) + } + return buildCmd + } + + if log != nil { + log.Info("Next >=16 detected with webpack config but no turbopack key — appending `-- --webpack` to build command so Turbopack is bypassed.") + } + return buildCmd + " -- --webpack" +} diff --git a/shared/nextcore/features.go b/shared/nextcore/features.go index c3de827..5f5bb41 100644 --- a/shared/nextcore/features.go +++ b/shared/nextcore/features.go @@ -97,7 +97,7 @@ func detectWellKnownService(f *DetectedFeatures, host string) { case strings.Contains(h, "youtube.com") || strings.Contains(h, "ytimg.com") || strings.Contains(h, "youtu.be"): f.HasYouTube = true - case strings.Contains(h, "googleapis.com") || strings.Contains(h, "gstatic.com") || strings.Contains(h, "fonts.google.com"): + case strings.Contains(h, "fonts.googleapis.com") || strings.Contains(h, "gstatic.com") || strings.Contains(h, "fonts.google.com"): f.HasGoogleFonts = true case strings.Contains(h, "google-analytics.com") || strings.Contains(h, "googletagmanager.com"): diff --git a/shared/nextcore/metadatafuncs.go b/shared/nextcore/metadatafuncs.go index 690da69..0dab705 100644 --- a/shared/nextcore/metadatafuncs.go +++ b/shared/nextcore/metadatafuncs.go @@ -9,31 +9,27 @@ import ( "strings" ) -func CollectBuildMetadata() (*NextBuildMetadata, error) { +// CollectBuildMetadata runs the Next.js build and reads the manifests it +// produces. It intentionally does NOT compute OutputMode — the canonical +// source is NextConfig.Output, and the caller threads that into the payload. +func CollectBuildMetadata(buildCmd string) (*NextBuildMetadata, error) { projectDir, err := os.Getwd() if err != nil { return nil, err } - NextCoreLogger.Debug("Build the next to generate build metadata") - PackageManager, err := DetectPackageManager(projectDir) - if err != nil { - PackageManager = "npm" - } - buildCommand, err := buildCommand(string(PackageManager)) - if err != nil { - return nil, err - } + NextCoreLogger.Debug("Building Next.js app to generate build metadata") + if err := os.MkdirAll(".nextdeploy", 0750); err != nil { return nil, fmt.Errorf("failed to create .nextdeploy directory: %w", err) } + // #nosec G204 - cmd := exec.Command("sh", "-c", buildCommand) + cmd := exec.Command("sh", "-c", buildCmd) cmd.Dir = projectDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stdout - if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("build failed:%w", err) + return nil, fmt.Errorf("build failed: %w", err) } nextDir := filepath.Join(projectDir, ".next") @@ -42,6 +38,7 @@ func CollectBuildMetadata() (*NextBuildMetadata, error) { if err != nil { return nil, fmt.Errorf("failed to read BUILD_ID: %w", err) } + readJSON := func(filename string) (interface{}, error) { // #nosec G304 data, err := os.ReadFile(filepath.Join(nextDir, filename)) @@ -65,34 +62,22 @@ func CollectBuildMetadata() (*NextBuildMetadata, error) { imagesManifest, _ := readJSON("images-manifest.json") appPathRoutesManifest, _ := readJSON("app-path-routes-manifest.json") reactLoadableManifest, _ := readJSON("react-loadable-manifest.json") + var diagnostics []string - diagnosticsDir := filepath.Join(nextDir, "diagnostics") - if files, err := os.ReadDir(diagnosticsDir); err == nil { + if files, err := os.ReadDir(filepath.Join(nextDir, "diagnostics")); err == nil { for _, file := range files { diagnostics = append(diagnostics, file.Name()) } } - outputMode := OutputModeDefault - if _, err := os.Stat(filepath.Join(nextDir, "standalone")); err == nil { - outputMode = OutputModeStandalone - } else if b, err := os.ReadFile(filepath.Join(projectDir, "next.config.js")); err == nil { // #nosec G304 - content := string(b) - if strings.Contains(content, "output: 'export'") || strings.Contains(content, "output: \"export\"") { - outputMode = OutputModeExport - } - } else if b, err := os.ReadFile(filepath.Join(projectDir, "next.config.mjs")); err == nil { // #nosec G304 - content := string(b) - if strings.Contains(content, "output: 'export'") || strings.Contains(content, "output: \"export\"") { - outputMode = OutputModeExport - } - } - hasAppRouter := appPathRoutesManifest != nil - if _, err := os.Stat(filepath.Join(projectDir, "app")); err == nil { - hasAppRouter = true - } else if _, err := os.Stat(filepath.Join(projectDir, "src", "app")); err == nil { - hasAppRouter = true + if !hasAppRouter { + for _, rel := range []string{"app", filepath.Join("src", "app")} { + if _, err := os.Stat(filepath.Join(projectDir, rel)); err == nil { + hasAppRouter = true + break + } + } } return &NextBuildMetadata{ @@ -105,8 +90,38 @@ func CollectBuildMetadata() (*NextBuildMetadata, error) { AppPathRoutesManifest: appPathRoutesManifest, ReactLoadableManifest: reactLoadableManifest, Diagnostics: diagnostics, - OutputMode: outputMode, HasAppRouter: hasAppRouter, }, nil +} +// detectOutputMode resolves the Next.js output mode using the parsed config +// first (the authoritative source) and falling back to filesystem / raw +// config-file scanning when the config couldn't be evaluated. +func detectOutputMode(projectDir string, nextConfig *NextConfig, distDir string) OutputMode { + if nextConfig != nil { + switch nextConfig.Output { + case "export": + return OutputModeExport + case "standalone": + return OutputModeStandalone + } + } + if distDir == "" { + distDir = ".next" + } + if _, err := os.Stat(filepath.Join(projectDir, distDir, "standalone")); err == nil { + return OutputModeStandalone + } + for _, name := range []string{"next.config.js", "next.config.mjs", "next.config.ts"} { + // #nosec G304 + b, err := os.ReadFile(filepath.Join(projectDir, name)) + if err != nil { + continue + } + s := string(b) + if strings.Contains(s, "output: 'export'") || strings.Contains(s, `output: "export"`) { + return OutputModeExport + } + } + return OutputModeDefault } diff --git a/shared/nextcore/nextconfig.go b/shared/nextcore/nextconfig.go index a47deed..bfdd4a3 100644 --- a/shared/nextcore/nextconfig.go +++ b/shared/nextcore/nextconfig.go @@ -220,6 +220,10 @@ func parseConfigObject(config map[string]interface{}) (*NextConfig, error) { result.Webpack = webpack } + if turbopack, ok := config["turbopack"]; ok { + result.Turbopack = turbopack + } + return result, nil } diff --git a/shared/nextcore/nextcore.go b/shared/nextcore/nextcore.go index c36163e..34f23f3 100644 --- a/shared/nextcore/nextcore.go +++ b/shared/nextcore/nextcore.go @@ -28,119 +28,82 @@ var ( ) func GenerateMetadata() (metadata NextCorePayload, err error) { - NextCoreLogger.Info("Generating metadata for Next.js application...") cfg, err := config.Load() if err != nil { NextCoreLogger.Error("Failed to load configuration: %v", err) return NextCorePayload{}, err } - AppName := cfg.App.Name - NextJsVersion, err := GetNextJsVersion("package.json") - if err != nil { - NextCoreLogger.Error("Failed to get Next.js version: %v", err) - return NextCorePayload{}, err - } - NextCoreLogger.Info("Next.js version: %s", NextJsVersion) cwd, err := os.Getwd() if err != nil { NextCoreLogger.Error("Error getting current working directory") return NextCorePayload{}, err } - configPath := filepath.Join(cwd, "next.config.mjs") - nextConfig, err := ParseNextConfigFile(configPath) + nextConfig, err := ParseNextConfigFile(filepath.Join(cwd, "next.config.mjs")) if err != nil { NextCoreLogger.Error("Failed to parse next config (non-fatal): %v", err) } features := DetectFeatures(nextConfig) - NextCoreLogger.Info("Collecting build metadata...") - buildMeta, err := CollectBuildMetadata() + packageManager, err := DetectPackageManager(cwd) if err != nil { - NextCoreLogger.Error("Failed to collect build metadata: %v", err) + NextCoreLogger.Error("Failed to detect package manager: %v", err) return NextCorePayload{}, err } - routeInfo, err := getRoutesFromManifests(buildMeta, features.DistDir) + buildCmd, err := buildCommand(packageManager.String()) if err != nil { + NextCoreLogger.Error("Failed to get build command: %v", err) return NextCorePayload{}, err } - packageManager, err := DetectPackageManager(cwd) - if err != nil { - NextCoreLogger.Error("Failed to detect package manager: %v", err) - return NextCorePayload{}, err - } - buildCommand, err := buildCommand(packageManager.String()) + nextVersion, _ := GetNextJsVersion(filepath.Join(cwd, "package.json")) + buildCmd = MaybeInjectWebpackFlag(buildCmd, cwd, nextConfig, nextVersion, NextCoreLogger) + + buildMeta, err := CollectBuildMetadata(buildCmd) if err != nil { - NextCoreLogger.Error("Failed to get build command: %v", err) + NextCoreLogger.Error("Failed to collect build metadata: %v", err) return NextCorePayload{}, err } - startCommand, err := startCommand(packageManager.String()) + outputMode := detectOutputMode(cwd, nextConfig, features.DistDir) + + routeInfo, err := getRoutesFromManifests(buildMeta, features.DistDir) if err != nil { - NextCoreLogger.Error("Failed to get start command: %v", err) return NextCorePayload{}, err } + imagesAssets, err := detectImageAssets(buildMeta, cwd, features.DistDir) - var HasImageAssets bool if err != nil { NextCoreLogger.Error("Failed to detect image assets: %v", err) return NextCorePayload{}, err } - if len(imagesAssets.PublicImages) == 0 && len(imagesAssets.OptimizedImages) == 0 && len(imagesAssets.StaticImports) == 0 { - NextCoreLogger.Info("No image assets found in the Next.js build") - } else { - HasImageAssets = true - } - domainName := cfg.App.Domain middlewareConfig, err := ParseMiddleware(cwd) - if err != nil { NextCoreLogger.Error("Failed to parse middleware configuration: %v", err) return NextCorePayload{}, err } - StaticAssets, err := ParseStaticAssets(cwd, features.DistDir) + staticAssets, err := ParseStaticAssets(cwd, features.DistDir) if err != nil { NextCoreLogger.Error("Failed to parse static assets: %v", err) return NextCorePayload{}, err } - gitCommt, err := git.GetCommitHash() + gitCommit, err := git.GetCommitHash() if err != nil { NextCoreLogger.Error("Failed to get git commit hash: %v", err) return NextCorePayload{}, err - } else { - NextCoreLogger.Debug("Git commit hash: %s", gitCommt) } - gitDiry := git.IsDirty() + NextCoreLogger.Debug("Git commit hash: %s", gitCommit) - PayloadPath, err := filepath.Abs(filepath.Join(cwd, MetadataFileName)) - if err != nil { - NextCoreLogger.Error("Failed to get payload path: %v", err) - return NextCorePayload{}, err - } - buildLockPath, err := filepath.Abs(filepath.Join(cwd, BuildLockFileName)) - if err != nil { - NextCoreLogger.Error("Failed to get build lock path: %v", err) - return NextCorePayload{}, err - } - AssetsOutputDir, err := filepath.Abs(filepath.Join(cwd, AssetsOutputDir)) - if err != nil { - NextCoreLogger.Error("Failed to get assets output directory: %v", err) - return NextCorePayload{}, err - } - // 4. Copy static assets if err := copyStaticAssets(); err != nil { NextCoreLogger.Error("Failed to copy static assets: %v", err) return NextCorePayload{}, fmt.Errorf("failed to copy static assets: %w", err) } - // 4. Track git state metadata = NextCorePayload{ - AppName: AppName, - NextVersion: NextJsVersion, + AppName: cfg.App.Name, NextBuildMetadata: *buildMeta, Config: config.SafeConfig{ AppName: cfg.App.Name, @@ -149,44 +112,29 @@ func GenerateMetadata() (metadata NextCorePayload, err error) { Environment: cfg.App.Environment, TargetType: cfg.ResolveTargetType(""), }, - BuildCommand: buildCommand, - StartCommand: startCommand, - Entrypoint: deriveEntrypoint(buildMeta.OutputMode, "."), - HasImageAssets: HasImageAssets, CDNEnabled: cfg.App.CDNEnabled, - Domain: domainName, + Domain: cfg.App.Domain, RouteInfo: *routeInfo, DetectedFeatures: features, DistDir: features.DistDir, ExportDir: features.ExportDir, - StaticRoutes: routeInfo.StaticRoutes, - DynamicRoutes: routeInfo.DynamicRoutes, Middleware: middlewareConfig, - StaticAssets: StaticAssets, - GitCommit: gitCommt, - GitDirty: gitDiry, + StaticAssets: staticAssets, + GitCommit: gitCommit, + GitDirty: git.IsDirty(), GeneratedAt: time.Now().Format(time.RFC3339), - MetadataFilePath: PayloadPath, - BuildLockFile: buildLockPath, - AssetsOutputDir: AssetsOutputDir, PackageManager: packageManager.String(), - RootDir: cwd, - WorkingDir: cwd, - OutputMode: buildMeta.OutputMode, + OutputMode: outputMode, ImageAssets: *imagesAssets, - NextBuild: NextBuild{ - HasAppRouter: buildMeta.HasAppRouter, - RootFiles: RootFiles{ - BuildManifest: filepath.Join(cwd, features.DistDir, "build-manifest.json"), - PackageJSON: filepath.Join(cwd, "package.json"), - LastBuildTimestamp: time.Now().Format(time.RFC3339), - }, - BuildMetadata: BuildMetadata{ - NextVersion: NextJsVersion, - BuildID: buildMeta.BuildID, - OutputMode: buildMeta.OutputMode, - }, - }, + } + + if len(metadata.RouteInfo.ISRDetail) > 0 { + tagMap := BuildTagMap(metadata.RouteInfo.ISRDetail) + if tagMapData, err := json.MarshalIndent(tagMap, "", " "); err == nil { + assetsDir := filepath.Join(cwd, AssetsOutputDir) + _ = os.MkdirAll(assetsDir, 0750) + _ = os.WriteFile(filepath.Join(assetsDir, "isr-tag-map.json"), tagMapData, 0600) + } } if err := createBuildLock(&metadata); err != nil { @@ -211,19 +159,6 @@ func LoadMetadata() (NextCorePayload, error) { return metadata, nil } -func deriveEntrypoint(outputMode OutputMode, releaseDir string) string { - switch outputMode { - case OutputModeStandalone: - return filepath.Join(releaseDir, "server.js") - case OutputModeDefault: - return filepath.Join(releaseDir, "node_modules", ".bin", "next start") - case OutputModeExport: - return "" // no process entrypoint - Caddy serves files directly - default: - return "" - } -} - func copyStaticAssets() error { srcDir := "public" dstDir := filepath.Join(".nextdeploy", "assets") @@ -288,55 +223,30 @@ func copyFile(src, dst string) error { return err } -// createBuildLock creates the build.lock file with git state +// createBuildLock writes the metadata payload and a build.lock using the git +// state already captured on the payload. func createBuildLock(metadata *NextCorePayload) error { - commitHash, err := git.GetCommitHash() - if err != nil { - NextCoreLogger.Error("Failed to get git commit hash: %v", err) - return fmt.Errorf("failed to get git commit hash: %w", err) - } - - dirty := git.IsDirty() - // Write metadata to json file - fileName := ".nextdeploy/metadata.json" - marshalledData, err := json.MarshalIndent(metadata, "", " ") + payloadData, err := json.MarshalIndent(metadata, "", " ") if err != nil { NextCoreLogger.Error("Failed to marshal metadata: %v", err) return err } - if err := os.WriteFile(fileName, marshalledData, 0600); err != nil { + if err := os.WriteFile(MetadataFileName, payloadData, 0600); err != nil { NextCoreLogger.Error("Failed to write metadata json: %v", err) return err } - buildLock := BuildLock{ - GitCommit: commitHash, - GitDirty: dirty, + lockData, err := json.MarshalIndent(BuildLock{ + GitCommit: metadata.GitCommit, + GitDirty: metadata.GitDirty, GeneratedAt: metadata.GeneratedAt, - Metadata: fileName, - } - - data, err := json.MarshalIndent(buildLock, "", " ") + Metadata: MetadataFileName, + }, "", " ") if err != nil { NextCoreLogger.Error("Failed to marshal build lock: %v", err) return err } - - return os.WriteFile(filepath.Join(".nextdeploy", "build.lock"), data, 0600) -} - -// getPublicEnvVars collects NEXT_PUBLIC_* environment variables -func getPublicEnvVars() map[string]string { - vars := make(map[string]string) - for _, env := range os.Environ() { - if strings.HasPrefix(env, "NEXT_PUBLIC_") { - parts := strings.SplitN(env, "=", 2) - if len(parts) == 2 { - vars[parts[0]] = parts[1] - } - } - } - return vars + return os.WriteFile(BuildLockFileName, lockData, 0600) } // ValidateBuildState checks if the current git state matches the build lock @@ -361,8 +271,6 @@ func ValidateBuildState() error { return fmt.Errorf("failed to get current git commit: %w", err) } //TODO: use this data to avoid unnecessary builds - NextCoreLogger.Info("Current git commit: %s", currentCommit) - NextCoreLogger.Info("Expected git commit: %s", lock.GitCommit) if currentCommit != lock.GitCommit { NextCoreLogger.Error("Git commit mismatch: expected %s, got %s", lock.GitCommit, currentCommit) return fmt.Errorf("git commit mismatch: expected %s, got %s", lock.GitCommit, currentCommit) @@ -567,6 +475,8 @@ func ParseMiddleware(projectDir string) (*MiddlewareConfig, error) { middlewarePaths := []string{ filepath.Join(projectDir, "middleware.ts"), filepath.Join(projectDir, "middleware.js"), + filepath.Join(projectDir, "proxy.ts"), + filepath.Join(projectDir, "proxy.js"), } var middlewareFile string @@ -620,11 +530,20 @@ func parseMiddlewareMatchers(content string) ([]MiddlewareRoute, error) { var matchers []MiddlewareRoute // First try to parse config object style - configObjRegex := regexp.MustCompile(`config\s*=\s*{([^}]*)}`) + configObjRegex := regexp.MustCompile(`(?:export\s+const\s+)?config\s*=\s*{([^}]*)}`) configMatches := configObjRegex.FindStringSubmatch(content) if len(configMatches) > 1 { - // Try to parse as JSON (with some cleaning) - cleaned := strings.ReplaceAll(configMatches[1], "'", `"`) + // Clean the content for pseudo-JSON parsing + body := configMatches[1] + // Convert ' to " + cleaned := strings.ReplaceAll(body, "'", `"`) + // Add quotes to keys if missing + keyRegex := regexp.MustCompile(`(\s*)([a-zA-Z0-9_]+):`) + cleaned = keyRegex.ReplaceAllString(cleaned, `$1"$2":`) + // Remove trailing commas before closing braces/brackets + trailingCommaRegex := regexp.MustCompile(`,(\s*[}\]])`) + cleaned = trailingCommaRegex.ReplaceAllString(cleaned, `$1`) + cleaned = strings.ReplaceAll(cleaned, "\n", "") cleaned = fmt.Sprintf("{%s}", cleaned) @@ -675,7 +594,7 @@ func parseMiddlewareMatchers(content string) ([]MiddlewareRoute, error) { } // Fallback to parsing individual matchers - matcherRegex := regexp.MustCompile(`matcher:\s*(\[[^\]]+\]|{[^}]+})`) + matcherRegex := regexp.MustCompile(`matcher:\s*(\[[^\]]+\]|{[^}]+}|"[^"]+"|'[^']+')`) matcherMatches := matcherRegex.FindStringSubmatch(content) if len(matcherMatches) > 1 { cleaned := strings.ReplaceAll(matcherMatches[1], "'", `"`) @@ -694,6 +613,15 @@ func parseMiddlewareMatchers(content string) ([]MiddlewareRoute, error) { } } + // Handle single path string + if strings.HasPrefix(cleaned, `"`) { + path := strings.Trim(cleaned, `"`) + matchers = append(matchers, MiddlewareRoute{ + Pathname: path, + }) + return matchers, nil + } + // Handle object matcher if strings.HasPrefix(cleaned, "{") { var matcher struct { @@ -734,62 +662,6 @@ func parseUnstableFlag(content string) string { return "" } -func parseImageConfig(images map[string]interface{}) ImageConfig { - result := ImageConfig{ - Loader: "default", - } - - if domains, ok := images["domains"].([]interface{}); ok { - for _, d := range domains { - if s, ok := d.(string); ok { - result.Domains = append(result.Domains, s) - } - } - } - - if formats, ok := images["formats"].([]interface{}); ok { - for _, f := range formats { - if s, ok := f.(string); ok { - result.Formats = append(result.Formats, s) - } - } - } - - if deviceSizes, ok := images["deviceSizes"].([]interface{}); ok { - for _, s := range deviceSizes { - if n, ok := s.(float64); ok { - result.DeviceSizes = append(result.DeviceSizes, int(n)) - } - } - } - - if imageSizes, ok := images["imageSizes"].([]interface{}); ok { - for _, s := range imageSizes { - if n, ok := s.(float64); ok { - result.ImageSizes = append(result.ImageSizes, int(n)) - } - } - } - - if loader, ok := images["loader"].(string); ok { - result.Loader = loader - } - - if path, ok := images["path"].(string); ok { - result.Path = path - } - - if ttl, ok := images["minimumCacheTTL"].(float64); ok { - result.MinimumCacheTTL = int(ttl) - } - - if unoptimized, ok := images["unoptimized"].(bool); ok { - result.Unoptimized = unoptimized - } - - return result -} - func detectImageAssets(buildMeta *NextBuildMetadata, projectDir string, distDir string) (*ImageAssets, error) { assets := &ImageAssets{} var err error @@ -818,41 +690,31 @@ func detectImageAssets(buildMeta *NextBuildMetadata, projectDir string, distDir } func findPublicImages(publicDir, projectDir string) ([]ImageAsset, error) { + _ = projectDir // retained for signature parity; paths are public-relative var images []ImageAsset - // Supported image extensions - imageExts := map[string]bool{ - ".jpg": true, - ".jpeg": true, - ".png": true, - ".webp": true, - ".gif": true, - ".avif": true, - ".svg": true, - } - err := filepath.Walk(publicDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - - if !info.IsDir() { - ext := strings.ToLower(filepath.Ext(path)) - if imageExts[ext] { - relPath, err := filepath.Rel(publicDir, path) - if err != nil { - return err - } - - images = append(images, ImageAsset{ - Path: relPath, - AbsolutePath: path, - PublicPath: filepath.Join("/", relPath), - Format: strings.TrimPrefix(ext, "."), - IsOptimized: false, - }) - } + if info.IsDir() { + return nil } + ext := strings.ToLower(filepath.Ext(path)) + if assetExtensions[ext] != "image" { + return nil + } + relPath, err := filepath.Rel(publicDir, path) + if err != nil { + return err + } + images = append(images, ImageAsset{ + Path: relPath, + AbsolutePath: path, + PublicPath: filepath.Join("/", relPath), + Format: strings.TrimPrefix(ext, "."), + IsOptimized: false, + }) return nil }) @@ -925,24 +787,6 @@ func parseStaticImageImports(buildManifest map[string]interface{}, projectDir st return images } -func startCommand(PackageManager string) (string, error) { - if PackageManager == "" { - return "", fmt.Errorf("no package manager provided") - } - switch PackageManager { - case "npm": - return "npm start", nil - case "yarn": - return "yarn start", nil - case "pnpm": - return "pnpm start", nil - case "bun": - return "bun start", nil - default: - return "npm start", fmt.Errorf("unsupported package manager: %s", PackageManager) - - } -} func buildCommand(PackageManager string) (string, error) { if PackageManager == "" { diff --git a/shared/nextcore/nextjs.go b/shared/nextcore/nextjs.go index 055e40c..fe4fb4c 100644 --- a/shared/nextcore/nextjs.go +++ b/shared/nextcore/nextjs.go @@ -66,7 +66,6 @@ func ValidateNextJSProject(cmd *cobra.Command, args []string) error { } return fmt.Errorf("directory doesn't appear to be a Next.js project") } - return nil } diff --git a/shared/nextcore/routes.go b/shared/nextcore/routes.go index 1e81d5d..d5e35b9 100644 --- a/shared/nextcore/routes.go +++ b/shared/nextcore/routes.go @@ -40,6 +40,24 @@ func getRoutesFromManifests(buildMeta *NextBuildMetadata, distDir string) (*Rout if initialRevalidate, ok := detailMap["initialRevalidateSeconds"].(float64); ok { if initialRevalidate > 0 { info.ISRRoutes[route] = filepath.Join(distDir, "server", detailMap["dataRoute"].(string)) + + // Extract extended ISR metadata + isrRoute := ISRRoute{ + Path: route, + Revalidate: int(initialRevalidate), + Tags: []string{route}, // Default to the route itself as a tag + } + + // If next.js suddenly adds tags to the prerender manifest, grab them + if routeTagsIfc, hasTags := detailMap["tags"].([]interface{}); hasTags { + for _, t := range routeTagsIfc { + if tagStr, isStr := t.(string); isStr { + isrRoute.Tags = append(isrRoute.Tags, tagStr) + } + } + } + + info.ISRDetail = append(info.ISRDetail, isrRoute) } else { info.SSGRoutes[route] = filepath.Join(distDir, "server", "pages", route+".html") } diff --git a/shared/nextcore/tagmap.go b/shared/nextcore/tagmap.go new file mode 100644 index 0000000..8ae7a94 --- /dev/null +++ b/shared/nextcore/tagmap.go @@ -0,0 +1,29 @@ +package nextcore + +type ISRRoute struct { + Path string `json:"path"` + Tags []string `json:"tags"` + Revalidate int `json:"revalidate"` +} + +type TagPathMap struct { + // tag -> list of cloudfront paths to invalidate + Tags map[string][]string `json:"tags"` + Intervals map[string]int `json:"intervals"` +} + +func BuildTagMap(routes []ISRRoute) TagPathMap { + m := TagPathMap{ + Tags: make(map[string][]string), + Intervals: make(map[string]int), + } + + for _, r := range routes { + m.Intervals[r.Path] = r.Revalidate + for _, tag := range r.Tags { + m.Tags[tag] = append(m.Tags[tag], r.Path) + } + } + + return m +} diff --git a/shared/nextcore/types.go b/shared/nextcore/types.go index df313ca..319aecb 100644 --- a/shared/nextcore/types.go +++ b/shared/nextcore/types.go @@ -14,13 +14,7 @@ const ( type NextCorePayload struct { AppName string `json:"app_name"` - NextVersion string `json:"next_version"` NextBuildMetadata NextBuildMetadata `json:"nextbuildmetadata"` - StaticRoutes []string `json:"static_routes"` - DynamicRoutes []string `json:"dynamic_routes"` - BuildCommand string `json:"build_command"` - StartCommand string `json:"start_command"` - HasImageAssets bool `json:"has_image_assets"` CDNEnabled bool `json:"cdn_enabled"` Domain string `json:"domain"` Middleware *MiddlewareConfig `json:"middleware"` @@ -28,9 +22,6 @@ type NextCorePayload struct { GitCommit string `json:"git_commit,omitempty"` GitDirty bool `json:"git_dirty,omitempty"` GeneratedAt string `json:"generated_at,omitempty"` - BuildLockFile string `json:"build_lock_file,omitempty"` - MetadataFilePath string `json:"metadata_file_path,omitempty"` - AssetsOutputDir string `json:"assets_output_dir,omitempty"` Config config.SafeConfig `json:"config,omitempty"` ImageAssets ImageAssets `json:"image_assets"` RouteInfo RouteInfo `json:"route_info"` @@ -38,19 +29,7 @@ type NextCorePayload struct { DistDir string `json:"dist_dir"` ExportDir string `json:"export_dir"` OutputMode OutputMode `json:"output_mode"` - NextBuild NextBuild `json:"next_build"` - WorkingDir string `json:"working_dir"` - RootDir string `json:"root_dir"` PackageManager string `json:"package_manager"` - Entrypoint string `json:"entrypoint"` -} - -type DeployMetadata struct { - GeneratedAt string `json:"generated_at"` - Routes RoutesManifest `json:"routes"` - BuildInfo BuildManifest `json:"build_info"` - Middleware []string `json:"middleware"` - EnvVars map[string]string `json:"env_vars"` } type BuildLock struct { @@ -60,22 +39,6 @@ type BuildLock struct { Metadata string `json:"metadata_file"` } -type RoutesManifest struct { - Version int `json:"version"` - Pages []string `json:"pages"` - DynamicRoutes []DynamicRoute `json:"dynamicRoutes"` -} - -type DynamicRoute struct { - Page string `json:"page"` - Regex string `json:"regex"` - RouteKeys map[string]string `json:"routeKeys"` -} - -type BuildManifest struct { - Pages map[string][]string `json:"pages"` -} - type StaticAsset struct { Path string `json:"path"` AbsolutePath string `json:"absolute_path"` @@ -141,6 +104,7 @@ type NextConfig struct { Compiler *CompilerConfig `json:"compiler,omitempty"` Webpack interface{} `json:"webpack,omitempty"` Webpack5 bool `json:"webpack5,omitempty"` + Turbopack interface{} `json:"turbopack,omitempty"` Experimental *ExperimentalConfig `json:"experimental,omitempty"` EdgeRegions []string `json:"edgeRegions,omitempty"` EdgeRuntime string `json:"edgeRuntime,omitempty"` @@ -249,7 +213,8 @@ type RouteInfo struct { DynamicRoutes []string `json:"dynamic_routes"` SSGRoutes map[string]string `json:"ssg_routes"` SSRRoutes []string `json:"ssr_routes"` - ISRRoutes map[string]string `json:"isr_routes"` + ISRRoutes map[string]string `json:"isr_routes"` // Route -> HTML File Path + ISRDetail []ISRRoute `json:"isr_detail"` // Extended tagging info for ISR APIRoutes []string `json:"api_routes"` FallbackRoutes map[string]string `json:"fallback_routes"` MiddlewareRoutes []string `json:"middleware_routes"` @@ -265,114 +230,5 @@ type NextBuildMetadata struct { AppPathRoutesManifest interface{} `json:"appPathRoutesManifest"` ReactLoadableManifest interface{} `json:"reactLoadableManifest"` Diagnostics []string `json:"diagnostics"` - OutputMode OutputMode `json:"outputMode"` HasAppRouter bool `json:"hasAppRouter"` } - -type NextBuild struct { - RootFiles RootFiles `json:"root_files"` - Cache Cache `json:"cache"` - Server Server `json:"server"` - Static Static `json:"static"` - HasAppRouter bool `json:"has_app_router"` - HasPagesRouter bool `json:"has_pages_router"` - BuildMetadata BuildMetadata `json:"build_metadata"` -} - -type RootFiles struct { - BuildManifest string `json:"build_manifest"` - AppBuildManifest string `json:"app_build_manifest"` - ReactLoadableManifest string `json:"react_loadable_manifest"` - PackageJSON string `json:"package_json"` - LastBuildTimestamp string `json:"last_build_timestamp"` - TraceFile string `json:"trace_file,omitempty"` -} - -type Cache struct { - Images []ImageCacheEntry `json:"images"` - Webpack WebpackCache `json:"webpack"` - SWC []string `json:"swc"` -} - -type ImageCacheEntry struct { - Hash string `json:"hash"` - Format string `json:"format"` - Width int `json:"width"` - Height int `json:"height"` - CachePath string `json:"cache_path"` -} - -type WebpackCache struct { - ClientDevelopment []string `json:"client_development"` - ClientProduction []string `json:"client_production"` - ServerDevelopment []string `json:"server_development"` - ServerProduction []string `json:"server_production"` - EdgeServerDevelopment []string `json:"edge_server_development"` - EdgeServerProduction []string `json:"edge_server_production"` -} - -type Server struct { - Manifests ServerManifests `json:"manifests"` - AppRoutes []AppRoute `json:"app_routes"` - VendorChunks []string `json:"vendor_chunks"` - Middleware Middleware `json:"middleware"` -} - -type ServerManifests struct { - AppPaths string `json:"app_paths"` - Middleware string `json:"middleware"` - Pages string `json:"pages"` - Font string `json:"font"` - ServerReference string `json:"server_reference"` -} - -type AppRoute struct { - RoutePath string `json:"route_path"` - PageJS string `json:"page_js"` - ClientReference string `json:"client_reference"` -} - -type Middleware struct { - Path string `json:"path"` - Matchers []string `json:"matchers"` -} - -type Static struct { - Chunks Chunks `json:"chunks"` - CSS []CSSFile `json:"css"` - Media []MediaFile `json:"media"` - Webpack []WebpackHotUpdate `json:"webpack"` -} - -type Chunks struct { - App []string `json:"app"` - Pages []string `json:"pages"` - Polyfills string `json:"polyfills"` - Webpack string `json:"webpack"` - Main string `json:"main"` -} - -type CSSFile struct { - Path string `json:"path"` - IsGlobal bool `json:"is_global"` -} - -type MediaFile struct { - Name string `json:"name"` - Type string `json:"type"` - Ext string `json:"ext"` -} - -type WebpackHotUpdate struct { - Name string `json:"name"` - Type string `json:"type"` -} - -type BuildMetadata struct { - NextVersion string `json:"next_version"` - BuildTarget string `json:"build_target"` - BuildID string `json:"build_id"` - HasTypeScript bool `json:"has_typescript"` - HasESLint bool `json:"has_eslint"` - OutputMode OutputMode `json:"output_mode"` -} diff --git a/shared/nextdeploy/types.go b/shared/nextdeploy/types.go index 2e331d2..f7513a9 100644 --- a/shared/nextdeploy/types.go +++ b/shared/nextdeploy/types.go @@ -76,10 +76,10 @@ type ContainerHealthcheck struct { // Database represents database configuration type Database struct { - Type string `yaml:"type"` - Host string `yaml:"host"` - Port int `yaml:"port"` - Username string `yaml:"username"` + Type string `yaml:"type"` + Host string `yaml:"host"` + Port int `yaml:"port"` + Username string `yaml:"username"` // #nosec G117 Password string `yaml:"password"` Name string `yaml:"name"` @@ -118,9 +118,9 @@ type Backup struct { } type Storage struct { - Provider string `yaml:"provider"` - Bucket string `yaml:"bucket"` - Region string `yaml:"region"` + Provider string `yaml:"provider"` + Bucket string `yaml:"bucket"` + Region string `yaml:"region"` // #nosec G117 AccessKey string `yaml:"access_key"` SecretKey string `yaml:"secret_key"` diff --git a/shared/secrets/secretmanager.go b/shared/secrets/secretmanager.go index 5d13f21..85c9e0a 100644 --- a/shared/secrets/secretmanager.go +++ b/shared/secrets/secretmanager.go @@ -94,6 +94,7 @@ func NewSecretManager(opts ...Option) (*SecretManager, error) { } if sm.cfg == nil { + //TODO: the secrets env should be loaded here not config cfg, err := config.Load() if err != nil { return nil, ErrConfigNotFound diff --git a/shared/sensitive/sensitive.go b/shared/sensitive/sensitive.go new file mode 100644 index 0000000..4c239e8 --- /dev/null +++ b/shared/sensitive/sensitive.go @@ -0,0 +1,141 @@ +// Package sensitive holds a process-wide registry of secret values that must +// never appear in any log line, error message, or stdout output. +// +// Usage: +// +// token := os.Getenv("CLOUDFLARE_API_TOKEN") +// sensitive.Register(token) // ← BEFORE token reaches any code path that may log +// scrubbed := sensitive.Scrub(message) // applied automatically by the shared Logger +// +// Any value passed to Register is replaced with "***" wherever it appears in +// strings passed through Scrub. Values shorter than minRedactLen are ignored +// to avoid scrubbing common short tokens (e.g. account IDs that may legitimately +// appear in URLs). +package sensitive + +import ( + "fmt" + "io" + "os" + "regexp" + "strings" + "sync" +) + +const ( + // minRedactLen is the shortest registered value we will scrub. Anything + // shorter is too likely to cause false positives in normal log output. + minRedactLen = 12 + // redaction is the replacement token. Short on purpose so log lines stay + // scannable. + redaction = "***" +) + +var ( + mu sync.RWMutex + registered = map[string]struct{}{} +) + +// Register adds value to the global redaction set. Safe for concurrent use. +// Values shorter than minRedactLen are silently ignored (avoids redacting +// short identifiers and breaking unrelated logs). +func Register(values ...string) { + mu.Lock() + defer mu.Unlock() + for _, v := range values { + if len(v) < minRedactLen { + continue + } + registered[v] = struct{}{} + } +} + +// Clear empties the registry. Intended for tests. +func Clear() { + mu.Lock() + defer mu.Unlock() + registered = map[string]struct{}{} +} + +// commonPatterns catches well-known credential shapes even when the producing +// code forgot to call Register. Defense in depth — order matters: more specific +// patterns first. +var commonPatterns = []*regexp.Regexp{ + // Bearer tokens in HTTP-style strings + regexp.MustCompile(`(?i)(Bearer\s+)[A-Za-z0-9._\-]{12,}`), + // AWS access key IDs (always start with AKIA/ASIA, exactly 20 chars) + regexp.MustCompile(`\b(AKIA|ASIA)[A-Z0-9]{16}\b`), + // Authorization header value forms + regexp.MustCompile(`(?i)(Authorization:\s*)[A-Za-z0-9._\-+=/]{12,}`), +} + +// Scrub returns s with every registered value replaced by "***" and every +// well-known credential pattern collapsed. Cheap when registry is empty. +func Scrub(s string) string { + mu.RLock() + defer mu.RUnlock() + + if len(registered) == 0 && !mightContainPattern(s) { + return s + } + + out := s + for v := range registered { + if v == "" { + continue + } + out = strings.ReplaceAll(out, v, redaction) + } + + out = commonPatterns[0].ReplaceAllString(out, "${1}"+redaction) + out = commonPatterns[1].ReplaceAllString(out, redaction) + out = commonPatterns[2].ReplaceAllString(out, "${1}"+redaction) + return out +} + +// mightContainPattern is a fast pre-check so we skip regex work on most +// log lines. Keep these substrings in lower-case. +func mightContainPattern(s string) bool { + lower := strings.ToLower(s) + return strings.Contains(lower, "bearer ") || + strings.Contains(lower, "authorization:") || + strings.Contains(s, "AKIA") || + strings.Contains(s, "ASIA") +} + +// ── Safe printers ─────────────────────────────────────────────────────────── +// +// Use these wherever you'd otherwise call fmt.Print* / fmt.Fprint* with content +// derived from external systems (errors from cloud APIs, subprocess output, +// daemon responses) — anything that might inadvertently echo back a token. +// The raw value is formatted, then Scrub'd, then written. + +// Print scrubs and writes to stdout. Mirrors fmt.Print signature. +func Print(a ...any) (int, error) { + return io.WriteString(os.Stdout, Scrub(fmt.Sprint(a...))) +} + +// Println scrubs and writes to stdout with a trailing newline. +func Println(a ...any) (int, error) { + return io.WriteString(os.Stdout, Scrub(fmt.Sprintln(a...))) +} + +// Printf scrubs and writes to stdout. Mirrors fmt.Printf signature. +func Printf(format string, a ...any) (int, error) { + return io.WriteString(os.Stdout, Scrub(fmt.Sprintf(format, a...))) +} + +// Fprint scrubs and writes to w. Mirrors fmt.Fprint signature. +func Fprint(w io.Writer, a ...any) (int, error) { + return io.WriteString(w, Scrub(fmt.Sprint(a...))) +} + +// Fprintln scrubs and writes to w with a trailing newline. +func Fprintln(w io.Writer, a ...any) (int, error) { + return io.WriteString(w, Scrub(fmt.Sprintln(a...))) +} + +// Fprintf scrubs and writes to w. Mirrors fmt.Fprintf signature. +func Fprintf(w io.Writer, format string, a ...any) (int, error) { + return io.WriteString(w, Scrub(fmt.Sprintf(format, a...))) +} diff --git a/shared/updater/updater.go b/shared/updater/updater.go index 2cf6524..f0e00a4 100644 --- a/shared/updater/updater.go +++ b/shared/updater/updater.go @@ -19,6 +19,8 @@ import ( "strings" "time" + "github.com/Golangcodes/nextdeploy/shared/sensitive" + "github.com/Golangcodes/nextdeploy/shared" ) @@ -384,7 +386,7 @@ func selfUpdateWithOptions(current, binaryBase string, opts *UpdateOptions) erro fmt.Printf(" Creating backup: %s\n", filepath.Base(backupBin)) if err := copyFileWithSudo(currentBin, backupBin); err != nil { - fmt.Printf("⚠️ Warning: failed to create backup: %v\n", err) + sensitive.Printf("⚠️ Warning: failed to create backup: %v\n", err) backupBin = "" } } @@ -414,7 +416,7 @@ func selfUpdateWithOptions(current, binaryBase string, opts *UpdateOptions) erro // 14. Set permissions if err := setPermissions(currentBin); err != nil { - fmt.Printf(" Warning: could not set permissions: %v\n", err) + sensitive.Printf(" Warning: could not set permissions: %v\n", err) } // 15. Verify installed version diff --git a/shared/utils/utils.go b/shared/utils/utils.go new file mode 100644 index 0000000..dbc312f --- /dev/null +++ b/shared/utils/utils.go @@ -0,0 +1,509 @@ +// This package stores all util functions +package utils + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "container/heap" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/Golangcodes/nextdeploy/shared/nextcore" +) + +const ( + maxInFlight = 64 + largeFileThreshold = 4 * 1024 * 1024 // 4MB +) + +var workerCount = func() int { + n := runtime.NumCPU() + if n > 8 { + return 8 + } + return n +}() + +type logger interface { + Info(msg string, args ...interface{}) + Warn(msg string, args ...interface{}) + Error(msg string, args ...interface{}) +} + +type fileJob struct { + index int + path string + relPath string + size int64 +} + +type fileResult struct { + job fileJob + info os.FileInfo + data []byte + err error +} + +type resultHeap []fileResult + +func (h resultHeap) Len() int { return len(h) } +func (h resultHeap) Less(i, j int) bool { return h[i].job.index < h[j].job.index } +func (h resultHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *resultHeap) Push(x interface{}) { *h = append(*h, x.(fileResult)) } +func (h *resultHeap) Pop() interface{} { + old := *h + n := len(old) + x := old[n-1] + *h = old[:n-1] + return x +} + +var webAllowedExts = map[string]struct{}{ + ".js": {}, ".jsx": {}, ".ts": {}, ".tsx": {}, ".mjs": {}, ".cjs": {}, + ".css": {}, ".scss": {}, ".sass": {}, ".less": {}, + ".html": {}, ".htm": {}, ".xml": {}, ".json": {}, ".yaml": {}, ".yml": {}, ".toml": {}, + ".png": {}, ".jpg": {}, ".jpeg": {}, ".gif": {}, ".webp": {}, ".avif": {}, + ".svg": {}, ".ico": {}, ".bmp": {}, ".tiff": {}, + ".woff": {}, ".woff2": {}, ".ttf": {}, ".otf": {}, ".eot": {}, + ".mp4": {}, ".webm": {}, ".ogg": {}, ".ogv": {}, ".mov": {}, + ".mp3": {}, ".wav": {}, ".aac": {}, ".opus": {}, + ".pdf": {}, ".txt": {}, ".md": {}, ".mdx": {}, + ".csv": {}, ".geojson": {}, + ".webmanifest": {}, ".map": {}, + "": {}, +} + +var excludedDirs = map[string]struct{}{ + "node_modules": {}, + ".git": {}, + ".nextdeploy": {}, + "cypress": {}, + "__tests__": {}, + "coverage": {}, + ".turbo": {}, + ".vercel": {}, +} + +func isTempTarball(name string) bool { + return strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tar") +} + +func shouldExcludeDir(name string) bool { + _, ok := excludedDirs[name] + return ok +} + +func shouldExcludeFile(name, relPath string) (bool, string) { + if isTempTarball(name) { + return true, "tarball" + } + if strings.Contains(relPath, ".env.") { + return true, ".env file" + } + ext := strings.ToLower(filepath.Ext(name)) + if _, ok := webAllowedExts[ext]; !ok { + return true, "ext '" + ext + "' not in whitelist" + } + return false, "" +} + +func CopyFile(src, dst string) error { + info, err := os.Stat(src) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("stat %s: %w", src, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%s is not a regular file", src) + } + // #nosec G304 + source, err := os.Open(src) + if err != nil { + return fmt.Errorf("open %s: %w", src, err) + } + defer source.Close() + + if err := os.MkdirAll(filepath.Dir(dst), 0o750); err != nil { + return fmt.Errorf("mkdir %s: %w", filepath.Dir(dst), err) + } + // #nosec G304 + destination, err := os.Create(dst) + if err != nil { + return fmt.Errorf("create %s: %w", dst, err) + } + defer destination.Close() + + if _, err := io.Copy(destination, source); err != nil { + return fmt.Errorf("copy %s → %s: %w", src, dst, err) + } + return nil +} + +func CopyDir(src, dst string) error { + if _, err := os.Stat(src); os.IsNotExist(err) { + return nil + } + return filepath.WalkDir(src, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + dstPath := filepath.Join(dst, relPath) + if d.IsDir() { + return os.MkdirAll(dstPath, 0o750) + } + return CopyFile(path, dstPath) + }) +} + +func writeTarEntry(tw *tar.Writer, r fileResult, log logger) error { + if r.info == nil { + log.Info("[tarball] skip (vanished): %s", r.job.relPath) + return nil + } + + var linkTarget string + if r.info.Mode()&os.ModeSymlink != 0 { + var err error + linkTarget, err = os.Readlink(r.job.path) + if err != nil { + return fmt.Errorf("readlink %s: %w", r.job.path, err) + } + } + + header, err := tar.FileInfoHeader(r.info, linkTarget) + if err != nil { + return fmt.Errorf("file info header %s: %w", r.job.relPath, err) + } + header.Name = r.job.relPath + + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("write header %s: %w", r.job.relPath, err) + } + + if !r.info.Mode().IsRegular() { + return nil + } + + if r.data != nil { + if _, err := tw.Write(r.data); err != nil { + return fmt.Errorf("write buffered %s: %w", r.job.relPath, err) + } + log.Info("[tarball] file → %s (%d bytes, buffered)", header.Name, len(r.data)) + return nil + } + f, err := os.Open(r.job.path) + if err != nil { + return fmt.Errorf("open large file %s: %w", r.job.path, err) + } + defer f.Close() + + written, err := io.CopyBuffer(tw, f, make([]byte, 32*1024)) + if err != nil { + return fmt.Errorf("stream %s: %w", r.job.path, err) + } + log.Info("[tarball] file → %s (%.1f MB, streamed)", header.Name, float64(written)/1024/1024) + return nil +} + +func fileCopyAndRemove(src, dst string) error { + // #nosec G304, G703 + source, err := os.Open(src) + if err != nil { + return fmt.Errorf("open %s: %w", src, err) + } + defer source.Close() + + // #nosec G304 + destination, err := os.Create(dst) + if err != nil { + return fmt.Errorf("create %s: %w", dst, err) + } + defer destination.Close() + + if _, err := io.Copy(destination, source); err != nil { + return fmt.Errorf("copy %s → %s: %w", src, dst, err) + } + // #nosec G703 + _ = os.Remove(src) + return nil +} + +func readFile(job fileJob, pool *sync.Pool, workerID int, log logger) fileResult { + info, err := os.Lstat(job.path) + if err != nil { + if os.IsNotExist(err) { + log.Info("[tarball] worker-%d file vanished: %s", workerID, job.relPath) + return fileResult{job: job} + } + return fileResult{job: job, err: fmt.Errorf("lstat: %w", err)} + } + + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return fileResult{job: job, info: info} + } + + if job.size > largeFileThreshold { + log.Info("[tarball] worker-%d deferred (large): %s (%.1f MB)", + workerID, job.relPath, float64(job.size)/1024/1024) + return fileResult{job: job, info: info, data: nil} + } + + f, err := os.Open(job.path) + if err != nil { + return fileResult{job: job, err: fmt.Errorf("open: %w", err)} + } + defer f.Close() + data := make([]byte, info.Size()) + if _, err := io.ReadFull(f, data); err != nil { + return fileResult{job: job, err: fmt.Errorf("read: %w", err)} + } + + log.Info("[tarball] worker-%d read: %s (%d bytes)", workerID, job.relPath, len(data)) + return fileResult{job: job, info: info, data: data} +} + +func CreateTarball(sourceDir, targetTar, targetType string, payload *nextcore.NextCorePayload, log logger) error { + outputMode := payload.OutputMode + log.Info("[tarball] Starting — source=%s target=%s mode=%s workers=%d", + sourceDir, targetTar, outputMode, workerCount) + + tarfile, err := os.CreateTemp(filepath.Dir(targetTar), "next-deploy-*.tar.gz") + if err != nil { + return fmt.Errorf("create temp file: %w", err) + } + tempName := tarfile.Name() + log.Info("[tarball] Temp file: %s", tempName) + + success := false + defer func() { + _ = tarfile.Close() + if !success { + log.Info("[tarball] Removing temp file after error: %s", tempName) + _ = os.Remove(tempName) + } + }() + + bufWriter := bufio.NewWriterSize(tarfile, 512*1024) + gzw, err := gzip.NewWriterLevel(bufWriter, gzip.BestSpeed) + if err != nil { + return fmt.Errorf("create gzip writer: %w", err) + } + tw := tar.NewWriter(gzw) + log.Info("[tarball] Phase 1: Walking %s...", sourceDir) + walkStart := time.Now() + + var jobs []fileJob + var dirHeaders []tar.Header + + err = filepath.WalkDir(sourceDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + if os.IsPermission(err) { + log.Info("[tarball] Skip (permission denied): %s", path) + return nil + } + return err + } + if isTempTarball(filepath.Base(path)) { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + + relPath, err := filepath.Rel(sourceDir, path) + if err != nil { + return fmt.Errorf("rel path for %s: %w", path, err) + } + if relPath == "." { + return nil + } + + if d.IsDir() { + if d.Name() == ".git" || d.Name() == ".nextdeploy" { + log.Info("[tarball] Skip dir (excluded): %s", relPath) + return filepath.SkipDir + } + + if d.Name() == payload.DistDir || d.Name() == payload.ExportDir { + if sourceDir == "." || sourceDir == "./" { + log.Info("[tarball] Skip build/export dir: %s", relPath) + return filepath.SkipDir + } + } + + if shouldExcludeDir(d.Name()) { + if d.Name() == "node_modules" { + if targetType == "vps" && (outputMode == nextcore.OutputModeStandalone || outputMode == nextcore.OutputModeDefault) { + log.Info("[tarball] VPS: Including node_modules (mode=%s): %s", outputMode, relPath) + } else if targetType == "serverless" && outputMode == nextcore.OutputModeStandalone { + log.Info("[tarball] Serverless: Including standalone node_modules: %s", relPath) + } else { + log.Info("[tarball] Skip dir (excluded): %s", relPath) + return filepath.SkipDir + } + } else { + log.Info("[tarball] Skip dir (excluded): %s", relPath) + return filepath.SkipDir + } + } + + info, err := d.Info() + if err != nil { + return nil + } + h, err := tar.FileInfoHeader(info, "") + if err != nil { + return nil + } + h.Name = filepath.ToSlash(relPath) + "/" + dirHeaders = append(dirHeaders, *h) + return nil + } + if outputMode == nextcore.OutputModeDefault { + if skip, reason := shouldExcludeFile(d.Name(), relPath); skip { + log.Info("[tarball] Skip file (%s): %s", reason, relPath) + return nil + } + } + + info, err := d.Info() + if err != nil { + return nil + } + + jobs = append(jobs, fileJob{ + index: len(jobs), + path: path, + relPath: filepath.ToSlash(relPath), + size: info.Size(), + }) + return nil + }) + if err != nil { + return fmt.Errorf("walk failed: %w", err) + } + + log.Info("[tarball] Walk done in %s — %d dirs, %d files to archive", + time.Since(walkStart).Round(time.Millisecond), len(dirHeaders), len(jobs)) + log.Info("[tarball] Phase 2: Writing %d directory entries...", len(dirHeaders)) + for _, h := range dirHeaders { + hCopy := h + if err := tw.WriteHeader(&hCopy); err != nil { + return fmt.Errorf("write dir header %s: %w", h.Name, err) + } + log.Info("[tarball] dir → %s", h.Name) + } + + log.Info("[tarball] Phase 3: Reading %d files (%d workers)...", len(jobs), workerCount) + pipelineStart := time.Now() + fileChan := make(chan fileJob, maxInFlight) + resultChan := make(chan fileResult, maxInFlight) + readBufPool := &sync.Pool{ + New: func() interface{} { return make([]byte, 32*1024) }, + } + + var wg sync.WaitGroup + for i := 0; i < workerCount; i++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + for job := range fileChan { + resultChan <- readFile(job, readBufPool, id, log) + } + }(i) + } + + go func() { + wg.Wait() + close(resultChan) + }() + + go func() { + for _, job := range jobs { + fileChan <- job + } + close(fileChan) + }() + + h := &resultHeap{} + heap.Init(h) + nextExpected := 0 + var processedFiles int64 + lastProgressLog := time.Now() + + for result := range resultChan { + if result.err != nil { + return fmt.Errorf("worker error for %s: %w", result.job.relPath, result.err) + } + + heap.Push(h, result) + + for h.Len() > 0 && (*h)[0].job.index == nextExpected { + r := heap.Pop(h).(fileResult) + + if err := writeTarEntry(tw, r, log); err != nil { + return err + } + + nextExpected++ + atomic.AddInt64(&processedFiles, 1) + if time.Since(lastProgressLog) > 2*time.Second { + pct := float64(atomic.LoadInt64(&processedFiles)) / float64(len(jobs)) * 100 + log.Info("[tarball] Progress: %d/%d files (%.1f%%)", + atomic.LoadInt64(&processedFiles), len(jobs), pct) + lastProgressLog = time.Now() + } + } + } + + log.Info("[tarball] Pipeline done in %s — %d files written", + time.Since(pipelineStart).Round(time.Millisecond), processedFiles) + + if err := tw.Close(); err != nil { + return fmt.Errorf("close tar: %w", err) + } + if err := gzw.Close(); err != nil { + return fmt.Errorf("close gzip: %w", err) + } + if err := bufWriter.Flush(); err != nil { + return fmt.Errorf("flush buffer: %w", err) + } + if err := tarfile.Close(); err != nil { + return fmt.Errorf("close temp file: %w", err) + } + + log.Info("[tarball] Renaming %s → %s", tempName, targetTar) + // #nosec G703 + if err := os.Rename(tempName, targetTar); err != nil { + if strings.Contains(err.Error(), "invalid cross-device link") { + log.Info("[tarball] Cross-device rename detected, falling back to copy...") + if copyErr := fileCopyAndRemove(tempName, targetTar); copyErr != nil { + return fmt.Errorf("cross-device copy: %w", copyErr) + } + } else { + return fmt.Errorf("rename: %w", err) + } + } + + success = true + + if fi, err := os.Stat(targetTar); err == nil { + log.Info("[tarball] Done — %d files, %.2f MB", processedFiles, float64(fi.Size())/1024/1024) + } + + return nil +} diff --git a/shared/version.go b/shared/version.go index 9a6e3f2..ec6ec27 100644 --- a/shared/version.go +++ b/shared/version.go @@ -1,3 +1,8 @@ package shared const Version = "v0.7.18" + +// Commit is the short git commit the binary was built from. Populated via +// ldflags at build time (see Makefile / magefile.go / .goreleaser.yml). +// Left as a var rather than const so -X can rewrite it. +var Commit = "unknown" diff --git a/shared/websocket/types.go b/shared/websocket/types.go deleted file mode 100644 index d37d906..0000000 --- a/shared/websocket/types.go +++ /dev/null @@ -1,31 +0,0 @@ -package websocket - -import ( - "crypto/ecdh" - "crypto/ecdsa" - "sync" - "time" - - "github.com/gorilla/websocket" -) - -type WSClient struct { - conn *websocket.Conn - mu sync.Mutex - agentID string - privateKey *ecdsa.PrivateKey - connected bool - pingPeriod time.Duration - writeWait time.Duration - pongWait time.Duration - authKey *ecdsa.PrivateKey - sessionKeys map[string]*ecdh.PrivateKey - upgrader websocket.Upgrader -} - -type SecureMessage struct { - IV []byte `json:"iv"` - Ciphertext []byte `json:"ciphertext"` - Tag []byte `json:"tag"` - Sequence uint64 `json:"sequence"` -} diff --git a/shared/websocket/websocket.go b/shared/websocket/websocket.go deleted file mode 100644 index 8292751..0000000 --- a/shared/websocket/websocket.go +++ /dev/null @@ -1,140 +0,0 @@ -package websocket - -import ( - "crypto/ecdsa" - "errors" - "fmt" - "net/http" - "time" - - "github.com/Golangcodes/nextdeploy/shared" - - "github.com/gorilla/websocket" -) - -var ( - websockerlogger = shared.PackageLogger("api", "🌐 API") -) - -func NewWSClient(agentID string, privateKey *ecdsa.PrivateKey) *WSClient { - return &WSClient{ - agentID: agentID, - privateKey: privateKey, - pingPeriod: 15 * time.Second, - writeWait: 10 * time.Second, - pongWait: 20 * time.Second, - } -} - -func (c *WSClient) ReceiveMessage() (shared.AgentMessage, error) { - c.mu.Lock() - defer c.mu.Unlock() - - if !c.connected { - websockerlogger.Error("WebSocket client is not connected") - return shared.AgentMessage{}, errors.New("websocket client is not connected") - } - - messsageType, message, err := c.conn.ReadMessage() - fmt.Printf("Received message type: %v", message) - fmt.Printf("Message type: %d\n", messsageType) - if err != nil { - websockerlogger.Error("Failed to read message from WebSocket error:%s", err) - return shared.AgentMessage{}, err - } - var agentMessage shared.AgentMessage - return agentMessage, nil -} -func (c *WSClient) Connect(url string, headers http.Header) error { - dialer := websocket.DefaultDialer - conn, _, err := dialer.Dial(url, headers) - if err != nil { - websockerlogger.Error("Failed to connect to WebSocket server:%s", err) - return err - } - - c.conn = conn - c.connected = true - go c.readPump() - go c.writePump() - - return nil - -} -func (c *WSClient) SendMessage(msg interface{}) error { - - c.mu.Lock() - defer c.mu.Unlock() - if !c.connected { - websockerlogger.Error("WebSocket client is not connected") - return errors.New("websocket client is not connected") - } - - return c.conn.WriteJSON(msg) - -} - -func (c *WSClient) Close() error { - c.mu.Lock() - defer c.mu.Unlock() - - if !c.connected { - websockerlogger.Error("WebSocket client is not connected") - return errors.New("websocket client is not connected") - } - - err := c.conn.Close() - if err != nil { - websockerlogger.Error("Failed to close WebSocket connection error:%s", err) - return err - } - - c.connected = false - return nil -} - -func (c *WSClient) readPump() { - defer c.Close() - - _ = c.conn.SetReadDeadline(time.Now().Add(c.pongWait)) - c.conn.SetPongHandler(func(string) error { - _ = c.conn.SetReadDeadline(time.Now().Add(c.pongWait)) - return nil - }) - - for { - _, message, err := c.conn.ReadMessage() - if err != nil { - if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) { - websockerlogger.Error("WebSocket read error:%s", err) - } - return - } - websockerlogger.Info("Received message from WebSocket server message:%s", string(message)) - } - -} - -func (c *WSClient) writePump() { - ticker := time.NewTicker(c.pingPeriod) - defer func() { - ticker.Stop() - _ = c.Close() - }() - - for { - select { - case <-ticker.C: - c.mu.Lock() - if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil { - websockerlogger.Error("Failed to send ping error:%s", err) - c.mu.Unlock() - return - } - c.mu.Unlock() - - default: - time.Sleep(1 * time.Second) // Avoid busy waiting - } - } -} diff --git a/tmp/air.log b/tmp/air.log deleted file mode 100644 index 5ecb6d6..0000000 --- a/tmp/air.log +++ /dev/null @@ -1 +0,0 @@ -exit status 2exit status 2 \ No newline at end of file diff --git a/tools/tools.go b/tools/tools.go new file mode 100644 index 0000000..b8023e0 --- /dev/null +++ b/tools/tools.go @@ -0,0 +1,11 @@ +//go:build tools + +// Package tools pins development tool versions so `go mod tidy` tracks them. +package tools + +import ( + _ "github.com/boyter/scc" + _ "github.com/golangci/golangci-lint/cmd/golangci-lint" + _ "github.com/securego/gosec/v2/cmd/gosec" + _ "golang.org/x/vuln/cmd/govulncheck" +)