diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index f8483a67..99be5621 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -16,6 +16,7 @@ jobs: scope: name: Detect affected runtimes runs-on: ubuntu-24.04 + timeout-minutes: 10 outputs: shared: ${{ steps.scope.outputs.shared }} web: ${{ steps.scope.outputs.web }} @@ -27,6 +28,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: fetch-depth: 0 + persist-credentials: false - name: Detect change scope id: scope shell: bash @@ -53,6 +55,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false - name: Set up Python 3.13 uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 with: @@ -86,9 +90,12 @@ jobs: needs: scope if: needs.scope.outputs.engine == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 20 steps: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false - name: Set up Python 3.13 uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 with: @@ -117,6 +124,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false - name: Set up Java 21 uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e15023c4..db13253c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,9 +18,12 @@ jobs: name: Build and record release provenance runs-on: ubuntu-24.04 environment: release + timeout-minutes: 30 steps: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false - name: Set up Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 with: diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index f41ad0da..767ffd4c 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -22,6 +22,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false - name: Set up Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 with: diff --git a/docs/development/README.md b/docs/development/README.md index 3f3c9583..abb037a2 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -10,7 +10,8 @@ Android companion, API, and Python engine. OpenTofu when infrastructure validation is needed. The repository pins the package manager and checks runtime versions. 2. Run `corepack pnpm repo:bootstrap`. -3. Run `corepack pnpm repo:check` and `corepack pnpm repo:build`. A clean +3. Run `corepack pnpm repo:check` and `corepack pnpm repo:build`. The check + includes non-applying local/AWS infrastructure validation. A clean checkout is the baseline; do not hide a failure with an untracked local configuration file. 4. Copy `infrastructure/local/.env.example` to a local-only `.env` if needed, diff --git a/docs/operations/engineering-foundation-verification.md b/docs/operations/engineering-foundation-verification.md index 3b74b1f0..f1d18326 100644 --- a/docs/operations/engineering-foundation-verification.md +++ b/docs/operations/engineering-foundation-verification.md @@ -12,13 +12,13 @@ environment limits that remain explicit. | Area | Command | Result | |---|---|---| -| JavaScript workspace | `corepack pnpm repo:check` | Pass; formatting, lint, typecheck, requirements, contracts, repository tests, and 19 Turborepo test tasks passed. | +| JavaScript workspace | `corepack pnpm repo:check` | Pass; formatting, lint, typecheck, requirements, contracts, infrastructure policy, repository tests, and 19 Turborepo test tasks passed. | | JavaScript builds | `corepack pnpm repo:build` | Pass; API, Web, Desktop, shared packages, and engine package build completed. | | Contracts | `corepack pnpm --filter @databreeze/contracts contract:check` | Pass; generated models, compatibility baseline, and 28-case TypeScript/Python/Kotlin parity passed. | | Engine | `uv sync --locked --offline`; engine test/lint/typecheck/build through the root gates | Pass; locked Python 3.13 environment and package build verified. | | Android | `apps/android/gradlew :app:testDebugUnitTest --no-daemon` | Pass; 27 Gradle tasks completed. | | Brand | `corepack pnpm brand:check` and design-token build checks | Pass; canonical legacy asset checksums and derivatives remain unchanged. | -| AWS static validation | `corepack pnpm infra:check` | Pass; static checks passed and no infrastructure was applied. | +| AWS/static infrastructure | `corepack pnpm infra:check`; `node tools/repo-cli/src/local-services.mjs config` | Pass; static AWS checks and daemon-free local Compose validation passed; no infrastructure or containers were applied. | The contract drift check ignores only `uv`/Hatch editable-install products under `generated/python/build` and `generated/python/*.egg-info`; a regression @@ -31,8 +31,10 @@ products are never committed. executed locally. CI must run them with the pinned OpenTofu release before an infrastructure change is approved. - Docker was not required for the repository gates and live local dependency - startup was not claimed. Run `corepack pnpm local:smoke` on a machine with a - healthy Docker daemon before dogfood acceptance. + startup was not claimed. Run `corepack pnpm local:services check` and then + `corepack pnpm local:services restart-check` on a machine with a healthy + Docker daemon before dogfood acceptance. See + `foundation-local-infrastructure-2026-08-02.md` for the FND-003 boundary. - Android instrumentation/emulator and signed release packaging are separate production-readiness gates; the shell unit test is the evidence recorded here. diff --git a/docs/operations/foundation-batch-handoff-2026-08-02.md b/docs/operations/foundation-batch-handoff-2026-08-02.md new file mode 100644 index 00000000..13946ceb --- /dev/null +++ b/docs/operations/foundation-batch-handoff-2026-08-02.md @@ -0,0 +1,78 @@ +# Foundation batch handoff + +Observed at (UTC): 2026-08-02 +Canonical repository: `databreeze-platform` +Branch: `feat/fnd003-local-infra-batch` +Base: `origin/dev` at `92b3e9a4d581f3a6947b7a2bf58c8334f4ae0c18` +Promotion base: `origin/main` at `a2fcba34037c1ffd77816be16be75453abfb16fa` + +## Active delivery boundary + +- FND-003 local infrastructure is `in-progress`. Static Compose, bootstrap, + lifecycle, port, Docker-diagnostic, disk-preflight, and daemon-free config + checks are committed. +- Live Docker health, port-collision simulation, disk-pressure threshold, and + restart-persistence evidence remain environment-gated because the Docker + daemon is unavailable on the current machine. +- The batch also hardens AWS static checks, content-safe telemetry handling, + infrastructure path-aware CI, and the updated 30–70 commit policy. These + changes are not marked as verified foundation tasks until their own gates are + recorded. + +## Verification already run + +- Local infrastructure, AWS infrastructure, CI policy, change-scope, telemetry, + orchestration, and diff checks pass in their scoped commands. +- `node tools/repo-cli/src/local-services.mjs config` passes without a Docker + daemon; `check` fails closed with a clear daemon-unavailable reason. +- Android lifecycle review fixes were promoted through PRs #17 and #18; the + existing PR #14 promotion was merged to `main` after hosted checks and its + single CodeRabbit review/incremental dispositions. + +## Git batching rule + +This is a normal feature batch. Keep atomic commits and do not open the feature +PR until the branch reaches at least 30 commits, targeting approximately 70 and +never reaching 100. The only small-PR exceptions are focused promotion-review +fixes required to close an already-open `dev`→`main` gate. + +At this checkpoint the branch is 72 commits ahead of `origin/dev`. The current +boundary is still coherent: FND-003 local lifecycle hardening is accompanied by +portable AWS safety, telemetry, CI/supply-chain, and evidence updates. Continue +with scoped foundation work until the final handoff boundary; do not manufacture +empty commits or open a small feature PR merely to reset the count. + +## Final scoped verification before the feature PR + +Passed from this worktree: + +- `corepack pnpm format:check` +- `corepack pnpm lint` +- `corepack pnpm typecheck` +- `corepack pnpm contracts:check` +- `corepack pnpm orchestration:check` +- `corepack pnpm requirements:check` +- `corepack pnpm test` (67 repository CLI tests plus all workspace suites) +- `corepack pnpm repo:build` (API, Web, Desktop, shared packages, and engine) +- `uv run --locked pytest`, Ruff, format, and mypy (91 engine tests) +- `apps/android/gradlew.bat :app:testDebugUnitTest --offline --no-daemon` + +Environment-gated and intentionally not claimed as verified: + +- OpenTofu format/init/validate because OpenTofu is not installed locally. +- Live Docker startup, health, port-collision, disk-pressure, Redis + persistence, and restart checks because the Docker daemon is unavailable. +- Android instrumentation/emulator and signed release packaging. + +## Safest next command + +```powershell +git status --short --branch +git fetch origin dev main +node tools/repo-cli/src/check-execution-orchestration.mjs +node --test tools/repo-cli/test/**/*.test.mjs +``` + +After the normal batch reaches its commit boundary, push this branch, open one +PR to `dev` without CodeRabbit, merge after hosted checks, then create the +separate `dev`→`main` promotion PR and invoke CodeRabbit exactly once there. diff --git a/docs/operations/foundation-ci-supply-chain-2026-08-02.md b/docs/operations/foundation-ci-supply-chain-2026-08-02.md new file mode 100644 index 00000000..f62c5d2b --- /dev/null +++ b/docs/operations/foundation-ci-supply-chain-2026-08-02.md @@ -0,0 +1,51 @@ +# FND-006 CI and supply-chain evidence + +Observed at (UTC): 2026-08-02 +Branch: `feat/fnd003-local-infra-batch` +Task: `FND-006 — Close CI and supply-chain gaps` + +## Implemented boundaries + +- Path-aware scope detection treats infrastructure, tooling, contracts, plans, + and workflows as shared quality-gate inputs. +- Every required workflow uses SHA-pinned actions, least-privilege top-level + permissions, discarded checkout credentials, bounded runner timeouts, and + no `pull_request_target` or long-lived AWS credentials. +- Artifact uploads fail when an expected output is missing. +- SBOM and provenance generators reject malformed output arguments; provenance + fails closed when a declared artifact is missing and records sorted SHA-256 + subjects. +- Container image, secret-pattern, license, SBOM, and provenance checks remain + non-deploying repository operations. + +## Verification + +Passed: + +- `node --test tools/repo-cli/test/ci-policy.test.mjs` +- `node --test tools/repo-cli/test/provenance.test.mjs` +- `node --test tools/repo-cli/test/sbom.test.mjs` +- `node tools/repo-cli/src/check-ci-policy.mjs` +- `node tools/repo-cli/src/check-container-policy.mjs` +- `node tools/repo-cli/src/check-secret-patterns.mjs` +- `node tools/repo-cli/src/check-license-policy.mjs` +- `node tools/repo-cli/src/generate-sbom.mjs --output ` +- `node tools/repo-cli/src/generate-provenance.mjs --output --artifact ` +- `git diff --check` + +Hosted CI remains authoritative for the complete dependency, SAST, container, +OpenTofu, build, and release-environment gates. The generators never write +runtime evidence inside the repository during these checks. + +The `release` environment's required reviewers and branch restrictions are +GitHub repository settings rather than workflow YAML. Before promoting to +`main`, an administrator must verify those settings and record the check in +the release evidence; the repository policy checker deliberately verifies the +workflow's environment reference but cannot infer external protection rules. + +## Rollback + +Revert the focused CI or generator commit that introduced the behavior, rerun +the scoped tests and `pnpm ci:policy`, then record the resulting gap before +merging. No cloud resource, credential, or customer data is changed by these +checks. diff --git a/docs/operations/foundation-local-infrastructure-2026-08-02.md b/docs/operations/foundation-local-infrastructure-2026-08-02.md new file mode 100644 index 00000000..90cf4adc --- /dev/null +++ b/docs/operations/foundation-local-infrastructure-2026-08-02.md @@ -0,0 +1,56 @@ +# FND-003 local infrastructure evidence + +Observed at (UTC): 2026-08-02 +Branch: `feat/fnd003-local-infra-batch` +Task: `FND-003 — Close local infrastructure gaps` + +## Implemented boundaries + +- PostgreSQL 17.5, Redis 7.4.5, MinIO, Mailpit, and the OpenTelemetry collector + remain pinned in `infrastructure/local/compose.yml`. +- PostgreSQL initialization creates every module-owned schema and contains no + credentials, roles, or secret literals. +- `tools/repo-cli/src/local-services.mjs` provides daemon-free `config` and + `preflight` commands plus `check`, `start`, `stop`, `reset`, + `restart-check`, `status`, bounded read-only `logs`, and legacy `smoke` + commands. +- Lifecycle commands preserve named volumes. `reset` uses Compose + `down --remove-orphans` without `--volumes`; data deletion is never implicit. +- Preflight reports missing Docker CLI/daemon, host port collisions, and + insufficient free disk space before starting containers. +- `restart-check` restarts the running stack and waits for every service health + check, providing the entry point for persistence evidence. +- `persistence-check` writes a five-minute Redis sentinel, restarts only Redis, + verifies the sentinel, and deletes it; it never flushes a database or volume. +- The documented lifecycle command set includes `config`, `preflight`, `check`, + `start`, `stop`, `reset`, `restart-check`, `persistence-check`, `status`, + `logs`, and the legacy `smoke` entry point. + +## Verification + +Passed: + +- `node --test tools/repo-cli/test/local-infrastructure.test.mjs` +- `node tools/repo-cli/src/local-services.mjs --help` +- `node tools/repo-cli/src/local-services-smoke.mjs --help` +- `node tools/repo-cli/src/local-services.mjs config` +- `node tools/repo-cli/src/local-services.mjs preflight --min-free-gib=0` +- `git diff --check` + +Environment-gated: + +- `node tools/repo-cli/src/local-services.mjs check` fails closed with + `Docker daemon is unavailable` because no Docker daemon is running here. +- The daemon-free `preflight` command completes Compose, port, and disk checks + without starting containers; the evidence run used a zero-GiB threshold so + it remains independent of the workstation's available disk headroom. +- Live `compose up`, health polling, port-collision simulation, disk-pressure + threshold validation, and restart-persistence checks (including + `persistence-check`) must run on a machine with Docker Desktop/Compose v2 + before FND-003 can become `verified`. + +## Rollback + +Revert the lifecycle commit and retain the prior static Compose checks. No +containers, named volumes, host files, or credentials are modified by the +repository changes. diff --git a/docs/operations/foundation-reconciliation-2026-08-02.md b/docs/operations/foundation-reconciliation-2026-08-02.md index 0f393ff2..e223229a 100644 --- a/docs/operations/foundation-reconciliation-2026-08-02.md +++ b/docs/operations/foundation-reconciliation-2026-08-02.md @@ -33,7 +33,7 @@ This record was created from the current merged `dev` checkpoint, not from an ea | Task 15 | reconciled | Electron main/preload/renderer shell, context isolation, CSP/navigation policy, and IPC boundary tests pass. | | Task 16 | reconciled | Python `uv` project, typed action registry, framed entry points, deterministic processor, and engine checks/build pass. | | Task 17 | reconciled | Kotlin/Compose shell, generated resources, Room/WorkManager/Keystore ports, network security, backup policy, and debug unit tests pass. | -| Task 18 | reconciled | PostgreSQL/Redis/MinIO/Mailpit/OpenTelemetry Compose definitions, health checks, and static readiness checks pass; live Docker startup is separately environment-gated. | +| Task 18 | reconciled-with-limit | PostgreSQL/Redis/MinIO/Mailpit/OpenTelemetry Compose definitions, health checks, explicit network/volume isolation, safe lifecycle commands, port/disk/Docker preflight, and daemon-free config checks pass; live Docker startup/restart persistence is separately environment-gated in FND-003. | | Task 19 | reconciled-with-limit | AWS OpenTofu modules, alpha composition, encryption/private-network/OIDC source checks, and non-applying infrastructure check pass; OpenTofu itself is not installed locally, so format/validate remain hosted-check obligations. | | Task 20 | reconciled | Content-safe telemetry package, correlation propagation, redaction allowlists, and hostile-attribute tests are present and covered by repository checks. | | Task 21 | reconciled | Path-aware quality/security/release workflows, pinned actions, least-privilege permissions, SBOM/provenance, and scan definitions are present and hosted checks pass. | @@ -59,7 +59,7 @@ The repository checks include generated-contract drift, brand checksum/derivativ ## Known environment limits - OpenTofu is not installed on this workstation. No AWS infrastructure was applied; hosted CI must run the pinned OpenTofu format/validate checks before an infrastructure PR is accepted. -- Docker is installed but live local dependency startup was not required for this reconciliation. Run `corepack pnpm local:smoke` on a healthy Docker daemon before dogfood acceptance. +- Docker is installed but its daemon is unavailable for this reconciliation. Run `corepack pnpm local:services check`, `start`, and `restart-check` on a healthy Docker daemon before dogfood acceptance. - Android instrumentation/emulator testing and signed release packaging remain Plan 400 gates; the debug unit suite passed with the local SDK path supplied through `ANDROID_HOME`/`ANDROID_SDK_ROOT`. - No customer data or credentials were used. Ignored dependency caches, virtual environments, build output, and Gradle state remain untracked. diff --git a/docs/operations/telemetry-safety.md b/docs/operations/telemetry-safety.md new file mode 100644 index 00000000..0d42b961 --- /dev/null +++ b/docs/operations/telemetry-safety.md @@ -0,0 +1,33 @@ +# Content-safe telemetry boundary + +DataBreeze telemetry is operational metadata, not a source-data transport. +Every runtime emits the versioned `@databreeze/telemetry/v1` shape and the +Python/Kotlin mirrors are checked against its allowlist. + +## Allowed data + +- correlation, trace, workspace, job, artifact, dataset, and device IDs +- bounded route/operation/outcome/reason/provider tokens +- status, duration, queue, retry, item, byte, and redaction counters +- an explicit sampling boolean + +## Prohibited data + +Paths, filenames, source values, formulas, document text, previews, evidence +snippets, questions/prompts, transcripts, contact data, secrets, tokens, +provider causes, and raw exception messages never enter ordinary telemetry. +Unknown attributes are dropped. Strict assertion helpers reject unsafe records +at adapter boundaries. JavaScript sanitization reads only own data properties, +so accessor-backed diagnostics cannot execute arbitrary getters during logging. + +## Failure behavior + +Malformed correlation or trace headers fail closed. Ambiguous duplicate headers +are rejected, including accessor-backed or non-string header values; hostile +header mappings fail with a generic unreadable-header reason. Invalid or +oversized values are omitted by the permissive sanitizer and rejected by strict mode. Providers and exporters remain +replaceable; a collector outage cannot become domain authority or block durable +jobs and audit writes. + +See `packages/telemetry/README.md` and the TypeScript/Python/Android parity +tests before adding an attribute or event. diff --git a/docs/plans/000-platform-program.md b/docs/plans/000-platform-program.md index 2811f164..b86015c7 100644 --- a/docs/plans/000-platform-program.md +++ b/docs/plans/000-platform-program.md @@ -48,7 +48,7 @@ docs/plans/requirement-traceability.json records all 611 IDs with requested trac - `main` contains stable releases. `dev` is the integration branch. - New capabilities use `feat/`; corrections use `fix/`; operational and documentation work use conventional prefixes when more accurate. - Commit one coherent tested unit at a time. Do not combine unrelated applications or domains merely to reduce commit count. -- Feature/fix pull requests target `dev`, preferably contain 30–50 commits, and must not exceed 60 commits. Run local and hosted checks, but do not invoke CodeRabbit on these integration pull requests. +- Feature/fix pull requests target `dev`, normally contain 30–70 commits, and must remain under 100 commits. Run local and hosted checks, but do not invoke CodeRabbit on these integration pull requests. Focused review-gate fixes are the only intentional small-PR exception. - Merge a green integration pull request while preserving its atomic rollback commits, then immediately open a separate `dev` → `main` promotion pull request for that completed batch. - Invoke CodeRabbit exactly once on the otherwise-ready promotion pull request and request a full review. Do not invoke it if the promotion exceeds 280 changed files; split or revert the batch first. - Reproduce every CodeRabbit claim against the reviewed commit. Fix valid findings in focused commits, document rejected findings with evidence, rerun all affected gates, and never request a second CodeRabbit review on that pull request. diff --git a/docs/plans/003-luna-handoff-runbook.md b/docs/plans/003-luna-handoff-runbook.md index b9b3c579..8d112214 100644 --- a/docs/plans/003-luna-handoff-runbook.md +++ b/docs/plans/003-luna-handoff-runbook.md @@ -93,7 +93,7 @@ For each `#### TASK-ID —` item in `002-complete-execution-orchestration.md`: ## Pull-request and CodeRabbit protocol -1. Branch from current `origin/dev` using `feat/` or `fix/`. Keep atomic commits; preferred PR size is 30–50 commits and hard maximum is 60. Cut earlier at a coherent boundary when risk, dependency ordering, or reviewability warrants it. +1. Branch from current `origin/dev` using `feat/` or `fix/`. Keep atomic commits; preferred PR size is 30–70 commits and hard maximum is 99. Cut earlier only for a coherent boundary, a mandatory promotion-gate fix, or a safety constraint. 2. Before a feature PR, ensure it targets `dev`, has no unrelated commits, and passes local gates. Open it with requirement/task/evidence/rollback notes. CodeRabbit must not be invoked on this PR. 3. Wait for hosted checks. Diagnose failures; do not merge red or missing required checks. Merge with history that preserves the atomic rollback units, normally `--no-ff`/merge commit rather than squash. 4. Immediately compare `dev` and `main`. If `dev` contains the reviewed batch and no incompatible promotion is open, create `dev` → `main`. If the diff exceeds 280 changed files, do not invoke CodeRabbit: split/revert the feature batch or request user direction first. @@ -171,7 +171,7 @@ Live verified checkpoint: branch [BRANCH], HEAD [HEAD], origin/dev [DEV], origin Resume task [TASK_ID] only after proving its dependency/entry gate. Follow test-first atomic delivery: canonical contracts when the interface changes, failing domain/state tests, PostgreSQL migration/tenant/transaction/outbox tests when durable state changes, implementation through ports, vertical client/adapter coverage when the task involves client behavior, safe telemetry/recovery, traceability evidence, scoped checks, repo:check, repo:build, diff review, and one reversible commit. For documentation-only or other non-durable/non-client tasks, record why those conditional tests do not apply. Do not mark merged code verified without all evidence. -Git flow is fixed: feat/* or fix/* → PR to dev with hosted checks and no CodeRabbit; merge preserving atomic commits; immediately open dev→main; request exactly one CodeRabbit full review there; reproduce every comment, fix only valid findings, document rejected ones, never request a second review on that PR. Prefer 30–50 commits, hard cap 60, and do not invoke the promotion review over 280 changed files. +Git flow is fixed: feat/* or fix/* → PR to dev with hosted checks and no CodeRabbit; merge preserving atomic commits; immediately open dev→main; request exactly one CodeRabbit full review there; reproduce every comment, fix only valid findings, document rejected ones, never request a second review on that PR. Prefer 30–70 commits, hard cap 99, and do not invoke the promotion review over 280 changed files. Keep PostgreSQL authoritative, Redis ephemeral, tenant scope explicit, originals/versions immutable, Hybrid default, Local content/path out of cloud, workers without database credentials, Desktop/sidecar allowlisted, Android background payloads content-free, Vietnamese and English complete, and external providers replaceable. Stop and record state for destructive migration risk, unknown overlapping changes, privacy/security fail-open behavior, missing production/signing authority, or a spec conflict. End with the exact handoff record from the runbook. ``` diff --git a/docs/plans/010-engineering-foundation.md b/docs/plans/010-engineering-foundation.md index 7cfa17bb..9dc207e5 100644 --- a/docs/plans/010-engineering-foundation.md +++ b/docs/plans/010-engineering-foundation.md @@ -91,7 +91,7 @@ Create the Gradle wrapper/version catalog and Kotlin/Compose application with bi ### Task 18: Local development infrastructure -Create Docker Compose definitions for PostgreSQL 17, Redis 7.4, MinIO, Mailpit, and an OpenTelemetry collector. Add health checks, named development volumes, `.env.example`, initialization scripts without credentials, and a smoke script that validates readiness. Commit as `feat(infra): add portable local dependencies`. +Create Docker Compose definitions for PostgreSQL 17, Redis 7.4, MinIO, Mailpit, and an OpenTelemetry collector. Add health checks, an isolated project network, named development volumes, `.env.example`, initialization scripts without credentials, and lifecycle commands for daemon-free config validation, safe start/stop/reset, port and disk preflight, status, and restart-persistence checks. Commit as `feat(infra): add portable local dependencies`. ### Task 19: AWS OpenTofu foundation diff --git a/docs/plans/execution-orchestration.json b/docs/plans/execution-orchestration.json index 0dcad477..84f853b2 100644 --- a/docs/plans/execution-orchestration.json +++ b/docs/plans/execution-orchestration.json @@ -15,11 +15,11 @@ } }, "checkpoint": { - "observedAt": "2026-08-02T10:08:12Z", - "remoteDev": "4d415494240abbc610574132678007a623c405e4", - "remoteMain": "d26e6be16ecadc07467b458b853eb8070940e846", - "lastFeaturePullRequest": 13, - "lastPromotionPullRequest": 11, + "observedAt": "2026-08-02T12:00:00Z", + "remoteDev": "92b3e9a4d581f3a6947b7a2bf58c8334f4ae0c18", + "remoteMain": "a2fcba34037c1ffd77816be16be75453abfb16fa", + "lastFeaturePullRequest": 18, + "lastPromotionPullRequest": 14, "openPullRequestsObserved": 0, "note": "Historical observation only; every session must fetch and recompute current state." }, @@ -38,8 +38,8 @@ }, "commitBudget": { "preferredMinimum": 30, - "preferredMaximum": 50, - "hardMaximum": 60 + "preferredMaximum": 70, + "hardMaximum": 99 } }, "statusVocabulary": [ @@ -77,6 +77,19 @@ "tools/repo-cli/test/android-shell.test.mjs" ], "note": "Room, WorkManager, Keystore, bilingual resources, generated contracts/tokens, backup/network policy, account isolation, and process-recreation evidence are complete. No product requirement status was promoted." + }, + "FND-003": { + "status": "in-progress", + "commit": "d9598a2", + "evidence": [ + "infrastructure/local/compose.yml", + "infrastructure/local/README.md", + "infrastructure/local/postgres/init/001-create-module-schemas.sql", + "tools/repo-cli/src/local-services.mjs", + "tools/repo-cli/test/local-infrastructure.test.mjs", + "docs/operations/foundation-local-infrastructure-2026-08-02.md" + ], + "note": "Static Compose/bootstrap/lifecycle checks, daemon-free config/preflight, bounded local diagnostics/log retention, AWS safety checks, telemetry redaction tests, and infrastructure path-aware CI pass. Live Docker health, port-collision, disk-pressure, and restart-persistence evidence remains pending because the Docker daemon is unavailable in this environment." } }, "plans": [ diff --git a/infrastructure/aws/README.md b/infrastructure/aws/README.md index ba101ea8..aa816b61 100644 --- a/infrastructure/aws/README.md +++ b/infrastructure/aws/README.md @@ -39,6 +39,22 @@ redundant API/worker capacity, PITR/backups, Multi-AZ data, NAT, and CloudFront explicitly. Production must use a remote encrypted state backend approved in a separate deployment ADR. +The production profile also enables RDS Performance Insights with the platform +KMS key; alpha keeps it disabled unless explicitly selected. + +Before a production plan, set `api_image` and `worker_image` to immutable +registry references ending in a 64-character SHA-256 digest (for example, +`ghcr.io/example/databreeze-api@sha256:`). Mutable tags are accepted +for alpha development only; the ECS task definitions have a production +precondition that rejects them. + The modules expose IDs and endpoints only as outputs. Database credentials are never output; the security module creates named Secrets Manager records for later provider-managed rotation. + +`pnpm infra:check` is intentionally non-applying. It checks module presence, +Singapore region defaults, private-network and encryption boundaries, OIDC +subject scoping, production recovery preconditions, and then runs OpenTofu +format/initialization/validation when the pinned tool is installed. Missing +OpenTofu is reported as an explicit environment gate rather than silently +treated as a production validation pass. diff --git a/infrastructure/aws/environments/alpha/README.md b/infrastructure/aws/environments/alpha/README.md index 74e33421..58c4f750 100644 --- a/infrastructure/aws/environments/alpha/README.md +++ b/infrastructure/aws/environments/alpha/README.md @@ -10,6 +10,11 @@ Before planning, install the pinned OpenTofu release required by the root planning. RDS manages its master password through AWS Secrets Manager. Never commit `.terraform/`, `*.tfstate`, `terraform.tfvars`, or provider credentials. +Use `tofu init -backend=false` followed by `tofu validate` for a credential-free +syntax check. A real `tofu plan` requires an approved account, region, state +backend, and injected credentials; no apply or destroy command belongs in this +alpha workflow. + Production changes require a separate reviewed variable set enabling at least: - two availability zones with one NAT gateway per AZ; diff --git a/infrastructure/aws/environments/alpha/main.tf b/infrastructure/aws/environments/alpha/main.tf index 8dd3efc2..21062cfa 100644 --- a/infrastructure/aws/environments/alpha/main.tf +++ b/infrastructure/aws/environments/alpha/main.tf @@ -66,5 +66,7 @@ module "compute" { private_egress_enabled = var.enable_nat_gateway api_desired_count = var.api_desired_count worker_desired_count = var.worker_desired_count + worker_cpu = var.worker_cpu + worker_memory = var.worker_memory tags = local.tags } diff --git a/infrastructure/aws/environments/alpha/variables.tf b/infrastructure/aws/environments/alpha/variables.tf index 37080882..370cf456 100644 --- a/infrastructure/aws/environments/alpha/variables.tf +++ b/infrastructure/aws/environments/alpha/variables.tf @@ -108,3 +108,13 @@ variable "worker_desired_count" { type = number default = 1 } + +variable "worker_cpu" { + type = number + default = 1024 +} + +variable "worker_memory" { + type = number + default = 2048 +} diff --git a/infrastructure/aws/modules/compute/main.tf b/infrastructure/aws/modules/compute/main.tf index 756b5593..7e9c14cb 100644 --- a/infrastructure/aws/modules/compute/main.tf +++ b/infrastructure/aws/modules/compute/main.tf @@ -1,5 +1,12 @@ locals { common_tags = merge(var.tags, { Component = "compute" }) + allowed_worker_memory_by_cpu = { + "256" = [512, 1024, 2048] + "512" = [1024, 2048, 3072, 4096] + "1024" = [2048, 3072, 4096, 5120, 6144, 7168, 8192] + "2048" = [4096, 5120, 6144, 7168, 8192, 9216, 10240, 11264, 12288, 13312, 14336, 15360, 16384] + "4096" = [8192, 9216, 10240, 11264, 12288, 13312, 14336, 15360, 16384, 17408, 18432, 19456, 20480, 21504, 22528, 23552, 24576, 25600, 26624, 27648, 28672, 29696, 30720] + } } resource "aws_ecs_cluster" "this" { @@ -65,24 +72,15 @@ resource "aws_iam_role" "task" { tags = merge(local.common_tags, { Name = "databreeze-${var.name}-ecs-task" }) } -resource "aws_iam_role_policy" "task" { - name = "databreeze-${var.name}-ecs-task-minimal" - role = aws_iam_role.task.id - policy = jsonencode({ - Version = "2012-10-17" - Statement = [{ - Effect = "Allow" - Action = ["secretsmanager:GetSecretValue", "kms:Decrypt"] - Resource = [var.database_secret_arn, var.application_secret_arn, var.kms_key_arn] - }] - }) -} - locals { api_container = { name = "api" image = var.api_image essential = true + readonlyRootFilesystem = true + privileged = false + user = "10001" + stopTimeout = 30 cpu = var.api_cpu memory = var.api_memory portMappings = [{ @@ -119,20 +117,31 @@ resource "aws_ecs_task_definition" "api" { task_role_arn = aws_iam_role.task.arn container_definitions = jsonencode([local.api_container]) tags = merge(local.common_tags, { Name = "databreeze-${var.name}-api" }) + + lifecycle { + precondition { + condition = var.environment != "production" || can(regex("@sha256:[0-9a-f]{64}$", var.api_image)) + error_message = "Production API deployments must use an immutable image digest." + } + } } resource "aws_ecs_task_definition" "worker" { family = "databreeze-${var.name}-worker" requires_compatibilities = ["FARGATE"] network_mode = "awsvpc" - cpu = "1024" - memory = "2048" + cpu = tostring(var.worker_cpu) + memory = tostring(var.worker_memory) execution_role_arn = aws_iam_role.execution.arn task_role_arn = aws_iam_role.task.arn container_definitions = jsonencode([{ name = "worker" image = var.worker_image essential = true + readonlyRootFilesystem = true + privileged = false + user = "10001" + stopTimeout = 30 logConfiguration = { logDriver = "awslogs" options = { @@ -143,6 +152,20 @@ resource "aws_ecs_task_definition" "worker" { } }]) tags = merge(local.common_tags, { Name = "databreeze-${var.name}-worker" }) + + lifecycle { + precondition { + condition = var.environment != "production" || can(regex("@sha256:[0-9a-f]{64}$", var.worker_image)) + error_message = "Production worker deployments must use an immutable image digest." + } + precondition { + condition = contains( + lookup(local.allowed_worker_memory_by_cpu, tostring(var.worker_cpu), []), + var.worker_memory, + ) + error_message = "worker_memory must be an AWS-supported Fargate size for worker_cpu." + } + } } resource "aws_ecs_service" "api" { diff --git a/infrastructure/aws/modules/compute/variables.tf b/infrastructure/aws/modules/compute/variables.tf index dca538c3..e54afd04 100644 --- a/infrastructure/aws/modules/compute/variables.tf +++ b/infrastructure/aws/modules/compute/variables.tf @@ -76,6 +76,28 @@ variable "api_memory" { default = 1024 } +variable "worker_cpu" { + type = number + description = "Fargate task CPU units reserved for the worker pool." + default = 1024 + + validation { + condition = contains([256, 512, 1024, 2048, 4096], var.worker_cpu) + error_message = "worker_cpu must be a supported Fargate CPU size." + } +} + +variable "worker_memory" { + type = number + description = "Fargate task memory (MiB) reserved for the worker pool." + default = 2048 + + validation { + condition = var.worker_memory >= 512 && var.worker_memory <= 30720 + error_message = "worker_memory must be between 512 and 30720 MiB." + } +} + variable "tags" { type = map(string) default = {} diff --git a/infrastructure/aws/modules/data/main.tf b/infrastructure/aws/modules/data/main.tf index b8a12a6b..dc731da4 100644 --- a/infrastructure/aws/modules/data/main.tf +++ b/infrastructure/aws/modules/data/main.tf @@ -38,6 +38,8 @@ resource "aws_db_instance" "postgres" { final_snapshot_identifier = "databreeze-${var.name}-final" auto_minor_version_upgrade = true copy_tags_to_snapshot = true + performance_insights_enabled = var.environment == "production" + performance_insights_kms_key_id = var.environment == "production" ? var.kms_key_arn : null tags = merge(local.common_tags, { Name = "databreeze-${var.name}" }) } diff --git a/infrastructure/aws/modules/web/main.tf b/infrastructure/aws/modules/web/main.tf index 7cf3001d..6937f755 100644 --- a/infrastructure/aws/modules/web/main.tf +++ b/infrastructure/aws/modules/web/main.tf @@ -3,8 +3,9 @@ locals { } resource "aws_s3_bucket" "web" { - bucket = "databreeze-${var.name}-web" - tags = merge(local.common_tags, { Name = "databreeze-${var.name}-web" }) + bucket = "databreeze-${var.name}-web" + force_destroy = false + tags = merge(local.common_tags, { Name = "databreeze-${var.name}-web" }) } resource "aws_s3_bucket_ownership_controls" "web" { @@ -29,6 +30,27 @@ resource "aws_s3_bucket_versioning" "web" { } } +resource "aws_s3_bucket_lifecycle_configuration" "web" { + bucket = aws_s3_bucket.web.id + + rule { + id = "bounded-version-retention" + status = "Enabled" + + filter {} + + noncurrent_version_expiration { + noncurrent_days = 90 + } + + abort_incomplete_multipart_upload { + days_after_initiation = 7 + } + } + + depends_on = [aws_s3_bucket_versioning.web] +} + resource "aws_s3_bucket_server_side_encryption_configuration" "web" { bucket = aws_s3_bucket.web.id rule { diff --git a/infrastructure/local/.env.example b/infrastructure/local/.env.example index 92729a75..4d0f8617 100644 --- a/infrastructure/local/.env.example +++ b/infrastructure/local/.env.example @@ -2,6 +2,7 @@ # These values are intentionally local placeholders, not deployable secrets. COMPOSE_PROJECT_NAME=databreeze-local +DATABREEZE_MIN_FREE_GIB=5 POSTGRES_IMAGE=docker.io/library/postgres:17.5-alpine POSTGRES_DB=databreeze diff --git a/infrastructure/local/README.md b/infrastructure/local/README.md index 371b774e..a94eee8e 100644 --- a/infrastructure/local/README.md +++ b/infrastructure/local/README.md @@ -9,17 +9,41 @@ through adapters, so the same contracts work with managed services later. 1. Copy `.env.example` to `.env` and change the local-only values if needed. 2. Start Docker Desktop (or another Docker Engine with Compose v2). -3. Run `pnpm local:smoke -- --start` from the repository root. +3. Run `pnpm local:services start` from the repository root. 4. Open Mailpit at and MinIO Console at when you need to inspect local data. The stack is defined in [`compose.yml`](compose.yml). All state is held in named volumes prefixed by the Compose project name; no repository directory is mounted for database, object, or mail data. The volumes are disposable and are -not removed by the smoke script. Use `docker compose --env-file -infrastructure/local/.env -f infrastructure/local/compose.yml down` to stop -the containers. Remove the named volumes only when you explicitly want to -discard local state. +not removed by the lifecycle commands. Remove the named volumes only when you +explicitly want to discard local state. +Every published port is bound to `127.0.0.1`, so the development credentials and +data endpoints are not reachable from other hosts on the local network. +Container JSON logs are capped at 10 MiB per file with three retained files so +diagnostics cannot silently consume the host disk. + +## Lifecycle commands + +Run these from the repository root: + +| Command | Effect | +| --- | --- | +| `pnpm local:services config` | Validate Compose syntax without requiring a running Docker daemon. | +| `pnpm local:services preflight` | Validate Compose, host ports, and disk headroom without starting containers. | +| `pnpm local:services check` | Validate Compose, Docker, host ports, and free disk without starting anything. | +| `pnpm local:services start` | Run preflight, start the stack, and wait for every health check. | +| `pnpm local:services stop` | Stop containers while preserving containers and named volumes. | +| `pnpm local:services reset` | Recreate containers/networks while preserving named volumes; it never passes `--volumes`. | +| `pnpm local:services restart-check` | Restart the running stack and verify health after restart. | +| `pnpm local:services persistence-check` | Restart Redis and verify a disposable sentinel survives. | +| `pnpm local:services status` | Print container/health state without changing it. | +| `pnpm local:services logs --tail=100` | Print bounded, read-only logs for known local services. | + +The older `pnpm local:smoke -- --start` form remains supported. Port collisions +can be resolved by copying `.env.example` to `.env` and changing the host port +variables. `check` fails closed when Docker is missing, the daemon is stopped, +or free space is below the configured threshold (`--min-free-gib=N`). ## Services @@ -44,10 +68,13 @@ package manager while still making readiness observable. - These images and credentials are for local development. Never copy `.env` into a deployment or commit it. - The Compose health checks are the readiness contract for local consumers. - `pnpm local:smoke` validates the Compose file and reports the first unhealthy - service. Add `--start` to bring the stack up before polling. -- If a previous run left a stopped container, rerun the smoke command; it is - idempotent and does not delete volumes. + `pnpm local:services status` reports the current health and + `pnpm local:services restart-check` verifies restart persistence. +- `pnpm local:services logs --service=postgres --tail=100` is read-only and + accepts only known service names. Logs are local diagnostics; review them + before sharing because provider messages can still contain development data. +- If a previous run left a stopped container, rerun `pnpm local:services start`; + it is idempotent and does not delete volumes. - If Docker is unavailable, the static infrastructure tests still validate the service definitions, image release lines, volume names, and credential-free initialization files. diff --git a/infrastructure/local/compose.yml b/infrastructure/local/compose.yml index 00868cea..85fd97de 100644 --- a/infrastructure/local/compose.yml +++ b/infrastructure/local/compose.yml @@ -1,14 +1,23 @@ name: ${COMPOSE_PROJECT_NAME:-databreeze-local} +x-default-logging: &default-logging + driver: json-file + options: + max-size: 10m + max-file: '3' + services: postgres: image: ${POSTGRES_IMAGE:-docker.io/library/postgres:17.5-alpine} + init: true environment: POSTGRES_DB: ${POSTGRES_DB:-databreeze} POSTGRES_USER: ${POSTGRES_USER:-databreeze} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-databreeze-local-change-me} ports: - - '${POSTGRES_PORT:-5432}:5432' + - '127.0.0.1:${POSTGRES_PORT:-5432}:5432' + networks: [local] + logging: *default-logging volumes: - postgres-data:/var/lib/postgresql/data - ./postgres/init:/docker-entrypoint-initdb.d:ro @@ -24,9 +33,12 @@ services: redis: image: ${REDIS_IMAGE:-docker.io/library/redis:7.4.5-alpine} + init: true command: ['redis-server', '--appendonly', 'yes'] ports: - - '${REDIS_PORT:-6379}:6379' + - '127.0.0.1:${REDIS_PORT:-6379}:6379' + networks: [local] + logging: *default-logging volumes: - redis-data:/data healthcheck: @@ -38,13 +50,16 @@ services: minio: image: ${MINIO_IMAGE:-quay.io/minio/minio:RELEASE.2025-06-13T11-33-47Z} + init: true command: ['server', '/data', '--console-address', ':9001'] environment: MINIO_ROOT_USER: ${MINIO_ROOT_USER:-databreeze} MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-databreeze-local-change-me} ports: - - '${MINIO_API_PORT:-9000}:9000' - - '${MINIO_CONSOLE_PORT:-9001}:9001' + - '127.0.0.1:${MINIO_API_PORT:-9000}:9000' + - '127.0.0.1:${MINIO_CONSOLE_PORT:-9001}:9001' + networks: [local] + logging: *default-logging volumes: - minio-data:/data healthcheck: @@ -57,9 +72,12 @@ services: minio-init: image: ${MINIO_MC_IMAGE:-quay.io/minio/mc:RELEASE.2025-06-13T11-33-47Z} + init: true depends_on: minio: condition: service_healthy + networks: [local] + logging: *default-logging environment: MINIO_ROOT_USER: ${MINIO_ROOT_USER:-databreeze} MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-databreeze-local-change-me} @@ -72,12 +90,15 @@ services: mailpit: image: ${MAILPIT_IMAGE:-axllent/mailpit:v1.21.8} + init: true environment: MP_MAX_MESSAGES: ${MAILPIT_MAX_MESSAGES:-5000} MP_DATABASE: /data/mailpit.db ports: - - '${MAILPIT_SMTP_PORT:-1025}:1025' - - '${MAILPIT_UI_PORT:-8025}:8025' + - '127.0.0.1:${MAILPIT_SMTP_PORT:-1025}:1025' + - '127.0.0.1:${MAILPIT_UI_PORT:-8025}:8025' + networks: [local] + logging: *default-logging volumes: - mailpit-data:/data healthcheck: @@ -89,11 +110,14 @@ services: otel-collector: image: ${OTEL_IMAGE:-otel/opentelemetry-collector-contrib:0.128.0} + init: true command: ['--config=/etc/otelcol-contrib/config.yaml'] ports: - - '${OTEL_GRPC_PORT:-4317}:4317' - - '${OTEL_HTTP_PORT:-4318}:4318' - - '${OTEL_HEALTH_PORT:-13133}:13133' + - '127.0.0.1:${OTEL_GRPC_PORT:-4317}:4317' + - '127.0.0.1:${OTEL_HTTP_PORT:-4318}:4318' + - '127.0.0.1:${OTEL_HEALTH_PORT:-13133}:13133' + networks: [local] + logging: *default-logging volumes: - ./otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro # The collector image is intentionally distroless and has no shell or @@ -109,9 +133,12 @@ services: otel-collector-health: image: ${HEALTHCHECK_IMAGE:-curlimages/curl:8.14.1} + init: true depends_on: otel-collector: condition: service_started + networks: [local] + logging: *default-logging command: ['sh', '-c', "trap 'exit 0' TERM INT; while :; do sleep 3600; done"] healthcheck: test: [CMD, curl, --fail, --silent, http://otel-collector:13133/] @@ -130,3 +157,7 @@ volumes: name: ${COMPOSE_PROJECT_NAME:-databreeze-local}-minio-data mailpit-data: name: ${COMPOSE_PROJECT_NAME:-databreeze-local}-mailpit-data + +networks: + local: + name: ${COMPOSE_PROJECT_NAME:-databreeze-local}-network diff --git a/package.json b/package.json index b46e0aba..fb21c49a 100644 --- a/package.json +++ b/package.json @@ -27,11 +27,12 @@ "format:check": "prettier --check .", "lint": "eslint . && node tools/repo-cli/src/check-dependency-boundaries.mjs", "local:smoke": "node tools/repo-cli/src/local-services-smoke.mjs", + "local:services": "node tools/repo-cli/src/local-services.mjs", "infra:check": "node tools/repo-cli/src/check-aws-infrastructure.mjs", "orchestration:check": "node tools/repo-cli/src/check-execution-orchestration.mjs", "repo:bootstrap": "corepack pnpm install --frozen-lockfile", "repo:build": "corepack pnpm build", - "repo:check": "corepack pnpm format:check && corepack pnpm lint && corepack pnpm typecheck && corepack pnpm orchestration:check && corepack pnpm requirements:check && corepack pnpm contracts:check && corepack pnpm test", + "repo:check": "corepack pnpm format:check && corepack pnpm lint && corepack pnpm typecheck && corepack pnpm orchestration:check && corepack pnpm requirements:check && corepack pnpm contracts:check && corepack pnpm infra:check && corepack pnpm test", "repo:dev": "turbo run dev --parallel", "repo:test": "corepack pnpm test", "requirements:check": "node tools/repo-cli/src/generate-requirement-index.mjs --check", @@ -50,6 +51,7 @@ "prettier": "3.6.2", "turbo": "2.5.6", "typescript": "5.9.2", - "typescript-eslint": "8.43.0" + "typescript-eslint": "8.43.0", + "yaml": "2.8.1" } } diff --git a/packages/telemetry/README.md b/packages/telemetry/README.md index 192da062..3f2157c1 100644 --- a/packages/telemetry/README.md +++ b/packages/telemetry/README.md @@ -14,6 +14,11 @@ Unknown keys are dropped. Sensitive keys and values (tokens, secrets, paths, filenames, source values, prompts, evidence, contact data, and content) are rejected by the strict assertion helper and never serialized. +Sanitization reads only own data properties. Accessor-backed or hostile +diagnostic fields and correlation headers are ignored (or rejected by the +strict helper) without executing their getters. Header arrays must contain +strings; provider mapping failures become a generic unreadable-header error. + Use `createStructuredLoggerV1` at runtime boundaries and pass only the correlation headers produced by `correlationHeadersV1`. A logger sink receives one JSON object per event; it never receives a source payload or an exception diff --git a/packages/telemetry/src/v1.ts b/packages/telemetry/src/v1.ts index 95118273..e74ea2e2 100644 --- a/packages/telemetry/src/v1.ts +++ b/packages/telemetry/src/v1.ts @@ -166,12 +166,33 @@ function safeScalar(key: string, value: unknown): TelemetryScalarV1 | undefined return safeString(key, value); } +function readOwnDataEntries(input: Record): { + entries: Array<[string, unknown]>; + readable: boolean; +} { + const entries: Array<[string, unknown]> = []; + try { + for (const key of Object.keys(input)) { + const descriptor = Object.getOwnPropertyDescriptor(input, key); + if (!descriptor || !('value' in descriptor)) return { entries: [], readable: false }; + entries.push([key, descriptor.value]); + } + } catch { + return { entries: [], readable: false }; + } + return { entries, readable: true }; +} + +function ownDataEntries(input: Record): Array<[string, unknown]> { + return readOwnDataEntries(input).entries; +} + export function sanitizeTelemetryAttributesV1( input: Record, ): SafeTelemetryAttributesV1 { if (input === null || typeof input !== 'object' || Array.isArray(input)) return {}; const output: SafeTelemetryAttributesV1 = {}; - for (const [key, value] of Object.entries(input)) { + for (const [key, value] of ownDataEntries(input)) { assertBoundedKey(key); if (!safeAttributeSet.has(key)) continue; const scalar = safeScalar(key, value); @@ -183,12 +204,14 @@ export function sanitizeTelemetryAttributesV1( export function assertSafeTelemetryAttributesV1( input: Record, ): asserts input is SafeTelemetryAttributesV1 { - for (const key of Object.keys(input ?? {})) { + const readable = readOwnDataEntries(input ?? {}); + if (!readable.readable) throw new UnsafeTelemetryAttributeErrorV1('unreadable'); + for (const [key, value] of readable.entries) { assertBoundedKey(key); if (forbiddenKeyPattern.test(key) || !safeAttributeSet.has(key)) { throw new UnsafeTelemetryAttributeErrorV1(key); } - if (safeScalar(key, input[key]) === undefined) { + if (safeScalar(key, value) === undefined) { throw new UnsafeTelemetryAttributeErrorV1(key); } } @@ -249,10 +272,48 @@ function readSingleHeader( name: string, ): string | undefined { const values: string[] = []; - for (const [key, value] of Object.entries(headers)) { + let keys: string[]; + try { + keys = Object.keys(headers); + } catch { + throw new Error(`Unreadable telemetry ${name} header`); + } + for (const key of keys) { + let descriptor: PropertyDescriptor | undefined; + try { + descriptor = Object.getOwnPropertyDescriptor(headers, key); + } catch { + throw new Error(`Unreadable telemetry ${name} header`); + } + if (!descriptor || !('value' in descriptor)) + throw new Error(`Unreadable telemetry ${name} header`); + const value = descriptor.value as string | string[] | undefined; if (key.toLowerCase() !== name) continue; - if (Array.isArray(value)) values.push(...value); - else if (value !== undefined) values.push(value); + let arrayValue = false; + try { + arrayValue = Array.isArray(value); + } catch { + throw new Error(`Unreadable telemetry ${name} header`); + } + if (arrayValue) { + let valid = false; + try { + valid = Array.prototype.every.call(value, (item: unknown) => typeof item === 'string'); + } catch { + throw new Error(`Unreadable telemetry ${name} header`); + } + if (!valid) { + throw new Error(`Unreadable telemetry ${name} header`); + } + try { + values.push(...(value as string[])); + } catch { + throw new Error(`Unreadable telemetry ${name} header`); + } + } else if (value !== undefined) { + if (typeof value !== 'string') throw new Error(`Unreadable telemetry ${name} header`); + values.push(value); + } } if (values.length > 1) throw new Error(`Ambiguous telemetry ${name} header`); if (values.length === 0) return undefined; diff --git a/packages/telemetry/test/telemetry-v1.test.mjs b/packages/telemetry/test/telemetry-v1.test.mjs index 3ff031d3..d4a53d02 100644 --- a/packages/telemetry/test/telemetry-v1.test.mjs +++ b/packages/telemetry/test/telemetry-v1.test.mjs @@ -55,6 +55,75 @@ test('strict assertions reject secrets, paths, content, and unbounded values', ( ); }); +test('telemetry never executes accessor-backed attributes', () => { + let accessed = false; + const hostile = {}; + Object.defineProperty(hostile, 'outcome', { + enumerable: true, + get() { + accessed = true; + throw new Error('hostile getter'); + }, + }); + + assert.deepEqual(sanitizeTelemetryAttributesV1(hostile), {}); + assert.throws(() => assertSafeTelemetryAttributesV1(hostile), UnsafeTelemetryAttributeErrorV1); + assert.equal(accessed, false); +}); + +test('telemetry never executes accessor-backed correlation headers', () => { + let accessed = false; + const hostile = {}; + Object.defineProperty(hostile, 'x-correlation-id', { + enumerable: true, + get() { + accessed = true; + throw new Error('hostile header getter'); + }, + }); + assert.throws(() => correlationFromHeadersV1(hostile), /Unreadable telemetry/u); + assert.equal(accessed, false); +}); + +test('telemetry rejects proxies that fail during reflection without exposing trap errors', () => { + const hostileAttributes = new Proxy( + {}, + { + ownKeys() { + throw new Error('attribute trap cause'); + }, + }, + ); + assert.deepEqual(sanitizeTelemetryAttributesV1(hostileAttributes), {}); + assert.throws( + () => assertSafeTelemetryAttributesV1(hostileAttributes), + (error) => { + assert.ok(error instanceof UnsafeTelemetryAttributeErrorV1); + assert.equal(error.key, 'unreadable'); + assert.equal(error.message, 'Telemetry attribute is not allowed: unreadable'); + assert.doesNotMatch(error.message, /attribute trap cause/u); + return true; + }, + ); + + const hostileHeaders = new Proxy( + {}, + { + ownKeys() { + throw new Error('header trap cause'); + }, + }, + ); + assert.throws( + () => correlationFromHeadersV1(hostileHeaders), + (error) => { + assert.equal(error.message, 'Unreadable telemetry x-correlation-id header'); + assert.doesNotMatch(error.message, /header trap cause/u); + return true; + }, + ); +}); + test('correlation headers round-trip without accepting malformed identifiers', () => { const context = createCorrelationContextV1({ correlationId, @@ -88,6 +157,13 @@ test('correlation headers round-trip without accepting malformed identifiers', ( }), ); assert.throws(() => correlationFromHeadersV1({})); + assert.throws( + () => correlationFromHeadersV1({ 'x-correlation-id': 1 }), + (error) => { + assert.equal(error.message, 'Unreadable telemetry x-correlation-id header'); + return true; + }, + ); assert.throws(() => correlationFromHeadersV1({ 'x-correlation-id': [correlationId, correlationId] }), ); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b99d5e1e..43cd5809 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -34,6 +34,9 @@ importers: typescript-eslint: specifier: 8.43.0 version: 8.43.0(eslint@9.36.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@5.9.2) + yaml: + specifier: 2.8.1 + version: 2.8.1 apps/desktop: dependencies: @@ -67,7 +70,7 @@ importers: version: 19.2.4(@types/react@19.2.18) '@vitejs/plugin-react': specifier: 6.0.5 - version: 6.0.5(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)) + version: 6.0.5(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1)) electron: specifier: 43.2.0 version: 43.2.0(supports-color@7.2.0) @@ -79,10 +82,10 @@ importers: version: 5.9.2 vite: specifier: 8.2.0 - version: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1) + version: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1) vitest: specifier: 4.1.10 - version: 4.1.10(@types/node@24.13.3)(jsdom@30.0.1)(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)) + version: 4.1.10(@types/node@24.13.3)(jsdom@30.0.1)(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1)) apps/web: dependencies: @@ -119,7 +122,7 @@ importers: version: 1.62.1 '@tailwindcss/vite': specifier: 4.3.3 - version: 4.3.3(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)) + version: 4.3.3(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1)) '@testing-library/react': specifier: 16.3.0 version: 16.3.0(@testing-library/dom@10.4.1)(@types/react-dom@19.2.4(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -137,7 +140,7 @@ importers: version: 19.2.4(@types/react@19.2.18) '@vitejs/plugin-react': specifier: 6.0.5 - version: 6.0.5(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)) + version: 6.0.5(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1)) jsdom: specifier: 30.0.1 version: 30.0.1 @@ -149,10 +152,10 @@ importers: version: 5.9.2 vite: specifier: 8.2.0 - version: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1) + version: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1) vitest: specifier: 4.1.10 - version: 4.1.10(@types/node@24.13.3)(jsdom@30.0.1)(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)) + version: 4.1.10(@types/node@24.13.3)(jsdom@30.0.1)(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1)) packages/config: dependencies: @@ -3061,6 +3064,11 @@ packages: xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + yaml@2.8.1: + resolution: {integrity: sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==} + engines: {node: '>= 14.6'} + hasBin: true + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -3835,12 +3843,12 @@ snapshots: '@tailwindcss/oxide-win32-arm64-msvc': 4.3.3 '@tailwindcss/oxide-win32-x64-msvc': 4.3.3 - '@tailwindcss/vite@4.3.3(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))': + '@tailwindcss/vite@4.3.3(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1))': dependencies: '@tailwindcss/node': 4.3.3 '@tailwindcss/oxide': 4.3.3 tailwindcss: 4.3.3 - vite: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1) + vite: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1) '@tanstack/query-core@5.101.4': {} @@ -4119,10 +4127,10 @@ snapshots: d3-time-format: 4.1.0 internmap: 2.0.3 - '@vitejs/plugin-react@6.0.5(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))': + '@vitejs/plugin-react@6.0.5(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1))': dependencies: '@rolldown/pluginutils': 1.0.1 - vite: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1) + vite: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1) '@vitest/expect@4.1.10': dependencies: @@ -4133,13 +4141,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.1 - '@vitest/mocker@4.1.10(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))': + '@vitest/mocker@4.1.10(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1))': dependencies: '@vitest/spy': 4.1.10 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1) + vite: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1) '@vitest/pretty-format@4.1.10': dependencies: @@ -5396,7 +5404,7 @@ snapshots: validator@13.15.35: {} - vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1): + vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1): dependencies: lightningcss: 1.33.0 picomatch: 4.0.5 @@ -5409,11 +5417,12 @@ snapshots: fsevents: 2.3.3 jiti: 2.7.0 tsx: 4.23.1 + yaml: 2.8.1 - vitest@4.1.10(@types/node@24.13.3)(jsdom@30.0.1)(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)): + vitest@4.1.10(@types/node@24.13.3)(jsdom@30.0.1)(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1)): dependencies: '@vitest/expect': 4.1.10 - '@vitest/mocker': 4.1.10(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)) + '@vitest/mocker': 4.1.10(vite@8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1)) '@vitest/pretty-format': 4.1.10 '@vitest/runner': 4.1.10 '@vitest/snapshot': 4.1.10 @@ -5430,7 +5439,7 @@ snapshots: tinyexec: 1.2.4 tinyglobby: 0.2.17 tinyrainbow: 3.1.1 - vite: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1) + vite: 8.2.0(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.8.1) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 24.13.3 @@ -5477,6 +5486,8 @@ snapshots: xmlchars@2.2.0: {} + yaml@2.8.1: {} + yocto-queue@0.1.0: {} zeptomatch@2.1.0: diff --git a/services/engine/src/databreeze_engine/telemetry.py b/services/engine/src/databreeze_engine/telemetry.py index e48c2606..b8418ab2 100644 --- a/services/engine/src/databreeze_engine/telemetry.py +++ b/services/engine/src/databreeze_engine/telemetry.py @@ -107,6 +107,10 @@ _LEVELS = frozenset({"debug", "info", "warn", "error"}) +class _LocalTelemetryValidationError(ValueError): + """A validation error raised by this module rather than a provider object.""" + + @dataclass(frozen=True) class CorrelationContext: correlation_id: str @@ -152,7 +156,7 @@ def _validate_key(key: object) -> str: or len(key) > 64 or not _KEY_PATTERN.fullmatch(key) ): - raise ValueError(f"invalid telemetry key: {key!r}") + raise _LocalTelemetryValidationError(f"invalid telemetry key: {key!r}") return key @@ -162,23 +166,33 @@ def sanitize_attributes(attributes: dict[str, Any]) -> dict[str, str | int | flo if not isinstance(attributes, Mapping): return {} safe: dict[str, str | int | float | bool] = {} - for raw_key, value in attributes.items(): - key = _validate_key(raw_key) - if key not in SAFE_ATTRIBUTE_KEYS: - continue - scalar = _safe_scalar(key, value) - if scalar is not None: - safe[key] = scalar + try: + for raw_key, value in attributes.items(): + key = _validate_key(raw_key) + if key not in SAFE_ATTRIBUTE_KEYS: + continue + scalar = _safe_scalar(key, value) + if scalar is not None: + safe[key] = scalar + except Exception: + # Diagnostics must fail closed without reflecting provider causes or + # executing a hostile Mapping implementation again. + return safe return safe def assert_safe_attributes(attributes: Mapping[str, Any]) -> None: """Raise when a record contains an unsafe or unknown attribute.""" - for raw_key, value in attributes.items(): - key = _validate_key(raw_key) - if key not in SAFE_ATTRIBUTE_KEYS or _safe_scalar(key, value) is None: - raise ValueError(f"telemetry attribute is not allowed: {key}") + try: + for raw_key, value in attributes.items(): + key = _validate_key(raw_key) + if key not in SAFE_ATTRIBUTE_KEYS or _safe_scalar(key, value) is None: + raise _LocalTelemetryValidationError(f"telemetry attribute is not allowed: {key}") + except _LocalTelemetryValidationError as error: + raise ValueError(str(error)) from None + except Exception: + raise ValueError("telemetry attributes are not readable") from None def _correlation_id(value: str) -> str: @@ -246,13 +260,24 @@ def correlation_headers(context: CorrelationContext) -> dict[str, str]: def _single_header(headers: Mapping[str, str | Sequence[str] | None], name: str) -> str | None: values: list[str] = [] - for key, value in headers.items(): - if key.lower() != name: - continue - if isinstance(value, str): - values.append(value) - elif value is not None: - values.extend(value) + try: + for key, value in headers.items(): + if not isinstance(key, str): + raise _LocalTelemetryValidationError("telemetry header name is not a string") + if key.lower() != name: + continue + if isinstance(value, str): + values.append(value) + elif value is not None: + if not isinstance(value, Sequence) or isinstance(value, (bytes, bytearray)): + raise _LocalTelemetryValidationError("telemetry header value is not readable") + if not all(isinstance(item, str) for item in value): + raise _LocalTelemetryValidationError("telemetry header value is not readable") + values.extend(value) + except _LocalTelemetryValidationError as error: + raise ValueError(str(error)) from None + except Exception: + raise ValueError("telemetry headers are not readable") from None if len(values) > 1: raise ValueError(f"ambiguous telemetry {name} header") if not values: diff --git a/services/engine/tests/test_telemetry.py b/services/engine/tests/test_telemetry.py index 2edc1968..2860e7a9 100644 --- a/services/engine/tests/test_telemetry.py +++ b/services/engine/tests/test_telemetry.py @@ -68,6 +68,50 @@ def test_engine_rejects_ambiguous_or_zero_trace_headers() -> None: ) +def test_engine_telemetry_does_not_execute_hostile_mapping_items() -> None: + class HostileMapping(dict[str, object]): + def items(self): # type: ignore[override] + raise RuntimeError("provider cause must not escape") + + assert sanitize_attributes(HostileMapping()) == {} + with pytest.raises(ValueError, match="not readable"): + assert_safe_attributes(HostileMapping()) + + +def test_engine_telemetry_normalizes_provider_value_errors() -> None: + class ValueErrorMapping(dict[str, object]): + def items(self): # type: ignore[override] + raise ValueError("provider value error must not escape") + + assert sanitize_attributes(ValueErrorMapping()) == {} + with pytest.raises(ValueError, match="not readable") as error: + assert_safe_attributes(ValueErrorMapping()) + assert "provider value error" not in str(error.value) + + +def test_engine_telemetry_rejects_hostile_or_non_string_header_values() -> None: + class HostileHeaders(dict[str, object]): + def items(self): # type: ignore[override] + raise RuntimeError("provider header cause must not escape") + + with pytest.raises(ValueError, match="not readable"): + correlation_from_headers(HostileHeaders()) + with pytest.raises(ValueError, match="not readable"): + correlation_from_headers( + { + "x-correlation-id": [1, 2], + } + ) + + class ValueErrorHeaders(dict[str, object]): + def items(self): # type: ignore[override] + raise ValueError("provider header value error must not escape") + + with pytest.raises(ValueError, match="not readable") as error: + correlation_from_headers(ValueErrorHeaders()) + assert "provider header value error" not in str(error.value) + + def test_engine_accepts_mixed_case_header_names() -> None: context = CorrelationContext( "00000000-0000-4000-8000-000000000001", diff --git a/tools/repo-cli/src/check-aws-infrastructure.mjs b/tools/repo-cli/src/check-aws-infrastructure.mjs index ab3dde11..5cbf42f2 100644 --- a/tools/repo-cli/src/check-aws-infrastructure.mjs +++ b/tools/repo-cli/src/check-aws-infrastructure.mjs @@ -3,6 +3,7 @@ import os from 'node:os'; import path from 'node:path'; import { spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; +import { balancedBlocks } from './terraform-safety.mjs'; const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); const infrastructureRoot = path.join(repositoryRoot, 'infrastructure', 'aws'); @@ -31,6 +32,7 @@ const allTerraform = requiredFiles .filter((relativePath) => relativePath.endsWith('.tf')) .map((relativePath) => readFileSync(path.join(infrastructureRoot, relativePath), 'utf8')) .join('\n'); + for (const requiredText of [ 'ap-southeast-1', 'hashicorp/aws', @@ -45,6 +47,34 @@ for (const requiredText of [ ]) { if (!allTerraform.includes(requiredText)) fail(`missing required declaration ${requiredText}`); } +for (const requiredBoundary of [ + 'block_public_policy', + 'versioning_configuration', + 'assign_public_ip = false', + 'deletion_protection', + 'backup_retention_period', + 'token.actions.githubusercontent.com:sub', + 'recovery_window_in_days = 30', + 'force_destroy = false', +]) { + if (!allTerraform.includes(requiredBoundary)) + fail(`missing required safety boundary ${requiredBoundary}`); +} +if ( + balancedBlocks(allTerraform, 'ingress').some((block) => + /\bcidr_blocks\s*=\s*\[[^\]]*"0\.0\.0\.0\/0"/u.test(block), + ) +) { + fail('a private service security group permits unrestricted ingress'); +} +if ( + /resource\s+"aws_s3_bucket_policy"[\s\S]*?Principal\s*=\s*"\*"/u.test(allTerraform) || + balancedBlocks(allTerraform, 'principals').some((block) => + /identifiers\s*=\s*\[[^\]]*"\*"/u.test(block), + ) +) { + fail('the Web bucket policy grants a wildcard principal'); +} if ( /AKIA[0-9A-Z]{16}|aws_secret_access_key\s*=|BEGIN (RSA|OPENSSH) PRIVATE KEY/.test(allTerraform) ) { diff --git a/tools/repo-cli/src/check-ci-policy.mjs b/tools/repo-cli/src/check-ci-policy.mjs index b67cb913..d75d426b 100644 --- a/tools/repo-cli/src/check-ci-policy.mjs +++ b/tools/repo-cli/src/check-ci-policy.mjs @@ -1,6 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { parse } from 'yaml'; const WORKFLOW_DIRECTORY = '.github/workflows'; const REQUIRED_WORKFLOWS = ['quality.yml', 'security.yml', 'release.yml']; @@ -12,39 +13,108 @@ function readWorkflow(root, name) { return fs.readFileSync(filename, 'utf8'); } -function assertPinnedActions(text, filename) { - for (const match of text.matchAll(/(^|\s)uses:\s*([^\s#]+)/gim)) { - const reference = match[2]; - if (reference.startsWith('./') || reference.startsWith('docker://')) continue; +function isRecord(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function parseWorkflow(text, filename) { + try { + const workflow = parse(text, { strict: true, uniqueKeys: true }); + if (!isRecord(workflow)) throw new Error('top-level document must be a mapping'); + return workflow; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`${filename} is not valid workflow YAML: ${detail}`); + } +} + +function walk(value, visit) { + if (Array.isArray(value)) { + for (const item of value) walk(item, visit); + return; + } + if (!isRecord(value)) return; + visit(value); + for (const child of Object.values(value)) walk(child, visit); +} + +function containsRunText(value, expected) { + let found = false; + walk(value, (node) => { + if (typeof node.run === 'string' && node.run.includes(expected)) found = true; + }); + return found; +} + +function assertPinnedActions(workflow, filename) { + walk(workflow, (node) => { + if (typeof node.uses !== 'string') return; + const reference = node.uses; + if (reference.startsWith('./') || reference.startsWith('docker://')) return; const at = reference.lastIndexOf('@'); if (at < 1 || !SHA_REFERENCE.test(reference.slice(at + 1))) { throw new Error(`${filename} uses an unpinned action: ${reference}`); } - } + }); } -function assertLeastPrivilege(text, filename) { - if (/permissions:\s*write-all/iu.test(text)) { +function assertLeastPrivilege(workflow, filename) { + const permissions = workflow.permissions; + if (permissions === 'write-all' || (isRecord(permissions) && permissions['write-all'])) { throw new Error(`${filename} grants write-all permissions`); } - if (!/^permissions:\s*$/im.test(text)) { + if (!isRecord(permissions)) { throw new Error(`${filename} must declare a top-level permissions block`); } - if (/pull_request_target:/iu.test(text)) { - throw new Error(`${filename} must not execute untrusted code from pull_request_target`); + if (permissions.contents !== 'read') { + throw new Error(`${filename} must grant contents: read explicitly`); } - if (/AWS_(?:ACCESS_KEY_ID|SECRET_ACCESS_KEY)\s*:/iu.test(text)) { - throw new Error(`${filename} must not define long-lived AWS key environment variables`); + walk(workflow, (node) => { + if (Object.hasOwn(node, 'pull_request_target')) { + throw new Error(`${filename} must not execute untrusted code from pull_request_target`); + } + if (Object.keys(node).some((key) => /^AWS_(?:ACCESS_KEY_ID|SECRET_ACCESS_KEY)$/u.test(key))) { + throw new Error(`${filename} must not define long-lived AWS key environment variables`); + } + if (typeof node.uses === 'string' && node.uses.startsWith('actions/checkout@')) { + if (!isRecord(node.with) || node.with['persist-credentials'] !== false) { + throw new Error(`${filename} must disable checkout credential persistence`); + } + } + }); +} + +function assertBoundedJobs(workflow, filename) { + if (!isRecord(workflow.jobs)) return; + for (const job of Object.values(workflow.jobs)) { + if (!isRecord(job) || !Object.hasOwn(job, 'runs-on')) continue; + if (!Number.isInteger(job['timeout-minutes']) || job['timeout-minutes'] < 1) { + throw new Error(`${filename} must bound every runner job with timeout-minutes`); + } } } +function assertArtifactOutputs(workflow, filename) { + walk(workflow, (node) => { + if (typeof node.uses !== 'string' || !node.uses.startsWith('actions/upload-artifact@')) return; + if (!isRecord(node.with) || node.with['if-no-files-found'] !== 'error') { + throw new Error(`${filename} artifact uploads must fail when an output is missing`); + } + }); +} + export function checkCiPolicy(root = process.cwd()) { const workflows = Object.fromEntries( - REQUIRED_WORKFLOWS.map((name) => [name, readWorkflow(root, name)]), + REQUIRED_WORKFLOWS.map((name) => { + const text = readWorkflow(root, name); + return [name, parseWorkflow(text, name)]; + }), ); - for (const [name, text] of Object.entries(workflows)) { - assertPinnedActions(text, name); - assertLeastPrivilege(text, name); + for (const [name, workflow] of Object.entries(workflows)) { + assertPinnedActions(workflow, name); + assertLeastPrivilege(workflow, name); + assertBoundedJobs(workflow, name); + assertArtifactOutputs(workflow, name); } const security = workflows['security.yml']; for (const required of [ @@ -54,15 +124,23 @@ export function checkCiPolicy(root = process.cwd()) { 'check-container-policy.mjs', 'generate-sbom.mjs', ]) { - if (!security.includes(required)) throw new Error(`security.yml is missing ${required}`); + if (!containsRunText(security, required)) + throw new Error(`security.yml is missing ${required}`); } const release = workflows['release.yml']; - if (!/id-token:\s*write/iu.test(release)) { + if (!isRecord(release.permissions) || release.permissions['id-token'] !== 'write') { throw new Error('release.yml must request OIDC id-token permission explicitly'); } - if (!release.includes('generate-provenance.mjs')) { + if (!containsRunText(release, 'generate-provenance.mjs')) { throw new Error('release.yml must generate a provenance record'); } + let hasReleaseEnvironment = false; + walk(release, (node) => { + if (node.environment === 'release') hasReleaseEnvironment = true; + }); + if (!hasReleaseEnvironment) { + throw new Error('release.yml must use the protected release environment'); + } return { workflowCount: REQUIRED_WORKFLOWS.length }; } diff --git a/tools/repo-cli/src/check-execution-orchestration.mjs b/tools/repo-cli/src/check-execution-orchestration.mjs index 15688116..5229f4e7 100644 --- a/tools/repo-cli/src/check-execution-orchestration.mjs +++ b/tools/repo-cli/src/check-execution-orchestration.mjs @@ -25,7 +25,7 @@ const expectedPlans = new Map([ ]); const expectedPriorityTotals = { P0: 444, P1: 154, P2: 13 }; const expectedReviewPolicy = { - commitBudget: { hardMaximum: 60, preferredMaximum: 50, preferredMinimum: 30 }, + commitBudget: { hardMaximum: 99, preferredMaximum: 70, preferredMinimum: 30 }, featurePullRequest: { base: 'dev', codeRabbit: false, mergeAfterHostedChecks: true }, promotionPullRequest: { base: 'main', diff --git a/tools/repo-cli/src/detect-change-scope.mjs b/tools/repo-cli/src/detect-change-scope.mjs index fe8bb452..d2f46294 100644 --- a/tools/repo-cli/src/detect-change-scope.mjs +++ b/tools/repo-cli/src/detect-change-scope.mjs @@ -14,6 +14,7 @@ const shared = matches([ 'tools/', 'docs/specs/', 'docs/plans/', + 'infrastructure/', '.github/', 'package.json', 'pnpm-lock.yaml', diff --git a/tools/repo-cli/src/generate-provenance.mjs b/tools/repo-cli/src/generate-provenance.mjs index 49026f34..ec8eadbf 100644 --- a/tools/repo-cli/src/generate-provenance.mjs +++ b/tools/repo-cli/src/generate-provenance.mjs @@ -2,21 +2,32 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; -const output = process.argv[process.argv.indexOf('--output') + 1] || 'artifacts/provenance.json'; -const artifactArguments = process.argv.reduce((result, value, index, values) => { - if (value === '--artifact' && values[index + 1]) result.push(values[index + 1]); - return result; -}, []); -const artifacts = artifactArguments - .filter((file) => fs.existsSync(file)) - .map((file) => { - const bytes = fs.readFileSync(file); - return { - path: path.relative(process.cwd(), file).replaceAll('\\', '/'), - sha256: crypto.createHash('sha256').update(bytes).digest('hex'), - size: bytes.length, - }; - }); +const outputIndex = process.argv.indexOf('--output'); +const output = outputIndex === -1 ? 'artifacts/provenance.json' : process.argv[outputIndex + 1]; +if (!output || output.startsWith('--')) { + throw new Error('--output requires a file path'); +} +const artifactArguments = []; +for (let index = 0; index < process.argv.length; index += 1) { + if (process.argv[index] !== '--artifact') continue; + const artifact = process.argv[index + 1]; + if (!artifact || artifact.startsWith('--')) { + throw new Error('--artifact requires a file path'); + } + artifactArguments.push(artifact); +} +const missingArtifacts = artifactArguments.filter((file) => !fs.existsSync(file)); +if (missingArtifacts.length > 0) { + throw new Error(`Provenance artifact(s) do not exist: ${missingArtifacts.join(', ')}`); +} +const artifacts = artifactArguments.map((file) => { + const bytes = fs.readFileSync(file); + return { + path: path.relative(process.cwd(), file).replaceAll('\\', '/'), + sha256: crypto.createHash('sha256').update(bytes).digest('hex'), + size: bytes.length, + }; +}); artifacts.sort((a, b) => a.path.localeCompare(b.path)); const epoch = Number(process.env.SOURCE_DATE_EPOCH || 0); const provenance = { diff --git a/tools/repo-cli/src/generate-sbom.mjs b/tools/repo-cli/src/generate-sbom.mjs index bf5cd75f..2536da38 100644 --- a/tools/repo-cli/src/generate-sbom.mjs +++ b/tools/repo-cli/src/generate-sbom.mjs @@ -2,7 +2,11 @@ import fs from 'node:fs'; import path from 'node:path'; const root = process.cwd(); -const output = process.argv[process.argv.indexOf('--output') + 1] || 'artifacts/sbom.cdx.json'; +const outputIndex = process.argv.indexOf('--output'); +const output = outputIndex === -1 ? 'artifacts/sbom.cdx.json' : process.argv[outputIndex + 1]; +if (!output || output.startsWith('--')) { + throw new Error('--output requires a file path'); +} const ignored = new Set([ 'node_modules', '.git', diff --git a/tools/repo-cli/src/local-services-smoke.mjs b/tools/repo-cli/src/local-services-smoke.mjs index 44b89dc6..53cb413a 100644 --- a/tools/repo-cli/src/local-services-smoke.mjs +++ b/tools/repo-cli/src/local-services-smoke.mjs @@ -1,113 +1,6 @@ -import { existsSync } from 'node:fs'; -import { spawnSync } from 'node:child_process'; -import { setTimeout as delay } from 'node:timers/promises'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { main } from './local-services.mjs'; -const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); -const composeFile = path.join(repositoryRoot, 'infrastructure', 'local', 'compose.yml'); -const exampleEnvFile = path.join(repositoryRoot, 'infrastructure', 'local', '.env.example'); -const localEnvFile = path.join(repositoryRoot, 'infrastructure', 'local', '.env'); -const services = [ - 'postgres', - 'redis', - 'minio', - 'mailpit', - 'otel-collector', - 'otel-collector-health', -]; - -function usage() { - console.log(`Usage: pnpm local:smoke [-- --start] [-- --wait-seconds=60] - -Validates the local Compose file and polls health checks for the five required -services. The command never removes containers or named volumes. - - --start run docker compose up -d before polling - --wait-seconds=N maximum readiness wait (default: 60) - --help show this help`); -} - -function runDocker(args, { allowFailure = false } = {}) { - const result = spawnSync('docker', args, { cwd: repositoryRoot, encoding: 'utf8' }); - if (!allowFailure && result.status !== 0) { - const detail = (result.stderr || result.stdout || '').trim(); - throw new Error(`docker ${args.join(' ')} failed${detail ? `: ${detail}` : ''}`); - } - return result; -} - -function composeArgs() { - const envFile = existsSync(localEnvFile) ? localEnvFile : exampleEnvFile; - return ['compose', '--env-file', envFile, '-f', composeFile]; -} - -function inspectHealth(service) { - const idResult = runDocker([...composeArgs(), 'ps', '-q', service], { allowFailure: true }); - const id = idResult.stdout.trim(); - if (!id) return { state: 'missing', detail: 'no container' }; - - const inspect = runDocker([ - 'inspect', - '--format', - '{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}no-health{{end}}', - id, - ]); - const [state, health] = inspect.stdout.trim().split('|'); - return { state, health, detail: `${state}/${health}` }; -} - -function parseArguments(argv) { - const options = { start: false, waitSeconds: 60 }; - for (const argument of argv) { - if (argument === '--help' || argument === '-h') { - usage(); - process.exit(0); - } - if (argument === '--start') { - options.start = true; - continue; - } - if (argument.startsWith('--wait-seconds=')) { - const value = Number(argument.slice('--wait-seconds='.length)); - if (!Number.isInteger(value) || value < 1 || value > 3600) { - throw new Error('--wait-seconds must be an integer from 1 to 3600'); - } - options.waitSeconds = value; - continue; - } - throw new Error(`unknown argument: ${argument}`); - } - return options; -} - -async function main() { - const options = parseArguments(process.argv.slice(2)); - runDocker([...composeArgs(), 'config', '--quiet']); - if (options.start) runDocker([...composeArgs(), 'up', '-d']); - - const deadline = Date.now() + options.waitSeconds * 1000; - let last = new Map(); - while (Date.now() <= deadline) { - last = new Map(services.map((service) => [service, inspectHealth(service)])); - if ( - [...last.values()].every(({ state, health }) => state === 'running' && health === 'healthy') - ) { - console.log(`Local services ready (${services.join(', ')}).`); - return; - } - const summary = services.map((service) => `${service}=${last.get(service).detail}`).join(' '); - process.stdout.write(`Waiting for local services: ${summary}\r`); - await delay(1000); - } - - console.error('\nLocal services did not become ready:'); - for (const service of services) - console.error(`- ${service}: ${last.get(service)?.detail ?? 'unknown'}`); - process.exitCode = 1; -} - -main().catch((error) => { +main(process.argv.slice(2)).catch((error) => { console.error(error instanceof Error ? error.message : error); process.exitCode = 1; }); diff --git a/tools/repo-cli/src/local-services.mjs b/tools/repo-cli/src/local-services.mjs new file mode 100644 index 00000000..1d8409a3 --- /dev/null +++ b/tools/repo-cli/src/local-services.mjs @@ -0,0 +1,458 @@ +import { existsSync, readFileSync, statfsSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import net from 'node:net'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; + +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); +const localRoot = path.join(repositoryRoot, 'infrastructure', 'local'); +const composeFile = path.join(localRoot, 'compose.yml'); +const exampleEnvFile = path.join(localRoot, '.env.example'); +const localEnvFile = path.join(localRoot, '.env'); +const services = [ + 'postgres', + 'redis', + 'minio', + 'mailpit', + 'otel-collector', + 'otel-collector-health', +]; +const logServices = [...services, 'minio-init']; +const hostPorts = [ + { service: 'postgres', key: 'POSTGRES_PORT', fallback: 5432 }, + { service: 'redis', key: 'REDIS_PORT', fallback: 6379 }, + { service: 'minio', key: 'MINIO_API_PORT', fallback: 9000 }, + { service: 'minio', key: 'MINIO_CONSOLE_PORT', fallback: 9001 }, + { service: 'mailpit', key: 'MAILPIT_SMTP_PORT', fallback: 1025 }, + { service: 'mailpit', key: 'MAILPIT_UI_PORT', fallback: 8025 }, + { service: 'otel-collector', key: 'OTEL_GRPC_PORT', fallback: 4317 }, + { service: 'otel-collector', key: 'OTEL_HTTP_PORT', fallback: 4318 }, + { service: 'otel-collector', key: 'OTEL_HEALTH_PORT', fallback: 13133 }, +]; + +function usage() { + console.log(`Usage: pnpm local:services [options] + +Commands (all preserve named volumes): + config validate Compose syntax without requiring a Docker daemon + preflight validate Compose, ports, and disk without a Docker daemon + check validate Compose, ports, Docker, and disk headroom + start validate, start services, and wait for healthy checks + stop stop containers without removing containers or volumes + reset recreate containers and networks, preserving volumes + restart-check restart running services and verify health/persistence + persistence-check restart Redis and verify a disposable sentinel survives + status print current container and health state + logs print bounded local container logs (read-only) + smoke legacy readiness command (use --start to start first) + +Options: + --start with smoke, start services before polling + --wait-seconds=N readiness timeout (default: 60, maximum: 3600) + --tail=N log lines per service (default: 100, maximum: 1000) + --service=NAME limit logs to one known local service + --min-free-gib=N minimum host free space (default: 5) + --help show this help + +Smoke never removes containers or named volumes. Lifecycle reset recreates +containers but never passes --volumes to Compose. Use Docker directly only when +you explicitly intend to discard local development data.`); +} + +function fail(message) { + throw new Error(`Local infrastructure: ${message}`); +} + +function parseEnvFile(file) { + if (!existsSync(file)) return new Map(); + const values = new Map(); + for (const line of readFileSync(file, 'utf8').split(/\r?\n/u)) { + const match = /^\s*([A-Z][A-Z0-9_]*)\s*=\s*(.*?)\s*$/u.exec(line); + if (!match) continue; + values.set(match[1], match[2].replace(/^(['"])(.*)\1$/u, '$2')); + } + return values; +} + +function environment() { + const fileValues = parseEnvFile(existsSync(localEnvFile) ? localEnvFile : exampleEnvFile); + if (process.env.COMPOSE_PROJECT_NAME !== undefined) { + fileValues.set('COMPOSE_PROJECT_NAME', process.env.COMPOSE_PROJECT_NAME); + } + for (const definition of hostPorts) { + if (process.env[definition.key] !== undefined) + fileValues.set(definition.key, process.env[definition.key]); + } + if (process.env.DATABREEZE_MIN_FREE_GIB !== undefined) { + fileValues.set('DATABREEZE_MIN_FREE_GIB', process.env.DATABREEZE_MIN_FREE_GIB); + } + return fileValues; +} + +function projectName(values) { + const value = + values.get('COMPOSE_PROJECT_NAME') || process.env.COMPOSE_PROJECT_NAME || 'databreeze-local'; + if (!/^[a-z0-9][a-z0-9_-]{0,62}$/u.test(value)) { + fail( + 'COMPOSE_PROJECT_NAME must start with a lowercase letter or digit and contain only lowercase letters, digits, hyphens, or underscores', + ); + } + return value; +} + +function portValue(definition, values) { + const value = Number(values.get(definition.key) ?? definition.fallback); + if (!Number.isInteger(value) || value < 1024 || value > 65535) { + fail(`${definition.key} must be an integer between 1024 and 65535`); + } + return value; +} + +function composeArgs(values = environment()) { + const envFile = existsSync(localEnvFile) ? localEnvFile : exampleEnvFile; + const project = projectName(values); + return ['compose', '--project-name', project, '--env-file', envFile, '-f', composeFile]; +} + +function runDocker(args, { allowFailure = false, capture = true, timeoutMs = 30_000 } = {}) { + const result = spawnSync( + 'docker', + args, + capture + ? { cwd: repositoryRoot, encoding: 'utf8', timeout: timeoutMs } + : { cwd: repositoryRoot, stdio: 'inherit', timeout: timeoutMs }, + ); + if (!allowFailure && (result.error || result.status !== 0)) { + if (result.error?.code === 'ENOENT') { + fail( + 'Docker CLI is not installed or not on PATH; start Docker Desktop before using this command', + ); + } + if (result.error?.code === 'ETIMEDOUT') { + fail(`docker ${args.join(' ')} timed out after ${timeoutMs}ms`); + } + const detail = (result.stderr || result.stdout || result.error?.message || '').trim(); + fail(`docker ${args.join(' ')} failed${detail ? `: ${detail}` : ''}`); + } + return result; +} + +function requireDocker() { + const result = spawnSync('docker', ['info', '--format', '{{.ServerVersion}}'], { + cwd: repositoryRoot, + encoding: 'utf8', + timeout: 15_000, + }); + if (result.error?.code === 'ENOENT') { + fail( + 'Docker CLI is not installed or not on PATH; start Docker Desktop before using this command', + ); + } + if (result.error?.code === 'ETIMEDOUT') { + fail('Docker CLI check timed out after 15000ms; verify Docker Desktop is responsive'); + } + if (result.status !== 0) { + fail( + 'Docker daemon is unavailable; start Docker Desktop or another Docker Engine before using this command', + ); + } +} + +function validateCompose(values) { + runDocker([...composeArgs(values), 'config', '--quiet']); +} + +function ensureDiskSpace(minFreeGib) { + if (typeof statfsSync !== 'function') { + console.warn('Local infrastructure: disk free-space check is unavailable on this Node runtime'); + return; + } + const stats = statfsSync(repositoryRoot); + const freeBytes = Number(stats.bavail) * Number(stats.bsize); + const requiredBytes = minFreeGib * 1024 ** 3; + if (freeBytes < requiredBytes) { + fail( + `host free space is ${(freeBytes / 1024 ** 3).toFixed(2)} GiB; at least ${minFreeGib} GiB is required`, + ); + } +} + +function containerRunning(service, values) { + const id = runDocker([...composeArgs(values), 'ps', '-q', service], { + allowFailure: true, + }).stdout.trim(); + if (!id) return false; + const state = runDocker(['inspect', '--format', '{{.State.Running}}', id], { + allowFailure: true, + }); + return state.stdout.trim() === 'true'; +} + +function portAvailable(port) { + return new Promise((resolve) => { + const server = net.createServer(); + const finish = (available) => { + server.removeAllListeners(); + if (server.listening) server.close(() => resolve(available)); + else resolve(available); + }; + server.once('error', () => finish(false)); + server.listen({ host: '127.0.0.1', port }, () => finish(true)); + }); +} + +async function ensurePorts(values) { + const collisions = []; + const configured = new Map(); + for (const definition of hostPorts) { + const port = portValue(definition, values); + const prior = configured.get(port); + if (prior && prior.key !== definition.key) { + collisions.push(`${prior.key}=${port} and ${definition.key}=${port}`); + continue; + } + configured.set(port, definition); + if (containerRunning(definition.service, values)) continue; + if (!(await portAvailable(port))) + collisions.push(`${definition.key}=${port} (${definition.service})`); + } + if (collisions.length > 0) { + fail( + `host ports are already in use: ${collisions.join(', ')}; set alternate ports in infrastructure/local/.env`, + ); + } +} + +function inspectHealth(service, values) { + const idResult = runDocker([...composeArgs(values), 'ps', '-q', service], { allowFailure: true }); + const id = idResult.stdout.trim(); + if (!id) return { state: 'missing', health: 'unknown', detail: 'no container' }; + const inspect = runDocker( + [ + 'inspect', + '--format', + '{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}no-health{{end}}', + id, + ], + { allowFailure: true }, + ); + const inspection = inspect.stdout?.trim(); + if (inspect.error || inspect.status !== 0 || !inspection) { + return { state: 'unknown', health: 'unknown', detail: 'unknown/unknown (inspect unavailable)' }; + } + const [state, health] = inspection.split('|'); + return { state, health, detail: `${state}/${health}` }; +} + +async function waitForReady(values, waitSeconds) { + const deadline = Date.now() + waitSeconds * 1000; + let last = new Map(); + while (Date.now() <= deadline) { + last = new Map(services.map((service) => [service, inspectHealth(service, values)])); + if ( + [...last.values()].every(({ state, health }) => state === 'running' && health === 'healthy') + ) { + console.log(`Local services ready (${services.join(', ')}).`); + return; + } + const summary = services.map((service) => `${service}=${last.get(service).detail}`).join(' '); + process.stdout.write(`Waiting for local services: ${summary}\r`); + await delay(1000); + } + console.error('\nLocal services did not become ready:'); + for (const service of services) + console.error(`- ${service}: ${last.get(service)?.detail ?? 'unknown'}`); + fail(`readiness timeout after ${waitSeconds}s`); +} + +function parseArguments(argv, values = environment()) { + let command = 'smoke'; + const argumentsToParse = [...argv]; + if (argumentsToParse[0] && !argumentsToParse[0].startsWith('-')) + command = argumentsToParse.shift(); + const options = { + start: false, + waitSeconds: 60, + tail: 100, + service: undefined, + minFreeGib: Number(values.get('DATABREEZE_MIN_FREE_GIB') ?? 5), + }; + for (const argument of argumentsToParse) { + if (argument === '--help' || argument === '-h') return { command: 'help', options }; + if (argument === '--start') { + options.start = true; + continue; + } + if (argument.startsWith('--wait-seconds=')) { + options.waitSeconds = Number(argument.slice('--wait-seconds='.length)); + continue; + } + if (argument.startsWith('--tail=')) { + options.tail = Number(argument.slice('--tail='.length)); + continue; + } + if (argument.startsWith('--service=')) { + options.service = argument.slice('--service='.length); + continue; + } + if (argument.startsWith('--min-free-gib=')) { + options.minFreeGib = Number(argument.slice('--min-free-gib='.length)); + continue; + } + fail(`unknown argument: ${argument}`); + } + if ( + !Number.isInteger(options.waitSeconds) || + options.waitSeconds < 1 || + options.waitSeconds > 3600 + ) { + fail('--wait-seconds must be an integer from 1 to 3600'); + } + if (!Number.isInteger(options.tail) || options.tail < 1 || options.tail > 1000) { + fail('--tail must be an integer from 1 to 1000'); + } + if (options.service !== undefined && !logServices.includes(options.service)) { + fail(`--service must name one of: ${logServices.join(', ')}`); + } + if (!Number.isFinite(options.minFreeGib) || options.minFreeGib < 0) { + fail('--min-free-gib must be a non-negative number'); + } + if ( + ![ + 'config', + 'preflight', + 'check', + 'start', + 'stop', + 'reset', + 'restart-check', + 'persistence-check', + 'status', + 'logs', + 'smoke', + ].includes(command) + ) { + fail(`unknown command: ${command}`); + } + return { command, options }; +} + +export async function main(argv = process.argv.slice(2)) { + const values = environment(); + const { command, options } = parseArguments(argv, values); + if (command === 'help') { + usage(); + return; + } + if (command === 'config') { + validateCompose(values); + console.log('Local Compose configuration is valid.'); + return; + } + if (command === 'preflight') { + validateCompose(values); + ensureDiskSpace(options.minFreeGib); + await ensurePorts(values); + console.log('Local Compose, port, and disk preflight passed without starting services.'); + return; + } + requireDocker(); + validateCompose(values); + + if (command === 'status') { + for (const service of services) + console.log(`${service}: ${inspectHealth(service, values).detail}`); + return; + } + if (command === 'logs') { + const selected = options.service ? [options.service] : logServices; + runDocker( + [...composeArgs(values), 'logs', '--no-color', `--tail=${options.tail}`, ...selected], + { capture: false, timeoutMs: 120_000 }, + ); + return; + } + if (command === 'stop') { + runDocker([...composeArgs(values), 'stop']); + console.log('Local services stopped; named volumes and containers were preserved.'); + return; + } + + const shouldStart = + command === 'start' || command === 'reset' || (command === 'smoke' && options.start); + if (shouldStart) { + ensureDiskSpace(options.minFreeGib); + await ensurePorts(values); + } + if (command === 'check') { + ensureDiskSpace(options.minFreeGib); + await ensurePorts(values); + console.log('Local Compose, Docker, port, and disk preflight passed.'); + return; + } + if (command === 'reset') { + runDocker([...composeArgs(values), 'down', '--remove-orphans']); + runDocker([...composeArgs(values), 'up', '-d']); + await waitForReady(values, options.waitSeconds); + console.log('Local services reset without removing named volumes.'); + return; + } + if (command === 'restart-check') { + runDocker([...composeArgs(values), 'restart']); + await waitForReady(values, options.waitSeconds); + console.log( + 'Local service restart and health checks passed. Use persistence-check for a Redis sentinel probe.', + ); + return; + } + if (command === 'persistence-check') { + const key = `databreeze:local:persistence-check:${process.pid}`; + const value = `${Date.now()}`; + let recovered = false; + try { + runDocker([ + ...composeArgs(values), + 'exec', + '-T', + 'redis', + 'redis-cli', + 'SET', + key, + value, + 'EX', + '300', + ]); + runDocker([...composeArgs(values), 'restart', 'redis']); + await waitForReady(values, options.waitSeconds); + const result = runDocker([ + ...composeArgs(values), + 'exec', + '-T', + 'redis', + 'redis-cli', + 'GET', + key, + ]); + if (result.stdout.trim() !== value) + fail('Redis persistence sentinel was not recovered after restart'); + recovered = true; + } finally { + runDocker([...composeArgs(values), 'exec', '-T', 'redis', 'redis-cli', 'DEL', key], { + allowFailure: true, + }); + } + if (!recovered) return; + console.log('Local Redis persistence check passed; sentinel was removed.'); + return; + } + if (shouldStart) runDocker([...composeArgs(values), 'up', '-d']); + await waitForReady(values, options.waitSeconds); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/tools/repo-cli/src/terraform-safety.mjs b/tools/repo-cli/src/terraform-safety.mjs new file mode 100644 index 00000000..f5aceed8 --- /dev/null +++ b/tools/repo-cli/src/terraform-safety.mjs @@ -0,0 +1,139 @@ +function maskTerraformNonCode(text) { + const masked = text.split(''); + let quoted = false; + let escaped = false; + let lineComment = false; + let blockComment = false; + let heredoc = undefined; + let lineStart = true; + + const mask = (index) => { + if (masked[index] !== '\n') masked[index] = ' '; + }; + + for (let index = 0; index < text.length; index += 1) { + const character = text[index]; + const next = text[index + 1]; + + if (heredoc !== undefined) { + if (lineStart) { + const lineEnd = text.indexOf('\n', index) === -1 ? text.length : text.indexOf('\n', index); + const rawLine = text.slice(index, lineEnd).replace(/\r$/u, ''); + const candidate = heredoc.allowIndent ? rawLine.trim() : rawLine; + for (let position = index; position < lineEnd; position += 1) mask(position); + if (candidate === heredoc.delimiter) heredoc = undefined; + lineStart = false; + index = lineEnd - 1; + continue; + } + mask(index); + if (character === '\n') lineStart = true; + continue; + } + + if (lineComment) { + mask(index); + if (character === '\n') { + lineComment = false; + lineStart = true; + } + continue; + } + + if (blockComment) { + mask(index); + if (character === '*' && next === '/') { + mask(index + 1); + index += 1; + blockComment = false; + } + lineStart = character === '\n'; + continue; + } + + if (quoted) { + if (escaped) escaped = false; + else if (character === '\\') escaped = true; + else if (character === '"') quoted = false; + if (character === '\n') lineStart = true; + else lineStart = false; + continue; + } + + if (character === '"') { + quoted = true; + lineStart = false; + continue; + } + if (character === '#') { + lineComment = true; + mask(index); + lineStart = false; + continue; + } + if (character === '/' && next === '/') { + lineComment = true; + mask(index); + mask(index + 1); + index += 1; + lineStart = false; + continue; + } + if (character === '/' && next === '*') { + blockComment = true; + mask(index); + mask(index + 1); + index += 1; + lineStart = false; + continue; + } + if (character === '<' && next === '<') { + const match = text.slice(index).match(/^<<(-?)([A-Za-z_][A-Za-z0-9_-]*)/u); + if (match) { + for (let position = index; position < index + match[0].length; position += 1) { + mask(position); + } + heredoc = { allowIndent: match[1] === '-', delimiter: match[2] }; + index += match[0].length - 1; + lineStart = false; + continue; + } + } + lineStart = character === '\n'; + } + return masked.join(''); +} + +export function balancedBlocks(text, keyword) { + const masked = maskTerraformNonCode(text); + const blocks = []; + const startPattern = new RegExp(`\\b${keyword}\\s*\\{`, 'g'); + for (const match of masked.matchAll(startPattern)) { + const openingBrace = masked.indexOf('{', match.index); + let depth = 0; + let quoted = false; + let escaped = false; + for (let index = openingBrace; index < masked.length; index += 1) { + const character = masked[index]; + if (quoted) { + if (escaped) escaped = false; + else if (character === '\\') escaped = true; + else if (character === '"') quoted = false; + continue; + } + if (character === '"') { + quoted = true; + continue; + } + if (character === '{') depth += 1; + if (character === '}') { + depth -= 1; + if (depth === 0) { + blocks.push(masked.slice(openingBrace, index + 1)); + break; + } + } + } + } + return blocks; +} diff --git a/tools/repo-cli/test/aws-infrastructure.test.mjs b/tools/repo-cli/test/aws-infrastructure.test.mjs index e8274103..3c3e5af8 100644 --- a/tools/repo-cli/test/aws-infrastructure.test.mjs +++ b/tools/repo-cli/test/aws-infrastructure.test.mjs @@ -65,6 +65,18 @@ test('AWS sources expose encryption, private data, and OIDC boundaries without s ]) assert.match(sources, new RegExp(token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))); assert.doesNotMatch(sources, /AKIA[0-9A-Z]{16}|BEGIN (RSA|OPENSSH) PRIVATE KEY/); + assert.doesNotMatch(sources, /ingress[\s\S]*?cidr_blocks\s*=\s*\["0\.0\.0\.0\/0"\]/u); + assert.doesNotMatch(sources, /principals[\s\S]*?identifiers\s*=\s*\[[^\]]*"\*"/u); + assert.match(sources, /assign_public_ip\s*=\s*false/u); + assert.match(sources, /token\.actions\.githubusercontent\.com:sub/u); + assert.match(sources, /repo:\$\{var\.github_repository\}:ref:refs\/heads\/dev/u); + assert.doesNotMatch(sources, /refs\/pull|refs\/tags|repo:\*\//u); + assert.match(sources, /aws_s3_bucket_lifecycle_configuration/u); + assert.match(sources, /noncurrent_version_expiration/u); + assert.match(sources, /abort_incomplete_multipart_upload/u); + assert.match(sources, /force_destroy\s*=\s*false/u); + assert.doesNotMatch(sources, /aws_iam_role_policy" task/u); + assert.doesNotMatch(sources, /ecs-task-minimal/u); }); test('AWS validation script is non-applying and reports missing OpenTofu clearly', () => { @@ -79,4 +91,57 @@ test('AWS validation script is non-applying and reports missing OpenTofu clearly assert.match(source, /init', '-backend=false/); assert.match(source, /validate', '-no-color/); assert.match(source, /process\.exitCode \?\? 0/); + assert.match(source, /missing required safety boundary/u); + assert.doesNotMatch(source, /tofu',\s*\['apply'/u); +}); + +test('AWS production profile enables recovery and prevents public data paths', () => { + const production = read('infrastructure/aws/environments/alpha/production.tfvars.example'); + const versions = read('infrastructure/aws/environments/alpha/versions.tf'); + const data = read('infrastructure/aws/modules/data/main.tf'); + const compute = read('infrastructure/aws/modules/compute/main.tf'); + assert.match(production, /backup_retention_period\s*=\s*7/u); + assert.match(production, /deletion_protection\s*=\s*true/u); + assert.match(production, /database_multi_az\s*=\s*true/u); + assert.match(production, /redis_automatic_failover_enabled\s*=\s*true/u); + assert.match(versions, /required_version\s*=\s*">= 1\.8\.0, < 2\.0\.0"/u); + assert.doesNotMatch(data, /publicly_accessible\s*=\s*true/u); + assert.doesNotMatch(data, /skip_final_snapshot\s*=\s*true/u); + assert.match(data, /performance_insights_enabled\s*=\s*var\.environment == "production"/u); + assert.match(data, /performance_insights_kms_key_id/u); + assert.match(compute, /assign_public_ip\s*=\s*false/u); + assert.equal((compute.match(/readonlyRootFilesystem\s*=\s*true/g) ?? []).length, 2); + assert.equal((compute.match(/privileged\s*=\s*false/g) ?? []).length, 2); + assert.equal((compute.match(/user\s*=\s*"10001"/g) ?? []).length, 2); + assert.match(compute, /Production API deployments must use an immutable image digest\./u); + assert.match(compute, /Production worker deployments must use an immutable image digest\./u); + assert.match(compute, /@sha256:\[0-9a-f\]\{64\}/u); + assert.match(compute, /cpu\s*=\s*tostring\(var\.worker_cpu\)/u); + assert.match(compute, /memory\s*=\s*tostring\(var\.worker_memory\)/u); + const computeVariables = read('infrastructure/aws/modules/compute/variables.tf'); + assert.match(computeVariables, /variable "worker_cpu"/u); + assert.match(computeVariables, /variable "worker_memory"/u); + assert.match(computeVariables, /supported Fargate CPU size/u); + assert.match(computeVariables, /between 512 and 30720 MiB/u); + assert.match(compute, /allowed_worker_memory_by_cpu/u); + assert.match(compute, /AWS-supported Fargate size for worker_cpu/u); +}); + +test('AWS foundation keeps state and apply outside the repository', () => { + const readme = read('infrastructure/aws/README.md'); + const sources = [ + read('infrastructure/aws/environments/alpha/main.tf'), + read('infrastructure/aws/environments/alpha/versions.tf'), + ].join('\n'); + assert.match(readme, /plan-only/u); + assert.match(readme, /remote encrypted state backend/u); + assert.doesNotMatch(sources, /^\s*backend\s+"/mu); + assert.doesNotMatch(sources, /terraform\.tfstate|\.tfstate\.backup/u); +}); + +test('AWS production guidance requires digest-pinned application images', () => { + const readme = read('infrastructure/aws/README.md'); + assert.match(readme, /api_image.*worker_image[\s\S]*immutable/u); + assert.match(readme, /64-character SHA-256 digest/u); + assert.match(readme, /Mutable tags are accepted\s+for alpha development only/u); }); diff --git a/tools/repo-cli/test/change-scope.test.mjs b/tools/repo-cli/test/change-scope.test.mjs new file mode 100644 index 00000000..ba376772 --- /dev/null +++ b/tools/repo-cli/test/change-scope.test.mjs @@ -0,0 +1,16 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); + +test('infrastructure changes are shared quality-gate inputs', () => { + const source = readFileSync( + path.join(repositoryRoot, 'tools', 'repo-cli', 'src', 'detect-change-scope.mjs'), + 'utf8', + ); + assert.match(source, /'infrastructure\/'/u); + assert.match(source, /infrastructure: shared \|\| matches\(\['infrastructure\/'\]\)/u); +}); diff --git a/tools/repo-cli/test/ci-policy.test.mjs b/tools/repo-cli/test/ci-policy.test.mjs index 7c1066af..b8e0b8df 100644 --- a/tools/repo-cli/test/ci-policy.test.mjs +++ b/tools/repo-cli/test/ci-policy.test.mjs @@ -10,6 +10,33 @@ test('repository workflows are present, pinned, and least privilege', () => { assert.deepEqual(checkCiPolicy(process.cwd()), { workflowCount: 3 }); }); +test('CI policy requires checkout credentials to be discarded', () => { + for (const name of ['quality.yml', 'security.yml', 'release.yml']) { + const workflow = fs.readFileSync(path.join(process.cwd(), '.github/workflows', name), 'utf8'); + assert.match(workflow, /persist-credentials:\s*false/u); + assert.match(workflow, /^\s+contents:\s*read\s*$/mu); + } +}); + +test('CI policy fails closed when an artifact output is missing', () => { + const workflow = fs.readFileSync( + path.join(process.cwd(), '.github/workflows/release.yml'), + 'utf8', + ); + assert.match( + workflow, + /actions\/upload-artifact@[0-9a-f]{40}[\s\S]*if-no-files-found:\s*error/iu, + ); +}); + +test('release workflow uses the protected release environment', () => { + const workflow = fs.readFileSync( + path.join(process.cwd(), '.github/workflows/release.yml'), + 'utf8', + ); + assert.match(workflow, /^\s+environment:\s*release\s*$/mu); +}); + test('CI policy rejects floating actions and pull request target execution', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'databreeze-ci-policy-')); fs.mkdirSync(path.join(root, '.github/workflows'), { recursive: true }); @@ -32,3 +59,156 @@ test('CI policy rejects floating actions and pull request target execution', () fs.writeFileSync(path.join(root, '.github/workflows', name), text); assert.throws(() => checkCiPolicy(root), /unpinned action/u); }); + +test('CI policy rejects runner jobs without a timeout', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'databreeze-ci-timeout-')); + fs.mkdirSync(path.join(root, '.github/workflows'), { recursive: true }); + const checkout = 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683'; + const workflows = { + 'quality.yml': [ + 'name: q', + 'permissions:', + ' contents: read', + 'jobs:', + ' check:', + ' runs-on: ubuntu-24.04', + ' steps:', + ` - uses: ${checkout}`, + ' with:', + ' persist-credentials: false', + ].join('\n'), + 'security.yml': [ + 'name: s', + 'permissions:', + ' contents: read', + 'jobs:', + ' scan:', + ' runs-on: ubuntu-24.04', + ' timeout-minutes: 10', + ' steps:', + ` - uses: ${checkout}`, + ' with:', + ' persist-credentials: false', + ' - run: pnpm audit && check-secret-patterns.mjs check-license-policy.mjs check-container-policy.mjs generate-sbom.mjs', + ].join('\n'), + 'release.yml': [ + 'name: r', + 'permissions:', + ' contents: read', + ' id-token: write', + 'jobs:', + ' release:', + ' runs-on: ubuntu-24.04', + ' timeout-minutes: 10', + ' steps:', + ` - uses: ${checkout}`, + ' with:', + ' persist-credentials: false', + ' - run: generate-provenance.mjs', + ].join('\n'), + }; + for (const [name, text] of Object.entries(workflows)) + fs.writeFileSync(path.join(root, '.github/workflows', name), text); + assert.throws(() => checkCiPolicy(root), /timeout-minutes/u); +}); + +test('CI policy rejects artifact steps without missing-output failure', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'databreeze-ci-artifact-')); + fs.mkdirSync(path.join(root, '.github/workflows'), { recursive: true }); + const checkout = 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683'; + const workflows = { + 'quality.yml': [ + 'name: q', + 'permissions:', + ' contents: read', + 'jobs:', + ' check:', + ' runs-on: ubuntu-24.04', + ' timeout-minutes: 10', + ' steps:', + ` - uses: ${checkout}`, + ' with:', + ' persist-credentials: false', + ].join('\n'), + 'security.yml': [ + 'name: s', + 'permissions:', + ' contents: read', + 'jobs:', + ' scan:', + ' runs-on: ubuntu-24.04', + ' timeout-minutes: 10', + ' steps:', + ` - uses: ${checkout}`, + ' with:', + ' persist-credentials: false', + ' - run: pnpm audit && check-secret-patterns.mjs check-license-policy.mjs check-container-policy.mjs generate-sbom.mjs', + ].join('\n'), + 'release.yml': [ + 'name: r', + 'permissions:', + ' contents: read', + ' id-token: write', + 'jobs:', + ' release:', + ' runs-on: ubuntu-24.04', + ' timeout-minutes: 10', + ' steps:', + ` - uses: ${checkout}`, + ' with:', + ' persist-credentials: false', + ' - run: generate-provenance.mjs', + ` - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02`, + ' with:', + ' path: output.json', + ].join('\n'), + }; + for (const [name, text] of Object.entries(workflows)) + fs.writeFileSync(path.join(root, '.github/workflows', name), text); + assert.throws(() => checkCiPolicy(root), /artifact uploads must fail/u); +}); + +test('CI policy requires security and release commands in run values', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'databreeze-ci-command-fields-')); + fs.mkdirSync(path.join(root, '.github/workflows'), { recursive: true }); + const checkout = 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683'; + const checkedOutJob = [ + 'jobs:', + ' check:', + ' runs-on: ubuntu-24.04', + ' timeout-minutes: 10', + ' steps:', + ` - uses: ${checkout}`, + ' with:', + ' persist-credentials: false', + ].join('\n'); + const security = [ + 'name: "pnpm audit check-secret-patterns.mjs check-license-policy.mjs check-container-policy.mjs generate-sbom.mjs"', + 'permissions:', + ' contents: read', + checkedOutJob, + ].join('\n'); + const release = [ + 'name: "generate-provenance.mjs"', + 'permissions:', + ' contents: read', + ' id-token: write', + 'jobs:', + ' release:', + ' runs-on: ubuntu-24.04', + ' timeout-minutes: 10', + ' environment: release', + ' steps:', + ` - uses: ${checkout}`, + ' with:', + ' persist-credentials: false', + ].join('\n'); + const quality = ['name: q', 'permissions:', ' contents: read', checkedOutJob].join('\n'); + for (const [name, text] of Object.entries({ + 'quality.yml': quality, + 'security.yml': security, + 'release.yml': release, + })) + fs.writeFileSync(path.join(root, '.github/workflows', name), text); + assert.throws(() => checkCiPolicy(root), /security\.yml is missing pnpm audit/u); +}); diff --git a/tools/repo-cli/test/execution-orchestration.test.mjs b/tools/repo-cli/test/execution-orchestration.test.mjs index cdef1148..5497c2e3 100644 --- a/tools/repo-cli/test/execution-orchestration.test.mjs +++ b/tools/repo-cli/test/execution-orchestration.test.mjs @@ -142,8 +142,8 @@ test('handoff policy preserves the requested dev and main review flow', () => { }); assert.deepEqual(ledger.reviewPolicy.commitBudget, { preferredMinimum: 30, - preferredMaximum: 50, - hardMaximum: 60, + preferredMaximum: 70, + hardMaximum: 99, }); }); @@ -177,7 +177,8 @@ test('repository checker validates the committed orchestration package', () => { test('ledger records verified task evidence before advancing the next task', () => { const ledger = readJson('docs/plans/execution-orchestration.json'); assert.equal(ledger.nextTaskId, 'FND-003'); - assert.equal(ledger.checkpoint.lastFeaturePullRequest, 13); + assert.equal(ledger.checkpoint.lastFeaturePullRequest, 18); + assert.equal(ledger.checkpoint.lastPromotionPullRequest, 14); assert.deepEqual(ledger.taskState?.['FND-001']?.status, 'verified'); assert.match(ledger.taskState?.['FND-001']?.commit ?? '', /^[0-9a-f]{40}$/u); assert.ok( diff --git a/tools/repo-cli/test/local-infrastructure.test.mjs b/tools/repo-cli/test/local-infrastructure.test.mjs index 1610c9fa..084cad42 100644 --- a/tools/repo-cli/test/local-infrastructure.test.mjs +++ b/tools/repo-cli/test/local-infrastructure.test.mjs @@ -10,6 +10,8 @@ const read = (relativePath) => readFileSync(path.join(repositoryRoot, relativePa test('local compose defines pinned, healthy disposable dependencies', () => { const compose = read('infrastructure/local/compose.yml'); + const envExample = read('infrastructure/local/.env.example'); + assert.match(envExample, /^DATABREEZE_MIN_FREE_GIB=5$/m); for (const service of [ 'postgres:', 'redis:', @@ -31,6 +33,21 @@ test('local compose defines pinned, healthy disposable dependencies', () => { assert.match(compose, new RegExp(`^ ${volume}:`, 'm')); } assert.equal((compose.match(/healthcheck:/g) ?? []).length, 6); + assert.equal((compose.match(/^\s{4}init: true$/gmu) ?? []).length, 7); + assert.match(compose, /minio-init:[\s\S]*depends_on:[\s\S]*condition: service_healthy/u); + assert.match(compose, /minio-init:[\s\S]*restart: 'no'/u); + assert.match(compose, /postgres-data:[\s\S]*name: \$\{COMPOSE_PROJECT_NAME/u); + assert.equal((compose.match(/networks: \[local\]/g) ?? []).length, 7); + assert.match(compose, /name: \$\{COMPOSE_PROJECT_NAME:-databreeze-local\}-network/u); + assert.match(compose, /x-default-logging: &default-logging/u); + assert.match(compose, /max-size: 10m/u); + assert.match(compose, /max-file: '3'/u); + assert.equal((compose.match(/logging: \*default-logging/g) ?? []).length, 7); + assert.equal((compose.match(/127\.0\.0\.1:\$\{/g) ?? []).length, 9); + assert.match( + read('infrastructure/local/README.md'), + /Every published port is bound to `127\.0\.0\.1`/u, + ); }); test('local bootstrap is credential-free and creates every owned module schema', () => { @@ -58,12 +75,47 @@ test('local bootstrap is credential-free and creates every owned module schema', ]; for (const schema of expectedSchemas) assert.match(sql, new RegExp(`'${schema}'`)); assert.doesNotMatch(sql, /password|secret|BEGIN\s+;|CREATE\s+ROLE/i); + assert.match(sql, /CREATE SCHEMA IF NOT EXISTS/u); + assert.doesNotMatch(sql, /DROP\s+SCHEMA|DROP\s+DATABASE|TRUNCATE/u); const bucketScript = read('infrastructure/local/minio/bootstrap-buckets.sh'); assert.match(bucketScript, /MINIO_ROOT_PASSWORD/); + assert.match(bucketScript, /mc mb --ignore-existing/u); + assert.match(bucketScript, /mc anonymous set none/u); assert.doesNotMatch(bucketScript, /databreeze-local-change-me/); }); +test('local OpenTelemetry collector keeps every signal on the bounded local pipeline', () => { + const collector = read('infrastructure/local/otel/collector.yaml'); + for (const [section, indentation] of [ + ['receivers:', ''], + ['processors:', ''], + ['exporters:', ''], + ['extensions:', ''], + ['service:', ''], + ['pipelines:', ' '], + ]) { + assert.match(collector, new RegExp(`^${indentation}${section}`, 'm')); + } + for (const signal of ['traces:', 'metrics:', 'logs:']) { + assert.match(collector, new RegExp(`^ ${signal}`, 'm')); + assert.match(collector, new RegExp(`${signal}[\\s\\S]*receivers: \\[otlp\\]`, 'u')); + assert.match( + collector, + new RegExp(`${signal}[\\s\\S]*processors: \\[memory_limiter, batch\\]`, 'u'), + ); + assert.match(collector, new RegExp(`${signal}[\\s\\S]*exporters: \\[debug\\]`, 'u')); + } + assert.match(collector, /health_check:[\s\S]*endpoint: 0\.0\.0\.0:13133/u); + assert.doesNotMatch(collector, /filelog|otlphttp|s3|https?:\/\//iu); +}); + +test('local infrastructure documents bounded diagnostic storage', () => { + const readme = read('infrastructure/local/README.md'); + assert.match(readme, /Container JSON logs are capped at 10 MiB per file/u); + assert.match(readme, /three retained files/u); +}); + test('readiness smoke script exposes a non-destructive help command', () => { const script = path.join(repositoryRoot, 'tools', 'repo-cli', 'src', 'local-services-smoke.mjs'); const result = spawnSync(process.execPath, [script, '--help'], { @@ -74,3 +126,119 @@ test('readiness smoke script exposes a non-destructive help command', () => { assert.match(result.stdout, /--start/); assert.match(result.stdout, /never removes containers or named volumes/i); }); + +test('local lifecycle commands fail safely around Docker, ports, disk, and volumes', () => { + const script = read('tools/repo-cli/src/local-services.mjs'); + const helpScript = path.join(repositoryRoot, 'tools', 'repo-cli', 'src', 'local-services.mjs'); + const result = spawnSync(process.execPath, [helpScript, '--help'], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr); + for (const command of [ + 'config', + 'preflight', + 'check', + 'start', + 'stop', + 'reset', + 'restart-check', + 'persistence-check', + 'status', + 'logs', + 'smoke', + ]) { + assert.match(result.stdout, new RegExp(`^ ${command}\\s`, 'm')); + } + assert.match(script, /statfsSync/u); + assert.match(script, /portAvailable/u); + assert.match(script, /configured = new Map/u); + assert.match(script, /Docker CLI is not installed/u); + assert.match(script, /Docker daemon is unavailable/u); + assert.match(script, /down', '--remove-orphans/u); + assert.doesNotMatch(script, /down'[^\n]*--volumes/u); + assert.doesNotMatch(script, /down\s+--volumes/u); + assert.doesNotMatch(script, /docker\s+(?:rm|volume\s+rm|system\s+prune)/iu); + + const invalidTimeout = spawnSync(process.execPath, [helpScript, 'check', '--wait-seconds=0'], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + assert.notEqual(invalidTimeout.status, 0); + assert.match( + `${invalidTimeout.stdout}\n${invalidTimeout.stderr}`, + /--wait-seconds must be an integer/u, + ); + + const invalidDisk = spawnSync(process.execPath, [helpScript, 'check', '--min-free-gib=-1'], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + assert.notEqual(invalidDisk.status, 0); + assert.match( + `${invalidDisk.stdout}\n${invalidDisk.stderr}`, + /--min-free-gib must be a non-negative number/u, + ); + + const invalidTail = spawnSync(process.execPath, [helpScript, 'logs', '--tail=0'], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + assert.notEqual(invalidTail.status, 0); + assert.match(`${invalidTail.stdout}\n${invalidTail.stderr}`, /--tail must be an integer/u); + + const invalidService = spawnSync(process.execPath, [helpScript, 'logs', '--service=unknown'], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + assert.notEqual(invalidService.status, 0); + assert.match(`${invalidService.stdout}\n${invalidService.stderr}`, /--service must name one of/u); + + const invalidProject = spawnSync(process.execPath, [helpScript, 'config'], { + cwd: repositoryRoot, + encoding: 'utf8', + env: { ...process.env, COMPOSE_PROJECT_NAME: '../unsafe-project' }, + }); + assert.notEqual(invalidProject.status, 0); + assert.match( + `${invalidProject.stdout}\n${invalidProject.stderr}`, + /COMPOSE_PROJECT_NAME must start/u, + ); + + const invalidEnvironmentDisk = spawnSync(process.execPath, [helpScript, 'preflight'], { + cwd: repositoryRoot, + encoding: 'utf8', + env: { ...process.env, DATABREEZE_MIN_FREE_GIB: 'not-a-number' }, + }); + assert.notEqual(invalidEnvironmentDisk.status, 0); + assert.match( + `${invalidEnvironmentDisk.stdout}\n${invalidEnvironmentDisk.stderr}`, + /--min-free-gib must be a non-negative number/u, + ); + + const composeConfig = spawnSync(process.execPath, [helpScript, 'config'], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + if (composeConfig.status === 0) { + assert.match(composeConfig.stdout, /Compose configuration is valid/u); + } else { + assert.match( + `${composeConfig.stdout}\n${composeConfig.stderr}`, + /Docker CLI is not installed or not on PATH/u, + ); + } + const preflight = spawnSync(process.execPath, [helpScript, 'preflight', '--min-free-gib=0'], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + if (preflight.status === 0) { + assert.match(preflight.stdout, /preflight passed without starting services/u); + } else { + assert.match( + `${preflight.stdout}\n${preflight.stderr}`, + /Docker CLI is not installed or not on PATH/u, + ); + } + assert.doesNotMatch(script, /redis-cli\s+FLUSH(?:ALL|DB)/iu); +}); diff --git a/tools/repo-cli/test/provenance.test.mjs b/tools/repo-cli/test/provenance.test.mjs new file mode 100644 index 00000000..70ebec9c --- /dev/null +++ b/tools/repo-cli/test/provenance.test.mjs @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import test from 'node:test'; + +const repositoryRoot = path.resolve(import.meta.dirname, '..', '..', '..'); +const script = path.join(repositoryRoot, 'tools/repo-cli/src/generate-provenance.mjs'); + +test('provenance generation records sorted artifact digests', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'databreeze-provenance-')); + try { + const first = path.join(directory, 'z-output.json'); + const second = path.join(directory, 'a-output.json'); + const output = path.join(directory, 'provenance.json'); + writeFileSync(first, 'z'); + writeFileSync(second, 'a'); + const result = spawnSync( + process.execPath, + [script, '--output', output, '--artifact', first, '--artifact', second], + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + assert.equal(result.status, 0, result.stderr); + const provenance = JSON.parse(readFileSync(output, 'utf8')); + assert.deepEqual( + provenance.subject.map((subject) => path.posix.basename(subject.path)), + ['a-output.json', 'z-output.json'], + ); + assert.deepEqual( + provenance.subject.map((subject) => subject.sha256), + [ + 'ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb', + '594e519ae499312b29433b7dd8a97ff068defcba9755b6d5d00e84c524d67b06', + ], + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test('provenance generation fails closed when an artifact is missing', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'databreeze-provenance-missing-')); + try { + const result = spawnSync( + process.execPath, + [ + script, + '--output', + path.join(directory, 'provenance.json'), + '--artifact', + path.join(directory, 'missing.json'), + ], + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + assert.notEqual(result.status, 0); + assert.match(`${result.stdout}\n${result.stderr}`, /Provenance artifact\(s\) do not exist/u); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test('provenance generation rejects an artifact flag without a path', () => { + const result = spawnSync(process.execPath, [script, '--artifact'], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + assert.notEqual(result.status, 0); + assert.match(`${result.stdout}\n${result.stderr}`, /--artifact requires a file path/u); +}); + +test('provenance generation rejects an output flag without a path', () => { + const result = spawnSync(process.execPath, [script, '--output'], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + assert.notEqual(result.status, 0); + assert.match(`${result.stdout}\n${result.stderr}`, /--output requires a file path/u); +}); diff --git a/tools/repo-cli/test/sbom.test.mjs b/tools/repo-cli/test/sbom.test.mjs new file mode 100644 index 00000000..72ec7d73 --- /dev/null +++ b/tools/repo-cli/test/sbom.test.mjs @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import test from 'node:test'; + +const repositoryRoot = path.resolve(import.meta.dirname, '..', '..', '..'); +const script = path.join(repositoryRoot, 'tools/repo-cli/src/generate-sbom.mjs'); + +test('SBOM generation rejects an output flag without a path', () => { + const result = spawnSync(process.execPath, [script, '--output'], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + assert.notEqual(result.status, 0); + assert.match(`${result.stdout}\n${result.stderr}`, /--output requires a file path/u); +}); + +test('SBOM generation writes to an explicit output path', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'databreeze-sbom-')); + try { + const output = path.join(directory, 'sbom.json'); + const result = spawnSync(process.execPath, [script, '--output', output], { + cwd: repositoryRoot, + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Wrote .*sbom\.json/u); + assert.equal(existsSync(output), true); + const sbom = JSON.parse(readFileSync(output, 'utf8')); + assert.equal(sbom.bomFormat, 'CycloneDX'); + assert.equal(sbom.specVersion, '1.5'); + assert.ok(Array.isArray(sbom.components)); + assert.ok(sbom.components.some((component) => component.name === '@databreeze/platform')); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/tools/repo-cli/test/terraform-safety.test.mjs b/tools/repo-cli/test/terraform-safety.test.mjs new file mode 100644 index 00000000..34f43d26 --- /dev/null +++ b/tools/repo-cli/test/terraform-safety.test.mjs @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { balancedBlocks } from '../src/terraform-safety.mjs'; + +test('Terraform block scanning ignores comments, heredocs, and braces in strings', () => { + const source = ` +# ingress { cidr_blocks = ["0.0.0.0/0"] } +// ingress { cidr_blocks = ["0.0.0.0/0"] } } +/* ingress { cidr_blocks = ["0.0.0.0/0"] } */ +locals { + description = <<-EOT + ingress { + cidr_blocks = ["0.0.0.0/0"] + } + braces: { }; + EOT +} +ingress { + description = "literal } brace" + ${' '.repeat(500)} + cidr_blocks = ["10.0.0.0/8"] +} +`; + const blocks = balancedBlocks(source, 'ingress'); + assert.equal(blocks.length, 1); + assert.match(blocks[0], /10\.0\.0\.0\/8/u); + assert.doesNotMatch(blocks[0], /0\.0\.0\.0\/0/u); +}); + +test('Terraform principal scanning ignores commented wildcard identifiers', () => { + const source = ` +/* principals { identifiers = ["*"] } */ +principals { + type = "Service" + identifiers = ["example.amazonaws.com"] +} +`; + const blocks = balancedBlocks(source, 'principals'); + assert.equal(blocks.length, 1); + assert.doesNotMatch(blocks[0], /identifiers\s*=\s*\[[^\]]*"\*"/u); +});