From ee38dadf4805d26685dc18c0067c95b96bc0b074 Mon Sep 17 00:00:00 2001 From: Ellie Melton Date: Sun, 12 Jul 2026 18:32:07 +1000 Subject: [PATCH] Check blob backend before API startup --- CHANGELOG.md | 4 ++ README.md | 2 + cmd/api/main.go | 7 +++ cmd/api/main_test.go | 70 ++++++++++++++++++++++++++ docs/README.md | 2 +- docs/cluster-backup-restore-runbook.md | 19 +++++++ docs/configuration.md | 7 +++ docs/production-cluster-scope.md | 17 ++++--- docs/threat-model.md | 5 ++ 9 files changed, 124 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f027ca6..5612799 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ preview readiness claim. `internal/evidencebundle`, preserving fail-closed bundle integrity checks, server-controlled ZIP entry names, and ciphertext-only bundle behavior. +- Added a fail-closed API startup check for the selected local or S3-compatible + blob backend before HTTP listeners begin serving, while keeping failure logs + limited to the existing sanitized startup stage and error category. + - Extracted shared JSON response and error helpers into a narrower `internal/httpapi` boundary, with focused tests preserving existing response shapes. diff --git a/README.md b/README.md index 723b694..bd96c2b 100644 --- a/README.md +++ b/README.md @@ -276,6 +276,8 @@ escrow. - SQLite metadata and local disk encrypted blob storage by default - Optional PostgreSQL metadata backend for new deployments - Optional S3-compatible encrypted blob storage for committed chunks +- Fail-closed startup checks for the selected local or S3-compatible blob + backend before the API listeners begin serving - Account-scoped committed encrypted blob quota, defaulting to 10 GB per owner account as a preview abuse/cost control - Local temp-upload staging quota, defaulting to 1 GB for both local and diff --git a/cmd/api/main.go b/cmd/api/main.go index 71edbde..2d5b7ee 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -80,6 +80,9 @@ func run(logger *slog.Logger, configFilePath string) error { if err != nil { return withStartupStage(startupStageBlobStoreOpen, err) } + if err := ensureBlobStoreReady(ctx, blobStore); err != nil { + return withStartupStage(startupStageBlobStoreOpen, err) + } if err := runTempUploadCleanup(ctx, logger, blobStore, cfg); err != nil { return withStartupStage(startupStageTempUploadCleanup, err) } @@ -327,6 +330,10 @@ func newBlobStore(cfg config.Config) (storage.BlobStore, error) { } } +func ensureBlobStoreReady(ctx context.Context, store storage.BlobStore) error { + return store.Check(ctx) +} + func runTempUploadCleanup(ctx context.Context, logger *slog.Logger, store storage.BlobStore, cfg config.Config) error { if cfg.TempUploadCleanupAge <= 0 { return nil diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 3bb7da1..086d349 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -91,6 +91,65 @@ func TestStartupErrorLogDoesNotExposeFilesystemPath(t *testing.T) { } } +func TestEnsureBlobStoreReady(t *testing.T) { + checkErr := errors.New("blob backend unavailable") + tests := []struct { + name string + err error + }{ + {name: "ready"}, + {name: "unavailable", err: checkErr}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := &startupCheckBlobStore{checkErr: tt.err} + + err := ensureBlobStoreReady(context.Background(), store) + + if !errors.Is(err, tt.err) { + t.Fatalf("ensureBlobStoreReady error = %v, want %v", err, tt.err) + } + if store.checkCalls != 1 { + t.Fatalf("blob store check calls = %d, want 1", store.checkCalls) + } + }) + } +} + +func TestStartupErrorLogDoesNotExposeBlobBackendDetails(t *testing.T) { + var logs bytes.Buffer + logger := slog.New(slog.NewTextHandler(&logs, nil)) + privateDetails := []string{ + "https://private-object-store.example.test", + "private-evidence-bucket", + "private-access-key", + "/private/blob/staging/path", + "private-object-key", + } + err := errors.New(strings.Join(privateDetails, " ")) + + logStartupError(logger, withStartupStage(startupStageBlobStoreOpen, err)) + + for _, want := range [][]byte{ + []byte("component=startup"), + []byte("startup_stage=blob_store_open"), + []byte("error_category=storage"), + } { + if !bytes.Contains(logs.Bytes(), want) { + t.Fatalf("startup log omitted %q: %s", want, logs.String()) + } + } + for _, privateDetail := range privateDetails { + if bytes.Contains(logs.Bytes(), []byte(privateDetail)) { + t.Fatalf("startup log exposed blob backend detail %q: %s", privateDetail, logs.String()) + } + } + if bytes.Contains(logs.Bytes(), []byte("safe_error_detail")) { + t.Fatalf("startup log exposed raw blob backend error detail: %s", logs.String()) + } +} + func TestStartupErrorLogRedactsSecretConfigNameAndUsesSafeDetail(t *testing.T) { var logs bytes.Buffer logger := slog.New(slog.NewTextHandler(&logs, nil)) @@ -327,3 +386,14 @@ func assertServerTimeouts(t *testing.T, server *http.Server, want config.HTTPTim ) } } + +type startupCheckBlobStore struct { + storage.BlobStore + checkErr error + checkCalls int +} + +func (s *startupCheckBlobStore) Check(context.Context) error { + s.checkCalls++ + return s.checkErr +} diff --git a/docs/README.md b/docs/README.md index e26ceb7..21495c6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -94,7 +94,7 @@ planning notes for client and protocol work only as repository-boundary context. ## Current Backend Scope -Proofline Server receives already-encrypted chunks, stores metadata in SQLite by default or optional PostgreSQL, stores encrypted blobs on local disk by default or in optional S3-compatible object storage, loads TOML config with `SAFE_*` environment and secret-file overrides, performs a startup check against optional Valkey/Redis-compatible coordination when explicitly configured, groups chunks into media streams, can issue configured short-lived regional relay upload and fanout capabilities for authorized open streams, exposes service-authenticated core relay preflight/commit/fanout authorization endpoints when relay-to-core auth is configured, includes a separate regional stream-ingress relay route for complete encrypted chunk upload/staging/forwarding, optimistic encrypted unconfirmed fanout, and bounded fanout state when configured, serves private admin JSON routes under `/admin/api/...` and a private admin web surface under `/admin`, applies app-level route-class rate limiting to main API routes, can use Valkey for short-lived complete-upload leases, supports optional browser cookie sessions with CSRF and credentialed CORS for future production web-client calls, supports disabled-by-default configurable account registration with SMTP-backed email verification for open self-hosted registration, supports email challenge, TOTP, and disabled-by-default WebAuthn/FIDO2 passkey or roaming security-key second-factor setup for account gating, supports private-admin assisted second-factor reset for lost-factor recovery, and exposes a token-scoped read-only incident viewer with app-level route-class rate limiting. New chunk uploads must use the accepted post-quantum payload envelope by default, and the Go simulator now produces that PQ envelope unless an explicit v1 compatibility flag is used. +Proofline Server receives already-encrypted chunks, stores metadata in SQLite by default or optional PostgreSQL, stores encrypted blobs on local disk by default or in optional S3-compatible object storage, loads TOML config with `SAFE_*` environment and secret-file overrides, checks the selected blob backend and optional Valkey/Redis-compatible coordination before API startup, groups chunks into media streams, can issue configured short-lived regional relay upload and fanout capabilities for authorized open streams, exposes service-authenticated core relay preflight/commit/fanout authorization endpoints when relay-to-core auth is configured, includes a separate regional stream-ingress relay route for complete encrypted chunk upload/staging/forwarding, optimistic encrypted unconfirmed fanout, and bounded fanout state when configured, serves private admin JSON routes under `/admin/api/...` and a private admin web surface under `/admin`, applies app-level route-class rate limiting to main API routes, can use Valkey for short-lived complete-upload leases, supports optional browser cookie sessions with CSRF and credentialed CORS for future production web-client calls, supports disabled-by-default configurable account registration with SMTP-backed email verification for open self-hosted registration, supports email challenge, TOTP, and disabled-by-default WebAuthn/FIDO2 passkey or roaming security-key second-factor setup for account gating, supports private-admin assisted second-factor reset for lost-factor recovery, and exposes a token-scoped read-only incident viewer with app-level route-class rate limiting. New chunk uploads must use the accepted post-quantum payload envelope by default, and the Go simulator now produces that PQ envelope unless an explicit v1 compatibility flag is used. The regional stream-ingress relay currently has a separate health/readiness route with safe aggregate readiness categories, backend-issued upload and diff --git a/docs/cluster-backup-restore-runbook.md b/docs/cluster-backup-restore-runbook.md index 51443d8..28d873d 100644 --- a/docs/cluster-backup-restore-runbook.md +++ b/docs/cluster-backup-restore-runbook.md @@ -273,6 +273,25 @@ or multipart design adds object-storage staging: - Preserve any cleanup design in security, retention, backup, restore, and threat-model docs before implementation. +### Blob Backend Unavailable At Startup + +If the selected local or S3-compatible blob backend fails its startup check: + +- Treat startup failure as expected fail-closed behavior; neither API listener + should begin serving. +- For local storage, restore writable access to the configured committed-blob + and temporary-staging directories. +- For S3-compatible storage, restore writable local temporary staging and + bucket reachability/access without placing endpoint, bucket, credential, + path, or object details in logs or public diagnostics. +- Do not bypass the failure by switching backends unless the operator + deliberately chooses and documents a different private deployment shape. + +The startup check is a point-in-time preflight, not continuous storage +readiness monitoring. Runtime storage failures must still fail safely and be +handled through the documented backup, restore, retry, and reconciliation +procedures. + ### Coordination Service Unavailable If Valkey/Redis-compatible coordination is configured but unavailable at diff --git a/docs/configuration.md b/docs/configuration.md index 96e16be..4452e31 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -420,6 +420,13 @@ go run ./cmd/api Values are matched case-insensitively after trimming surrounding whitespace. Unsupported names fail startup with a clear configuration error. +Before either API listener begins serving, the server checks the selected blob +backend. Local storage requires writable committed-blob and temporary-staging +directories. S3-compatible storage requires writable local temporary staging +and a reachable, accessible configured bucket. A failed check stops startup; +logs retain only the sanitized `blob_store_open` stage and safe error category, +without backend endpoints, bucket names, credentials, paths, or object keys. + PostgreSQL metadata is implemented as an optional backend for new deployments. Prefer a secret file for the DSN: diff --git a/docs/production-cluster-scope.md b/docs/production-cluster-scope.md index c762249..1a0016f 100644 --- a/docs/production-cluster-scope.md +++ b/docs/production-cluster-scope.md @@ -3,12 +3,12 @@ This document records the production-cluster expansion path for Proofline Server. It is a planning and scope document for cluster-related work. Optional -PostgreSQL metadata, optional S3-compatible object storage, optional -Valkey/Redis-compatible coordination for startup checks, route-class counters, -and short-lived complete-upload leases, and local `/v1` account/session -authentication are implemented. Public product API deployment, public account -workflows, cloud deployment automation, production hardening, and resumable or -partial upload sessions are not implemented. +PostgreSQL metadata, optional startup-checked S3-compatible object storage, +optional Valkey/Redis-compatible coordination for startup checks, route-class +counters, and short-lived complete-upload leases, and local `/v1` +account/session authentication are implemented. Public product API deployment, +public account workflows, cloud deployment automation, production hardening, +and resumable or partial upload sessions are not implemented. ## Current Local-First Scope @@ -17,7 +17,8 @@ The current backend remains local-first and experimental: - SQLite metadata remains supported and remains the default. - Optional PostgreSQL metadata is available only when explicitly configured. - Local filesystem encrypted blob storage remains supported. -- Optional S3-compatible encrypted blob storage is available only when explicitly configured. +- Optional S3-compatible encrypted blob storage is available only when + explicitly configured and is checked before the API listeners begin serving. - No coordination backend remains the default; Valkey/Redis-compatible coordination is available only when explicitly configured. - The simulator and local development flow remain supported. @@ -37,7 +38,7 @@ Planned optional cluster backends: | Capability | Local/default backend | Planned cluster backend | |---|---|---| | Metadata | SQLite | PostgreSQL, implemented as an optional backend | -| Committed encrypted chunks | Local filesystem | S3-compatible object storage, implemented as an optional backend | +| Committed encrypted chunks | Local filesystem | S3-compatible object storage, implemented as an optional startup-checked backend | | Short-lived coordination | None | Valkey/Redis-compatible coordination, implemented as an optional startup-checked backend | These backends should be additive. They must not remove or weaken SQLite and local filesystem support. diff --git a/docs/threat-model.md b/docs/threat-model.md index 25eb91e..44fbf79 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -204,6 +204,11 @@ encrypted evidence bundles. mismatched field names and do not return uploaded bytes, stored paths, object keys, plaintext, raw keys, raw tokens, request bodies, private deployment details, or conflicting stored values. +- The selected local or S3-compatible blob backend is checked before the API + listeners begin serving. Local storage requires writable committed-blob and + temporary-staging directories; S3-compatible storage requires writable local + temporary staging plus a reachable, accessible configured bucket. Failures + stop startup and use only sanitized startup stages and error categories. - Optional Valkey/Redis-compatible coordination fails closed at startup when explicitly configured but unavailable. - TOML configuration can reference selected secret files. Secret files are read