Check blob backend before API startup

CHANGELOG.md

@@ -11,6 +11,10 @@ preview readiness claim.
   `internal/evidencebundle`, preserving fail-closed bundle integrity checks,
   server-controlled ZIP entry names, and ciphertext-only bundle behavior.
 
+- Added a fail-closed API startup check for the selected local or S3-compatible
+  blob backend before HTTP listeners begin serving, while keeping failure logs
+  limited to the existing sanitized startup stage and error category.
+
 - Extracted shared JSON response and error helpers into a narrower
   `internal/httpapi` boundary, with focused tests preserving existing response
   shapes.

README.md

@@ -276,6 +276,8 @@ escrow.
 - SQLite metadata and local disk encrypted blob storage by default
 - Optional PostgreSQL metadata backend for new deployments
 - Optional S3-compatible encrypted blob storage for committed chunks
+- Fail-closed startup checks for the selected local or S3-compatible blob
+  backend before the API listeners begin serving
 - Account-scoped committed encrypted blob quota, defaulting to 10 GB per owner
   account as a preview abuse/cost control
 - Local temp-upload staging quota, defaulting to 1 GB for both local and

cmd/api/main.go

@@ -80,6 +80,9 @@ func run(logger *slog.Logger, configFilePath string) error {
 	if err != nil {
 		return withStartupStage(startupStageBlobStoreOpen, err)
 	}
+	if err := ensureBlobStoreReady(ctx, blobStore); err != nil {
+		return withStartupStage(startupStageBlobStoreOpen, err)
+	}
 	if err := runTempUploadCleanup(ctx, logger, blobStore, cfg); err != nil {
 		return withStartupStage(startupStageTempUploadCleanup, err)
 	}
@@ -327,6 +330,10 @@ func newBlobStore(cfg config.Config) (storage.BlobStore, error) {
 	}
 }
 
+func ensureBlobStoreReady(ctx context.Context, store storage.BlobStore) error {
+	return store.Check(ctx)
+}
+
 func runTempUploadCleanup(ctx context.Context, logger *slog.Logger, store storage.BlobStore, cfg config.Config) error {
 	if cfg.TempUploadCleanupAge <= 0 {
 		return nil

cmd/api/main_test.go

@@ -91,6 +91,65 @@ func TestStartupErrorLogDoesNotExposeFilesystemPath(t *testing.T) {
 	}
 }
 
+func TestEnsureBlobStoreReady(t *testing.T) {
+	checkErr := errors.New("blob backend unavailable")
+	tests := []struct {
+		name string
+		err  error
+	}{
+		{name: "ready"},
+		{name: "unavailable", err: checkErr},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			store := &startupCheckBlobStore{checkErr: tt.err}
+
+			err := ensureBlobStoreReady(context.Background(), store)
+
+			if !errors.Is(err, tt.err) {
+				t.Fatalf("ensureBlobStoreReady error = %v, want %v", err, tt.err)
+			}
+			if store.checkCalls != 1 {
+				t.Fatalf("blob store check calls = %d, want 1", store.checkCalls)
+			}
+		})
+	}
+}
+
+func TestStartupErrorLogDoesNotExposeBlobBackendDetails(t *testing.T) {
+	var logs bytes.Buffer
+	logger := slog.New(slog.NewTextHandler(&logs, nil))
+	privateDetails := []string{
+		"https://private-object-store.example.test",
+		"private-evidence-bucket",
+		"private-access-key",
+		"/private/blob/staging/path",
+		"private-object-key",
+	}
+	err := errors.New(strings.Join(privateDetails, " "))
+
+	logStartupError(logger, withStartupStage(startupStageBlobStoreOpen, err))
+
+	for _, want := range [][]byte{
+		[]byte("component=startup"),
+		[]byte("startup_stage=blob_store_open"),
+		[]byte("error_category=storage"),
+	} {
+		if !bytes.Contains(logs.Bytes(), want) {
+			t.Fatalf("startup log omitted %q: %s", want, logs.String())
+		}
+	}
+	for _, privateDetail := range privateDetails {
+		if bytes.Contains(logs.Bytes(), []byte(privateDetail)) {
+			t.Fatalf("startup log exposed blob backend detail %q: %s", privateDetail, logs.String())
+		}
+	}
+	if bytes.Contains(logs.Bytes(), []byte("safe_error_detail")) {
+		t.Fatalf("startup log exposed raw blob backend error detail: %s", logs.String())
+	}
+}
+
 func TestStartupErrorLogRedactsSecretConfigNameAndUsesSafeDetail(t *testing.T) {
 	var logs bytes.Buffer
 	logger := slog.New(slog.NewTextHandler(&logs, nil))
@@ -327,3 +386,14 @@ func assertServerTimeouts(t *testing.T, server *http.Server, want config.HTTPTim
 		)
 	}
 }
+
+type startupCheckBlobStore struct {
+	storage.BlobStore
+	checkErr   error
+	checkCalls int
+}
+
+func (s *startupCheckBlobStore) Check(context.Context) error {
+	s.checkCalls++
+	return s.checkErr
+}

docs/README.md

@@ -94,7 +94,7 @@ planning notes for client and protocol work only as repository-boundary context.
 
 ## Current Backend Scope
 
-Proofline Server receives already-encrypted chunks, stores metadata in SQLite by default or optional PostgreSQL, stores encrypted blobs on local disk by default or in optional S3-compatible object storage, loads TOML config with `SAFE_*` environment and secret-file overrides, performs a startup check against optional Valkey/Redis-compatible coordination when explicitly configured, groups chunks into media streams, can issue configured short-lived regional relay upload and fanout capabilities for authorized open streams, exposes service-authenticated core relay preflight/commit/fanout authorization endpoints when relay-to-core auth is configured, includes a separate regional stream-ingress relay route for complete encrypted chunk upload/staging/forwarding, optimistic encrypted unconfirmed fanout, and bounded fanout state when configured, serves private admin JSON routes under `/admin/api/...` and a private admin web surface under `/admin`, applies app-level route-class rate limiting to main API routes, can use Valkey for short-lived complete-upload leases, supports optional browser cookie sessions with CSRF and credentialed CORS for future production web-client calls, supports disabled-by-default configurable account registration with SMTP-backed email verification for open self-hosted registration, supports email challenge, TOTP, and disabled-by-default WebAuthn/FIDO2 passkey or roaming security-key second-factor setup for account gating, supports private-admin assisted second-factor reset for lost-factor recovery, and exposes a token-scoped read-only incident viewer with app-level route-class rate limiting. New chunk uploads must use the accepted post-quantum payload envelope by default, and the Go simulator now produces that PQ envelope unless an explicit v1 compatibility flag is used.
+Proofline Server receives already-encrypted chunks, stores metadata in SQLite by default or optional PostgreSQL, stores encrypted blobs on local disk by default or in optional S3-compatible object storage, loads TOML config with `SAFE_*` environment and secret-file overrides, checks the selected blob backend and optional Valkey/Redis-compatible coordination before API startup, groups chunks into media streams, can issue configured short-lived regional relay upload and fanout capabilities for authorized open streams, exposes service-authenticated core relay preflight/commit/fanout authorization endpoints when relay-to-core auth is configured, includes a separate regional stream-ingress relay route for complete encrypted chunk upload/staging/forwarding, optimistic encrypted unconfirmed fanout, and bounded fanout state when configured, serves private admin JSON routes under `/admin/api/...` and a private admin web surface under `/admin`, applies app-level route-class rate limiting to main API routes, can use Valkey for short-lived complete-upload leases, supports optional browser cookie sessions with CSRF and credentialed CORS for future production web-client calls, supports disabled-by-default configurable account registration with SMTP-backed email verification for open self-hosted registration, supports email challenge, TOTP, and disabled-by-default WebAuthn/FIDO2 passkey or roaming security-key second-factor setup for account gating, supports private-admin assisted second-factor reset for lost-factor recovery, and exposes a token-scoped read-only incident viewer with app-level route-class rate limiting. New chunk uploads must use the accepted post-quantum payload envelope by default, and the Go simulator now produces that PQ envelope unless an explicit v1 compatibility flag is used.
 
 The regional stream-ingress relay currently has a separate health/readiness
 route with safe aggregate readiness categories, backend-issued upload and

docs/cluster-backup-restore-runbook.md

@@ -273,6 +273,25 @@ or multipart design adds object-storage staging:
 - Preserve any cleanup design in security, retention, backup, restore, and
   threat-model docs before implementation.
 
+### Blob Backend Unavailable At Startup
+
+If the selected local or S3-compatible blob backend fails its startup check:
+
+- Treat startup failure as expected fail-closed behavior; neither API listener
+  should begin serving.
+- For local storage, restore writable access to the configured committed-blob
+  and temporary-staging directories.
+- For S3-compatible storage, restore writable local temporary staging and
+  bucket reachability/access without placing endpoint, bucket, credential,
+  path, or object details in logs or public diagnostics.
+- Do not bypass the failure by switching backends unless the operator
+  deliberately chooses and documents a different private deployment shape.
+
+The startup check is a point-in-time preflight, not continuous storage
+readiness monitoring. Runtime storage failures must still fail safely and be
+handled through the documented backup, restore, retry, and reconciliation
+procedures.
+
 ### Coordination Service Unavailable
 
 If Valkey/Redis-compatible coordination is configured but unavailable at

docs/configuration.md

@@ -420,6 +420,13 @@ go run ./cmd/api
 
 Values are matched case-insensitively after trimming surrounding whitespace. Unsupported names fail startup with a clear configuration error.
 
+Before either API listener begins serving, the server checks the selected blob
+backend. Local storage requires writable committed-blob and temporary-staging
+directories. S3-compatible storage requires writable local temporary staging
+and a reachable, accessible configured bucket. A failed check stops startup;
+logs retain only the sanitized `blob_store_open` stage and safe error category,
+without backend endpoints, bucket names, credentials, paths, or object keys.
+
 PostgreSQL metadata is implemented as an optional backend for new deployments.
 Prefer a secret file for the DSN:
 

docs/production-cluster-scope.md

@@ -3,12 +3,12 @@
 This document records the production-cluster expansion path for Proofline Server.
 
 It is a planning and scope document for cluster-related work. Optional
-PostgreSQL metadata, optional S3-compatible object storage, optional
-Valkey/Redis-compatible coordination for startup checks, route-class counters,
-and short-lived complete-upload leases, and local `/v1` account/session
-authentication are implemented. Public product API deployment, public account
-workflows, cloud deployment automation, production hardening, and resumable or
-partial upload sessions are not implemented.
+PostgreSQL metadata, optional startup-checked S3-compatible object storage,
+optional Valkey/Redis-compatible coordination for startup checks, route-class
+counters, and short-lived complete-upload leases, and local `/v1`
+account/session authentication are implemented. Public product API deployment,
+public account workflows, cloud deployment automation, production hardening,
+and resumable or partial upload sessions are not implemented.
 
 ## Current Local-First Scope
 
@@ -17,7 +17,8 @@ The current backend remains local-first and experimental:
 - SQLite metadata remains supported and remains the default.
 - Optional PostgreSQL metadata is available only when explicitly configured.
 - Local filesystem encrypted blob storage remains supported.
-- Optional S3-compatible encrypted blob storage is available only when explicitly configured.
+- Optional S3-compatible encrypted blob storage is available only when
+  explicitly configured and is checked before the API listeners begin serving.
 - No coordination backend remains the default; Valkey/Redis-compatible
   coordination is available only when explicitly configured.
 - The simulator and local development flow remain supported.
@@ -37,7 +38,7 @@ Planned optional cluster backends:
 | Capability | Local/default backend | Planned cluster backend |
 |---|---|---|
 | Metadata | SQLite | PostgreSQL, implemented as an optional backend |
-| Committed encrypted chunks | Local filesystem | S3-compatible object storage, implemented as an optional backend |
+| Committed encrypted chunks | Local filesystem | S3-compatible object storage, implemented as an optional startup-checked backend |
 | Short-lived coordination | None | Valkey/Redis-compatible coordination, implemented as an optional startup-checked backend |
 
 These backends should be additive. They must not remove or weaken SQLite and local filesystem support.

docs/threat-model.md

@@ -204,6 +204,11 @@ encrypted evidence bundles.
   mismatched field names and do not return uploaded bytes, stored paths, object
   keys, plaintext, raw keys, raw tokens, request bodies, private deployment
   details, or conflicting stored values.
+- The selected local or S3-compatible blob backend is checked before the API
+  listeners begin serving. Local storage requires writable committed-blob and
+  temporary-staging directories; S3-compatible storage requires writable local
+  temporary staging plus a reachable, accessible configured bucket. Failures
+  stop startup and use only sanitized startup stages and error categories.
 - Optional Valkey/Redis-compatible coordination fails closed at startup when
   explicitly configured but unavailable.
 - TOML configuration can reference selected secret files. Secret files are read