Authentication & authorization across every Candybox channel (SASL + TLS + per-Box/object ACLs + S3 SigV4)

.github/workflows/s3-compat.yml

@@ -57,16 +57,30 @@ jobs:
         run: docker compose up -d --no-build
 
       # Drive a real PUT bucket through the gateway until the full path (gateway -> node -> bookies)
-      # answers, so the suite never races a not-yet-serving cluster.
+      # answers, so the suite never races a not-yet-serving cluster. The gateway verifies SigV4 in
+      # CI, so the probe signs with the provisioned `candybox` account (the runner's preinstalled
+      # AWS CLI); if the CLI is ever missing, an unsigned probe answering 403 still proves the
+      # gateway is up and authenticating (the suite itself covers the signed path).
       - name: Wait for the gateway to serve
+        env:
+          AWS_ACCESS_KEY_ID: candybox
+          AWS_SECRET_ACCESS_KEY: candybox
+          AWS_DEFAULT_REGION: us-east-1
         run: |
           for i in $(seq 1 60); do
-            code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT "$S3_ENDPOINT/ci-preflight" || true)
-            if [ "$code" = "200" ]; then
-              curl -s -o /dev/null -X DELETE "$S3_ENDPOINT/ci-preflight" || true
-              echo "gateway is serving (PUT bucket -> $code)"; exit 0
+            if command -v aws >/dev/null; then
+              if aws --endpoint-url "$S3_ENDPOINT" s3api create-bucket --bucket ci-preflight >/dev/null 2>&1; then
+                aws --endpoint-url "$S3_ENDPOINT" s3api delete-bucket --bucket ci-preflight >/dev/null 2>&1 || true
+                echo "gateway is serving (signed PUT bucket succeeded)"; exit 0
+              fi
+              echo "attempt $i: signed PUT bucket failed; retrying"; sleep 5
+            else
+              code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT "$S3_ENDPOINT/ci-preflight" || true)
+              if [ "$code" = "200" ] || [ "$code" = "403" ]; then
+                echo "gateway is serving (unsigned PUT bucket -> $code)"; exit 0
+              fi
+              echo "attempt $i: PUT bucket -> ${code:-no-response}; retrying"; sleep 5
             fi
-            echo "attempt $i: PUT bucket -> ${code:-no-response}; retrying"; sleep 5
           done
           echo "gateway did not become ready in time"; exit 1
 

OPERATIONS.md

@@ -149,6 +149,119 @@ v1 has **no authentication** (matching the rest of candybox today); the deploy a
 trusted network. The single mutating dashboard operation in v1 — deleting an object — goes through
 the S3 gateway from the browser, not through this API. See `WEB_DASHBOARD_PLAN.md`.
 
+## Security
+
+Candybox secures every channel with SASL authentication, per-Box/per-object ACLs, and in-process
+TLS. The full design lives in [`AUTH_PLAN.md`](AUTH_PLAN.md); this section is the operator view.
+All keys below follow the usual convention: any key can come from the environment as
+`CANDYBOX_<KEY>` (dots to underscores, upper-cased).
+
+### Clients / S3 gateway / admin API → storage nodes (SASL + TLS)
+
+On each node:
+
+```properties
+auth.enabled=true                  # SASL on the TCP listener
+auth.required=true                 # false = unauthenticated connections pass as anonymous
+auth.sasl.mechanisms=PLAIN,SCRAM-SHA-256
+auth.credentials.file=/etc/candybox/credentials.properties
+auth.super.users=Gateway:s3-gw,Admin:admin-api
+
+tls.enabled=true                   # PEM paths; the key must be UNENCRYPTED PKCS#8
+tls.cert.path=/etc/candybox/tls/tls.crt
+tls.key.path=/etc/candybox/tls/tls.key
+tls.ca.path=/etc/candybox/tls/ca.crt
+tls.client.auth=false              # true = demand a client certificate (mTLS)
+```
+
+Generate credential-file entries with `candybox make-credentials <user>`; the file is reloaded on
+change (rotation needs no restart), and `plain:` verifiers are accepted for dev fixtures. Choose
+**PLAIN only on TLS listeners** (the password crosses the wire; the server stores a one-way hash);
+**SCRAM-SHA-256** never sends the password and mutually authenticates, so it is the safer choice
+when TLS terminates elsewhere. Dev certificates: `examples/security/gen-dev-certs.sh`.
+
+Anything dialing the nodes uses the client side of the same surface (gateway, admin API, CLI):
+
+```properties
+auth.client.mechanism=PLAIN
+auth.client.username=s3-gw
+auth.client.password.file=/etc/candybox/s3-gw.password
+tls.enabled=true
+tls.ca.path=/etc/candybox/tls/ca.crt
+```
+
+The CLI takes the same via `--user/--password-file/--mechanism/--tls/--tls-ca` or
+`CANDYBOX_AUTH_*`/`CANDYBOX_TLS*` env vars.
+
+### Authorization (ACLs)
+
+Every request is authorized against the caller's principal (`User:alice`, `Gateway:s3-gw`, ...).
+A Box's ACL document (owner + additive grants, stored in ZooKeeper) governs READ / WRITE /
+READ_ACP / WRITE_ACP / ADMIN; the creator owns the Box, and `ListBoxes` only reveals READable
+Boxes. Objects additionally carry their own owner + grants inside the `CandyLocator` (the S3
+union rule: an object grant can open READ on a single object inside a private Box). Manage with:
+
+```
+candybox acl get|set|grant|revoke <box> [--object <key>]
+e.g. candybox acl grant photos AllUsers:READ          # public-read Box
+```
+
+`auth.super.users` principals bypass ACLs entirely — list the S3 gateway and admin API service
+accounts there. A Box created before authorization (no document) falls back to
+authenticated-full-access.
+
+### Candybox → ZooKeeper
+
+```properties
+zookeeper.auth.scheme=digest
+zookeeper.auth.credentials=candybox:change-me
+zookeeper.acl.enabled=true         # defaults to true once a scheme is set
+```
+
+With ACLs on, every znode candybox creates is `CREATOR_ALL` — so **all candybox processes of one
+cluster must authenticate as the same ZooKeeper identity**. For SASL (Kerberos/DIGEST-MD5) set
+`CANDYBOX_JAAS_CONF` to a JAAS file with a `Client` section (see
+`examples/security/jaas.conf.example`); that one section also authenticates BookKeeper's internal
+ZooKeeper client. ZooKeeper client TLS uses ZooKeeper's standard system properties via
+`CANDYBOX_EXTRA_OPTS` (`-Dzookeeper.client.secure=true ...`).
+
+### Candybox → BookKeeper
+
+Everything goes through the verbatim `bookkeeper.client.*` passthrough (BK's own camelCase keys —
+via env, the suffix keeps its case: `CANDYBOX_BOOKKEEPER_CLIENT_tlsProvider`):
+
+```properties
+bookkeeper.client.clientAuthProviderFactoryClass=org.apache.bookkeeper.sasl.SASLClientProviderFactory
+bookkeeper.client.tlsProvider=OpenSSL
+bookkeeper.client.tlsTrustStoreType=PEM
+bookkeeper.client.tlsTrustStore=/etc/candybox/tls/ca.crt
+```
+
+The bookie side (`bookieAuthProviderFactoryClass`, bookie TLS, the bookies' own ZooKeeper auth) is
+configured in each bookie's `bookkeeper.conf` per the BookKeeper security docs — Candybox only
+brings the client half.
+
+### Admin API & metrics endpoints
+
+Set `CANDYBOX_ADMIN_AUTH_TOKEN` on the admin API: every `/api/*` route (cluster state, box
+browsing, uploads) then demands `Authorization: Bearer <token>`; `/healthz`, `/readyz` and the
+static SPA shell stay open. The dashboard captures the token once from `/ui/?token=<token>`
+(stored in localStorage, stripped from the URL; `/ui/?token=` clears it). Give the admin API an
+explicit `CANDYBOX_ADMIN_CORS` origin instead of `*` when it serves anything but same-origin.
+The listener serves HTTPS when the `tls.*` keys are set in its environment.
+
+Node and gateway `/metrics` accept an optional guard: set `metrics.auth.token` (or
+`CANDYBOX_METRICS_AUTH_TOKEN`) on the node/gateway, and give the admin API's scraper the same
+value via `CANDYBOX_ADMIN_SCRAPE_TOKEN` (Prometheus: `authorization.credentials`). The
+`/healthz`/`/readyz` probes stay unauthenticated plain-HTTP — they reveal only a boolean, and
+Kubernetes probes them directly.
+
+### Ledger password vs. authentication
+
+`ledger.password` predates all of the above: it is BookKeeper's per-ledger access password, a
+shared secret among the candybox nodes — keep it secret, but do not mistake it for user
+authentication.
+
 ## Known limitations / deferred work
 
 - **Streaming on the wire (Phase 2.5):** large objects are buffered in a single frame (≤ 16 MiB);

candybox-admin-api/src/main/java/me/predatorray/candybox/admin/AdminApiMain.java

@@ -23,9 +23,13 @@
 import me.predatorray.candybox.client.CandyboxClient;
 import me.predatorray.candybox.common.SystemClock;
 import me.predatorray.candybox.common.config.CandyboxConfig;
+import me.predatorray.candybox.common.config.SecurityConfig;
 import me.predatorray.candybox.coordination.CoordinationService;
+import me.predatorray.candybox.coordination.zk.ZkAuth;
 import me.predatorray.candybox.coordination.zk.ZooKeeperCoordinationService;
+import me.predatorray.candybox.protocol.auth.AuthenticatingTransport;
 import me.predatorray.candybox.protocol.transport.TcpTransport;
+import me.predatorray.candybox.protocol.transport.Transport;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,14 +67,25 @@ public static void main(String[] args) {
         AdminApiConfig config = fromEnv(System.getenv());
         // LIFO close: every component pushes itself; the shutdown hook pops them in reverse.
         Deque<AutoCloseable> stack = new ArrayDeque<>();
-        DashboardData data = buildDataSource(System.getenv("CANDYBOX_ADMIN_ZK"), stack);
+        DashboardData data =
+                buildDataSource(System.getenv("CANDYBOX_ADMIN_ZK"), stack, System.getenv());
 
         MetricsScraper scraper = buildScraper(System.getenv());
         if (scraper != null) {
             stack.push(scraper);
             scraper.start();
         }
-        AdminApiServer server = new AdminApiServer(config, () -> true, data, scraper);
+        String authToken = orDefault(System.getenv("CANDYBOX_ADMIN_AUTH_TOKEN"), null);
+        SecurityConfig listenerSecurity = SecurityConfig.resolve(key -> java.util.Optional
+                .ofNullable(System.getenv("CANDYBOX_" + key.toUpperCase().replace('.', '_')))
+                .filter(v -> !v.isBlank()).map(String::trim));
+        AdminApiServer server = new AdminApiServer(config, () -> true, data, scraper, authToken,
+                listenerSecurity.serverSslContext());
+        if (authToken == null) {
+            LOG.warn("CANDYBOX_ADMIN_AUTH_TOKEN is not set — the admin API (including box "
+                    + "browsing and uploads) is open to anyone who can reach port {}",
+                    config.port());
+        }
         stack.push(server);
         Runtime.getRuntime().addShutdownHook(new Thread(() -> closeAll(stack), "admin-api-shutdown"));
         server.start();
@@ -110,18 +125,31 @@ static MetricsScraper buildScraper(java.util.Map<String, String> env) {
         }
         long interval = parseIntOr(env.get("CANDYBOX_ADMIN_SCRAPE_INTERVAL_MS"), 5000);
         int window = parseIntOr(env.get("CANDYBOX_ADMIN_SCRAPE_WINDOW"), 60);
-        return new MetricsScraper(targets, interval, window);
+        return new MetricsScraper(targets, interval, window,
+                orDefault(env.get("CANDYBOX_ADMIN_SCRAPE_TOKEN"), null));
     }
 
-    private static DashboardData buildDataSource(String zk, Deque<AutoCloseable> closeStack) {
+    private static DashboardData buildDataSource(String zk, Deque<AutoCloseable> closeStack,
+                                                 java.util.Map<String, String> env) {
         if (zk == null || zk.isBlank()) {
             LOG.info("No CANDYBOX_ADMIN_ZK set — running with empty data source (UI-only demo).");
             return new EmptyDashboardData();
         }
         LOG.info("Wiring live data source against ZooKeeper {}", zk);
-        TcpTransport transport = new TcpTransport();
+        // The shared auth.*/tls.*/zookeeper.auth.* surface, resolved from CANDYBOX_* env vars
+        // (the admin API is env-configured) — how this process dials nodes and ZooKeeper.
+        SecurityConfig security = SecurityConfig.resolve(key -> java.util.Optional
+                .ofNullable(env.get("CANDYBOX_" + key.toUpperCase().replace('.', '_')))
+                .filter(s -> !s.isBlank()).map(String::trim));
+        Transport tcp = new TcpTransport(new me.predatorray.candybox.protocol.FrameCodec(),
+                security.clientSslContext(), security.tlsVerifyEndpoint());
+        Transport transport = security.clientUsername() == null ? tcp
+                : new AuthenticatingTransport(tcp, security.clientMechanism(),
+                        security.clientUsername(), security.clientPassword());
         closeStack.push(transport);
-        CoordinationService coordination = new ZooKeeperCoordinationService(zk, SystemClock.INSTANCE);
+        CoordinationService coordination = new ZooKeeperCoordinationService(zk,
+                SystemClock.INSTANCE, new ZkAuth(security.zkAuthScheme(),
+                        security.zkAuthCredentials(), security.zkAclEnabled()));
         closeStack.push(coordination);
         CandyboxClient client =
                 new CandyboxClient(transport, coordination, CandyboxConfig.builder().build());

candybox-admin-api/src/main/java/me/predatorray/candybox/admin/AdminApiServer.java

@@ -74,18 +74,41 @@ public final class AdminApiServer implements AutoCloseable {
     private final AdminApiConfig config;
     private final DashboardData data;
     private final MetricsScraper metrics;
+    private final String authToken;
 
     public AdminApiServer(AdminApiConfig config, BooleanSupplier ready, DashboardData data) {
         this(config, ready, data, null);
     }
 
     public AdminApiServer(AdminApiConfig config, BooleanSupplier ready, DashboardData data,
                           MetricsScraper metrics) {
+        this(config, ready, data, metrics, null, null);
+    }
+
+    /**
+     * @param authToken when non-null, every {@code /api/*} route demands
+     *                  {@code Authorization: Bearer <token>} (health probes and the static SPA
+     *                  stay open); the SPA stores the token after a one-time {@code /ui/?token=...}
+     * @param ssl       when non-null, the listener serves HTTPS with this context (PEM via
+     *                  {@code tls.*})
+     */
+    public AdminApiServer(AdminApiConfig config, BooleanSupplier ready, DashboardData data,
+                          MetricsScraper metrics, String authToken,
+                          javax.net.ssl.SSLContext ssl) {
         this.config = config;
         this.data = data;
         this.metrics = metrics;
+        this.authToken = authToken;
         try {
-            this.http = HttpServer.create(new InetSocketAddress(config.bindAddress(), config.port()), 0);
+            if (ssl != null) {
+                com.sun.net.httpserver.HttpsServer https = com.sun.net.httpserver.HttpsServer
+                        .create(new InetSocketAddress(config.bindAddress(), config.port()), 0);
+                https.setHttpsConfigurator(new com.sun.net.httpserver.HttpsConfigurator(ssl));
+                this.http = https;
+            } else {
+                this.http = HttpServer.create(
+                        new InetSocketAddress(config.bindAddress(), config.port()), 0);
+            }
         } catch (IOException e) {
             throw new UncheckedIOException(
                     "Failed to bind admin API on " + config.bindAddress() + ":" + config.port(), e);
@@ -111,7 +134,28 @@ public AdminApiServer(AdminApiConfig config, BooleanSupplier ready, DashboardDat
     }
 
     private void register(String path, HttpHandler handler) {
-        http.createContext(path, withCors(handler));
+        http.createContext(path, withCors(withAuth(path, handler)));
+    }
+
+    /** The bearer-token guard on {@code /api/*}; probes and the static SPA shell stay open. */
+    private HttpHandler withAuth(String path, HttpHandler delegate) {
+        if (authToken == null || !path.startsWith("/api")) {
+            return delegate;
+        }
+        return exchange -> {
+            String header = exchange.getRequestHeaders().getFirst("Authorization");
+            String presented = header != null && header.startsWith("Bearer ")
+                    ? header.substring("Bearer ".length()).trim() : null;
+            if (presented == null || !java.security.MessageDigest.isEqual(
+                    presented.getBytes(java.nio.charset.StandardCharsets.UTF_8),
+                    authToken.getBytes(java.nio.charset.StandardCharsets.UTF_8))) {
+                jsonRespond(exchange, 401, JsonWriter.write(Map.of(
+                        "error", "Unauthorized",
+                        "message", "This admin API requires Authorization: Bearer <token>")));
+                return;
+            }
+            delegate.handle(exchange);
+        };
     }
 
     public void start() {
@@ -527,7 +571,7 @@ private void applyCorsHeaders(Headers headers) {
         }
         headers.set("Access-Control-Allow-Origin", config.corsAllowOrigin());
         headers.set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS");
-        headers.set("Access-Control-Allow-Headers", "Content-Type, If-Match, If-None-Match");
+        headers.set("Access-Control-Allow-Headers", "Content-Type, Authorization, If-Match, If-None-Match");
         headers.set("Access-Control-Max-Age", "600");
     }
 

candybox-admin-api/src/main/java/me/predatorray/candybox/admin/MetricsScraper.java

@@ -64,9 +64,17 @@ final class MetricsScraper implements AutoCloseable {
     private final HttpClient http;
     private final ScheduledExecutorService scheduler;
     private final Map<SeriesKey, Deque<Sample>> series = new ConcurrentHashMap<>();
+    private final String scrapeToken;
     private volatile String latestText = "";
 
     MetricsScraper(List<URI> targets, long pollIntervalMillis, int windowSamples) {
+        this(targets, pollIntervalMillis, windowSamples, null);
+    }
+
+    /** @param scrapeToken sent as {@code Authorization: Bearer} to token-guarded /metrics targets */
+    MetricsScraper(List<URI> targets, long pollIntervalMillis, int windowSamples,
+                   String scrapeToken) {
+        this.scrapeToken = scrapeToken;
         this.targets = List.copyOf(targets);
         this.pollIntervalMillis = pollIntervalMillis;
         this.windowSamples = windowSamples;
@@ -136,9 +144,12 @@ private void scrapeOnce() {
         long ingestMillis = System.currentTimeMillis();
         for (URI target : targets) {
             try {
-                HttpResponse<String> resp = http.send(
-                        HttpRequest.newBuilder(target).timeout(Duration.ofSeconds(3))
-                                .header("Accept", "text/plain").GET().build(),
+                HttpRequest.Builder rb = HttpRequest.newBuilder(target)
+                        .timeout(Duration.ofSeconds(3)).header("Accept", "text/plain").GET();
+                if (scrapeToken != null) {
+                    rb.header("Authorization", "Bearer " + scrapeToken);
+                }
+                HttpResponse<String> resp = http.send(rb.build(),
                         HttpResponse.BodyHandlers.ofString());
                 if (resp.statusCode() != 200) {
                     LOG.debug("metrics target {} returned {}", target, resp.statusCode());