From 8c659ae45a196b8815572507ae3215b99d119ded Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 05:29:29 +0000 Subject: [PATCH 1/2] Partition Boxes by key hash with balanced, fenced per-partition ownership Removes the single-node write bottleneck per Box: every Box is split into a creation-time-fixed number of hash partitions (CRC32C of the key, mod the count in the new ZK BoxDescriptor; default 8), each an independent LSM engine (WAL/memtable/manifest/SSTables/Syrups) under its own fenced ZooKeeper ownership lease, so one Box's writes spread across the cluster. Design and decisions are captured in BOX_PARTITIONING_PLAN.md. - Coordination: per-partition owner/manifest keys plus boxes//meta (immutable descriptor) and cluster/{balancer,assignment}; new CoordinationService.children() (fake + ZK + contract test). - Server: BoxOwnership becomes PartitionOwnership; CandyboxNode keys ownership/maintenance by (box, partition); createBox publishes the descriptor and owns all partitions initially; deleteBox takes over or (force) defers to the owners' deleted-box sweep; boxExists/listBoxes now answer cluster-wide from coordination. - Balancer (new): every node runs rounds; the cluster/balancer lease holder is the elected coordinator publishing a sticky, failover-eager assignment with rate-limited moves away from live owners (balancer.max.moves.per.round); all nodes converge on it, releasing with a pre-handover flush and acquiring once leases free up. - Protocol: partition fields on list/delete-range/list-uploads requests, CreateBox partitionCount, new BoxInfo request/response; MOVED now names the requested partition's owner. - Client: routes keyed ops (incl. all multipart ops) to the key's partition owner, caches descriptors, scatter-gathers listings in global key order, fans out range deletes, and falls back to a byte copy for cross-partition copy/rename/uploadPartCopy. - Fix: recovering a partition whose WAL is empty observed Hlc.MIN and tripped an overflowing skew check in HybridLogicalClock.observe; skip the observe on empty replay and make the check overflow-safe. - Tests: Partitioning/BoxDescriptor/PartitionBalancer units, the fakes-based PartitionedBoxIT (multi-node partitioned Box through the cluster client), and updates across the existing suites; docs/conf updated (DESIGN.md, OPERATIONS.md, README, candybox.properties). https://claude.ai/code/session_015ZRRX63Hie7PDKxsH4Zosn --- BOX_PARTITIONING_PLAN.md | 176 +++++++++ DESIGN.md | 136 +++++-- OPERATIONS.md | 47 ++- README.md | 24 +- .../candybox/admin/LiveDashboardData.java | 78 ++-- .../candybox/admin/LiveDashboardDataTest.java | 20 +- .../candybox/client/CandyboxCli.java | 8 +- .../candybox/client/CandyboxClient.java | 245 +++++++++---- .../candybox/client/ClusterRouter.java | 34 +- .../candybox/client/DirectRouter.java | 2 +- .../predatorray/candybox/client/Router.java | 6 +- .../candybox/client/CandyboxClientTest.java | 4 +- .../candybox/client/ClusterRouterTest.java | 30 +- .../candybox/common/HybridLogicalClock.java | 5 +- .../candybox/common/Partitioning.java | 46 +++ .../common/config/CandyboxConfig.java | 45 +++ .../candybox/common/PartitioningTest.java | 65 ++++ .../candybox/coordination/BoxDescriptor.java | 57 +++ .../candybox/coordination/CandyboxKeys.java | 31 +- .../coordination/CoordinationService.java | 7 + .../fake/InMemoryCoordinationService.java | 23 ++ .../zk/ZooKeeperCoordinationService.java | 13 + .../coordination/BoxDescriptorTest.java | 53 +++ .../CoordinationServiceContract.java | 16 + .../src/conf/candybox.properties.example | 16 + .../candybox/it/BoxClusterHandoverIT.java | 4 +- .../candybox/it/BoxHandoverIT.java | 6 +- .../candybox/it/CompactionGcCycleIT.java | 10 +- .../candybox/it/PartitionedBoxIT.java | 269 ++++++++++++++ .../candybox/it/ServerClientLifecycleIT.java | 7 +- .../candybox/lsm/engine/BoxEngine.java | 8 +- .../candybox/protocol/Message.java | 51 ++- .../candybox/protocol/MessageCodec.java | 23 +- .../predatorray/candybox/protocol/Opcode.java | 4 +- .../candybox/protocol/MessageCodecTest.java | 31 +- .../candybox/server/CandyboxNode.java | 340 ++++++++++++++---- .../candybox/server/NodeRequestHandler.java | 158 ++++++-- .../candybox/server/PartitionAssignment.java | 85 +++++ .../candybox/server/PartitionBalancer.java | 251 +++++++++++++ ...Ownership.java => PartitionOwnership.java} | 99 +++-- .../candybox/server/ServerConfig.java | 3 + .../candybox/server/BoxHandoverTest.java | 22 +- .../candybox/server/CandyboxNodeTest.java | 32 +- .../server/NodeRequestHandlerTest.java | 16 +- .../server/PartitionBalancerTest.java | 210 +++++++++++ 45 files changed, 2398 insertions(+), 418 deletions(-) create mode 100644 BOX_PARTITIONING_PLAN.md create mode 100644 candybox-common/src/main/java/me/predatorray/candybox/common/Partitioning.java create mode 100644 candybox-common/src/test/java/me/predatorray/candybox/common/PartitioningTest.java create mode 100644 candybox-coordination/src/main/java/me/predatorray/candybox/coordination/BoxDescriptor.java create mode 100644 candybox-coordination/src/test/java/me/predatorray/candybox/coordination/BoxDescriptorTest.java create mode 100644 candybox-integration-tests/src/test/java/me/predatorray/candybox/it/PartitionedBoxIT.java create mode 100644 candybox-server/src/main/java/me/predatorray/candybox/server/PartitionAssignment.java create mode 100644 candybox-server/src/main/java/me/predatorray/candybox/server/PartitionBalancer.java rename candybox-server/src/main/java/me/predatorray/candybox/server/{BoxOwnership.java => PartitionOwnership.java} (57%) create mode 100644 candybox-server/src/test/java/me/predatorray/candybox/server/PartitionBalancerTest.java diff --git a/BOX_PARTITIONING_PLAN.md b/BOX_PARTITIONING_PLAN.md new file mode 100644 index 0000000..1376e0c --- /dev/null +++ b/BOX_PARTITIONING_PLAN.md @@ -0,0 +1,176 @@ +# Box Partitioning — Implementation Plan + +## Goal + +Remove the single-node write bottleneck per Box. A Box is split into a **fixed number of hash +partitions**; each partition is an independent LSM engine (its own WAL / memtable / manifest / +SSTables / Syrups) with its own ZooKeeper-fenced ownership lease, so the partitions of one Box are +owned and served by **multiple nodes concurrently**. An elected coordinator spreads partition +ownership evenly across the cluster (rate-limited when moving partitions away from live owners), and +clients route each request to the owner of the key's partition, re-routing on `MOVED`. + +Decisions locked with the project owner: + +1. **Hash partitioning** (CRC32C of the key's UTF-8 bytes, mod partition count). +2. **Partition count fixed at Box creation**, with a configurable cluster default (8). +3. **Elected coordinator** computes the desired assignment; every node acts on it. +4. **Rate-limited migration** — at most `balancer.max.moves.per.round` partitions are moved away + from live owners per balancing round (configurable). +5. Cross-partition `copy`/`rename` degrade to a **client-side byte copy** (rename additionally + deletes the source afterwards, i.e. it is no longer atomic across partitions). Same-partition + copy/rename keep the zero-copy server-side path. +6. **No migration / backward compatibility** with the pre-partitioning ZK layout or wire format + (not in production yet). +7. Scope is **write scaling only** — reads still go to each partition's owner; compaction still + runs on each partition's owner. + +## Design + +### Consistency + +Unchanged in substance: every key maps to exactly one partition, each partition has exactly one +fenced owner, so there is still a single fenced writer per key — per-key linearizability on the +owner, HLC/LWW semantics, and the handover replay + `observe` sequence all carry over verbatim at +partition granularity. What changes is *which* keys share a writer. + +Operations that used to be one engine call under one lock and now span partitions are weaker, by +design: + +- `deleteRange` / `deleteRangeByPrefix` — one range tombstone **per partition** (fanned out by the + client); not atomic across partitions, but idempotent and LWW-safe to retry. +- cross-partition `rename` — copy then delete; a crash in between leaves both keys live. +- `listCandies` / `listMultipartUploads` — scatter-gather merge in the client; each partition's + page is internally consistent, the merged page is not a snapshot. + +### Coordination layout (ZooKeeper) + +``` +boxes//meta versioned KV: BoxDescriptor {formatVersion, partitionCount} +boxes//partitions/

/owner ownership lease of partition p (fenced, TTL'd) — was boxes//owner +boxes//partitions/

/manifest manifest-ledger pointer of partition p — was boxes//manifest +cluster/balancer the coordinator election lease +cluster/assignment versioned KV: desired PartitionAssignment table +members/ unchanged +``` + +The Box descriptor is immutable after creation (the partition count cannot change in v1) and is the +routing source of truth for servers and clients. `CoordinationService` gains one operation, +`children(path)` (ZK `getChildren`; prefix scan in the in-memory fake), used to enumerate Boxes for +the balancer and for cluster-wide `listBoxes`. + +### Partition mapping + +`Partitioning.partitionOf(key, n) = (crc32c(utf8(key)) & 0x7fffffff) % n` — deterministic across +JVMs/nodes/clients (CRC32C is already a dependency-free primitive in `candybox-common`). + +### Ownership & engines (server) + +- `BoxOwnership` becomes `PartitionOwnership` — the same lease + recover + pointer-CAS sequence, + keyed by `(box, partition)` and using the per-partition ZK keys. `BoxEngine` is reused as the + per-partition engine unchanged (it is already fully self-contained). +- `CandyboxNode` keeps a `ConcurrentMap<(box,partition), PartitionOwnership>`: + - `createBox(box, partitionCount)`: CAS-create `boxes//meta` (conflict ⇒ + `BoxAlreadyExists`), then create all partitions' engines locally (the balancer spreads them + afterwards). Partial failure ⇒ best-effort rollback of created partitions + the descriptor. + - `openPartition` / `releasePartition`: per-partition takeover and graceful release (release + flushes the memtable first so the next owner's WAL replay is small). + - `engine(box, key)`: descriptor → `partitionOf(key)` → locally owned engine or throw. + - `deleteBox`: takes over every partition it does not own, checks emptiness across all + partitions (unless `force`), deletes each manifest pointer, then the descriptor. + - `boxExists`/`listBoxes` answer from coordination (descriptor existence / `children("boxes")`), + so any node can answer and listing is finally cluster-wide. + - Lease heartbeat, compaction, GC, and the multipart TTL sweeper iterate partitions — + mechanical. + +### Assignment & balancing (new) + +`PartitionBalancer`, scheduled on **every** node (`balancer.interval.millis`, 0 disables): + +1. **Coordinate** — try to acquire/renew the `cluster/balancer` lease; the holder enumerates all + `(box, partition)` pairs and live members, computes a target assignment, and CAS-writes it to + `cluster/assignment`: + - capacity = ⌈partitions / members⌉; current live holders keep their partitions up to capacity + (stickiness — no gratuitous shuffling); + - unowned partitions (new Box, dead node, over-capacity overflow) go to the least-loaded + members; + - at most `balancer.max.moves.per.round` partitions whose current holder is **live** are + reassigned per round (rate-limited migration); partitions with no live holder are failover, + not moves, and are never rate-limited. +2. **Apply** — every node reads the assignment table: + - owned here but assigned elsewhere ⇒ flush + release (the handover drain); + - assigned here but not owned ⇒ acquire once the lease is free (the previous owner released or + expired), via the normal fenced recover path. + +Convergence is via polling rounds: a move takes one round for the old owner to release and the next +round (or the same one, ordering permitting) for the new owner to acquire. Fencing safety never +depends on the balancer — it only decides who *tries* to acquire. + +### Protocol + +No back-compat constraints, so fields are added in place (no version bump): + +- `CreateBoxRequest` gains `partitionCount` (0 ⇒ server default). +- `ListCandiesRequest`, `DeleteRangeRequest`, `ListMultipartUploadsRequest` gain `partition` — the + client fans these out per partition. +- New `BoxInfoRequest(box)` → `BoxInfoResponse(partitionCount)`: served from the descriptor by any + node; the client uses it to learn (and cache) a Box's partition count. +- `MovedResponse(ownerNodeId)` is unchanged — the client already knows which partition it asked + for. +- Keyed requests are unchanged on the wire; the server derives the partition from the key. + +### Client routing + +- `Router.callBox(box, …)` becomes `callPartition(box, partition, …)`. `ClusterRouter` resolves + `boxes//partitions/

/owner` → member address, caches per `(box, partition)` with the + existing TTL, and re-routes on `MOVED` exactly as before. `DirectRouter` ignores the partition + (single-node). +- `CandyboxClient` caches each Box's partition count (immutable ⇒ cache forever; invalidated on + Box deletion/not-found) and: + - routes keyed ops (put/get/head/delete/range-get and **all multipart ops** — every multipart + request already carries the object key) to `partitionOf(key)`; + - fans out `deleteRange`/`deleteRangeByPrefix` to every partition; + - scatter-gathers `listCandies` (merge by unsigned-UTF-8 key order, honoring reverse; the + `lastKey` continuation token still works — each page re-fans-out) and + `listMultipartUploads` (merge by `(key, uploadId)`); + - same-partition copy/rename/uploadPartCopy stay server-side zero-copy; cross-partition fall + back to get+put (+delete for rename) / range-get+uploadPart. + +### Configuration + +| Key | Default | Meaning | +|---|---|---| +| `partitions.per.box.default` | 8 | Partition count for `createBox` when the caller passes 0. | +| `balancer.interval.millis` | 0 (off) | Balancing round period on every node; enabled in the shipped conf (5000). | +| `balancer.max.moves.per.round` | 4 | Max partitions moved away from live owners per round. | + +The default partition count of 8 bounds per-Box resource amplification (each partition is a full +engine: WAL + manifest ledgers, a 4 MiB memtable, an open Syrup) while spreading writes across up +to 8 nodes. The balancer interval defaults to 0 in `CandyboxConfig` so the in-JVM tests retain +deterministic, manually-driven ownership; the packaged `candybox.properties` enables it. + +## Work items + +1. **common** — `Partitioning` (hash), `CandyboxConfig` keys (`partitionsPerBoxDefault`, + `balancerIntervalMillis`, `balancerMaxMovesPerRound`). +2. **coordination** — partitioned `CandyboxKeys` + `cluster/*` keys; `BoxDescriptor` + encode/decode; `CoordinationService.children` in the fake, ZK impl, and the shared contract + test. +3. **protocol** — message/codec changes above + codec tests. +4. **server** — `PartitionOwnership`; `CandyboxNode` partition map, descriptor cache, + create/open/release/delete per partition, coordination-backed `boxExists`/`listBoxes`; + `PartitionAssignment` table; `PartitionBalancer` (coordinate + apply, rate-limited); + `NodeRequestHandler` partition resolution and per-partition `MOVED`; `ServerConfig` keys. +5. **client** — `Router.callPartition`, `ClusterRouter` per-partition resolution/caching, + `CandyboxClient` partition-count cache, fan-out list/delete-range/list-uploads merge, + cross-partition copy/rename/uploadPartCopy fallback, `createBox(box, partitionCount)`. +6. **tests** — update coordination contract, codec, server (handover/node/handler), client + (router/client) tests for the partitioned layout; new tests: `Partitioning` determinism + + spread, `BoxDescriptor` round-trip, `PartitionBalancer` (even spread, failover, rate limit, + stickiness, non-coordinator no-op), cross-partition list/delete-range/copy/rename through the + loopback transport, per-partition `MOVED` re-routing. +7. **docs/conf** — DESIGN.md (consistency §3, ownership §7, new partitioning section, drop the + §12 "no sub-Box partitioning" simplification), shipped `candybox.properties` examples, + OPERATIONS.md note on the balancer. + +Out of scope (unchanged v1 simplifications): read scaling off owners, cross-node compaction +scheduling, watch-based coordination (the balancer polls), Syrup defragmentation, streaming. diff --git a/DESIGN.md b/DESIGN.md index cd55f1c..e5ab59f 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -6,10 +6,10 @@ on **Apache BookKeeper** ledgers. This document captures the architecture, on-le decisions made (with rationale), and the deliberate v1 simplifications. It is written against the implementation in this repository: **Phases 0–4 are implemented and -tested** — the LSM engine, ZooKeeper-backed coordination, fenced Box ownership/handover, the framed -TCP protocol with cluster membership + client-side routing, multi-level leveled compaction, and -reference-counted GC all run, with the storage node, CLI, and a packaged distribution (Docker / -Kubernetes) on top. A handful of deliberate v1 simplifications remain (true on-the-wire streaming, +tested** — the LSM engine, ZooKeeper-backed coordination, hash-partitioned Boxes with fenced +per-partition ownership/handover and an elected-coordinator balancer, the framed TCP protocol with +cluster membership + client-side routing, multi-level leveled compaction, and reference-counted GC +all run, with the storage node, CLI, and a packaged distribution (Docker / Kubernetes) on top. A handful of deliberate v1 simplifications remain (true on-the-wire streaming, distributed cross-node compaction scheduling, watch-based coordination, a GC enumeration backstop); they are called out inline and collected in §12–§13, and a few are still marked `// TODO(phase-N)` in code. @@ -52,13 +52,22 @@ Dependency rule: `candybox-lsm` depends **only** on the two SPIs (`bookkeeper`, nodeId)`. Reads resolve a key to the **highest HLC** (LWW); the deterministic final tiebreaker is `nodeId`. HLC beats a raw wall clock (a fast clock cannot permanently win or resurrect deletes) and beats a global counter (no coordination bottleneck). -- **Leased single-owner-per-Box.** At any instant one node owns a Box and is the sole writer of its - WAL, memtable flushes, and manifest. Ownership is a movable, ZK-leased, **fenced** assignment. +- **Hash-partitioned Boxes, leased single-owner-per-partition.** Every Box is split into a + creation-time-fixed number of **hash partitions** (`Partitioning`: CRC32C of the key's UTF-8 bytes + mod the count in the Box's `BoxDescriptor`; default 8, `partitions.per.box.default`). Each + partition is an independent LSM engine — its own WAL, memtable, manifest, SSTables, and Syrups — + and at any instant one node owns a partition and is the sole writer of it. Ownership is a movable, + ZK-leased, **fenced** per-partition assignment, so one Box's write load spreads across the + cluster (see §7a for the balancer). - **The owner stamps the HLC at ingest** (never the client), so client clock skew is irrelevant. -- Because a single fenced owner serializes all writes for a Box, the system is **effectively per-key - linearizable on the owner**. The eventual-consistency window is confined to ownership handover (a - fenced old owner may briefly serve a stale memtable read, but its *writes* fail). This is the - guarantee Candybox documents — not general eventual consistency. +- Because every key maps to exactly one partition and a single fenced owner serializes all writes + for a partition, the system is **effectively per-key linearizable on the owner**. The + eventual-consistency window is confined to ownership handover (a fenced old owner may briefly + serve a stale memtable read, but its *writes* fail). Operations spanning partitions are weaker by + construction: `deleteRange` is one tombstone per partition (idempotent, not atomic across them), + listings are scatter-gather merges (each partition's page is consistent; the merged page is not a + snapshot), and a cross-partition `rename` is copy-then-delete (not atomic). This is the guarantee + Candybox documents — not general eventual consistency. ### HLC recovery on handover (critical correctness point) @@ -178,19 +187,26 @@ consults range tombstones across all tables rather than pruning by point range. SSTables at every level (range + bloom pre-filter); a point tombstone, a covering **range tombstone** newer than that locator, or absence ⇒ `CandyNotFound`. `getCandy` then streams bytes from Syrups and validates the whole-object crc. -- **list / scan**: a `MergingIterator` over the memtable + SSTables (LWW, tombstones suppressed), - driven by a `ScanQuery` — an optional `[start, end)` window, optional prefix, **forward or reverse** - direction, page size, and a continuation cursor (`lastKey`, exclusive in the scan direction). Keys - covered by a newer range tombstone are suppressed too (the union of range tombstones across the - memtable and all SSTables is gathered per scan; they are few). +- **list / scan**: per partition, a `MergingIterator` over the memtable + SSTables (LWW, tombstones + suppressed), driven by a `ScanQuery` — an optional `[start, end)` window, optional prefix, + **forward or reverse** direction, page size, and a continuation cursor (`lastKey`, exclusive in + the scan direction). Keys covered by a newer range tombstone are suppressed too (the union of + range tombstones across the memtable and all SSTables is gathered per scan; they are few). The + **client scatter-gathers** every partition for each page and merge-sorts by key; the `lastKey` + cursor still works because every partition is re-queried past it. - **deleteRange**: stamp one HLC and append a single `RangeTombstone` `[start, end)` to the WAL + - memtable — an O(1) delete of the whole interval. Shadowed keys are reclaimed lazily at compaction. - Either bound may be unbounded; `deleteRangeByPrefix` maps `prefix → [prefix, prefixSuccessor)`. -- **copy / rename**: write a fresh PUT at the destination reusing the source locator's *parts* - verbatim (zero byte copy and zero locator rebuild — a multipart source becomes a multipart-shaped - copy); `rename` also tombstones the source. Both commit atomically under the single owner's - write lock (same Box only). Because GC is reference-counted by Syrup id, several keys may share - one segment set safely. + memtable — an O(1) delete of the whole interval *per partition* (the client fans the request out; + idempotent to retry, not atomic across partitions). Shadowed keys are reclaimed lazily at + compaction. Either bound may be unbounded; `deleteRangeByPrefix` maps + `prefix → [prefix, prefixSuccessor)`. +- **copy / rename**: when source and destination hash to the **same partition**, write a fresh PUT + at the destination reusing the source locator's *parts* verbatim (zero byte copy and zero locator + rebuild — a multipart source becomes a multipart-shaped copy); `rename` also tombstones the + source. Both commit atomically under the partition owner's write lock (same Box only). Because GC + is reference-counted by Syrup id *within one manifest*, several keys may share one segment set + safely — which is also why a **cross-partition** copy/rename cannot share segments (the source + partition's GC would not see the destination's references) and instead degrades to a client-side + byte copy (`rename`: copy then delete, not atomic). - **range get**: `getCandyRange(key, firstByte, lastByte)` over HTTP `Range: bytes=…` semantics (inclusive on both ends). Prefix-sums the part lengths to find the first/last parts, then indexes into chunks by `byteWithinPart / part.chunkSize`. Chunks at the slice boundary are read whole — @@ -214,9 +230,9 @@ dedicated `RESPONSE_BUSY` opcode. ## 7. Manifest, fencing & ownership -The manifest is an **append-only metadata ledger**; a Box's manifest has exactly one writer (the -owner). `ManifestLog` appends serialized edits; `Manifest` keeps an in-memory `ManifestState` in -lock-step (append durably, then advance state). Handover (`Manifest.recover`): +The manifest is an **append-only metadata ledger**; a partition's manifest has exactly one writer +(the partition's owner). `ManifestLog` appends serialized edits; `Manifest` keeps an in-memory +`ManifestState` in lock-step (append durably, then advance state). Handover (`Manifest.recover`): 1. **recover-open** (fence + seal) the prior manifest ledger; 2. replay its edits to rebuild `ManifestState`; @@ -231,10 +247,40 @@ live in the `CoordinationService` SPI: an in-memory fake that models lease expir CAS conflicts (so these tests are not vacuous — `InMemoryCoordinationServiceTest`), and the real `ZooKeeperCoordinationService`. Both run the shared `CoordinationServiceContract` (as a `*ContractTest` against the fake and a `*ContractIT` against a live ZooKeeper), so the fast tests are -a faithful stand-in. `BoxOwnership` (server) ties the lease, fencing token, and manifest pointer -together; `CandyboxNode.createBox`/`openBox` are the take-ownership and failover/handover entry +a faithful stand-in. `PartitionOwnership` (server) ties the lease, fencing token, and manifest +pointer together for one partition; `CandyboxNode.createBox` (descriptor + all partitions on the +creating node) and `openPartition` (per-partition failover/handover) are the take-ownership entry points, with a background heartbeat renewing leases within the TTL. +The coordination layout: + +``` +boxes//meta BoxDescriptor {partitionCount} — immutable, the routing truth +boxes//partitions/

/owner partition p's ownership lease (fenced, TTL'd) +boxes//partitions/

/manifest partition p's manifest-ledger pointer (versioned CAS) +cluster/balancer the balancer's coordinator-election lease +cluster/assignment the desired partition→node assignment table (versioned CAS) +members/ membership (advertised host:port) +``` + +### 7a. Partition balancing (elected coordinator) + +`PartitionBalancer` runs a round on every node (`balancer.interval.millis`; 0 disables). Whichever +node holds the `cluster/balancer` lease is the **elected coordinator**: it enumerates every +`(box, partition)` (via `CoordinationService.children` + descriptors) and the live members, computes +a target assignment, and CAS-publishes it at `cluster/assignment`. The computation is **sticky** +(a live owner keeps its partitions up to the fair-share capacity ⌈partitions/members⌉), +**failover-eager** (unowned partitions — a new Box, a dead node — go to the least-loaded members +without limit), and **rate-limited** (at most `balancer.max.moves.per.round` partitions are taken +away from live owners per round, so a node join migrates load gradually). Every node — coordinator +or not — then converges on the table: partitions assigned elsewhere are released after a +**pre-handover flush** (shrinking the next owner's WAL replay to ~nothing), and partitions assigned +to it are acquired once their lease is observed free. The table is advisory: safety always rests on +the per-partition fenced lease, never on the balancer. Each round also sweeps locally owned +partitions whose Box descriptor is gone (the convergence path of a force `deleteBox` issued +elsewhere). Requests that land on the wrong node get `RESPONSE_MOVED` naming the partition's +current owner, and the client re-routes. + ## 8. Compaction model `CompactionStrategy` is a pluggable SPI (Cassandra-style) with **LevelDB-style leveled compaction** as @@ -328,6 +374,8 @@ Per-role BK quorum (E/Qw/Qa) defaults: **WAL 3/3/2, Manifest 3/3/2, SSTable 3/2/ | Leveled level base / multiplier | 10 MiB / 10× | LevelDB-style growing per-level byte budgets for L≥1. | | HLC logical width / skew bound | 32-bit logical; 5-min skew rejection | Overflow needs ~2.1e9 events/ms; bound stops a wild clock dragging us forward. | | Ownership lease TTL / renew / fencing token | 10 s TTL; 3 s renew; monotonic per-resource counter (ZK version) | Short TTL for quick failover; renew well inside it; strictly increasing tokens fence zombies. | +| Partitions per Box | 8 (fixed at create; `createBox` may override) | Spreads one Box's writes over up to 8 nodes while bounding per-Box engine cost (each partition is a full WAL/memtable/manifest). Immutable: re-hashing would re-home every key. | +| Balancer round interval / move rate | 5 s in shipped conf (0 = off, the unit-test default) / 4 moves per round | Frequent enough to converge quickly after joins/failures; the move cap keeps a node join from stampeding handovers (failover is never rate-limited). | | Compaction + GC worker interval | configurable; 0 disables | Background maintenance cadence on the owner. | | Tombstone-GC time bound | 24 h | Covers in-flight late writes before a delete is reclaimable. | | Ledger-GC grace | 5 min | Margin for in-flight readers / continuation tokens before a physical delete. | @@ -338,12 +386,20 @@ Per-role BK quorum (E/Qw/Qa) defaults: **WAL 3/3/2, Manifest 3/3/2, SSTable 3/2/ ## 12. Deliberate v1 simplifications (escape hatches) -- **Single owner per Box, no sub-Box partitioning** — caps a Box's write throughput at one node and - makes it unavailable during the handover fence+replay window. Manifest/ownership structures are kept - **partition-shaped** (one partition per Box now) so key-range tablets can be added later. -- **Reads served by the Box owner** — simple and correct. Sealed SSTables/Syrups are immutable and - replicated, so any node *could* serve reads of flushed data; only unflushed-memtable read-your-writes - needs the owner. Not built in v1. +- **Fixed hash partitioning, no key-range tablets** — a Box's partition count is set at creation and + never changes (changing it would re-home every key), and hash partitioning makes every ordered + listing a scatter-gather fan-out (each page queries all partitions). Key-range partitions with + splits, and re-partitioning, are future work. A partition is still unavailable during its own + handover fence+replay window (shrunk by the pre-handover flush). +- **Cross-partition copy/rename byte-copies through the client** — zero-copy segment sharing is only + safe within one partition's manifest (§6); a cross-partition `rename` is copy-then-delete and not + atomic. S3 semantics (CopyObject, no rename) are unaffected. +- **Box-level `deleteBox` needs takeover or the balancer** — the deleting node takes over partitions + whose leases are free; partitions held by other live nodes fail a non-force delete, while a force + delete removes the descriptor and lets each owner's balancer sweep drop its partitions. +- **Reads served by the partition owner** — simple and correct. Sealed SSTables/Syrups are immutable + and replicated, so any node *could* serve reads of flushed data; only unflushed-memtable + read-your-writes needs the owner. Not built in v1. - **No small-object inlining** — all bytes go to Syrups (an extra BK round trip + fragmentation for tiny Candies). The 256 KiB locator cap leaves room to inline small bytes later. - **No Syrup defragmentation** — see §9(d). Syrup compaction (copying survivors into a fresh Syrup) @@ -360,11 +416,15 @@ Per-role BK quorum (E/Qw/Qa) defaults: **WAL 3/3/2, Manifest 3/3/2, SSTable 3/2/ **Implemented (Phases 0–4):** the LSM engine (memtable/WAL/SSTable/Syrup, merged multi-level read path, range tombstones, zero-copy copy/rename); ZooKeeper-backed coordination (`ZooKeeperCoordinationService`) with the manifest ZK pointer under versioned CAS and fenced -ownership/handover; cluster membership + client-side routing over the framed TCP protocol -(`RESPONSE_MOVED`); multi-level leveled compaction on a background worker; reference-counted GC of -SSTable/Syrup/WAL ledgers; the storage node with health/metrics, the `candybox` CLI, and a packaged -distribution (Docker image + Compose + Kubernetes manifests). Fault-injection / failure-path -hardening and operational docs (`OPERATIONS.md`) landed in Phase 4. +ownership/handover; **hash-partitioned Boxes** — per-partition fenced ownership spread across the +cluster by the elected-coordinator `PartitionBalancer` (sticky, failover-eager, rate-limited +moves), with partition-aware client routing (keyed ops to the key's partition owner; scatter-gather +listing; fanned-out range deletes; cross-partition copy fallback); cluster membership + client-side +routing over the framed TCP protocol (`RESPONSE_MOVED`); multi-level leveled compaction on a +background worker; reference-counted GC of SSTable/Syrup/WAL ledgers; the storage node with +health/metrics, the `candybox` CLI, and a packaged distribution (Docker image + Compose + +Kubernetes manifests). Fault-injection / failure-path hardening and operational docs +(`OPERATIONS.md`) landed in Phase 4; Box partitioning is documented in `BOX_PARTITIONING_PLAN.md`. **Remaining future work:** diff --git a/OPERATIONS.md b/OPERATIONS.md index b814add..c53480f 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -8,11 +8,20 @@ For architecture and on-ledger formats see [`DESIGN.md`](DESIGN.md); for build/t A Candybox cluster is a set of **storage nodes** over a shared **Apache BookKeeper** ensemble and a shared **ZooKeeper** ensemble: -- **BookKeeper** stores all ledgers — WALs, SSTables, Syrups (Candy bytes), and the per-Box manifest. -- **ZooKeeper** (via the coordination SPI) holds cluster membership, per-Box ownership **leases**, and - the versioned pointer to each Box's current manifest ledger. -- Each **Box** is owned by exactly one node at a time (a fenced, movable lease). The owner is the sole - writer of that Box's WAL, memtable flushes, and manifest. Clients route requests to the owner. +- **BookKeeper** stores all ledgers — WALs, SSTables, Syrups (Candy bytes), and the per-partition + manifests. +- **ZooKeeper** (via the coordination SPI) holds cluster membership, each Box's **descriptor** + (its creation-time-fixed partition count), per-partition ownership **leases**, the versioned + pointer to each partition's current manifest ledger, and the balancer's election lease + + assignment table. +- Each **Box** is split into hash **partitions** (`partitions.per.box.default`, default 8); each + partition is owned by exactly one node at a time (a fenced, movable lease) and is a full LSM + engine. The owner is the sole writer of that partition's WAL, memtable flushes, and manifest. + Clients route each request to the owner of the key's partition, so one Box's writes spread across + the cluster. The **balancer** (`balancer.interval.millis` > 0; one node coordinates under the + `cluster/balancer` lease) keeps partition ownership evenly distributed, fails over dead nodes' + partitions, and rate-limits migrations away from live owners + (`balancer.max.moves.per.round`). A node is `CandyboxNode(nodeId, config, ledgerStore, coordination, clock, advertisedAddress)` wired behind a `TcpTransportServer`; the advertised `host:port` is published to membership so the @@ -35,9 +44,12 @@ All knobs are configurable; pick defaults unless a workload demands otherwise. | `memtableFlushThresholdBytes` | 4 MiB | Memtable size that triggers a flush to an L0 SSTable. | | `syrupRolloverBytes` | 1 GiB | Open Syrup size before rolling to a fresh one. | | `maxFrameSizeBytes` | 16 MiB | Protocol frame cap; oversized/malformed frames are rejected pre-allocation. | -| `ownershipLeaseTtlMillis` | 10 s | Box ownership lease TTL; must be renewed within it. | +| `ownershipLeaseTtlMillis` | 10 s | Partition ownership lease TTL; must be renewed within it. | | `leaseRenewIntervalMillis` | 3 s | Lease heartbeat interval; `0` disables the background heartbeat. | -| `routerCacheTtlMillis` | 5 s | Client Box→owner routing-cache TTL. | +| `routerCacheTtlMillis` | 5 s | Client partition→owner routing-cache TTL. | +| `partitionsPerBoxDefault` | 8 | Hash-partition count for a new Box when the creator passes none; fixed for the Box's lifetime. | +| `balancerIntervalMillis` | 0 (disabled) | Partition-balancing round period; **set > 0 in production** (shipped conf: 5 s). | +| `balancerMaxMovesPerRound` | 4 | Max partitions migrated away from live owners per round (failover is unlimited). | | `compactionIntervalMillis` | 0 (disabled) | Background compaction+GC tick; **set > 0 in production**. | | `l0CompactionTrigger` | 4 | L0 SSTable count that triggers a compaction. | | `l0StallThreshold` | 12 | L0 SSTable count at which writes are rejected with `BUSY`. | @@ -55,9 +67,12 @@ level N's budget is `levelBaseBytes × multiplier^(N-1)`. ## Consistency & backpressure -- **Per-key linearizable on the owner.** A single fenced owner serializes all writes for a Box and - stamps a Hybrid Logical Clock at ingest; reads resolve to the highest HLC (LWW, nodeId tiebreaker). - The only eventual-consistency window is ownership handover. +- **Per-key linearizable on the owner.** Every key maps to one partition, and a single fenced owner + serializes all writes for a partition and stamps a Hybrid Logical Clock at ingest; reads resolve + to the highest HLC (LWW, nodeId tiebreaker). The only per-key eventual-consistency window is + ownership handover. Cross-partition operations are weaker: listings are scatter-gather merges, + `deleteRange` is one tombstone per partition (idempotent, not atomic), and a cross-partition + rename is copy-then-delete. - **Backpressure.** When a Box accumulates `l0StallThreshold` L0 SSTables, writes return a retriable `BUSY` (protocol `RESPONSE_BUSY`) instead of blocking. Clients should back off and retry; the stall clears once compaction drains L0. @@ -74,8 +89,10 @@ level N's budget is `levelBaseBytes × multiplier^(N-1)`. | **Client hits the wrong node** | The node replies `RESPONSE_MOVED(ownerNodeId)`; the client re-resolves via coordination and retries. | | **Clock skew** | HLC ordering survives a regressing wall clock; an observed HLC leading local time by more than `maxClockSkewMillis` is rejected. | -Recovery currently requires some node to take over an unowned Box (`CandyboxNode.openBox`); automatic -failover (watch lease loss → elect a new owner) is future work. +With the balancer enabled (`balancerIntervalMillis > 0`), a dead node's partitions are reassigned +and taken over automatically within a round or two of its leases expiring. With it disabled, +recovery requires some node to take over unowned partitions manually +(`CandyboxNode.openPartition` / `openBox`). ## Garbage collection @@ -136,8 +153,10 @@ the S3 gateway from the browser, not through this API. See `WEB_DASHBOARD_PLAN.m - **Streaming on the wire (Phase 2.5):** large objects are buffered in a single frame (≤ 16 MiB); chunked multi-frame PUT/GET is not yet implemented. -- **Automatic failover:** a crashed owner's Box is served again only when some node calls `openBox`. -- **Cluster-wide `listBoxes`:** returns the contacted node's Boxes only. +- **Automatic failover needs the balancer:** with `balancerIntervalMillis = 0`, a crashed owner's + partitions are served again only when some node calls `openPartition`/`openBox`. +- **Fixed partition count per Box:** set at creation, no re-partitioning; ordered listings + scatter-gather every partition per page. - **Distributed compaction offload:** the owner produces and commits its own compactions; offloading output production to non-owners under a ZK task lease is future work. - **GC enumeration backstop** and **Syrup defragmentation** (see above). diff --git a/README.md b/README.md index 2ae162a..2035dc3 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,8 @@ replicated across a cluster. Under the hood it is a **distributed LSM tree built on [Apache BookKeeper](https://bookkeeper.apache.org/)**: object data and index live in BookKeeper's replicated, append-only ledgers, and a single fenced owner -per bucket keeps reads and writes consistent during failover. +per bucket partition keeps reads and writes consistent during failover, with partitions spread +evenly across the cluster. > **Vocabulary** > @@ -125,7 +126,7 @@ operations an S3-style store cannot do cheaply: ```bash candybox list photos --start a --end m --reverse # bounded, reverse-order range scan candybox copy photos cat.jpg cat-copy.jpg # zero-copy: shares the stored bytes -candybox rename photos cat.jpg pets/cat.jpg # zero-copy move (atomic, same Box) +candybox rename photos cat.jpg pets/cat.jpg # zero-copy move (same Box) candybox delete-range photos thumbnails/ # one O(1) range tombstone, not N deletes candybox delete-range photos --start a --end m # delete a half-open [start, end) key window ``` @@ -133,7 +134,9 @@ candybox delete-range photos --start a --end m # delete a half-open [start - **Bounded / reverse range scans** walk a `[start, end)` window in either direction (`list --start K --end K --reverse`), paging with `--start-after`. - **Zero-copy `copy` / `rename`** point a new key at the *same* stored bytes — no data is moved — and - `rename` removes the source atomically (same Box). + `rename` removes the source atomically (same Box; when source and destination land in different + hash partitions the client transparently falls back to a byte copy, and the rename is no longer + atomic). - **`delete-range`** deletes a whole prefix or key window with a single range tombstone (constant work regardless of how many keys it covers); the bytes are reclaimed lazily by compaction. @@ -208,7 +211,7 @@ Candybox blends three well-known designs: flowchart TB client([Client]) - subgraph owner["Box owner (single, fenced node)"] + subgraph owner["Partition owner (single, fenced node)"] direction TB wal[Write-ahead log] memtable[Memtable] @@ -250,11 +253,14 @@ flowchart TB BookKeeper ledger — append-only, replicated, and self-fencing. Candybox never mutates data in place; updates and deletes are new appends (with tombstones), Apache-Pulsar-style. -Consistency rests on **single, fenced ownership**: at any moment exactly one node owns a Box, holding -a ZooKeeper lease with a *fencing token*. Every state-changing operation carries that token, so if -ownership moves during a failure, a stale former owner can no longer corrupt the Box. Each write is -stamped with a hybrid logical clock for last-writer-wins ordering across nodes. The full record -formats and the reasoning behind the fencing/handover protocol are in [`DESIGN.md`](DESIGN.md). +Consistency rests on **single, fenced ownership per partition**: every Box is split into a fixed +number of hash partitions, and at any moment exactly one node owns a partition, holding a ZooKeeper +lease with a *fencing token*. Every state-changing operation carries that token, so if ownership +moves during a failure, a stale former owner can no longer corrupt the partition. An elected +balancer spreads partition ownership evenly across the cluster, so one Box's writes are served by +many nodes. Each write is stamped with a hybrid logical clock for last-writer-wins ordering across +nodes. The full record formats and the reasoning behind the fencing/handover protocol are in +[`DESIGN.md`](DESIGN.md); partitioning is described in [`BOX_PARTITIONING_PLAN.md`](BOX_PARTITIONING_PLAN.md). ## Building from source diff --git a/candybox-admin-api/src/main/java/me/predatorray/candybox/admin/LiveDashboardData.java b/candybox-admin-api/src/main/java/me/predatorray/candybox/admin/LiveDashboardData.java index 44862cc..04222ae 100644 --- a/candybox-admin-api/src/main/java/me/predatorray/candybox/admin/LiveDashboardData.java +++ b/candybox-admin-api/src/main/java/me/predatorray/candybox/admin/LiveDashboardData.java @@ -23,6 +23,7 @@ import java.util.Optional; import me.predatorray.candybox.client.BoxClient; import me.predatorray.candybox.client.CandyboxClient; +import me.predatorray.candybox.coordination.BoxDescriptor; import me.predatorray.candybox.coordination.CandyboxKeys; import me.predatorray.candybox.coordination.CoordinationService; import me.predatorray.candybox.coordination.LeaseInfo; @@ -59,13 +60,20 @@ public LiveDashboardData(CoordinationService coordination, BoxClient client) { @Override public ClusterSnapshot cluster() { List boxes = safeListBoxes(); - Map ownedCount = new HashMap<>(); - List ownerless = new ArrayList<>(); + Map ownedCount = new HashMap<>(); // owned *partitions* per node + List ownerless = new ArrayList<>(); // boxes with at least one ownerless partition for (String box : boxes) { - Optional holder = coordination.leaseHolder(CandyboxKeys.ownerResource(box)); - if (holder.isPresent()) { - ownedCount.merge(holder.get().ownerNodeId(), 1, Integer::sum); - } else { + boolean allOwned = true; + for (int p = 0; p < partitionCountOf(box); p++) { + Optional holder = + coordination.leaseHolder(CandyboxKeys.ownerResource(box, p)); + if (holder.isPresent()) { + ownedCount.merge(holder.get().ownerNodeId(), 1, Integer::sum); + } else { + allOwned = false; + } + } + if (!allOwned) { ownerless.add(box); } } @@ -109,19 +117,21 @@ public List lsm() { List boxes = safeListBoxes(); List rows = new ArrayList<>(boxes.size()); for (String box : boxes) { - String owner = ownerOf(box); - long fencing = coordination.leaseHolder(CandyboxKeys.ownerResource(box)) - .map(LeaseInfo::fencingToken) - .orElse(-1L); - // TODO(phase-5.5): the manifest pointer's coordination version is a proxy for - // "manifest revision" — bumps every CAS on the pointer, so it's monotonic per box. - // Per-box runtime fields (ledger counts, compactions, GC) require a new per-node - // JSON endpoint on HealthServer to fan out against; deferred to keep this commit - // contained to the admin API + UI. - long manifestVersion = coordination.get(CandyboxKeys.manifestKey(box)) - .map(v -> v.version()) - .orElse(-1L); - rows.add(LsmRow.coordinationOnly(box, owner, manifestVersion, fencing)); + for (int p = 0; p < partitionCountOf(box); p++) { + Optional holder = + coordination.leaseHolder(CandyboxKeys.ownerResource(box, p)); + String owner = holder.map(h -> String.valueOf(h.ownerNodeId())).orElse(null); + long fencing = holder.map(LeaseInfo::fencingToken).orElse(-1L); + // TODO(phase-5.5): the manifest pointer's coordination version is a proxy for + // "manifest revision" — bumps every CAS on the pointer, so it's monotonic per + // partition. Per-partition runtime fields (ledger counts, compactions, GC) require + // a new per-node JSON endpoint on HealthServer to fan out against; deferred to keep + // this commit contained to the admin API + UI. + long manifestVersion = coordination.get(CandyboxKeys.manifestKey(box, p)) + .map(v -> v.version()) + .orElse(-1L); + rows.add(LsmRow.coordinationOnly(box + "/" + p, owner, manifestVersion, fencing)); + } } return rows; } @@ -173,9 +183,33 @@ private List safeListBoxes() { } } + /** The Box's partition count from its descriptor, or 0 if the descriptor is missing. */ + private int partitionCountOf(String box) { + return coordination.get(CandyboxKeys.boxMetaKey(box)) + .map(v -> BoxDescriptor.decode(v.value()).partitionCount()) + .orElse(0); + } + + /** + * Summarizes who owns a Box: the single node id when one node owns every partition, a + * comma-joined list when ownership is spread, {@code null} when no partition has an owner. + */ private String ownerOf(String box) { - return coordination.leaseHolder(CandyboxKeys.ownerResource(box)) - .map(h -> String.valueOf(h.ownerNodeId())) - .orElse(null); + java.util.TreeSet owners = new java.util.TreeSet<>(); + for (int p = 0; p < partitionCountOf(box); p++) { + coordination.leaseHolder(CandyboxKeys.ownerResource(box, p)) + .ifPresent(h -> owners.add(h.ownerNodeId())); + } + if (owners.isEmpty()) { + return null; + } + StringBuilder sb = new StringBuilder(); + for (Integer owner : owners) { + if (sb.length() > 0) { + sb.append(','); + } + sb.append(owner); + } + return sb.toString(); } } diff --git a/candybox-admin-api/src/test/java/me/predatorray/candybox/admin/LiveDashboardDataTest.java b/candybox-admin-api/src/test/java/me/predatorray/candybox/admin/LiveDashboardDataTest.java index 3359114..1ff93f7 100644 --- a/candybox-admin-api/src/test/java/me/predatorray/candybox/admin/LiveDashboardDataTest.java +++ b/candybox-admin-api/src/test/java/me/predatorray/candybox/admin/LiveDashboardDataTest.java @@ -24,6 +24,7 @@ import java.util.Map; import me.predatorray.candybox.client.BoxClient; import me.predatorray.candybox.client.CandyboxClient; +import me.predatorray.candybox.coordination.BoxDescriptor; import me.predatorray.candybox.coordination.CandyboxKeys; import me.predatorray.candybox.coordination.fake.InMemoryCoordinationService; import org.junit.jupiter.api.Test; @@ -44,9 +45,12 @@ void clusterReportsOwnersAndOrphans() { // two boxes and node 2 owns one — the snapshot's ownedBoxCount needs to count accordingly. coord.registerMember(1, "host-a:9709".getBytes(StandardCharsets.UTF_8)); coord.registerMember(2, "host-b:9709".getBytes(StandardCharsets.UTF_8)); - coord.tryAcquireLease(CandyboxKeys.ownerResource("photos"), 1, 60_000); - coord.tryAcquireLease(CandyboxKeys.ownerResource("docs"), 1, 60_000); - coord.tryAcquireLease(CandyboxKeys.ownerResource("backups"), 2, 60_000); + for (String box : List.of("photos", "docs", "backups", "orphan")) { + coord.create(CandyboxKeys.boxMetaKey(box), new BoxDescriptor(1).encode()); + } + coord.tryAcquireLease(CandyboxKeys.ownerResource("photos", 0), 1, 60_000); + coord.tryAcquireLease(CandyboxKeys.ownerResource("docs", 0), 1, 60_000); + coord.tryAcquireLease(CandyboxKeys.ownerResource("backups", 0), 2, 60_000); FakeBoxClient client = new FakeBoxClient(); client.boxes.addAll(List.of("photos", "docs", "backups", "orphan")); @@ -104,10 +108,12 @@ void safeListBoxesDegradesGracefullyOnClientFailure() { @Test void boxesAndLsmReadOwnerAndManifestVersion() throws Exception { InMemoryCoordinationService coord = new InMemoryCoordinationService(); - coord.tryAcquireLease(CandyboxKeys.ownerResource("photos"), 4, 60_000); + coord.create(CandyboxKeys.boxMetaKey("photos"), new BoxDescriptor(1).encode()); + coord.create(CandyboxKeys.boxMetaKey("orphan"), new BoxDescriptor(1).encode()); + coord.tryAcquireLease(CandyboxKeys.ownerResource("photos", 0), 4, 60_000); // Bumping the manifest pointer twice gives us a deterministic version (2 after create + CAS). - long v0 = coord.create(CandyboxKeys.manifestKey("photos"), new byte[] {1}); - long v1 = coord.compareAndSet(CandyboxKeys.manifestKey("photos"), new byte[] {2}, v0); + long v0 = coord.create(CandyboxKeys.manifestKey("photos", 0), new byte[] {1}); + long v1 = coord.compareAndSet(CandyboxKeys.manifestKey("photos", 0), new byte[] {2}, v0); FakeBoxClient client = new FakeBoxClient(); client.boxes.add("photos"); @@ -131,7 +137,7 @@ void boxesAndLsmReadOwnerAndManifestVersion() throws Exception { List lsm = data.lsm(); assertThat(lsm).hasSize(2); DashboardData.LsmRow photos = lsm.get(0); - assertThat(photos.box()).isEqualTo("photos"); + assertThat(photos.box()).isEqualTo("photos/0"); assertThat(photos.owner()).isEqualTo("4"); assertThat(photos.manifestVersion()).isEqualTo(v1); assertThat(photos.fencingToken()).isGreaterThan(0); diff --git a/candybox-client/src/main/java/me/predatorray/candybox/client/CandyboxCli.java b/candybox-client/src/main/java/me/predatorray/candybox/client/CandyboxCli.java index f965228..e8fbf3a 100644 --- a/candybox-client/src/main/java/me/predatorray/candybox/client/CandyboxCli.java +++ b/candybox-client/src/main/java/me/predatorray/candybox/client/CandyboxCli.java @@ -36,7 +36,7 @@ * candybox [-s host:port] <command> [args] * * list-boxes - * create-box <box> + * create-box <box> [partitions] * delete-box <box> [--force] * head-box <box> * put <box> <key> [file] # file or stdin; --content-type T, --meta k=v (repeatable) @@ -122,7 +122,9 @@ private static int dispatch(String command, List args, CandyboxClient cl return 0; } case "create-box" -> { - client.createBox(requireArg(args, 0, "box")); + String box = requireArg(args, 0, "box"); + int partitions = args.size() > 1 ? Integer.parseInt(args.get(1)) : 0; + client.createBox(box, partitions); return 0; } case "delete-box" -> { @@ -302,7 +304,7 @@ private static void printUsage(PrintStream out) { Commands: list-boxes - create-box + create-box [partitions] delete-box [--force] head-box put [file] file or stdin; --content-type T, --meta k=v diff --git a/candybox-client/src/main/java/me/predatorray/candybox/client/CandyboxClient.java b/candybox-client/src/main/java/me/predatorray/candybox/client/CandyboxClient.java index 3b6dba0..5c9a78c 100644 --- a/candybox-client/src/main/java/me/predatorray/candybox/client/CandyboxClient.java +++ b/candybox-client/src/main/java/me/predatorray/candybox/client/CandyboxClient.java @@ -19,14 +19,19 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import me.predatorray.candybox.common.BoxName; import me.predatorray.candybox.common.CandyKey; +import me.predatorray.candybox.common.Partitioning; import me.predatorray.candybox.common.SystemClock; import me.predatorray.candybox.common.Validation; import me.predatorray.candybox.common.config.CandyboxConfig; import me.predatorray.candybox.common.config.SizeLimits; +import me.predatorray.candybox.common.exception.BoxNotFoundException; import me.predatorray.candybox.common.exception.BusyException; import me.predatorray.candybox.common.exception.CandyNotFoundException; import me.predatorray.candybox.common.exception.CandyboxException; @@ -41,15 +46,22 @@ * typed {@link Message}s, hands them to a {@link Router}, and maps responses back to results or the * Candybox exception hierarchy. Client-side size validation fails fast before a request is sent. * + *

Every Box is hash-partitioned (its partition count is fixed at creation and cached here after a + * {@code BoxInfo} lookup). Keyed operations route to the owner of the key's partition; + * {@code deleteRange} and the listings fan out across every partition (listings merge pages in key + * order); cross-partition {@code copy}/{@code rename}/{@code uploadPartCopy} fall back to a + * client-side byte copy because the zero-copy path is only safe within one partition's manifest. + * *

Construct with a {@code host:port} for a single node ({@link DirectRouter}), or with a - * {@link CoordinationService} for a cluster ({@link ClusterRouter}, which resolves each Box's owner and - * re-routes on {@code MOVED}). Large objects are buffered in memory for now; chunked streaming over the - * wire is TODO(phase-2.5). + * {@link CoordinationService} for a cluster ({@link ClusterRouter}, which resolves each partition's + * owner and re-routes on {@code MOVED}). Large objects are buffered in memory for now; chunked + * streaming over the wire is TODO(phase-2.5). */ public final class CandyboxClient implements BoxClient, AutoCloseable { private final Router router; private final SizeLimits limits; + private final ConcurrentMap partitionCounts = new ConcurrentHashMap<>(); /** Single-node client talking directly to {@code host:port}. */ public CandyboxClient(Transport transport, String host, int port) { @@ -71,12 +83,48 @@ public CandyboxClient(Transport transport, CoordinationService coordination, Can // ---- Box admin ------------------------------------------------------------------------- + /** Creates a Box with the server's default partition count. */ public void createBox(String box) { - expectOk(router.callAny(new Message.CreateBoxRequest(BoxName.of(box).value()))); + createBox(box, 0); + } + + /** Creates a Box with an explicit partition count ({@code 0} = the server's default). */ + public void createBox(String box, int partitionCount) { + expectOk(router.callAny(new Message.CreateBoxRequest(BoxName.of(box).value(), + partitionCount))); } public void deleteBox(String box, boolean force) { - expectOk(router.callBox(box, new Message.DeleteBoxRequest(BoxName.of(box).value(), force))); + expectOk(router.callAny(new Message.DeleteBoxRequest(BoxName.of(box).value(), force))); + partitionCounts.remove(box); + } + + // ---- partition routing ------------------------------------------------------------------- + + /** The Box's partition count, fetched once via {@code BoxInfo} and cached (it is immutable). */ + private int partitionCount(String box) { + Integer cached = partitionCounts.get(box); + if (cached != null) { + return cached; + } + Message response = router.callAny(new Message.BoxInfoRequest(BoxName.of(box).value())); + if (response instanceof Message.BoxInfoResponse info) { + partitionCounts.put(box, info.partitionCount()); + return info.partitionCount(); + } + if (response instanceof Message.NotFoundResponse) { + throw new BoxNotFoundException(box); + } + throw mapResponse(response); + } + + private int partitionFor(String box, String key) { + return Partitioning.partitionOf(key, partitionCount(box)); + } + + /** Routes a keyed request to the owner of the key's partition. */ + private Message callKey(String box, String key, Message request) { + return router.callPartition(box, partitionFor(box, key), request); } // ---- Candy ops ------------------------------------------------------------------------- @@ -87,7 +135,7 @@ public void putCandy(String box, String key, byte[] data, String contentType, Validation.checkCandyKey(candyKey, limits); Validation.checkUserMetadata(userMetadata, limits); Validation.checkCandySize(data.length, limits); - expectOk(router.callBox(box, new Message.PutCandyRequest(BoxName.of(box).value(), + expectOk(callKey(box, key, new Message.PutCandyRequest(BoxName.of(box).value(), candyKey.value(), contentType, userMetadata == null ? Map.of() : userMetadata, idempotencyToken, data))); } @@ -99,7 +147,7 @@ public void putCandy(String box, String key, InputStream data, String contentTyp } public byte[] getCandy(String box, String key) { - Message response = router.callBox(box, new Message.GetCandyRequest(BoxName.of(box).value(), + Message response = callKey(box, key, new Message.GetCandyRequest(BoxName.of(box).value(), CandyKey.of(key).value())); if (response instanceof Message.CandyDataResponse data) { return data.data(); @@ -127,8 +175,8 @@ public void getCandy(String box, String key, OutputStream out) { * */ public RangeBytes getCandyRange(String box, String key, long firstByte, long lastByte) { - Message response = router.callBox(box, new Message.RangeGetCandyRequest(BoxName.of(box).value(), - CandyKey.of(key).value(), firstByte, lastByte)); + Message response = callKey(box, key, new Message.RangeGetCandyRequest( + BoxName.of(box).value(), CandyKey.of(key).value(), firstByte, lastByte)); if (response instanceof Message.CandyDataResponse data) { // For range responses, contentLength is the slice length and totalLength is the whole // object; the resolved start byte is implicit: totalLength - sliceLength may differ from @@ -140,7 +188,7 @@ public RangeBytes getCandyRange(String box, String key, long firstByte, long las } public CandyInfo headCandy(String box, String key) { - Message response = router.callBox(box, new Message.HeadCandyRequest(BoxName.of(box).value(), + Message response = callKey(box, key, new Message.HeadCandyRequest(BoxName.of(box).value(), CandyKey.of(key).value())); if (response instanceof Message.HeadCandyResponse head) { return new CandyInfo(head.contentLength(), head.contentType(), head.userMetadata(), @@ -149,7 +197,7 @@ public CandyInfo headCandy(String box, String key) { throw mapUnexpected(response, box, key); } - /** Lists the Boxes known to the contacted node. (Cluster-wide listing is a later refinement.) */ + /** Lists every Box in the cluster (from the contacted node's coordination view). */ public List listBoxes() { Message response = router.callAny(new Message.ListBoxesRequest()); if (response instanceof Message.ListBoxesResponse boxes) { @@ -158,14 +206,9 @@ public List listBoxes() { throw mapResponse(response); } - /** Returns whether the Box exists (has a current owner). */ + /** Returns whether the Box exists. */ public boolean headBox(String box) { - Message response; - try { - response = router.callBox(box, new Message.HeadBoxRequest(BoxName.of(box).value())); - } catch (NotOwnerException noOwner) { - return false; // no current owner ⇒ not currently a live Box - } + Message response = router.callAny(new Message.HeadBoxRequest(BoxName.of(box).value())); if (response instanceof Message.OkResponse) { return true; } @@ -176,7 +219,7 @@ public boolean headBox(String box) { } public void deleteCandy(String box, String key) { - expectOk(router.callBox(box, + expectOk(callKey(box, key, new Message.DeleteCandyRequest(BoxName.of(box).value(), CandyKey.of(key).value()))); } @@ -188,7 +231,7 @@ public String createMultipartUpload(String box, String key, String contentType, CandyKey candyKey = CandyKey.of(key); Validation.checkCandyKey(candyKey, limits); Validation.checkUserMetadata(userMetadata, limits); - Message response = router.callBox(box, new Message.CreateMultipartUploadRequest( + Message response = callKey(box, key, new Message.CreateMultipartUploadRequest( BoxName.of(box).value(), candyKey.value(), contentType, userMetadata == null ? Map.of() : userMetadata)); if (response instanceof Message.CreateMultipartUploadResponse cr) { @@ -206,7 +249,7 @@ public PartUploadInfo uploadPart(String box, String key, String uploadId, int pa CandyKey candyKey = CandyKey.of(key); Validation.checkCandyKey(candyKey, limits); Validation.checkCandySize(data.length, limits); - Message response = router.callBox(box, new Message.UploadPartRequest(BoxName.of(box).value(), + Message response = callKey(box, key, new Message.UploadPartRequest(BoxName.of(box).value(), candyKey.value(), uploadId, partNumber, data)); if (response instanceof Message.UploadPartResponse up) { return new PartUploadInfo(partNumber, up.crc32c(), up.partLength()); @@ -227,7 +270,7 @@ public CandyInfo completeMultipartUpload(String box, String key, String uploadId for (PartUploadInfo p : parts) { wire.add(new Message.CompletedPart(p.partNumber(), p.crc32c())); } - Message response = router.callBox(box, new Message.CompleteMultipartUploadRequest( + Message response = callKey(box, key, new Message.CompleteMultipartUploadRequest( BoxName.of(box).value(), candyKey.value(), uploadId, wire, idempotencyToken)); if (response instanceof Message.HeadCandyResponse head) { return new CandyInfo(head.contentLength(), head.contentType(), head.userMetadata(), @@ -238,7 +281,7 @@ public CandyInfo completeMultipartUpload(String box, String key, String uploadId /** Aborts a multipart upload; idempotent (a missing upload is a no-op, matching S3 behavior). */ public void abortMultipartUpload(String box, String key, String uploadId) { - expectOk(router.callBox(box, new Message.AbortMultipartUploadRequest(BoxName.of(box).value(), + expectOk(callKey(box, key, new Message.AbortMultipartUploadRequest(BoxName.of(box).value(), CandyKey.of(key).value(), uploadId))); } @@ -246,14 +289,24 @@ public void abortMultipartUpload(String box, String key, String uploadId) { * Copies a byte range of {@code srcKey} into a part slot of an in-flight upload (same Box only). * Mirrors S3 {@code UploadPartCopy}. Bounds follow the HTTP Range convention; {@code -1} for * either bound means "open-ended" — pass {@code (-1, -1)} to copy the whole source. + * + *

When the source key lives in a different partition than the upload's target key, the copy + * degrades to a client-side read + part upload (the zero-copy path needs both in one manifest). */ public PartUploadInfo uploadPartCopy(String box, String key, String uploadId, int partNumber, String srcKey, long firstByte, long lastByte) { Validation.checkCandyKey(CandyKey.of(key), limits); Validation.checkCandyKey(CandyKey.of(srcKey), limits); - Message response = router.callBox(box, new Message.UploadPartCopyRequest(BoxName.of(box).value(), - CandyKey.of(key).value(), uploadId, partNumber, CandyKey.of(srcKey).value(), - firstByte, lastByte)); + int n = partitionCount(box); + if (Partitioning.partitionOf(srcKey, n) != Partitioning.partitionOf(key, n)) { + byte[] bytes = (firstByte < 0 && lastByte < 0) + ? getCandy(box, srcKey) + : getCandyRange(box, srcKey, firstByte, lastByte).data(); + return uploadPart(box, key, uploadId, partNumber, bytes); + } + Message response = callKey(box, key, new Message.UploadPartCopyRequest( + BoxName.of(box).value(), CandyKey.of(key).value(), uploadId, partNumber, + CandyKey.of(srcKey).value(), firstByte, lastByte)); if (response instanceof Message.UploadPartResponse up) { return new PartUploadInfo(partNumber, up.crc32c(), up.partLength()); } @@ -263,22 +316,36 @@ public PartUploadInfo uploadPartCopy(String box, String key, String uploadId, in /** Lists in-flight multipart uploads in {@code box}, narrowed by an optional key prefix. */ public MultipartListing listMultipartUploads(String box, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) { - Message response = router.callBox(box, new Message.ListMultipartUploadsRequest( - BoxName.of(box).value(), prefix, keyMarker, uploadIdMarker, maxUploads)); - if (response instanceof Message.ListMultipartUploadsResponse list) { - List rows = new ArrayList<>(); - for (Message.InProgressUpload u : list.uploads()) { - rows.add(new UploadEntry(u.uploadId(), u.key(), u.createdAtMillis())); + int n = partitionCount(box); + int limit = maxUploads <= 0 ? 1000 : maxUploads; + List all = new ArrayList<>(); + boolean anyTruncated = false; + for (int p = 0; p < n; p++) { + Message response = router.callPartition(box, p, new Message.ListMultipartUploadsRequest( + BoxName.of(box).value(), p, prefix, keyMarker, uploadIdMarker, limit)); + if (!(response instanceof Message.ListMultipartUploadsResponse page)) { + throw mapResponse(response); + } + for (Message.InProgressUpload u : page.uploads()) { + all.add(new UploadEntry(u.uploadId(), u.key(), u.createdAtMillis())); + } + if (page.nextKeyMarker() != null) { + anyTruncated = true; } - return new MultipartListing(rows, list.nextKeyMarker(), list.nextUploadIdMarker()); } - throw mapResponse(response); + all.sort(Comparator.comparing(UploadEntry::key).thenComparing(UploadEntry::uploadId)); + boolean truncated = anyTruncated || all.size() > limit; + List rows = all.size() > limit ? new ArrayList<>(all.subList(0, limit)) : all; + UploadEntry last = rows.isEmpty() ? null : rows.get(rows.size() - 1); + return new MultipartListing(rows, + truncated && last != null ? last.key() : null, + truncated && last != null ? last.uploadId() : null); } /** Lists the parts recorded for one in-flight upload, paged by {@code partNumberMarker}. */ public PartListing listParts(String box, String key, String uploadId, int partNumberMarker, int maxParts) { - Message response = router.callBox(box, new Message.ListPartsRequest(BoxName.of(box).value(), + Message response = callKey(box, key, new Message.ListPartsRequest(BoxName.of(box).value(), CandyKey.of(key).value(), uploadId, partNumberMarker, maxParts)); if (response instanceof Message.ListPartsResponse list) { List rows = new ArrayList<>(); @@ -294,25 +361,36 @@ public PartListing listParts(String box, String key, String uploadId, int partNu } /** - * Zero-copy server-side copy of {@code srcKey} to {@code dstKey} within the same Box: the new key - * reuses the source's stored bytes (no data is transferred). Returns the destination's metadata. + * Server-side copy of {@code srcKey} to {@code dstKey} within the same Box. When both keys hash + * to the same partition the copy is zero-copy (the new key reuses the stored bytes); across + * partitions it degrades to a client-side byte copy. Returns the destination's metadata. */ public CandyInfo copyCandy(String box, String srcKey, String dstKey, String idempotencyToken) { - return copyOrRename(box, srcKey, new Message.CopyCandyRequest(BoxName.of(box).value(), - CandyKey.of(srcKey).value(), CandyKey.of(dstKey).value(), idempotencyToken)); + if (partitionFor(box, srcKey) == partitionFor(box, dstKey)) { + return copyOrRename(box, srcKey, new Message.CopyCandyRequest(BoxName.of(box).value(), + CandyKey.of(srcKey).value(), CandyKey.of(dstKey).value(), idempotencyToken)); + } + return byteCopy(box, srcKey, dstKey, idempotencyToken); } /** - * Zero-copy server-side rename/move of {@code srcKey} to {@code dstKey} within the same Box: the - * bytes never move; the source key is atomically removed. Returns the destination's metadata. + * Rename/move of {@code srcKey} to {@code dstKey} within the same Box. Same-partition renames + * are zero-copy and atomic; a cross-partition rename is a byte copy followed by a delete of the + * source (not atomic — a failure in between can leave both keys live). Returns the destination's + * metadata. */ public CandyInfo renameCandy(String box, String srcKey, String dstKey, String idempotencyToken) { - return copyOrRename(box, srcKey, new Message.RenameCandyRequest(BoxName.of(box).value(), - CandyKey.of(srcKey).value(), CandyKey.of(dstKey).value(), idempotencyToken)); + if (partitionFor(box, srcKey) == partitionFor(box, dstKey)) { + return copyOrRename(box, srcKey, new Message.RenameCandyRequest(BoxName.of(box).value(), + CandyKey.of(srcKey).value(), CandyKey.of(dstKey).value(), idempotencyToken)); + } + CandyInfo copied = byteCopy(box, srcKey, dstKey, idempotencyToken); + deleteCandy(box, srcKey); + return copied; } private CandyInfo copyOrRename(String box, String srcKey, Message request) { - Message response = router.callBox(box, request); + Message response = callKey(box, srcKey, request); if (response instanceof Message.HeadCandyResponse head) { return new CandyInfo(head.contentLength(), head.contentType(), head.userMetadata(), head.crc32c(), head.createdAtMillis()); @@ -320,52 +398,81 @@ private CandyInfo copyOrRename(String box, String srcKey, Message request) { throw mapUnexpected(response, box, srcKey); } + /** The cross-partition copy fallback: read the source whole, re-put it at the destination. */ + private CandyInfo byteCopy(String box, String srcKey, String dstKey, String idempotencyToken) { + Message response = callKey(box, srcKey, new Message.GetCandyRequest(BoxName.of(box).value(), + CandyKey.of(srcKey).value())); + if (!(response instanceof Message.CandyDataResponse data)) { + throw mapUnexpected(response, box, srcKey); + } + putCandy(box, dstKey, data.data(), data.contentType(), data.userMetadata(), idempotencyToken); + return headCandy(box, dstKey); + } + /** - * Deletes every Candy whose key starts with {@code prefix} using a single server-side range - * tombstone (O(1), not a list-then-delete). An empty/null prefix deletes the whole Box's contents. + * Deletes every Candy whose key starts with {@code prefix} using one server-side range tombstone + * per partition (fanned out; not atomic across partitions, but idempotent to retry). An + * empty/null prefix deletes the whole Box's contents. */ public void deleteRangeByPrefix(String box, String prefix) { - expectOk(router.callBox(box, - new Message.DeleteRangeRequest(BoxName.of(box).value(), prefix == null ? "" : prefix, - null, null))); + int n = partitionCount(box); + for (int p = 0; p < n; p++) { + expectOk(router.callPartition(box, p, new Message.DeleteRangeRequest( + BoxName.of(box).value(), p, prefix == null ? "" : prefix, null, null))); + } } /** - * Deletes every Candy whose key falls in {@code [startKey, endKey)} (either bound nullable) using a - * single server-side range tombstone. + * Deletes every Candy whose key falls in {@code [startKey, endKey)} (either bound nullable) using + * one server-side range tombstone per partition (fanned out; idempotent to retry). */ public void deleteRange(String box, String startKey, String endKey) { - expectOk(router.callBox(box, - new Message.DeleteRangeRequest(BoxName.of(box).value(), null, startKey, endKey))); + int n = partitionCount(box); + for (int p = 0; p < n; p++) { + expectOk(router.callPartition(box, p, new Message.DeleteRangeRequest( + BoxName.of(box).value(), p, null, startKey, endKey))); + } } public Listing listCandies(String box, String prefix, String startAfter, int maxKeys) { - return listCandies(box, new Message.ListCandiesRequest(BoxName.of(box).value(), prefix, - startAfter, maxKeys)); + return listCandies(box, prefix, null, null, startAfter, false, maxKeys); } /** * Range/directional listing: lists live Candies over the half-open window {@code [startKey, endKey)} * (either bound nullable), optionally narrowed by {@code prefix}, walked forward or in reverse, and * paged via {@code startAfter} (the previous page's {@code nextStartAfter}, exclusive in the scan - * direction). + * direction). Internally scatter-gathers every partition and merges pages in key order. */ public Listing listCandies(String box, String prefix, String startKey, String endKey, String startAfter, boolean reverse, int maxKeys) { - return listCandies(box, new Message.ListCandiesRequest(BoxName.of(box).value(), prefix, - startAfter, maxKeys, startKey, endKey, reverse)); - } - - private Listing listCandies(String box, Message.ListCandiesRequest request) { - Message response = router.callBox(box, request); - if (response instanceof Message.ListCandiesResponse list) { - List entries = new ArrayList<>(); - for (Message.ListedCandy c : list.entries()) { - entries.add(new Listing.Entry(c.key(), c.contentLength(), c.createdAtMillis())); + int n = partitionCount(box); + List all = new ArrayList<>(); + boolean anyTruncated = false; + for (int p = 0; p < n; p++) { + Message response = router.callPartition(box, p, new Message.ListCandiesRequest( + BoxName.of(box).value(), p, prefix, startAfter, maxKeys, startKey, endKey, + reverse)); + if (!(response instanceof Message.ListCandiesResponse page)) { + throw mapResponse(response); + } + for (Message.ListedCandy c : page.entries()) { + all.add(new Listing.Entry(c.key(), c.contentLength(), c.createdAtMillis())); + } + if (page.nextStartAfter() != null) { + anyTruncated = true; } - return new Listing(entries, list.nextStartAfter()); } - throw mapResponse(response); + // Each partition page holds its first maxKeys matches past the cursor, so the merged first + // maxKeys are globally correct: anything a truncated partition withheld sorts after its + // page's last (returned) key and therefore after the merged page too. + Comparator byKey = Comparator.comparing(e -> CandyKey.of(e.key())); + all.sort(reverse ? byKey.reversed() : byKey); + boolean overflow = maxKeys > 0 && all.size() > maxKeys; + List entries = overflow ? new ArrayList<>(all.subList(0, maxKeys)) : all; + boolean truncated = anyTruncated || overflow; + String next = truncated && !entries.isEmpty() ? entries.get(entries.size() - 1).key() : null; + return new Listing(entries, next); } @Override @@ -392,7 +499,7 @@ private CandyboxException mapResponse(Message response) { return new CandyboxException("Not found"); } if (response instanceof Message.MovedResponse moved) { - // TODO(phase-2 WS5): re-route to moved.ownerNodeId() via the ClusterRouter and retry. + // Only reachable via the DirectRouter (the ClusterRouter re-routes MOVED internally). return new NotOwnerException("owned by node " + moved.ownerNodeId()); } return new CandyboxException("Unexpected response: " + response.opcode()); diff --git a/candybox-client/src/main/java/me/predatorray/candybox/client/ClusterRouter.java b/candybox-client/src/main/java/me/predatorray/candybox/client/ClusterRouter.java index 97b8497..a9d0c63 100644 --- a/candybox-client/src/main/java/me/predatorray/candybox/client/ClusterRouter.java +++ b/candybox-client/src/main/java/me/predatorray/candybox/client/ClusterRouter.java @@ -32,10 +32,11 @@ import me.predatorray.candybox.protocol.transport.Transport; /** - * A cluster-aware {@link Router}: it resolves a Box to its owning node via the coordination lease, maps - * the owner's node id to its advertised {@code host:port} via membership, connects there, and re-routes - * on a {@code MOVED} response (using the named owner). Box→address resolutions are cached with a TTL - * and invalidated on redirect; one connection is kept per node address. + * A cluster-aware {@link Router}: it resolves a (Box, partition) to its owning node via the + * per-partition coordination lease, maps the owner's node id to its advertised {@code host:port} via + * membership, connects there, and re-routes on a {@code MOVED} response (using the named owner). + * Partition→address resolutions are cached with a TTL and invalidated on redirect; one connection is + * kept per node address. */ final class ClusterRouter implements Router { @@ -47,7 +48,7 @@ final class ClusterRouter implements Router { private final Clock clock; private final MessageCodec codec = new MessageCodec(); - private final ConcurrentMap boxCache = new ConcurrentHashMap<>(); + private final ConcurrentMap partitionCache = new ConcurrentHashMap<>(); private final ConcurrentMap connections = new ConcurrentHashMap<>(); ClusterRouter(Transport transport, CoordinationService coordination, long cacheTtlMillis, @@ -59,18 +60,21 @@ final class ClusterRouter implements Router { } @Override - public Message callBox(String box, Message request) { - NodeAddress address = resolveOwner(box); + public Message callPartition(String box, int partition, Message request) { + String cacheKey = box + "#" + partition; + NodeAddress address = resolveOwner(box, partition, cacheKey); for (int attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { Message response = send(address, request); if (response instanceof Message.MovedResponse moved) { address = addressOfNode(moved.ownerNodeId()); - boxCache.put(box, new CachedAddress(address, clock.currentTimeMillis() + cacheTtlMillis)); + partitionCache.put(cacheKey, + new CachedAddress(address, clock.currentTimeMillis() + cacheTtlMillis)); continue; } return response; } - throw new NotOwnerException("box " + box + ": exceeded routing attempts"); + throw new NotOwnerException("box " + box + " partition " + partition + + ": exceeded routing attempts"); } @Override @@ -78,15 +82,17 @@ public Message callAny(Message request) { return send(anyMember(), request); } - private NodeAddress resolveOwner(String box) { - CachedAddress cached = boxCache.get(box); + private NodeAddress resolveOwner(String box, int partition, String cacheKey) { + CachedAddress cached = partitionCache.get(cacheKey); if (cached != null && clock.currentTimeMillis() < cached.expiry()) { return cached.address(); } - LeaseInfo holder = coordination.leaseHolder(CandyboxKeys.ownerResource(box)) - .orElseThrow(() -> new NotOwnerException("box " + box + " has no current owner")); + LeaseInfo holder = coordination.leaseHolder(CandyboxKeys.ownerResource(box, partition)) + .orElseThrow(() -> new NotOwnerException("box " + box + " partition " + partition + + " has no current owner")); NodeAddress address = addressOfNode(holder.ownerNodeId()); - boxCache.put(box, new CachedAddress(address, clock.currentTimeMillis() + cacheTtlMillis)); + partitionCache.put(cacheKey, + new CachedAddress(address, clock.currentTimeMillis() + cacheTtlMillis)); return address; } diff --git a/candybox-client/src/main/java/me/predatorray/candybox/client/DirectRouter.java b/candybox-client/src/main/java/me/predatorray/candybox/client/DirectRouter.java index c92655b..f0861cf 100644 --- a/candybox-client/src/main/java/me/predatorray/candybox/client/DirectRouter.java +++ b/candybox-client/src/main/java/me/predatorray/candybox/client/DirectRouter.java @@ -36,7 +36,7 @@ final class DirectRouter implements Router { } @Override - public Message callBox(String box, Message request) { + public Message callPartition(String box, int partition, Message request) { return send(request); } diff --git a/candybox-client/src/main/java/me/predatorray/candybox/client/Router.java b/candybox-client/src/main/java/me/predatorray/candybox/client/Router.java index 56bda55..4aa48aa 100644 --- a/candybox-client/src/main/java/me/predatorray/candybox/client/Router.java +++ b/candybox-client/src/main/java/me/predatorray/candybox/client/Router.java @@ -24,10 +24,10 @@ */ interface Router extends AutoCloseable { - /** Routes a Box-scoped request to that Box's current owner. */ - Message callBox(String box, Message request); + /** Routes a partition-scoped request to that partition's current owner. */ + Message callPartition(String box, int partition, Message request); - /** Routes a cluster-wide request (createBox, listBoxes) to any reachable node. */ + /** Routes a cluster-wide request (createBox, boxInfo, listBoxes) to any reachable node. */ Message callAny(Message request); @Override diff --git a/candybox-client/src/test/java/me/predatorray/candybox/client/CandyboxClientTest.java b/candybox-client/src/test/java/me/predatorray/candybox/client/CandyboxClientTest.java index 71f8fdc..8fd19ce 100644 --- a/candybox-client/src/test/java/me/predatorray/candybox/client/CandyboxClientTest.java +++ b/candybox-client/src/test/java/me/predatorray/candybox/client/CandyboxClientTest.java @@ -41,7 +41,9 @@ class CandyboxClientTest { private static final RequestHandler HANDLER = request -> { Message message = CODEC.decode(request); Message response; - if (message instanceof Message.HeadCandyRequest) { + if (message instanceof Message.BoxInfoRequest) { + response = new Message.BoxInfoResponse(1); // single partition keeps the stub canned + } else if (message instanceof Message.HeadCandyRequest) { response = new Message.HeadCandyResponse(42, "text/plain", Map.of("a", "b"), 7, 123); } else if (message instanceof Message.ListBoxesRequest) { response = new Message.ListBoxesResponse(List.of("alpha", "beta")); diff --git a/candybox-client/src/test/java/me/predatorray/candybox/client/ClusterRouterTest.java b/candybox-client/src/test/java/me/predatorray/candybox/client/ClusterRouterTest.java index 63956be..655154b 100644 --- a/candybox-client/src/test/java/me/predatorray/candybox/client/ClusterRouterTest.java +++ b/candybox-client/src/test/java/me/predatorray/candybox/client/ClusterRouterTest.java @@ -81,11 +81,11 @@ private InMemoryCoordinationService coordinationWithMembers() { @Test void routesToTheLeaseHolder() { InMemoryCoordinationService coordination = coordinationWithMembers(); - coordination.tryAcquireLease(CandyboxKeys.ownerResource("b"), 2, 10_000); // owner = node 2 + coordination.tryAcquireLease(CandyboxKeys.ownerResource("b", 0), 2, 10_000); // owner = node 2 RecordingTransport transport = new RecordingTransport(); try (ClusterRouter router = new ClusterRouter(transport, coordination, 5_000, SystemClock.INSTANCE)) { - Message resp = router.callBox("b", new Message.GetCandyRequest("b", "k")); + Message resp = router.callPartition("b", 0, new Message.GetCandyRequest("b", "k")); assertThat(resp).isInstanceOf(Message.OkResponse.class); assertThat(transport.contacted).containsExactly(2002); } @@ -94,16 +94,16 @@ void routesToTheLeaseHolder() { @Test void followsMovedRedirectAndCachesTheNewOwner() { InMemoryCoordinationService coordination = coordinationWithMembers(); - coordination.tryAcquireLease(CandyboxKeys.ownerResource("b"), 1, 10_000); // stale owner = node 1 + coordination.tryAcquireLease(CandyboxKeys.ownerResource("b", 0), 1, 10_000); // stale owner = node 1 RecordingTransport transport = new RecordingTransport(); try (ClusterRouter router = new ClusterRouter(transport, coordination, 5_000, SystemClock.INSTANCE)) { - Message first = router.callBox("b", new Message.GetCandyRequest("b", "k")); + Message first = router.callPartition("b", 0, new Message.GetCandyRequest("b", "k")); assertThat(first).isInstanceOf(Message.OkResponse.class); assertThat(transport.contacted).containsExactly(1001, 2002); // redirected to node 2 // The new owner is cached: a second call goes straight to node 2. - router.callBox("b", new Message.GetCandyRequest("b", "k")); + router.callPartition("b", 0, new Message.GetCandyRequest("b", "k")); assertThat(transport.contacted).containsExactly(1001, 2002, 2002); } } @@ -113,7 +113,7 @@ void throwsWhenBoxHasNoOwner() { InMemoryCoordinationService coordination = coordinationWithMembers(); RecordingTransport transport = new RecordingTransport(); try (ClusterRouter router = new ClusterRouter(transport, coordination, 5_000, SystemClock.INSTANCE)) { - assertThatThrownBy(() -> router.callBox("ghost", new Message.GetCandyRequest("ghost", "k"))) + assertThatThrownBy(() -> router.callPartition("ghost", 0, new Message.GetCandyRequest("ghost", "k"))) .isInstanceOf(NotOwnerException.class); } } @@ -143,10 +143,10 @@ void callAnyThrowsWhenThereAreNoMembers() { @Test void throwsWhenOwnerNodeIsNotRegistered() { InMemoryCoordinationService coordination = coordinationWithMembers(); - coordination.tryAcquireLease(CandyboxKeys.ownerResource("b"), 99, 10_000); // node 99 unknown + coordination.tryAcquireLease(CandyboxKeys.ownerResource("b", 0), 99, 10_000); // node 99 unknown RecordingTransport transport = new RecordingTransport(); try (ClusterRouter router = new ClusterRouter(transport, coordination, 5_000, SystemClock.INSTANCE)) { - assertThatThrownBy(() -> router.callBox("b", new Message.GetCandyRequest("b", "k"))) + assertThatThrownBy(() -> router.callPartition("b", 0, new Message.GetCandyRequest("b", "k"))) .isInstanceOf(NotOwnerException.class) .hasMessageContaining("not registered"); } @@ -156,10 +156,10 @@ void throwsWhenOwnerNodeIsNotRegistered() { void throwsOnUnparseableMemberAddress() { InMemoryCoordinationService coordination = new InMemoryCoordinationService(); coordination.registerMember(5, "no-colon-here".getBytes(StandardCharsets.UTF_8)); - coordination.tryAcquireLease(CandyboxKeys.ownerResource("b"), 5, 10_000); + coordination.tryAcquireLease(CandyboxKeys.ownerResource("b", 0), 5, 10_000); RecordingTransport transport = new RecordingTransport(); try (ClusterRouter router = new ClusterRouter(transport, coordination, 5_000, SystemClock.INSTANCE)) { - assertThatThrownBy(() -> router.callBox("b", new Message.GetCandyRequest("b", "k"))) + assertThatThrownBy(() -> router.callPartition("b", 0, new Message.GetCandyRequest("b", "k"))) .isInstanceOf(NotOwnerException.class) .hasMessageContaining("Unroutable"); } @@ -168,7 +168,7 @@ void throwsOnUnparseableMemberAddress() { @Test void exceedingRoutingAttemptsThrowsNotOwner() { InMemoryCoordinationService coordination = coordinationWithMembers(); - coordination.tryAcquireLease(CandyboxKeys.ownerResource("b"), 2, 10_000); + coordination.tryAcquireLease(CandyboxKeys.ownerResource("b", 0), 2, 10_000); // A transport that always redirects: the router exhausts MAX_ATTEMPTS and gives up. Transport alwaysMoved = new Transport() { @Override @@ -190,7 +190,7 @@ public void close() { } }; try (ClusterRouter router = new ClusterRouter(alwaysMoved, coordination, 5_000, SystemClock.INSTANCE)) { - assertThatThrownBy(() -> router.callBox("b", new Message.GetCandyRequest("b", "k"))) + assertThatThrownBy(() -> router.callPartition("b", 0, new Message.GetCandyRequest("b", "k"))) .isInstanceOf(NotOwnerException.class) .hasMessageContaining("exceeded routing attempts"); } @@ -199,7 +199,7 @@ public void close() { @Test void brokenConnectionIsDroppedAndReopenedOnRetry() { InMemoryCoordinationService coordination = coordinationWithMembers(); - coordination.tryAcquireLease(CandyboxKeys.ownerResource("b"), 2, 10_000); + coordination.tryAcquireLease(CandyboxKeys.ownerResource("b", 0), 2, 10_000); // First call on each freshly-opened connection throws; the router must drop it and rethrow. Transport flaky = new Transport() { @Override @@ -221,11 +221,11 @@ public void close() { } }; try (ClusterRouter router = new ClusterRouter(flaky, coordination, 5_000, SystemClock.INSTANCE)) { - assertThatThrownBy(() -> router.callBox("b", new Message.GetCandyRequest("b", "k"))) + assertThatThrownBy(() -> router.callPartition("b", 0, new Message.GetCandyRequest("b", "k"))) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("connection reset"); // A subsequent call reopens the connection and fails the same way (not a stale-handle error). - assertThatThrownBy(() -> router.callBox("b", new Message.GetCandyRequest("b", "k"))) + assertThatThrownBy(() -> router.callPartition("b", 0, new Message.GetCandyRequest("b", "k"))) .isInstanceOf(IllegalStateException.class); } } diff --git a/candybox-common/src/main/java/me/predatorray/candybox/common/HybridLogicalClock.java b/candybox-common/src/main/java/me/predatorray/candybox/common/HybridLogicalClock.java index 3b93e3c..1851aea 100644 --- a/candybox-common/src/main/java/me/predatorray/candybox/common/HybridLogicalClock.java +++ b/candybox-common/src/main/java/me/predatorray/candybox/common/HybridLogicalClock.java @@ -100,7 +100,10 @@ public synchronized Hlc tick() { */ public synchronized void observe(Hlc observed) { long localPhysical = clock.currentTimeMillis(); - if (observed.physicalMillis() - localPhysical > maxAcceptableSkewMillis) { + // Only a timestamp *ahead* of the local clock can violate the skew bound; checking the sign + // first also keeps the subtraction from overflowing on extreme inputs (e.g. Hlc.MIN). + if (observed.physicalMillis() > localPhysical + && observed.physicalMillis() - localPhysical > maxAcceptableSkewMillis) { throw new IllegalStateException("Observed HLC physical time " + observed.physicalMillis() + " leads local clock " + localPhysical + " by more than the accepted skew of " + maxAcceptableSkewMillis + "ms"); diff --git a/candybox-common/src/main/java/me/predatorray/candybox/common/Partitioning.java b/candybox-common/src/main/java/me/predatorray/candybox/common/Partitioning.java new file mode 100644 index 0000000..be2dda2 --- /dev/null +++ b/candybox-common/src/main/java/me/predatorray/candybox/common/Partitioning.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package me.predatorray.candybox.common; + +import java.nio.charset.StandardCharsets; +import me.predatorray.candybox.common.checksum.Crc32c; + +/** + * The Box hash-partitioning function, shared by servers (engine selection) and clients (routing) so + * both always agree on which partition a key lives in. CRC32C over the key's UTF-8 bytes keeps it + * deterministic across JVMs and architectures and dependency-free. + * + *

A Box's partition count is fixed at creation (see {@code BoxDescriptor}), so the mapping for a + * given Box never changes. + */ +public final class Partitioning { + + private Partitioning() { + } + + /** The partition of {@code key} in a Box with {@code partitionCount} partitions. */ + public static int partitionOf(String key, int partitionCount) { + return partitionOf(key.getBytes(StandardCharsets.UTF_8), partitionCount); + } + + /** The partition of a key given as UTF-8 bytes. */ + public static int partitionOf(byte[] keyUtf8, int partitionCount) { + if (partitionCount <= 0) { + throw new IllegalArgumentException("partitionCount must be positive: " + partitionCount); + } + return (Crc32c.of(keyUtf8) & 0x7fffffff) % partitionCount; + } +} diff --git a/candybox-common/src/main/java/me/predatorray/candybox/common/config/CandyboxConfig.java b/candybox-common/src/main/java/me/predatorray/candybox/common/config/CandyboxConfig.java index 85a7f09..234f7b8 100644 --- a/candybox-common/src/main/java/me/predatorray/candybox/common/config/CandyboxConfig.java +++ b/candybox-common/src/main/java/me/predatorray/candybox/common/config/CandyboxConfig.java @@ -45,6 +45,9 @@ public final class CandyboxConfig { private final int multipartMaxParts; private final long multipartUploadTtlMillis; private final int multipartMaxConcurrentUploadsPerBox; + private final int partitionsPerBoxDefault; + private final long balancerIntervalMillis; + private final int balancerMaxMovesPerRound; private CandyboxConfig(Builder b) { this.sizeLimits = b.sizeLimits; @@ -66,6 +69,9 @@ private CandyboxConfig(Builder b) { this.multipartMaxParts = b.multipartMaxParts; this.multipartUploadTtlMillis = b.multipartUploadTtlMillis; this.multipartMaxConcurrentUploadsPerBox = b.multipartMaxConcurrentUploadsPerBox; + this.partitionsPerBoxDefault = b.partitionsPerBoxDefault; + this.balancerIntervalMillis = b.balancerIntervalMillis; + this.balancerMaxMovesPerRound = b.balancerMaxMovesPerRound; } public static CandyboxConfig defaults() { @@ -162,6 +168,21 @@ public int multipartMaxConcurrentUploadsPerBox() { return multipartMaxConcurrentUploadsPerBox; } + /** Partition count given to a new Box when the creator does not specify one. */ + public int partitionsPerBoxDefault() { + return partitionsPerBoxDefault; + } + + /** How often a node runs a partition-balancing round. {@code 0} disables the balancer. */ + public long balancerIntervalMillis() { + return balancerIntervalMillis; + } + + /** Max partitions moved away from live owners per balancing round (failover is not limited). */ + public int balancerMaxMovesPerRound() { + return balancerMaxMovesPerRound; + } + public static final class Builder { private SizeLimits sizeLimits = SizeLimits.defaults(); private Map quorums = new EnumMap<>(QuorumConfig.defaults()); @@ -182,6 +203,9 @@ public static final class Builder { private int multipartMaxParts = 10_000; // S3 cap private long multipartUploadTtlMillis = 7L * 24 * 3600 * 1000; // 7 days private int multipartMaxConcurrentUploadsPerBox = 10_000; // defensive ceiling + private int partitionsPerBoxDefault = 8; // write spread vs. per-engine cost + private long balancerIntervalMillis = 0L; // balancing round; 0 disables + private int balancerMaxMovesPerRound = 4; // migration rate limit public Builder sizeLimits(SizeLimits v) { this.sizeLimits = v; @@ -278,6 +302,21 @@ public Builder multipartMaxConcurrentUploadsPerBox(int v) { return this; } + public Builder partitionsPerBoxDefault(int v) { + this.partitionsPerBoxDefault = v; + return this; + } + + public Builder balancerIntervalMillis(long v) { + this.balancerIntervalMillis = v; + return this; + } + + public Builder balancerMaxMovesPerRound(int v) { + this.balancerMaxMovesPerRound = v; + return this; + } + public CandyboxConfig build() { if (l0StallThreshold < l0CompactionTrigger) { throw new IllegalArgumentException("l0StallThreshold must be >= l0CompactionTrigger"); @@ -288,6 +327,12 @@ public CandyboxConfig build() { if (multipartMaxParts < 1) { throw new IllegalArgumentException("multipartMaxParts must be positive"); } + if (partitionsPerBoxDefault < 1) { + throw new IllegalArgumentException("partitionsPerBoxDefault must be positive"); + } + if (balancerMaxMovesPerRound < 1) { + throw new IllegalArgumentException("balancerMaxMovesPerRound must be positive"); + } return new CandyboxConfig(this); } } diff --git a/candybox-common/src/test/java/me/predatorray/candybox/common/PartitioningTest.java b/candybox-common/src/test/java/me/predatorray/candybox/common/PartitioningTest.java new file mode 100644 index 0000000..eedaa82 --- /dev/null +++ b/candybox-common/src/test/java/me/predatorray/candybox/common/PartitioningTest.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package me.predatorray.candybox.common; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class PartitioningTest { + + @Test + void deterministicAndInRange() { + for (int n : new int[] {1, 2, 8, 17}) { + for (int i = 0; i < 200; i++) { + String key = "key/" + i; + int p = Partitioning.partitionOf(key, n); + assertThat(p).isBetween(0, n - 1); + // Stable across repeated calls and across the String/byte[] overloads (clients and + // servers must always agree). + assertThat(Partitioning.partitionOf(key, n)).isEqualTo(p); + assertThat(Partitioning.partitionOf(key.getBytes(StandardCharsets.UTF_8), n)) + .isEqualTo(p); + } + } + } + + @Test + void spreadsKeysAcrossPartitions() { + Set used = new HashSet<>(); + for (int i = 0; i < 200; i++) { + used.add(Partitioning.partitionOf("key/" + i, 8)); + } + assertThat(used).hasSize(8); // 200 keys over 8 buckets must touch every bucket + } + + @Test + void singlePartitionMapsEverythingToZero() { + assertThat(Partitioning.partitionOf("anything", 1)).isZero(); + } + + @Test + void rejectsNonPositivePartitionCount() { + assertThatThrownBy(() -> Partitioning.partitionOf("k", 0)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> Partitioning.partitionOf("k", -3)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/BoxDescriptor.java b/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/BoxDescriptor.java new file mode 100644 index 0000000..d24685a --- /dev/null +++ b/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/BoxDescriptor.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package me.predatorray.candybox.coordination; + +import me.predatorray.candybox.common.Partitioning; +import me.predatorray.candybox.common.serial.BinaryReader; +import me.predatorray.candybox.common.serial.BinaryWriter; + +/** + * A Box's immutable metadata record, stored at {@link CandyboxKeys#boxMetaKey} when the Box is + * created. The partition count is fixed for the Box's lifetime — it is the routing source of truth + * for servers and clients, so changing it would silently re-home every key. + */ +public record BoxDescriptor(int partitionCount) { + + private static final int FORMAT_VERSION = 1; + + public BoxDescriptor { + if (partitionCount < 1) { + throw new IllegalArgumentException("partitionCount must be positive: " + partitionCount); + } + } + + /** The partition the given key lives in, under this descriptor. */ + public int partitionOf(String key) { + return Partitioning.partitionOf(key, partitionCount); + } + + public byte[] encode() { + return new BinaryWriter(8) + .writeByte(FORMAT_VERSION) + .writeVarInt(partitionCount) + .toByteArray(); + } + + public static BoxDescriptor decode(byte[] data) { + BinaryReader r = new BinaryReader(data); + int version = r.readByte(); + if (version != FORMAT_VERSION) { + throw new CoordinationException("Unsupported BoxDescriptor version: " + version); + } + return new BoxDescriptor(r.readVarInt()); + } +} diff --git a/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/CandyboxKeys.java b/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/CandyboxKeys.java index 9e9fecf..3f81bf3 100644 --- a/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/CandyboxKeys.java +++ b/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/CandyboxKeys.java @@ -16,21 +16,36 @@ package me.predatorray.candybox.coordination; /** - * Canonical coordination key/resource names, shared by the server (which owns Boxes) and the client - * (which routes to owners) so both agree on where the ownership lease and manifest pointer live. + * Canonical coordination key/resource names, shared by the server (which owns Box partitions) and + * the client (which routes to owners) so both agree on where the Box descriptor, the per-partition + * ownership lease and manifest pointer, and the cluster-wide balancer state live. */ public final class CandyboxKeys { + /** The root under which every Box's coordination state lives. */ + public static final String BOXES_ROOT = "boxes"; + + /** The coordinator-election lease for the partition balancer. */ + public static final String BALANCER_RESOURCE = "cluster/balancer"; + + /** The versioned key holding the desired partition→node assignment table. */ + public static final String ASSIGNMENT_KEY = "cluster/assignment"; + private CandyboxKeys() { } - /** The ownership-lease resource for a Box. The lease holder is the Box's owner. */ - public static String ownerResource(String boxName) { - return "boxes/" + boxName + "/owner"; + /** The versioned key holding a Box's immutable descriptor (partition count). */ + public static String boxMetaKey(String boxName) { + return BOXES_ROOT + "/" + boxName + "/meta"; + } + + /** The ownership-lease resource for one partition of a Box. The lease holder is its owner. */ + public static String ownerResource(String boxName, int partition) { + return BOXES_ROOT + "/" + boxName + "/partitions/" + partition + "/owner"; } - /** The versioned key holding the pointer to a Box's current manifest ledger. */ - public static String manifestKey(String boxName) { - return "boxes/" + boxName + "/manifest"; + /** The versioned key holding the pointer to one partition's current manifest ledger. */ + public static String manifestKey(String boxName, int partition) { + return BOXES_ROOT + "/" + boxName + "/partitions/" + partition + "/manifest"; } } diff --git a/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/CoordinationService.java b/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/CoordinationService.java index b0ebf71..968fa1f 100644 --- a/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/CoordinationService.java +++ b/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/CoordinationService.java @@ -65,6 +65,13 @@ public interface CoordinationService extends AutoCloseable { */ void delete(String key, long expectedVersion); + /** + * Lists the direct child names under {@code path}, considering every key and lease resource as a + * {@code /}-separated path (e.g. {@code children("boxes")} returns the Box names that have any + * coordination state). Sorted ascending; empty if the path has no children. + */ + List children(String path); + // ---- leases / leader election ----------------------------------------------------------- /** diff --git a/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/fake/InMemoryCoordinationService.java b/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/fake/InMemoryCoordinationService.java index ba7440c..fe1c978 100644 --- a/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/fake/InMemoryCoordinationService.java +++ b/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/fake/InMemoryCoordinationService.java @@ -99,6 +99,29 @@ public synchronized void delete(String key, long expectedVersion) { kv.remove(key); } + @Override + public synchronized List children(String path) { + String prefix = path.endsWith("/") ? path : path + "/"; + java.util.TreeSet names = new java.util.TreeSet<>(); + for (String key : kv.keySet()) { + childOf(key, prefix).ifPresent(names::add); + } + for (String resource : leases.keySet()) { + childOf(resource, prefix).ifPresent(names::add); + } + return new ArrayList<>(names); + } + + private static Optional childOf(String key, String prefix) { + if (!key.startsWith(prefix)) { + return Optional.empty(); + } + String rest = key.substring(prefix.length()); + int slash = rest.indexOf('/'); + String child = slash < 0 ? rest : rest.substring(0, slash); + return child.isEmpty() ? Optional.empty() : Optional.of(child); + } + // ---- leases ---------------------------------------------------------------------------- @Override diff --git a/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/zk/ZooKeeperCoordinationService.java b/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/zk/ZooKeeperCoordinationService.java index 316b127..d35ea7d 100644 --- a/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/zk/ZooKeeperCoordinationService.java +++ b/candybox-coordination/src/main/java/me/predatorray/candybox/coordination/zk/ZooKeeperCoordinationService.java @@ -157,6 +157,19 @@ public void delete(String key, long expectedVersion) { } } + @Override + public List children(String path) { + try { + List names = new ArrayList<>(client.getChildren().forPath(path(path))); + names.sort(String::compareTo); + return names; + } catch (KeeperException.NoNodeException e) { + return List.of(); + } catch (Exception e) { + throw wrap("children", path, e); + } + } + private long currentVersion(String key) { Stat stat = new Stat(); try { diff --git a/candybox-coordination/src/test/java/me/predatorray/candybox/coordination/BoxDescriptorTest.java b/candybox-coordination/src/test/java/me/predatorray/candybox/coordination/BoxDescriptorTest.java new file mode 100644 index 0000000..bbc5b73 --- /dev/null +++ b/candybox-coordination/src/test/java/me/predatorray/candybox/coordination/BoxDescriptorTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package me.predatorray.candybox.coordination; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import me.predatorray.candybox.common.Partitioning; +import org.junit.jupiter.api.Test; + +class BoxDescriptorTest { + + @Test + void encodeDecodeRoundTrips() { + for (int count : new int[] {1, 8, 1024}) { + BoxDescriptor descriptor = new BoxDescriptor(count); + assertThat(BoxDescriptor.decode(descriptor.encode())).isEqualTo(descriptor); + } + } + + @Test + void partitionOfMatchesTheSharedHashFunction() { + BoxDescriptor descriptor = new BoxDescriptor(8); + assertThat(descriptor.partitionOf("some/key")) + .isEqualTo(Partitioning.partitionOf("some/key", 8)); + } + + @Test + void rejectsNonPositivePartitionCount() { + assertThatThrownBy(() -> new BoxDescriptor(0)).isInstanceOf(IllegalArgumentException.class); + } + + @Test + void rejectsUnknownFormatVersion() { + byte[] encoded = new BoxDescriptor(4).encode(); + encoded[0] = 99; + assertThatThrownBy(() -> BoxDescriptor.decode(encoded)) + .isInstanceOf(CoordinationException.class); + } +} diff --git a/candybox-coordination/src/test/java/me/predatorray/candybox/coordination/CoordinationServiceContract.java b/candybox-coordination/src/test/java/me/predatorray/candybox/coordination/CoordinationServiceContract.java index 5925c1d..07f4fc6 100644 --- a/candybox-coordination/src/test/java/me/predatorray/candybox/coordination/CoordinationServiceContract.java +++ b/candybox-coordination/src/test/java/me/predatorray/candybox/coordination/CoordinationServiceContract.java @@ -99,6 +99,22 @@ void deleteRejectsStaleVersionAndRemovesOnMatch() { assertThat(service.get("k")).isEmpty(); } + @Test + void childrenListsDirectChildNamesAcrossKeysAndLeases() { + assertThat(service.children("boxes")).isEmpty(); + + service.create("boxes/alpha/meta", b("d")); + service.create("boxes/alpha/partitions/0/manifest", b("m")); + service.create("boxes/beta/meta", b("d")); + service.tryAcquireLease("boxes/gamma/partitions/0/owner", 1, 5_000).orElseThrow(); + + // Direct child names only, deduplicated, sorted — and lease resources count as paths too. + assertThat(service.children("boxes")).containsExactly("alpha", "beta", "gamma"); + assertThat(service.children("boxes/alpha")).containsExactly("meta", "partitions"); + assertThat(service.children("boxes/alpha/partitions")).containsExactly("0"); + assertThat(service.children("elsewhere")).isEmpty(); + } + @Test void leaseBlocksOthersThenExpiresAndTokenIncreases() { Lease ownerA = service.tryAcquireLease("owner", 1, 5_000).orElseThrow(); diff --git a/candybox-dist/src/conf/candybox.properties.example b/candybox-dist/src/conf/candybox.properties.example index 5261d3e..90f3b41 100644 --- a/candybox-dist/src/conf/candybox.properties.example +++ b/candybox-dist/src/conf/candybox.properties.example @@ -78,6 +78,22 @@ data.dir=./data # l0.compaction.trigger=4 # l0.stall.threshold=12 +# --------------------------------------------------------------------------- +# Box partitioning & ownership balancing. +# partitions.per.box.default: hash-partition count given to a new Box when the creator does not +# specify one. Fixed for the Box's lifetime; each partition is a full LSM engine whose +# ownership can sit on a different node, so this is the Box's write-parallelism ceiling. +# balancer.interval.millis: how often each node runs a balancing round (the cluster/balancer +# lease holder publishes the desired assignment; every node converges on it). 0 disables the +# balancer — partitions then stay where they were created until moved manually. +# balancer.max.moves.per.round: at most this many partitions are migrated away from live owners +# per round (failover of a dead node's partitions is never rate-limited). +# --------------------------------------------------------------------------- + +# partitions.per.box.default=8 +balancer.interval.millis=5000 +# balancer.max.moves.per.round=4 + # --------------------------------------------------------------------------- # Multipart upload tuning (S3-style): # min-part-bytes: minimum size of every part except the last (S3 default 5 MiB). diff --git a/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/BoxClusterHandoverIT.java b/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/BoxClusterHandoverIT.java index 94626dc..0dc3942 100644 --- a/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/BoxClusterHandoverIT.java +++ b/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/BoxClusterHandoverIT.java @@ -106,8 +106,8 @@ void clientReRoutesToTheNewOwnerAfterFailover() { TcpTransport transport = new TcpTransport(); CandyboxClient client = new CandyboxClient(transport, coordClient, config); try { - // createBox lands on a node (node 1, lowest member) which acquires ownership. - client.createBox(box.value()); + // createBox lands on a node (node 1, lowest member) which owns all partitions initially. + client.createBox(box.value(), 2); client.putCandy(box.value(), "k", bytes("v1"), "text/plain", Map.of(), null); assertThat(client.getCandy(box.value(), "k")).isEqualTo(bytes("v1")); diff --git a/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/BoxHandoverIT.java b/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/BoxHandoverIT.java index e1f828b..16e439b 100644 --- a/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/BoxHandoverIT.java +++ b/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/BoxHandoverIT.java @@ -80,15 +80,15 @@ void ownershipMovesBetweenNodesOnRealBackends() { CandyboxNode nodeA = new CandyboxNode(1, config, storeA, coordA, SystemClock.INSTANCE); CandyboxNode nodeB = new CandyboxNode(2, config, storeB, coordB, SystemClock.INSTANCE) ) { - nodeA.createBox(box); - nodeA.engine(box).putCandy(CandyKey.of("k"), bytes("v1"), "text/plain", Map.of(), null); + nodeA.createBox(box, 1); + nodeA.enginePartition(box, 0).putCandy(CandyKey.of("k"), bytes("v1"), "text/plain", Map.of(), null); // A relinquishes ownership (releases the lease, closes its engine). nodeA.releaseBox(box); // B takes over: acquires the lease, recovers the manifest + WAL, advances the pointer. nodeB.openBox(box); - assertThat(nodeB.engine(box).getCandy(CandyKey.of("k"))).isEqualTo(bytes("v1")); + assertThat(nodeB.enginePartition(box, 0).getCandy(CandyKey.of("k"))).isEqualTo(bytes("v1")); } } } diff --git a/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/CompactionGcCycleIT.java b/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/CompactionGcCycleIT.java index c7d092c..d97e796 100644 --- a/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/CompactionGcCycleIT.java +++ b/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/CompactionGcCycleIT.java @@ -80,11 +80,11 @@ void compactionThenGcReclaimsLedgersOnRealBookKeeper() { new ZooKeeperCoordinationService(zookeeper.getConnectString(), SystemClock.INSTANCE); CandyboxNode node = new CandyboxNode(1, config, store, coordination, SystemClock.INSTANCE); try { - node.createBox(box); + node.createBox(box, 1); for (int i = 0; i < 4; i++) { - node.engine(box).putCandy(CandyKey.of("k" + i), bytes("v"), null, Map.of(), null); + node.enginePartition(box, 0).putCandy(CandyKey.of("k" + i), bytes("v"), null, Map.of(), null); } - node.engine(box).putCandy(CandyKey.of("k0"), bytes("v-new"), null, Map.of(), null); // overwrite + node.enginePartition(box, 0).putCandy(CandyKey.of("k0"), bytes("v-new"), null, Map.of(), null); // overwrite int ledgersBefore = store.listLedgers().size(); @@ -96,9 +96,9 @@ void compactionThenGcReclaimsLedgersOnRealBookKeeper() { assertThat(ledgersAfter).isLessThan(ledgersBefore); // The live data is intact after compaction + GC. - assertThat(node.engine(box).getCandy(CandyKey.of("k0"))).isEqualTo(bytes("v-new")); + assertThat(node.enginePartition(box, 0).getCandy(CandyKey.of("k0"))).isEqualTo(bytes("v-new")); for (int i = 1; i < 4; i++) { - assertThat(node.engine(box).getCandy(CandyKey.of("k" + i))).isEqualTo(bytes("v")); + assertThat(node.enginePartition(box, 0).getCandy(CandyKey.of("k" + i))).isEqualTo(bytes("v")); } } finally { node.close(); diff --git a/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/PartitionedBoxIT.java b/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/PartitionedBoxIT.java new file mode 100644 index 0000000..aac199a --- /dev/null +++ b/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/PartitionedBoxIT.java @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package me.predatorray.candybox.it; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import me.predatorray.candybox.bookkeeper.fake.InMemoryLedgerStore; +import me.predatorray.candybox.client.CandyboxClient; +import me.predatorray.candybox.common.BoxName; +import me.predatorray.candybox.common.ManualClock; +import me.predatorray.candybox.common.Partitioning; +import me.predatorray.candybox.common.config.CandyboxConfig; +import me.predatorray.candybox.common.exception.CandyNotFoundException; +import me.predatorray.candybox.coordination.fake.InMemoryCoordinationService; +import me.predatorray.candybox.protocol.Frame; +import me.predatorray.candybox.protocol.FrameCodec; +import me.predatorray.candybox.protocol.transport.Connection; +import me.predatorray.candybox.protocol.transport.RequestHandler; +import me.predatorray.candybox.protocol.transport.Transport; +import me.predatorray.candybox.server.CandyboxNode; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * End-to-end behaviour of a hash-partitioned Box whose partitions are owned by two different + * nodes, driven through the cluster-aware {@link CandyboxClient}: keyed routing, scatter-gather + * listing with pagination, fanned-out range deletes, cross-partition copy/rename byte-copy + * fallbacks, and multipart uploads. Runs entirely on the in-memory fakes (no BookKeeper/ZooKeeper / + * sockets) with a port-routing in-JVM transport, so it is fast yet exercises the real codec, request + * handler, and routing stack. + */ +class PartitionedBoxIT { + + private static final int PARTITIONS = 4; + private static final String BOX = "parted-box"; + + /** An in-JVM {@link Transport} that routes by port to the matching node's request handler. */ + private static final class RoutingTransport implements Transport { + private final Map handlers; + private final FrameCodec codec = new FrameCodec(); + + RoutingTransport(Map handlers) { + this.handlers = handlers; + } + + @Override + public Connection connect(String host, int port) { + RequestHandler handler = handlers.get(port); + return new Connection() { + @Override + public Frame call(Frame request) { + Frame onWire = codec.decode(codec.encode(request)); + return codec.decode(codec.encode(handler.handle(onWire))); + } + + @Override + public void close() { + } + }; + } + + @Override + public void close() { + } + } + + private InMemoryLedgerStore store; + private InMemoryCoordinationService coordination; + private CandyboxNode nodeA; + private CandyboxNode nodeB; + private CandyboxClient client; + + @BeforeEach + void setUp() { + ManualClock clock = new ManualClock(1_000); + store = new InMemoryLedgerStore(); + coordination = new InMemoryCoordinationService(clock); + CandyboxConfig config = CandyboxConfig.builder() + .leaseRenewIntervalMillis(0) + .multipartMinPartBytes(0) + .build(); + nodeA = new CandyboxNode(1, config, store, coordination, clock, "127.0.0.1:1"); + nodeB = new CandyboxNode(2, config, store, coordination, clock, "127.0.0.1:2"); + + Map handlers = new HashMap<>(); + handlers.put(1, nodeA.requestHandler()); + handlers.put(2, nodeB.requestHandler()); + client = new CandyboxClient(new RoutingTransport(handlers), coordination, config); + + // Create the Box on node 1 (all partitions land there), then hand partitions 2 and 3 over + // to node 2 so the Box's ownership genuinely spans two nodes. + client.createBox(BOX, PARTITIONS); + BoxName box = BoxName.of(BOX); + nodeA.releasePartition(box, 2); + nodeA.releasePartition(box, 3); + nodeB.openPartition(box, 2); + nodeB.openPartition(box, 3); + } + + @AfterEach + void tearDown() { + client.close(); + nodeA.close(); + nodeB.close(); + store.close(); + } + + private static byte[] bytes(String s) { + return s.getBytes(StandardCharsets.UTF_8); + } + + /** A key that hashes to the given partition. */ + private static String keyIn(int partition, String tag) { + for (int i = 0; i < 10_000; i++) { + String candidate = tag + "-" + i; + if (Partitioning.partitionOf(candidate, PARTITIONS) == partition) { + return candidate; + } + } + throw new AssertionError("no key found for partition " + partition); + } + + @Test + void keyedOperationsRouteToTheOwningNodeOfEachPartition() { + for (int p = 0; p < PARTITIONS; p++) { + String key = keyIn(p, "k"); + client.putCandy(BOX, key, bytes("v" + p), "text/plain", Map.of("p", "" + p), null); + assertThat(client.getCandy(BOX, key)).isEqualTo(bytes("v" + p)); + assertThat(client.headCandy(BOX, key).userMetadata()).containsEntry("p", "" + p); + } + // Each node only holds the keys of its partitions. + assertThat(nodeA.ownedBoxStats().keySet()).containsExactly(BOX + "/0", BOX + "/1"); + assertThat(nodeB.ownedBoxStats().keySet()).containsExactly(BOX + "/2", BOX + "/3"); + } + + @Test + void listingMergesPartitionsInGlobalKeyOrderWithPagination() { + TreeSet expected = new TreeSet<>(); + for (int i = 0; i < 23; i++) { + String key = "item/" + String.format("%02d", i); + client.putCandy(BOX, key, bytes("v"), null, Map.of(), null); + expected.add(key); + } + + // Forward, paged: walking pages of 7 yields every key exactly once, globally sorted. + List walked = new ArrayList<>(); + String cursor = null; + do { + CandyboxClient.Listing page = client.listCandies(BOX, "item/", cursor, 7); + for (CandyboxClient.Listing.Entry e : page.entries()) { + walked.add(e.key()); + } + cursor = page.nextStartAfter(); + } while (cursor != null); + assertThat(walked).containsExactlyElementsOf(expected); + + // Reverse listing comes back in descending order. + CandyboxClient.Listing reversed = client.listCandies(BOX, "item/", null, null, null, true, 100); + assertThat(reversed.entries()).extracting(CandyboxClient.Listing.Entry::key) + .containsExactlyElementsOf(expected.descendingSet()); + } + + @Test + void deleteRangeFansOutToEveryPartition() { + for (int i = 0; i < 12; i++) { + client.putCandy(BOX, "logs/" + i, bytes("v"), null, Map.of(), null); + } + client.putCandy(BOX, "other/key", bytes("keep"), null, Map.of(), null); + + client.deleteRangeByPrefix(BOX, "logs/"); + + assertThat(client.listCandies(BOX, "logs/", null, 100).entries()).isEmpty(); + assertThat(client.getCandy(BOX, "other/key")).isEqualTo(bytes("keep")); + } + + @Test + void crossPartitionCopyAndRenameFallBackToByteCopy() { + String src = keyIn(0, "src"); // owned by node 1 + String dstCopy = keyIn(3, "copy"); // owned by node 2 + String dstMove = keyIn(2, "move"); // owned by node 2 + client.putCandy(BOX, src, bytes("payload"), "text/plain", Map.of("m", "x"), null); + + CandyboxClient.CandyInfo copied = client.copyCandy(BOX, src, dstCopy, null); + assertThat(copied.contentLength()).isEqualTo(7); + assertThat(client.getCandy(BOX, dstCopy)).isEqualTo(bytes("payload")); + assertThat(client.headCandy(BOX, dstCopy).userMetadata()).containsEntry("m", "x"); + assertThat(client.getCandy(BOX, src)).isEqualTo(bytes("payload")); // copy keeps the source + + CandyboxClient.CandyInfo moved = client.renameCandy(BOX, src, dstMove, null); + assertThat(moved.contentLength()).isEqualTo(7); + assertThat(client.getCandy(BOX, dstMove)).isEqualTo(bytes("payload")); + assertThatThrownBy(() -> client.getCandy(BOX, src)) + .isInstanceOf(CandyNotFoundException.class); // rename removed the source + } + + @Test + void samePartitionCopyStaysServerSide() { + String src = keyIn(1, "zsrc"); + String dst = keyIn(1, "zdst"); + client.putCandy(BOX, src, bytes("zero-copy"), null, Map.of(), null); + client.copyCandy(BOX, src, dst, null); + assertThat(client.getCandy(BOX, dst)).isEqualTo(bytes("zero-copy")); + } + + @Test + void multipartUploadRoutesByTargetKeyAndListsAcrossPartitions() { + String keyOnB = keyIn(3, "mp"); // the upload lives on node 2's partition + String uploadId = client.createMultipartUpload(BOX, keyOnB, "text/plain", Map.of()); + + CandyboxClient.PartUploadInfo part1 = client.uploadPart(BOX, keyOnB, uploadId, 1, + bytes("hello ")); + CandyboxClient.PartUploadInfo part2 = client.uploadPart(BOX, keyOnB, uploadId, 2, + bytes("world")); + + // The merged cross-partition listing surfaces the in-flight upload. + CandyboxClient.MultipartListing uploads = + client.listMultipartUploads(BOX, null, null, null, 100); + assertThat(uploads.uploads()).extracting(CandyboxClient.UploadEntry::uploadId) + .containsExactly(uploadId); + + client.completeMultipartUpload(BOX, keyOnB, uploadId, List.of(part1, part2), null); + assertThat(client.getCandy(BOX, keyOnB)).isEqualTo(bytes("hello world")); + } + + @Test + void uploadPartCopyAcrossPartitionsDegradesToClientSideCopy() { + String src = keyIn(0, "upcsrc"); // node 1 + String target = keyIn(2, "upcdst"); // node 2 + client.putCandy(BOX, src, bytes("0123456789"), null, Map.of(), null); + + String uploadId = client.createMultipartUpload(BOX, target, null, Map.of()); + CandyboxClient.PartUploadInfo part = client.uploadPartCopy(BOX, target, uploadId, 1, + src, 2, 5); + assertThat(part.partLength()).isEqualTo(4); + + client.completeMultipartUpload(BOX, target, uploadId, List.of(part), null); + assertThat(client.getCandy(BOX, target)).isEqualTo(bytes("2345")); + } + + @Test + void boxInfoHeadBoxAndListBoxesAreClusterWide() { + // Both nodes answer Box existence/descriptor queries from coordination, so even a node that + // owns nothing about the Box can serve them (callAny picks the first member). + assertThat(client.headBox(BOX)).isTrue(); + assertThat(client.headBox("no-such-box")).isFalse(); + assertThat(client.listBoxes()).containsExactly(BOX); + } +} diff --git a/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/ServerClientLifecycleIT.java b/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/ServerClientLifecycleIT.java index e322a1a..0846105 100644 --- a/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/ServerClientLifecycleIT.java +++ b/candybox-integration-tests/src/test/java/me/predatorray/candybox/it/ServerClientLifecycleIT.java @@ -72,6 +72,7 @@ static void start() throws Exception { CandyboxConfig config = CandyboxConfig.builder() .leaseRenewIntervalMillis(0) .multipartMinPartBytes(0) + .partitionsPerBoxDefault(2) // small but real partition spread over embedded BK .build(); int port = freePort(); store = new BookKeeperLedgerStore(bookKeeper.clientConfiguration(), bytes("candybox")); @@ -336,7 +337,7 @@ void boxOwnershipReleaseAndReopenReplaysState() { node.releaseBox(boxName); node.openBox(boxName); - assertThat(node.currentOwner(boxName)).contains(node.nodeId()); + assertThat(node.currentOwner(boxName, 0)).contains(node.nodeId()); assertThat(client.getCandy(box, "durable")).isEqualTo(bytes("survives-handover")); } @@ -354,9 +355,9 @@ void nodeMaintenanceOperationsRunOverOwnedBoxes() { BoxName boxName = BoxName.of(box); assertThat(node.boxExists(boxName)).isTrue(); - assertThat(node.currentOwner(boxName)).contains(node.nodeId()); + assertThat(node.currentOwner(boxName, 0)).contains(node.nodeId()); assertThat(node.listBoxes()).contains(box); - assertThat(node.ownedBoxStats()).containsKey(box); + assertThat(node.ownedBoxStats()).containsKeys(box + "/0", box + "/1"); // The three periodic maintenance passes should run without error and report a non-negative // count of units processed. diff --git a/candybox-lsm/src/main/java/me/predatorray/candybox/lsm/engine/BoxEngine.java b/candybox-lsm/src/main/java/me/predatorray/candybox/lsm/engine/BoxEngine.java index 7fb65bb..15aee79 100644 --- a/candybox-lsm/src/main/java/me/predatorray/candybox/lsm/engine/BoxEngine.java +++ b/candybox-lsm/src/main/java/me/predatorray/candybox/lsm/engine/BoxEngine.java @@ -196,8 +196,12 @@ public static BoxEngine recover(BoxName box, CandyboxConfig config, LedgerStore } } // The current WAL always holds the most recent mutations (it is rotated on flush), so its - // max HLC dominates the flushed SSTables — observing it suffices for LWW correctness. - hlc.observe(replay.maxHlc()); + // max HLC dominates the flushed SSTables — observing it suffices for LWW correctness. An + // empty WAL (prior owner flushed before handing over) reports Hlc.MIN: nothing to observe, + // and the SSTables' HLCs are in the past relative to any clock the prior owner stamped by. + if (!replay.entries().isEmpty()) { + hlc.observe(replay.maxHlc()); + } } WriteAheadLog newWal = WriteAheadLog.create(ledgerStore, diff --git a/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/Message.java b/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/Message.java index 2f5308b..ba2d28b 100644 --- a/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/Message.java +++ b/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/Message.java @@ -33,10 +33,16 @@ public sealed interface Message { // ---- Box admin requests ---------------------------------------------------------------- - record CreateBoxRequest(String box) implements Message { + /** Creates a Box. {@code partitionCount == 0} means "use the server's configured default". */ + record CreateBoxRequest(String box, int partitionCount) implements Message { public Opcode opcode() { return Opcode.CREATE_BOX; } + + /** Creates with the server's default partition count. */ + public CreateBoxRequest(String box) { + this(box, 0); + } } record DeleteBoxRequest(String box, boolean force) implements Message { @@ -57,6 +63,13 @@ public Opcode opcode() { } } + /** Asks any node for a Box's descriptor (currently just its partition count). */ + record BoxInfoRequest(String box) implements Message { + public Opcode opcode() { + return Opcode.BOX_INFO; + } + } + // ---- Candy requests -------------------------------------------------------------------- record PutCandyRequest(String box, String key, String contentType, @@ -113,12 +126,13 @@ public Opcode opcode() { } /** - * Deletes a key range with a single range tombstone. Exactly one of: a {@code prefix}, or a + * Deletes a key range with a single range tombstone in one partition; the client fans + * the request out to every partition of the Box. Exactly one of: a {@code prefix}, or a * {@code [startKey, endKey)} window (either bound nullable). {@code prefix} non-null selects the * prefix form. */ - record DeleteRangeRequest(String box, String prefix, String startKey, String endKey) - implements Message { + record DeleteRangeRequest(String box, int partition, String prefix, String startKey, + String endKey) implements Message { public Opcode opcode() { return Opcode.DELETE_RANGE; } @@ -158,8 +172,11 @@ public Opcode opcode() { } } - /** Lists in-flight multipart uploads in a Box, narrowed by an optional key prefix. */ - record ListMultipartUploadsRequest(String box, String prefix, String keyMarker, + /** + * Lists in-flight multipart uploads in one partition of a Box, narrowed by an optional key + * prefix; the client fans out across partitions and merges. + */ + record ListMultipartUploadsRequest(String box, int partition, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) implements Message { public Opcode opcode() { return Opcode.LIST_MULTIPART_UPLOADS; @@ -186,15 +203,18 @@ public Opcode opcode() { } } - record ListCandiesRequest(String box, String prefix, String startAfter, int maxKeys, - String startKey, String endKey, boolean reverse) implements Message { + /** Lists live Candies in one partition of a Box; the client fans out and merge-sorts pages. */ + record ListCandiesRequest(String box, int partition, String prefix, String startAfter, + int maxKeys, String startKey, String endKey, boolean reverse) + implements Message { public Opcode opcode() { return Opcode.LIST_CANDIES; } - /** A plain forward prefix/startAfter listing (the classic three-arg form). */ - public ListCandiesRequest(String box, String prefix, String startAfter, int maxKeys) { - this(box, prefix, startAfter, maxKeys, null, null, false); + /** A plain forward prefix/startAfter listing of one partition. */ + public ListCandiesRequest(String box, int partition, String prefix, String startAfter, + int maxKeys) { + this(box, partition, prefix, startAfter, maxKeys, null, null, false); } } @@ -264,13 +284,20 @@ public Opcode opcode() { } } - /** Tells the client which node now owns the Box, so it can re-route (Phase 2 routing). */ + /** Tells the client which node owns the requested partition, so it can re-route. */ record MovedResponse(int ownerNodeId) implements Message { public Opcode opcode() { return Opcode.RESPONSE_MOVED; } } + /** A Box's descriptor: its (creation-time-fixed) partition count. */ + record BoxInfoResponse(int partitionCount) implements Message { + public Opcode opcode() { + return Opcode.RESPONSE_BOX_INFO; + } + } + record ListBoxesResponse(List boxes) implements Message { public Opcode opcode() { return Opcode.RESPONSE_BOX_LIST; diff --git a/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/MessageCodec.java b/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/MessageCodec.java index 06a01d4..566bbeb 100644 --- a/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/MessageCodec.java +++ b/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/MessageCodec.java @@ -35,6 +35,9 @@ public Frame encode(Message message) { w.writeByte(BODY_VERSION); if (message instanceof Message.CreateBoxRequest m) { w.writeString(m.box()); + w.writeVarInt(m.partitionCount()); + } else if (message instanceof Message.BoxInfoRequest m) { + w.writeString(m.box()); } else if (message instanceof Message.DeleteBoxRequest m) { w.writeString(m.box()); w.writeBoolean(m.force()); @@ -71,6 +74,7 @@ public Frame encode(Message message) { writeNullable(w, m.idempotencyToken()); } else if (message instanceof Message.DeleteRangeRequest m) { w.writeString(m.box()); + w.writeVarInt(m.partition()); writeNullable(w, m.prefix()); writeNullable(w, m.startKey()); writeNullable(w, m.endKey()); @@ -101,6 +105,7 @@ public Frame encode(Message message) { w.writeString(m.uploadId()); } else if (message instanceof Message.ListMultipartUploadsRequest m) { w.writeString(m.box()); + w.writeVarInt(m.partition()); writeNullable(w, m.prefix()); writeNullable(w, m.keyMarker()); writeNullable(w, m.uploadIdMarker()); @@ -143,6 +148,7 @@ public Frame encode(Message message) { w.writeVarInt(m.nextPartNumberMarker()); } else if (message instanceof Message.ListCandiesRequest m) { w.writeString(m.box()); + w.writeVarInt(m.partition()); writeNullable(w, m.prefix()); writeNullable(w, m.startAfter()); w.writeInt(m.maxKeys()); @@ -181,6 +187,8 @@ public Frame encode(Message message) { w.writeVarLong(Math.max(0, m.createdAtMillis())); } else if (message instanceof Message.MovedResponse m) { w.writeInt(m.ownerNodeId()); + } else if (message instanceof Message.BoxInfoResponse m) { + w.writeVarInt(m.partitionCount()); } else if (message instanceof Message.ListBoxesResponse m) { w.writeVarInt(m.boxes().size()); for (String box : m.boxes()) { @@ -199,7 +207,8 @@ public Message decode(Frame frame) { throw new ProtocolException("Unsupported message body version: " + version); } return switch (frame.opcode()) { - case CREATE_BOX -> new Message.CreateBoxRequest(r.readString()); + case CREATE_BOX -> new Message.CreateBoxRequest(r.readString(), r.readVarInt()); + case BOX_INFO -> new Message.BoxInfoRequest(r.readString()); case DELETE_BOX -> new Message.DeleteBoxRequest(r.readString(), r.readBoolean()); case LIST_BOXES -> new Message.ListBoxesRequest(); case HEAD_BOX -> new Message.HeadBoxRequest(r.readString()); @@ -214,10 +223,11 @@ public Message decode(Frame frame) { r.readString(), readNullable(r)); case RENAME_CANDY -> new Message.RenameCandyRequest(r.readString(), r.readString(), r.readString(), readNullable(r)); - case DELETE_RANGE -> new Message.DeleteRangeRequest(r.readString(), readNullable(r), - readNullable(r), readNullable(r)); - case LIST_CANDIES -> new Message.ListCandiesRequest(r.readString(), readNullable(r), - readNullable(r), r.readInt(), readNullable(r), readNullable(r), r.readBoolean()); + case DELETE_RANGE -> new Message.DeleteRangeRequest(r.readString(), r.readVarInt(), + readNullable(r), readNullable(r), readNullable(r)); + case LIST_CANDIES -> new Message.ListCandiesRequest(r.readString(), r.readVarInt(), + readNullable(r), readNullable(r), r.readInt(), readNullable(r), readNullable(r), + r.readBoolean()); case CREATE_MULTIPART_UPLOAD -> new Message.CreateMultipartUploadRequest(r.readString(), r.readString(), readNullable(r), readMetadata(r)); case UPLOAD_PART -> new Message.UploadPartRequest(r.readString(), r.readString(), @@ -226,7 +236,7 @@ public Message decode(Frame frame) { case ABORT_MULTIPART_UPLOAD -> new Message.AbortMultipartUploadRequest(r.readString(), r.readString(), r.readString()); case LIST_MULTIPART_UPLOADS -> new Message.ListMultipartUploadsRequest(r.readString(), - readNullable(r), readNullable(r), readNullable(r), r.readVarInt()); + r.readVarInt(), readNullable(r), readNullable(r), readNullable(r), r.readVarInt()); case LIST_PARTS -> new Message.ListPartsRequest(r.readString(), r.readString(), r.readString(), r.readVarInt(), r.readVarInt()); case UPLOAD_PART_COPY -> new Message.UploadPartCopyRequest(r.readString(), r.readString(), @@ -245,6 +255,7 @@ public Message decode(Frame frame) { case RESPONSE_HEAD -> new Message.HeadCandyResponse(r.readVarLong(), readNullable(r), readMetadata(r), r.readInt(), r.readVarLong()); case RESPONSE_MOVED -> new Message.MovedResponse(r.readInt()); + case RESPONSE_BOX_INFO -> new Message.BoxInfoResponse(r.readVarInt()); case RESPONSE_BOX_LIST -> decodeBoxList(r); }; } diff --git a/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/Opcode.java b/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/Opcode.java index 5fd85ca..1fc300c 100644 --- a/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/Opcode.java +++ b/candybox-protocol/src/main/java/me/predatorray/candybox/protocol/Opcode.java @@ -41,6 +41,7 @@ public enum Opcode { LIST_MULTIPART_UPLOADS(33), LIST_PARTS(34), UPLOAD_PART_COPY(35), + BOX_INFO(36), RESPONSE_OK(20), RESPONSE_ERROR(21), @@ -55,7 +56,8 @@ public enum Opcode { RESPONSE_CREATE_MULTIPART(40), RESPONSE_UPLOAD_PART(41), RESPONSE_LIST_MULTIPART_UPLOADS(42), - RESPONSE_LIST_PARTS(43); + RESPONSE_LIST_PARTS(43), + RESPONSE_BOX_INFO(44); private final int code; diff --git a/candybox-protocol/src/test/java/me/predatorray/candybox/protocol/MessageCodecTest.java b/candybox-protocol/src/test/java/me/predatorray/candybox/protocol/MessageCodecTest.java index 53f35ad..784fe6b 100644 --- a/candybox-protocol/src/test/java/me/predatorray/candybox/protocol/MessageCodecTest.java +++ b/candybox-protocol/src/test/java/me/predatorray/candybox/protocol/MessageCodecTest.java @@ -68,12 +68,13 @@ void copyRenameAndDeleteRangeRequestsRoundTrip() { assertThat(rename.idempotencyToken()).isNull(); Message.DeleteRangeRequest byPrefix = (Message.DeleteRangeRequest) roundTrip( - new Message.DeleteRangeRequest("box", "logs/", null, null)); + new Message.DeleteRangeRequest("box", 3, "logs/", null, null)); + assertThat(byPrefix.partition()).isEqualTo(3); assertThat(byPrefix.prefix()).isEqualTo("logs/"); assertThat(byPrefix.startKey()).isNull(); Message.DeleteRangeRequest byWindow = (Message.DeleteRangeRequest) roundTrip( - new Message.DeleteRangeRequest("box", null, "b", "e")); + new Message.DeleteRangeRequest("box", 1, null, "b", "e")); assertThat(byWindow.prefix()).isNull(); assertThat(byWindow.startKey()).isEqualTo("b"); assertThat(byWindow.endKey()).isEqualTo("e"); @@ -81,9 +82,10 @@ void copyRenameAndDeleteRangeRequestsRoundTrip() { @Test void listCandiesRequestRoundTripsRangeAndDirection() { - Message.ListCandiesRequest req = new Message.ListCandiesRequest("box", "p/", "p/cursor", 50, + Message.ListCandiesRequest req = new Message.ListCandiesRequest("box", 2, "p/", "p/cursor", 50, "p/a", "p/z", true); Message.ListCandiesRequest out = (Message.ListCandiesRequest) roundTrip(req); + assertThat(out.partition()).isEqualTo(2); assertThat(out.startKey()).isEqualTo("p/a"); assertThat(out.endKey()).isEqualTo("p/z"); assertThat(out.reverse()).isTrue(); @@ -156,6 +158,19 @@ void movedResponseRoundTrips() { assertThat(out.ownerNodeId()).isEqualTo(7); } + @Test + void boxInfoRequestAndResponseRoundTrip() { + Message.BoxInfoRequest req = (Message.BoxInfoRequest) roundTrip( + new Message.BoxInfoRequest("my-box")); + assertThat(req.box()).isEqualTo("my-box"); + assertThat(req.opcode()).isEqualTo(Opcode.BOX_INFO); + + Message.BoxInfoResponse resp = (Message.BoxInfoResponse) roundTrip( + new Message.BoxInfoResponse(8)); + assertThat(resp.partitionCount()).isEqualTo(8); + assertThat(resp.opcode()).isEqualTo(Opcode.RESPONSE_BOX_INFO); + } + @Test void listBoxesResponseRoundTrips() { Message.ListBoxesResponse out = (Message.ListBoxesResponse) roundTrip( @@ -168,8 +183,13 @@ void boxRequestsRoundTrip() { Message.CreateBoxRequest create = (Message.CreateBoxRequest) roundTrip( new Message.CreateBoxRequest("my-box")); assertThat(create.box()).isEqualTo("my-box"); + assertThat(create.partitionCount()).isZero(); // convenience ctor = server default assertThat(create.opcode()).isEqualTo(Opcode.CREATE_BOX); + Message.CreateBoxRequest partitioned = (Message.CreateBoxRequest) roundTrip( + new Message.CreateBoxRequest("my-box", 16)); + assertThat(partitioned.partitionCount()).isEqualTo(16); + Message.DeleteBoxRequest forced = (Message.DeleteBoxRequest) roundTrip( new Message.DeleteBoxRequest("my-box", true)); assertThat(forced.box()).isEqualTo("my-box"); @@ -226,7 +246,7 @@ void candyDataResponseHandlesNullContentTypeAndEmptyBody() { @Test void plainListCandiesRequestConstructorRoundTrips() { - Message.ListCandiesRequest req = new Message.ListCandiesRequest("box", "p/", "p/x", 25); + Message.ListCandiesRequest req = new Message.ListCandiesRequest("box", 0, "p/", "p/x", 25); Message.ListCandiesRequest out = (Message.ListCandiesRequest) roundTrip(req); assertThat(out.prefix()).isEqualTo("p/"); assertThat(out.startAfter()).isEqualTo("p/x"); @@ -306,7 +326,8 @@ void uploadPartCopyRequestRoundTripsRangeBounds() { void multipartListingRequestsRoundTrip() { Message.ListMultipartUploadsRequest uploads = (Message.ListMultipartUploadsRequest) roundTrip(new Message.ListMultipartUploadsRequest( - "box", "p/", "key-m", "upload-m", 200)); + "box", 4, "p/", "key-m", "upload-m", 200)); + assertThat(uploads.partition()).isEqualTo(4); assertThat(uploads.prefix()).isEqualTo("p/"); assertThat(uploads.keyMarker()).isEqualTo("key-m"); assertThat(uploads.uploadIdMarker()).isEqualTo("upload-m"); diff --git a/candybox-server/src/main/java/me/predatorray/candybox/server/CandyboxNode.java b/candybox-server/src/main/java/me/predatorray/candybox/server/CandyboxNode.java index ee66722..d517705 100644 --- a/candybox-server/src/main/java/me/predatorray/candybox/server/CandyboxNode.java +++ b/candybox-server/src/main/java/me/predatorray/candybox/server/CandyboxNode.java @@ -18,6 +18,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; @@ -28,26 +29,30 @@ import me.predatorray.candybox.common.Clock; import me.predatorray.candybox.common.SystemClock; import me.predatorray.candybox.common.config.CandyboxConfig; +import me.predatorray.candybox.common.exception.BoxAlreadyExistsException; import me.predatorray.candybox.common.exception.BoxNotEmptyException; import me.predatorray.candybox.common.exception.BoxNotFoundException; import me.predatorray.candybox.common.exception.FencedException; import me.predatorray.candybox.common.exception.NotOwnerException; +import me.predatorray.candybox.coordination.BoxDescriptor; +import me.predatorray.candybox.coordination.CandyboxKeys; +import me.predatorray.candybox.coordination.CasConflictException; import me.predatorray.candybox.coordination.CoordinationService; import me.predatorray.candybox.coordination.VersionedValue; import me.predatorray.candybox.lsm.engine.BoxEngine; import me.predatorray.candybox.protocol.transport.RequestHandler; +import me.predatorray.candybox.server.PartitionAssignment.BoxPartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * A Candybox storage node. It owns a set of Boxes under fenced ZooKeeper leases (via - * {@link BoxOwnership}), serving each from its local {@link BoxEngine}. {@link #createBox} takes - * ownership of a brand-new Box; {@link #openBox} takes over an existing Box (the failover/handover - * path), recovering its manifest and WAL. A background heartbeat renews the leases. - * - *

Still scaffolded for later: request routing across the cluster (a non-owner currently - * surfaces {@link me.predatorray.candybox.common.exception.NotOwnerException}; WS5 turns that into a - * {@code MOVED} response), and background compaction/GC workers (Phase 3). + * A Candybox storage node. Every Box is split into a creation-time-fixed number of hash partitions + * (its {@link BoxDescriptor}); the node owns a set of partitions under fenced ZooKeeper + * leases (via {@link PartitionOwnership}), serving each from its own {@link BoxEngine}, so the + * write load of one Box spreads across the cluster. {@link #createBox} creates the descriptor and + * takes initial ownership of every partition; the {@link PartitionBalancer} then spreads ownership + * evenly and {@link #openPartition} is the per-partition failover/takeover path. A background + * heartbeat renews the leases. */ public final class CandyboxNode implements AutoCloseable { @@ -58,13 +63,17 @@ public final class CandyboxNode implements AutoCloseable { private final LedgerStore ledgerStore; private final CoordinationService coordination; private final Clock clock; - private final ConcurrentMap boxes = new ConcurrentHashMap<>(); + private final ConcurrentMap partitions = + new ConcurrentHashMap<>(); + private final ConcurrentMap descriptorCache = new ConcurrentHashMap<>(); private final ScheduledExecutorService leaseHeartbeat; private final ScheduledExecutorService compactionWorker; + private final ScheduledExecutorService balancerWorker; private final CompactionService compactionService; private final GarbageCollector garbageCollector; + private final PartitionBalancer balancer; - /** Bounded compaction passes per Box per worker tick, so one Box cannot starve the others. */ + /** Bounded compaction passes per partition per worker tick, so one cannot starve the others. */ private static final int MAX_COMPACTIONS_PER_TICK = 8; public CandyboxNode(int nodeId, CandyboxConfig config, LedgerStore ledgerStore, @@ -91,6 +100,7 @@ public CandyboxNode(int nodeId, CandyboxConfig config, LedgerStore ledgerStore, coordination.registerMember(nodeId, advertisedAddress.getBytes(StandardCharsets.UTF_8)); this.compactionService = new CompactionService(ledgerStore, config, clock); this.garbageCollector = new GarbageCollector(ledgerStore, config.ledgerGcGraceMillis(), clock); + this.balancer = new PartitionBalancer(this, coordination, config); long renewInterval = config.leaseRenewIntervalMillis(); if (renewInterval > 0) { @@ -109,11 +119,20 @@ public CandyboxNode(int nodeId, CandyboxConfig config, LedgerStore ledgerStore, } else { this.compactionWorker = null; } + + long balancerInterval = config.balancerIntervalMillis(); + if (balancerInterval > 0) { + this.balancerWorker = daemonScheduler("candybox-balancer-" + nodeId); + this.balancerWorker.scheduleWithFixedDelay(this::runBalancerOnce, balancerInterval, + balancerInterval, TimeUnit.MILLISECONDS); + } else { + this.balancerWorker = null; + } } /** - * One background maintenance tick: compact owned Boxes, GC their obsoleted ledgers, and sweep any - * abandoned in-flight multipart uploads (older than {@code multipartUploadTtlMillis}). + * One background maintenance tick: compact owned partitions, GC their obsoleted ledgers, and sweep + * any abandoned in-flight multipart uploads (older than {@code multipartUploadTtlMillis}). */ private void runMaintenance() { compactOwnedBoxesOnce(); @@ -133,145 +152,314 @@ public int nodeId() { return nodeId; } - /** Creates and takes ownership of a brand-new Box on this node. */ + /** Creates a Box with the configured default partition count and owns all partitions here. */ public void createBox(BoxName box) { - boxes.compute(box.value(), (name, existing) -> { - if (existing != null && existing.isOwner()) { - throw new me.predatorray.candybox.common.exception.BoxAlreadyExistsException(name); + createBox(box, 0); + } + + /** + * Creates a brand-new Box: publishes its descriptor ({@code partitionCount}, or the configured + * default if {@code 0}) and takes initial ownership of every partition on this node — the + * balancer spreads them across the cluster afterwards. + */ + public void createBox(BoxName box, int partitionCount) { + int count = partitionCount > 0 ? partitionCount : config.partitionsPerBoxDefault(); + BoxDescriptor descriptor = new BoxDescriptor(count); + try { + coordination.create(CandyboxKeys.boxMetaKey(box.value()), descriptor.encode()); + } catch (CasConflictException exists) { + throw new BoxAlreadyExistsException(box.value()); + } + LOG.info("Creating box {} with {} partitions on node {}", box, count, nodeId); + List created = new ArrayList<>(count); + try { + for (int p = 0; p < count; p++) { + PartitionOwnership ownership = PartitionOwnership.createNew(box, p, config, + ledgerStore, coordination, nodeId, clock); + created.add(ownership); + partitions.put(new BoxPartition(box.value(), p), ownership); } - LOG.info("Creating box {} on node {}", name, nodeId); - return BoxOwnership.createNew(box, config, ledgerStore, coordination, nodeId, clock); - }); + descriptorCache.put(box.value(), descriptor); + } catch (RuntimeException e) { + // Roll back the half-created Box so a retry starts clean. + for (PartitionOwnership ownership : created) { + dropPartition(ownership); + } + deleteMetaQuietly(box); + throw e; + } } /** - * Takes over ownership of an existing Box (failover/handover): acquires the lease, recovers the - * manifest + WAL from the current pointer, and advances the pointer. + * Takes over ownership of one existing partition (failover/handover): acquires its lease, + * recovers the manifest + WAL from the current pointer, and advances the pointer. */ - public void openBox(BoxName box) { - boxes.compute(box.value(), (name, existing) -> { + public void openPartition(BoxName box, int partition) { + partitions.compute(new BoxPartition(box.value(), partition), (bp, existing) -> { if (existing != null && existing.isOwner()) { return existing; // already owned here } - LOG.info("Opening (taking over) box {} on node {}", name, nodeId); - return BoxOwnership.recover(box, config, ledgerStore, coordination, nodeId, clock); + LOG.info("Opening (taking over) box {} partition {} on node {}", bp.box(), + bp.partition(), nodeId); + return PartitionOwnership.recover(box, partition, config, ledgerStore, coordination, + nodeId, clock); }); } - /** Relinquishes ownership of a Box (releases the lease, closes the engine); does not delete data. */ - public void releaseBox(BoxName box) { - BoxOwnership ownership = boxes.remove(box.value()); + /** Takes over ownership of every partition of an existing Box (test/operational convenience). */ + public void openBox(BoxName box) { + BoxDescriptor descriptor = descriptor(box); + for (int p = 0; p < descriptor.partitionCount(); p++) { + openPartition(box, p); + } + } + + /** Relinquishes one partition (releases the lease, closes the engine); does not delete data. */ + public void releasePartition(BoxName box, int partition) { + PartitionOwnership ownership = partitions.remove(new BoxPartition(box.value(), partition)); if (ownership != null) { ownership.close(); } } - /** Deletes a Box. Requires this node to own it and (unless {@code force}) for it to be empty. */ + /** Like {@link #releasePartition} but flushes first so the next owner's WAL replay is small. */ + void releasePartitionForHandover(BoxName box, int partition) { + PartitionOwnership ownership = partitions.remove(new BoxPartition(box.value(), partition)); + if (ownership != null) { + ownership.closeForHandover(); + } + } + + /** Relinquishes every locally owned partition of a Box; does not delete data. */ + public void releaseBox(BoxName box) { + for (BoxPartition bp : ownedPartitionsOf(box)) { + releasePartition(box, bp.partition()); + } + } + + /** + * Deletes a Box: takes over every partition this node does not own (their leases must be free — + * with the balancer running, other owners release within a round of the descriptor disappearing), + * checks emptiness across all partitions unless {@code force}, drops every manifest pointer, and + * finally the descriptor. + * + * @throws NotOwnerException if {@code !force} and another live node still owns a partition + */ public void deleteBox(BoxName box, boolean force) { - BoxOwnership ownership = require(box); - if (!force && !ownership.engine().listCandies("", null, 1).entries().isEmpty()) { - throw new BoxNotEmptyException(box.value()); + BoxDescriptor descriptor = descriptor(box); + List owned = new ArrayList<>(descriptor.partitionCount()); + for (int p = 0; p < descriptor.partitionCount(); p++) { + PartitionOwnership ownership = partitions.get(new BoxPartition(box.value(), p)); + if (ownership != null && ownership.isOwner()) { + owned.add(ownership); + continue; + } + try { + openPartition(box, p); + owned.add(partitions.get(new BoxPartition(box.value(), p))); + } catch (RuntimeException e) { + if (!force) { + throw new NotOwnerException(box.value()); + } + // force: the live owner cleans its partition up once it sees the descriptor gone. + } + } + if (!force) { + for (PartitionOwnership ownership : owned) { + if (!ownership.engine().listCandies("", null, 1).entries().isEmpty()) { + throw new BoxNotEmptyException(box.value()); + } + } + } + for (PartitionOwnership ownership : owned) { + partitions.remove(new BoxPartition(box.value(), ownership.partition())); + dropPartition(ownership); } - boxes.remove(box.value()); - // Drop the manifest pointer (CAS on its version) so the Box no longer exists, then release. + descriptorCache.remove(box.value()); + deleteMetaQuietly(box); + // TODO(phase-3): reference-counted GC of the Box's SSTable/Syrup/WAL/manifest ledgers. + } + + /** Drops a partition's manifest pointer (so it no longer exists) and closes its engine/lease. */ + private void dropPartition(PartitionOwnership ownership) { ownership.currentPointer().ifPresent(p -> { try { - coordination.delete(BoxOwnership.manifestKey(box), p.version()); + coordination.delete( + PartitionOwnership.manifestKey(ownership.box(), ownership.partition()), + p.version()); } catch (RuntimeException e) { - LOG.warn("Failed to delete manifest pointer for box {}: {}", box, e.getMessage()); + LOG.warn("Failed to delete manifest pointer for box {} partition {}: {}", + ownership.box(), ownership.partition(), e.getMessage()); } }); ownership.close(); - // TODO(phase-3): reference-counted GC of the Box's SSTable/Syrup/WAL/manifest ledgers. } + private void deleteMetaQuietly(BoxName box) { + try { + Optional meta = coordination.get(CandyboxKeys.boxMetaKey(box.value())); + if (meta.isPresent()) { + coordination.delete(CandyboxKeys.boxMetaKey(box.value()), meta.get().version()); + } + } catch (RuntimeException e) { + LOG.warn("Failed to delete descriptor for box {}: {}", box, e.getMessage()); + } + } + + /** Every existing Box in the cluster (descriptor present), sorted by name. */ public List listBoxes() { - List names = new ArrayList<>(boxes.keySet()); + List names = new ArrayList<>(); + for (String boxName : coordination.children(CandyboxKeys.BOXES_ROOT)) { + if (coordination.get(CandyboxKeys.boxMetaKey(boxName)).isPresent()) { + names.add(boxName); + } + } names.sort(String::compareTo); return names; } /** * A point-in-time snapshot of {@link me.predatorray.candybox.lsm.engine.BoxEngineStats} for every - * Box this node currently owns, keyed by Box name. Used by the health/metrics endpoint; a Box that - * has just lost ownership is simply omitted rather than failing the whole snapshot. + * partition this node currently owns, keyed by {@code box/partition}. Used by the health/metrics + * endpoint; a partition that has just lost ownership is omitted rather than failing the snapshot. */ public java.util.Map ownedBoxStats() { java.util.Map out = new java.util.TreeMap<>(); - for (java.util.Map.Entry e : boxes.entrySet()) { - BoxOwnership ownership = e.getValue(); + for (java.util.Map.Entry e : partitions.entrySet()) { + PartitionOwnership ownership = e.getValue(); if (!ownership.isOwner()) { continue; } try { - out.put(e.getKey(), ownership.engine().stats()); + out.put(e.getKey().box() + "/" + e.getKey().partition(), ownership.engine().stats()); } catch (RuntimeException ignore) { - // Ownership lost between the check and the read; drop this Box from the snapshot. + // Ownership lost between the check and the read; drop this partition. } } return out; } + /** Whether the Box exists in the cluster (its descriptor is present in coordination). */ public boolean boxExists(BoxName box) { - BoxOwnership o = boxes.get(box.value()); - return o != null && o.isOwner(); + return findDescriptor(box).isPresent(); + } + + /** The Box's descriptor, or throws {@link BoxNotFoundException}. Cached (it is immutable). */ + public BoxDescriptor descriptor(BoxName box) { + return findDescriptor(box).orElseThrow(() -> new BoxNotFoundException(box.value())); } - /** The node id currently owning {@code box} (from the coordination lease), if any. */ - public java.util.Optional currentOwner(BoxName box) { - return coordination.leaseHolder(BoxOwnership.ownerResource(box)) + private Optional findDescriptor(BoxName box) { + BoxDescriptor cached = descriptorCache.get(box.value()); + if (cached != null) { + return Optional.of(cached); + } + Optional loaded = coordination.get(CandyboxKeys.boxMetaKey(box.value())) + .map(v -> BoxDescriptor.decode(v.value())); + loaded.ifPresent(d -> descriptorCache.put(box.value(), d)); + return loaded; + } + + /** The node currently owning one partition of {@code box} (from the lease), if any. */ + public Optional currentOwner(BoxName box, int partition) { + return coordination.leaseHolder(PartitionOwnership.ownerResource(box, partition)) .map(me.predatorray.candybox.coordination.LeaseInfo::ownerNodeId); } + /** Whether this node currently owns the given partition. */ + boolean ownsPartition(String box, int partition) { + PartitionOwnership ownership = partitions.get(new BoxPartition(box, partition)); + return ownership != null && ownership.isOwner(); + } + /** - * Returns the engine for a Box this node owns. Throws {@link BoxNotFoundException} if this node has - * no ownership object for the Box, or {@link me.predatorray.candybox.common.exception.NotOwnerException} - * if its lease is no longer valid. + * Returns the engine of the partition holding {@code key}. Throws {@link BoxNotFoundException} + * if the Box does not exist or this node has no ownership object for the partition, or + * {@link NotOwnerException} if its lease is no longer valid. */ - public BoxEngine engine(BoxName box) { - return require(box).engine(); + public BoxEngine engine(BoxName box, String key) { + return enginePartition(box, descriptor(box).partitionOf(key)); } - private BoxOwnership require(BoxName box) { - BoxOwnership ownership = boxes.get(box.value()); + /** Returns the engine of one partition this node owns (see {@link #engine}). */ + public BoxEngine enginePartition(BoxName box, int partition) { + PartitionOwnership ownership = partitions.get(new BoxPartition(box.value(), partition)); if (ownership == null) { throw new BoxNotFoundException(box.value()); } - return ownership; + return ownership.engine(); + } + + /** + * Drops locally owned partitions whose Box descriptor no longer exists — the convergence path of + * a (force) {@link #deleteBox} issued on another node while this one owned some partitions. + */ + void sweepDeletedBoxes() { + for (BoxPartition bp : partitions.keySet()) { + if (coordination.get(CandyboxKeys.boxMetaKey(bp.box())).isEmpty()) { + PartitionOwnership ownership = partitions.remove(bp); + if (ownership != null) { + LOG.info("Dropping box {} partition {} on node {}: the Box was deleted", + bp.box(), bp.partition(), nodeId); + dropPartition(ownership); + } + descriptorCache.remove(bp.box()); + } + } + } + + private List ownedPartitionsOf(BoxName box) { + List owned = new ArrayList<>(); + for (BoxPartition bp : partitions.keySet()) { + if (bp.box().equals(box.value())) { + owned.add(bp); + } + } + return owned; } CandyboxConfig config() { return config; } - /** A request handler that serves this node's Boxes; wire it into a Transport server or loopback. */ + /** A request handler that serves this node's partitions; wire it into a Transport or loopback. */ public RequestHandler requestHandler() { return new NodeRequestHandler(this); } + /** + * Runs one partition-balancing round (coordinate if elected, then converge on the assignment). + * Driven by the background worker when {@code balancerIntervalMillis > 0}; exposed so tests and + * operational tooling can drive it manually. + */ + public void runBalancerOnce() { + balancer.runOnce(); + } + private void renewLeases() { - for (BoxOwnership ownership : boxes.values()) { + for (PartitionOwnership ownership : partitions.values()) { try { ownership.renew(); } catch (RuntimeException e) { - LOG.warn("Lease renewal error for a box on node {}", nodeId, e); + LOG.warn("Lease renewal error for a partition on node {}", nodeId, e); } } } /** - * Runs one bounded round of compaction over every Box this node still owns. The commit is gated on - * the owner's fencing token by the manifest, so a Box whose ownership was lost mid-round fails the - * commit ({@link FencedException}) and is simply skipped — a zombie owner cannot corrupt state. + * Runs one bounded round of compaction over every partition this node still owns. The commit is + * gated on the owner's fencing token by the manifest, so a partition whose ownership was lost + * mid-round fails the commit ({@link FencedException}) and is simply skipped — a zombie owner + * cannot corrupt state. * *

Exposed so it can be driven manually (tests, operational triggers) without the scheduler. * - * @return the number of compactions performed across all Boxes + * @return the number of compactions performed across all partitions */ public int compactOwnedBoxesOnce() { int performed = 0; - for (BoxOwnership ownership : boxes.values()) { + for (PartitionOwnership ownership : partitions.values()) { if (!ownership.isOwner()) { continue; } @@ -284,7 +472,7 @@ public int compactOwnedBoxesOnce() { performed++; } } catch (FencedException | NotOwnerException lostOwnership) { - LOG.info("Stopping compaction of a box on node {}: {}", nodeId, + LOG.info("Stopping compaction of a partition on node {}: {}", nodeId, lostOwnership.getMessage()); } catch (RuntimeException e) { LOG.warn("Compaction error on node {}", nodeId, e); @@ -294,21 +482,22 @@ public int compactOwnedBoxesOnce() { } /** - * Runs one GC pass over every Box this node owns, deleting SSTable ledgers obsoleted by committed + * Runs one GC pass over every partition this node owns, deleting ledgers obsoleted by committed * compactions past the grace period. Exposed for manual/operational triggering. * * @return the number of ledgers deleted */ public int collectGarbageOnce() { int deleted = 0; - for (BoxOwnership ownership : boxes.values()) { + for (PartitionOwnership ownership : partitions.values()) { if (!ownership.isOwner()) { continue; } try { deleted += garbageCollector.collect(ownership.engine()); } catch (NotOwnerException lostOwnership) { - LOG.info("Skipping GC of a box on node {}: {}", nodeId, lostOwnership.getMessage()); + LOG.info("Skipping GC of a partition on node {}: {}", nodeId, + lostOwnership.getMessage()); } catch (RuntimeException e) { LOG.warn("GC error on node {}", nodeId, e); } @@ -321,7 +510,7 @@ public int collectGarbageOnce() { * already in the past. Reuses the engine's {@link me.predatorray.candybox.lsm.engine.BoxEngine * #abortMultipartUpload} (fencing-gated; orphaned Syrup segments enter the normal reclaim path). * - * @return the number of uploads aborted across all owned Boxes + * @return the number of uploads aborted across all owned partitions */ public int sweepStaleMultipartUploadsOnce() { long ttl = config.multipartUploadTtlMillis(); @@ -330,7 +519,7 @@ public int sweepStaleMultipartUploadsOnce() { } long cutoff = clock.currentTimeMillis() - ttl; int aborted = 0; - for (BoxOwnership ownership : boxes.values()) { + for (PartitionOwnership ownership : partitions.values()) { if (!ownership.isOwner()) { continue; } @@ -343,7 +532,7 @@ public int sweepStaleMultipartUploadsOnce() { } } } catch (NotOwnerException lostOwnership) { - LOG.info("Skipping multipart TTL sweep on a box on node {}: {}", nodeId, + LOG.info("Skipping multipart TTL sweep on a partition on node {}: {}", nodeId, lostOwnership.getMessage()); } catch (RuntimeException e) { LOG.warn("Multipart TTL sweep error on node {}", nodeId, e); @@ -360,10 +549,13 @@ public void close() { if (compactionWorker != null) { compactionWorker.shutdownNow(); } - for (BoxOwnership ownership : boxes.values()) { + if (balancerWorker != null) { + balancerWorker.shutdownNow(); + } + for (PartitionOwnership ownership : partitions.values()) { ownership.close(); } - boxes.clear(); + partitions.clear(); coordination.unregisterMember(nodeId); } } diff --git a/candybox-server/src/main/java/me/predatorray/candybox/server/NodeRequestHandler.java b/candybox-server/src/main/java/me/predatorray/candybox/server/NodeRequestHandler.java index 5a52608..30d325d 100644 --- a/candybox-server/src/main/java/me/predatorray/candybox/server/NodeRequestHandler.java +++ b/candybox-server/src/main/java/me/predatorray/candybox/server/NodeRequestHandler.java @@ -27,6 +27,7 @@ import me.predatorray.candybox.common.exception.CandyboxException; import me.predatorray.candybox.common.exception.NotOwnerException; import me.predatorray.candybox.common.exception.ValidationException; +import me.predatorray.candybox.coordination.BoxDescriptor; import me.predatorray.candybox.lsm.engine.BoxEngine; import me.predatorray.candybox.lsm.engine.CandyMetadata; import me.predatorray.candybox.common.Part; @@ -46,8 +47,10 @@ * results (and the Candybox exception hierarchy) back into response frames — including the retriable * {@code BUSY} response under write-stall. * - *

Candy CRUD and list, plus createBox/deleteBox, are fully wired. listBoxes/headBox and large-object - * streaming are TODO(phase-2). + *

Every keyed request is dispatched to the engine of the key's hash partition; partition-scoped + * requests (list, delete-range, list-uploads — fanned out by the client) carry an explicit partition. + * A request landing on a node that does not own the target partition gets a {@code MOVED} response + * naming the partition's current owner. */ final class NodeRequestHandler implements RequestHandler { @@ -75,8 +78,8 @@ public Frame handle(Frame request) { } catch (CandyNotFoundException e) { return codec.encode(new Message.NotFoundResponse()); } catch (NotOwnerException | BoxNotFoundException e) { - // We don't own this Box. If another node does, tell the client to re-route; else not found. - return codec.encode(movedOrNotFound(message)); + // We don't own this partition. If another node does, tell the client to re-route. + return codec.encode(movedOrNotFound(message, e)); } catch (CandyboxException e) { return codec.encode(new Message.ErrorResponse(e.getClass().getSimpleName(), safe(e.getMessage()))); } catch (RuntimeException e) { @@ -85,18 +88,24 @@ public Frame handle(Frame request) { } } - private Message movedOrNotFound(Message message) { + private Message movedOrNotFound(Message message, RuntimeException cause) { + Integer partition = partitionOf(message); String box = boxOf(message); - if (box != null) { - Optional owner = node.currentOwner(BoxName.of(box)); + if (box != null && partition != null) { + Optional owner = node.currentOwner(BoxName.of(box), partition); if (owner.isPresent() && owner.get() != node.nodeId()) { return new Message.MovedResponse(owner.get()); } + return new Message.NotFoundResponse(); + } + if (cause instanceof NotOwnerException) { + // A Box-level request (e.g. deleteBox) that needs partitions another node still owns. + return new Message.ErrorResponse("NotOwnerException", safe(cause.getMessage())); } return new Message.NotFoundResponse(); } - /** The target Box of a Box-routed request, or {@code null} for cluster-wide requests. */ + /** The target Box of a Box- or partition-routed request, or {@code null} for cluster-wide ones. */ private static String boxOf(Message message) { if (message instanceof Message.PutCandyRequest m) { return m.box(); @@ -116,10 +125,6 @@ private static String boxOf(Message message) { return m.box(); } else if (message instanceof Message.ListCandiesRequest m) { return m.box(); - } else if (message instanceof Message.DeleteBoxRequest m) { - return m.box(); - } else if (message instanceof Message.HeadBoxRequest m) { - return m.box(); } else if (message instanceof Message.CreateMultipartUploadRequest m) { return m.box(); } else if (message instanceof Message.UploadPartRequest m) { @@ -138,26 +143,97 @@ private static String boxOf(Message message) { return null; } + /** + * The partition a request targets: the key's hash partition for keyed requests, the explicit + * partition for fanned-out ones, {@code null} for Box-level/cluster-wide requests (no single + * owner to redirect to). + */ + private Integer partitionOf(Message message) { + Integer explicit = explicitPartitionOf(message); + if (explicit != null) { + return explicit; + } + String box = boxOf(message); + String key = routingKeyOf(message); + if (box == null || key == null) { + return null; + } + try { + return node.descriptor(BoxName.of(box)).partitionOf(key); + } catch (BoxNotFoundException gone) { + return null; + } + } + + private static Integer explicitPartitionOf(Message message) { + if (message instanceof Message.ListCandiesRequest m) { + return m.partition(); + } else if (message instanceof Message.DeleteRangeRequest m) { + return m.partition(); + } else if (message instanceof Message.ListMultipartUploadsRequest m) { + return m.partition(); + } + return null; + } + + /** The key whose hash partition a keyed request routes by, or {@code null}. */ + private static String routingKeyOf(Message message) { + if (message instanceof Message.PutCandyRequest m) { + return m.key(); + } else if (message instanceof Message.GetCandyRequest m) { + return m.key(); + } else if (message instanceof Message.RangeGetCandyRequest m) { + return m.key(); + } else if (message instanceof Message.HeadCandyRequest m) { + return m.key(); + } else if (message instanceof Message.DeleteCandyRequest m) { + return m.key(); + } else if (message instanceof Message.CopyCandyRequest m) { + return m.dstKey(); + } else if (message instanceof Message.RenameCandyRequest m) { + return m.dstKey(); + } else if (message instanceof Message.CreateMultipartUploadRequest m) { + return m.key(); + } else if (message instanceof Message.UploadPartRequest m) { + return m.key(); + } else if (message instanceof Message.CompleteMultipartUploadRequest m) { + return m.key(); + } else if (message instanceof Message.AbortMultipartUploadRequest m) { + return m.key(); + } else if (message instanceof Message.ListPartsRequest m) { + return m.key(); + } else if (message instanceof Message.UploadPartCopyRequest m) { + return m.key(); + } + return null; + } + private Message dispatch(Message message) { if (message instanceof Message.CreateBoxRequest m) { - node.createBox(BoxName.of(m.box())); + node.createBox(BoxName.of(m.box()), m.partitionCount()); return new Message.OkResponse(); + } else if (message instanceof Message.BoxInfoRequest m) { + BoxName box = BoxName.of(m.box()); + if (!node.boxExists(box)) { + return new Message.NotFoundResponse(); + } + return new Message.BoxInfoResponse(node.descriptor(box).partitionCount()); } else if (message instanceof Message.DeleteBoxRequest m) { node.deleteBox(BoxName.of(m.box()), m.force()); return new Message.OkResponse(); } else if (message instanceof Message.PutCandyRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = node.engine(BoxName.of(m.box()), m.key()); engine.putCandy(CandyKey.of(m.key()), m.data(), m.contentType(), m.userMetadata(), m.idempotencyToken()); return new Message.OkResponse(); } else if (message instanceof Message.GetCandyRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = node.engine(BoxName.of(m.box()), m.key()); ByteArrayOutputStream out = new ByteArrayOutputStream(); CandyMetadata meta = engine.getCandy(CandyKey.of(m.key()), out); return new Message.CandyDataResponse(meta.contentLength(), meta.contentType(), meta.userMetadata(), meta.crc32c(), out.toByteArray()); } else if (message instanceof Message.RangeGetCandyRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = node.engine(BoxName.of(m.box()), m.key()); ByteArrayOutputStream out = new ByteArrayOutputStream(); BoxEngine.RangeReadResult result; try { @@ -170,25 +246,27 @@ private Message dispatch(Message message) { return new Message.CandyDataResponse(result.contentLength(), result.totalLength(), meta.contentType(), meta.userMetadata(), meta.crc32c(), out.toByteArray()); } else if (message instanceof Message.HeadCandyRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = node.engine(BoxName.of(m.box()), m.key()); CandyMetadata meta = engine.headCandy(CandyKey.of(m.key())); return new Message.HeadCandyResponse(meta.contentLength(), meta.contentType(), meta.userMetadata(), meta.crc32c(), meta.createdAtMillis()); } else if (message instanceof Message.DeleteCandyRequest m) { - node.engine(BoxName.of(m.box())).deleteCandy(CandyKey.of(m.key())); + node.engine(BoxName.of(m.box()), m.key()).deleteCandy(CandyKey.of(m.key())); return new Message.OkResponse(); } else if (message instanceof Message.CopyCandyRequest m) { - CandyMetadata meta = node.engine(BoxName.of(m.box())).copyCandy(CandyKey.of(m.srcKey()), - CandyKey.of(m.dstKey()), m.idempotencyToken()); + CandyMetadata meta = samePartitionEngine(m.box(), m.srcKey(), m.dstKey()) + .copyCandy(CandyKey.of(m.srcKey()), CandyKey.of(m.dstKey()), + m.idempotencyToken()); return new Message.HeadCandyResponse(meta.contentLength(), meta.contentType(), meta.userMetadata(), meta.crc32c(), meta.createdAtMillis()); } else if (message instanceof Message.RenameCandyRequest m) { - CandyMetadata meta = node.engine(BoxName.of(m.box())).renameCandy(CandyKey.of(m.srcKey()), - CandyKey.of(m.dstKey()), m.idempotencyToken()); + CandyMetadata meta = samePartitionEngine(m.box(), m.srcKey(), m.dstKey()) + .renameCandy(CandyKey.of(m.srcKey()), CandyKey.of(m.dstKey()), + m.idempotencyToken()); return new Message.HeadCandyResponse(meta.contentLength(), meta.contentType(), meta.userMetadata(), meta.crc32c(), meta.createdAtMillis()); } else if (message instanceof Message.DeleteRangeRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = node.enginePartition(BoxName.of(m.box()), m.partition()); if (m.prefix() != null) { engine.deleteRangeByPrefix(m.prefix()); } else { @@ -198,7 +276,7 @@ private Message dispatch(Message message) { } return new Message.OkResponse(); } else if (message instanceof Message.ListCandiesRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = node.enginePartition(BoxName.of(m.box()), m.partition()); ListResult result = engine.scanCandies(toScanQuery(m)); List entries = new ArrayList<>(); for (ListResult.ListEntry e : result.entries()) { @@ -207,23 +285,22 @@ private Message dispatch(Message message) { } return new Message.ListCandiesResponse(entries, result.nextStartAfter()); } else if (message instanceof Message.ListBoxesRequest) { - // Boxes owned by this node. Cluster-wide listing needs a coordination list op (later). return new Message.ListBoxesResponse(node.listBoxes()); } else if (message instanceof Message.HeadBoxRequest m) { return node.boxExists(BoxName.of(m.box())) ? new Message.OkResponse() : new Message.NotFoundResponse(); } else if (message instanceof Message.CreateMultipartUploadRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = node.engine(BoxName.of(m.box()), m.key()); String uploadId = engine.createMultipartUpload(CandyKey.of(m.key()), m.contentType(), m.userMetadata()); return new Message.CreateMultipartUploadResponse(uploadId); } else if (message instanceof Message.UploadPartRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = node.engine(BoxName.of(m.box()), m.key()); BoxEngine.PartUploadResult r = engine.uploadPart(m.uploadId(), m.partNumber(), m.data()); return new Message.UploadPartResponse(r.crc32c(), r.partLength()); } else if (message instanceof Message.CompleteMultipartUploadRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = node.engine(BoxName.of(m.box()), m.key()); java.util.List parts = new java.util.ArrayList<>(m.parts().size()); for (Message.CompletedPart p : m.parts()) { parts.add(new BoxEngine.PartCompletion(p.partNumber(), p.crc32c())); @@ -232,10 +309,10 @@ private Message dispatch(Message message) { return new Message.HeadCandyResponse(meta.contentLength(), meta.contentType(), meta.userMetadata(), meta.crc32c(), meta.createdAtMillis()); } else if (message instanceof Message.AbortMultipartUploadRequest m) { - node.engine(BoxName.of(m.box())).abortMultipartUpload(m.uploadId()); + node.engine(BoxName.of(m.box()), m.key()).abortMultipartUpload(m.uploadId()); return new Message.OkResponse(); } else if (message instanceof Message.ListMultipartUploadsRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = node.enginePartition(BoxName.of(m.box()), m.partition()); java.util.List rows = new java.util.ArrayList<>(); String prefix = m.prefix() == null ? "" : m.prefix(); int limit = m.maxUploads() <= 0 ? 1000 : m.maxUploads(); @@ -262,7 +339,7 @@ private Message dispatch(Message message) { } return new Message.ListMultipartUploadsResponse(rows, nextKey, nextUpl); } else if (message instanceof Message.ListPartsRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = node.engine(BoxName.of(m.box()), m.key()); MultipartUploadState upload = engine.multipartUpload(m.uploadId()); if (upload == null) { return new Message.NotFoundResponse(); @@ -285,7 +362,7 @@ private Message dispatch(Message message) { } return new Message.ListPartsResponse(rows, next); } else if (message instanceof Message.UploadPartCopyRequest m) { - BoxEngine engine = node.engine(BoxName.of(m.box())); + BoxEngine engine = samePartitionEngine(m.box(), m.srcKey(), m.key()); try { BoxEngine.PartUploadResult r = engine.uploadPartCopy(m.uploadId(), m.partNumber(), CandyKey.of(m.srcKey()), m.firstByte(), m.lastByte()); @@ -298,6 +375,23 @@ private Message dispatch(Message message) { "Not implemented in this phase: " + message.opcode()); } + /** + * The engine shared by two keys of the same Box — the zero-copy copy/rename/upload-part-copy + * path is only valid when both keys hash to the same partition (one manifest tracks the shared + * Syrup segments); the client falls back to a byte copy otherwise. + */ + private BoxEngine samePartitionEngine(String box, String srcKey, String dstKey) { + BoxDescriptor descriptor = node.descriptor(BoxName.of(box)); + int srcPartition = descriptor.partitionOf(srcKey); + int dstPartition = descriptor.partitionOf(dstKey); + if (srcPartition != dstPartition) { + throw new ValidationException("Cross-partition server-side copy is not supported " + + "(src partition " + srcPartition + ", dst partition " + dstPartition + + "); the client must byte-copy"); + } + return node.enginePartition(BoxName.of(box), dstPartition); + } + private static ScanQuery toScanQuery(Message.ListCandiesRequest m) { CandyKey start = m.startKey() == null ? null : CandyKey.of(m.startKey()); CandyKey end = m.endKey() == null ? null : CandyKey.of(m.endKey()); diff --git a/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionAssignment.java b/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionAssignment.java new file mode 100644 index 0000000..a8cfb13 --- /dev/null +++ b/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionAssignment.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package me.predatorray.candybox.server; + +import java.util.LinkedHashMap; +import java.util.Map; +import me.predatorray.candybox.common.serial.BinaryReader; +import me.predatorray.candybox.common.serial.BinaryWriter; + +/** + * The desired (box, partition) → node assignment table the elected balancer publishes at + * {@code cluster/assignment}. It is advisory: nodes acquire/release ownership to converge on it, but + * correctness always rests on the per-partition fenced lease, never on this table. + */ +final class PartitionAssignment { + + private static final int FORMAT_VERSION = 1; + + /** One partition of one Box. */ + record BoxPartition(String box, int partition) implements Comparable { + @Override + public int compareTo(BoxPartition o) { + int c = box.compareTo(o.box); + return c != 0 ? c : Integer.compare(partition, o.partition); + } + } + + private final Map targets; + + PartitionAssignment(Map targets) { + this.targets = new LinkedHashMap<>(targets); + } + + static PartitionAssignment empty() { + return new PartitionAssignment(Map.of()); + } + + Map targets() { + return java.util.Collections.unmodifiableMap(targets); + } + + /** The node assigned to a partition, or {@code null} if the table does not mention it. */ + Integer targetNode(String box, int partition) { + return targets.get(new BoxPartition(box, partition)); + } + + byte[] encode() { + BinaryWriter w = new BinaryWriter(64); + w.writeByte(FORMAT_VERSION); + w.writeVarInt(targets.size()); + for (Map.Entry e : targets.entrySet()) { + w.writeString(e.getKey().box()); + w.writeVarInt(e.getKey().partition()); + w.writeInt(e.getValue()); + } + return w.toByteArray(); + } + + static PartitionAssignment decode(byte[] data) { + BinaryReader r = new BinaryReader(data); + int version = r.readByte(); + if (version != FORMAT_VERSION) { + throw new IllegalArgumentException("Unsupported PartitionAssignment version: " + version); + } + int count = r.readVarInt(); + Map targets = new LinkedHashMap<>(Math.max(4, count * 2)); + for (int i = 0; i < count; i++) { + targets.put(new BoxPartition(r.readString(), r.readVarInt()), r.readInt()); + } + return new PartitionAssignment(targets); + } +} diff --git a/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionBalancer.java b/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionBalancer.java new file mode 100644 index 0000000..e81b286 --- /dev/null +++ b/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionBalancer.java @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package me.predatorray.candybox.server; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import me.predatorray.candybox.common.BoxName; +import me.predatorray.candybox.common.config.CandyboxConfig; +import me.predatorray.candybox.coordination.BoxDescriptor; +import me.predatorray.candybox.coordination.CandyboxKeys; +import me.predatorray.candybox.coordination.CasConflictException; +import me.predatorray.candybox.coordination.CoordinationService; +import me.predatorray.candybox.coordination.Lease; +import me.predatorray.candybox.coordination.LeaseInfo; +import me.predatorray.candybox.coordination.VersionedValue; +import me.predatorray.candybox.server.PartitionAssignment.BoxPartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Spreads partition ownership evenly across the cluster. Every node runs a balancing round on a + * timer; whichever node holds the {@code cluster/balancer} lease acts as the elected + * coordinator and publishes the desired assignment table, and every node (coordinator or + * not) then converges on it: releasing partitions assigned elsewhere (with a pre-handover flush) and + * acquiring partitions assigned to it once their lease is free. + * + *

The target computation is sticky and rate-limited: + *

    + *
  • a live owner keeps its partitions up to the fair-share capacity + * (⌈partitions/members⌉) — no gratuitous shuffling;
    • + *
  • unowned partitions (a new Box, a dead node's partitions) are failover and are assigned to + * the least-loaded members without limit;
    • + *
  • at most {@link CandyboxConfig#balancerMaxMovesPerRound()} partitions are taken away from a + * live owner per round, so a node join migrates load gradually.
    • + *
+ * + *

The table is advisory; safety always rests on the per-partition fenced lease. A move converges + * over polling rounds: the old owner releases in one round, the new owner acquires once it observes + * the lease free. + */ +final class PartitionBalancer { + + private static final Logger LOG = LoggerFactory.getLogger(PartitionBalancer.class); + + private final CandyboxNode node; + private final CoordinationService coordination; + private final CandyboxConfig config; + + PartitionBalancer(CandyboxNode node, CoordinationService coordination, CandyboxConfig config) { + this.node = node; + this.coordination = coordination; + this.config = config; + } + + /** One balancing round: coordinate (if elected) then converge on the published assignment. */ + void runOnce() { + try { + node.sweepDeletedBoxes(); + } catch (RuntimeException e) { + LOG.warn("Deleted-box sweep failed on node {}", node.nodeId(), e); + } + try { + coordinateIfElected(); + } catch (RuntimeException e) { + LOG.warn("Balancer coordination round failed on node {}", node.nodeId(), e); + } + try { + applyAssignment(); + } catch (RuntimeException e) { + LOG.warn("Balancer apply round failed on node {}", node.nodeId(), e); + } + } + + // ---- coordinator side -------------------------------------------------------------------- + + private void coordinateIfElected() { + Optional lease = coordination.tryAcquireLease(CandyboxKeys.BALANCER_RESOURCE, + node.nodeId(), config.ownershipLeaseTtlMillis()); + if (lease.isEmpty()) { + return; // another node coordinates + } + List members = coordination.members(); + List partitions = allPartitions(); + if (members.isEmpty() || partitions.isEmpty()) { + return; + } + PartitionAssignment current = readAssignment().map(v -> PartitionAssignment.decode(v.value())) + .orElse(PartitionAssignment.empty()); + PartitionAssignment target = computeTarget(partitions, members); + if (!target.targets().equals(current.targets())) { + publish(target); + } + } + + /** Every partition of every existing Box (descriptor present), in deterministic order. */ + private List allPartitions() { + List all = new ArrayList<>(); + for (String boxName : coordination.children(CandyboxKeys.BOXES_ROOT)) { + // A deleted Box can leave lease znodes behind; only the descriptor makes it real. + Optional meta = coordination.get(CandyboxKeys.boxMetaKey(boxName)); + if (meta.isEmpty()) { + continue; + } + int count = BoxDescriptor.decode(meta.get().value()).partitionCount(); + for (int p = 0; p < count; p++) { + all.add(new BoxPartition(boxName, p)); + } + } + all.sort(BoxPartition::compareTo); + return all; + } + + private PartitionAssignment computeTarget(List partitions, List members) { + int capacity = (partitions.size() + members.size() - 1) / members.size(); + Map load = new TreeMap<>(); + for (int member : members) { + load.put(member, 0); + } + + Map targets = new LinkedHashMap<>(); + List unowned = new ArrayList<>(); + Map overflow = new LinkedHashMap<>(); // candidate moves: bp -> holder + + for (BoxPartition bp : partitions) { + Integer holder = liveHolder(bp); + if (holder == null || !load.containsKey(holder)) { + unowned.add(bp); // failover / brand new: not a "move", never rate-limited + } else if (load.get(holder) < capacity) { + targets.put(bp, holder); // sticky: keep the live owner within its fair share + load.merge(holder, 1, Integer::sum); + } else { + overflow.put(bp, holder); // live owner above capacity: moving this away counts + } + } + + for (BoxPartition bp : unowned) { + int member = leastLoaded(load); + targets.put(bp, member); + load.merge(member, 1, Integer::sum); + } + + int movesLeft = config.balancerMaxMovesPerRound(); + for (Map.Entry e : overflow.entrySet()) { + if (movesLeft > 0) { + int member = leastLoaded(load); + targets.put(e.getKey(), member); + load.merge(member, 1, Integer::sum); + movesLeft--; + } else { + // Rate limit reached: the live owner keeps it this round, even above capacity. + targets.put(e.getKey(), e.getValue()); + load.merge(e.getValue(), 1, Integer::sum); + } + } + return new PartitionAssignment(targets); + } + + private Integer liveHolder(BoxPartition bp) { + return coordination.leaseHolder(CandyboxKeys.ownerResource(bp.box(), bp.partition())) + .map(LeaseInfo::ownerNodeId) + .orElse(null); + } + + private static int leastLoaded(Map load) { + int best = -1; + int bestLoad = Integer.MAX_VALUE; + for (Map.Entry e : load.entrySet()) { + if (e.getValue() < bestLoad) { + best = e.getKey(); + bestLoad = e.getValue(); + } + } + return best; + } + + private Optional readAssignment() { + return coordination.get(CandyboxKeys.ASSIGNMENT_KEY); + } + + private void publish(PartitionAssignment assignment) { + try { + Optional current = readAssignment(); + if (current.isEmpty()) { + coordination.create(CandyboxKeys.ASSIGNMENT_KEY, assignment.encode()); + } else { + coordination.compareAndSet(CandyboxKeys.ASSIGNMENT_KEY, assignment.encode(), + current.get().version()); + } + } catch (CasConflictException raced) { + LOG.debug("Assignment publish lost a race; will retry next round"); + } + } + + // ---- every-node side --------------------------------------------------------------------- + + private void applyAssignment() { + Optional stored = readAssignment(); + if (stored.isEmpty()) { + return; + } + PartitionAssignment assignment = PartitionAssignment.decode(stored.get().value()); + List members = coordination.members(); + + // Release first, so our partitions free up for their new owners within this round. + for (Map.Entry e : assignment.targets().entrySet()) { + BoxPartition bp = e.getKey(); + int target = e.getValue(); + if (target != node.nodeId() && node.ownsPartition(bp.box(), bp.partition()) + && members.contains(target)) { + LOG.info("Node {} handing over box {} partition {} to node {}", node.nodeId(), + bp.box(), bp.partition(), target); + node.releasePartitionForHandover(BoxName.of(bp.box()), bp.partition()); + } + } + for (Map.Entry e : assignment.targets().entrySet()) { + BoxPartition bp = e.getKey(); + if (e.getValue() != node.nodeId() || node.ownsPartition(bp.box(), bp.partition())) { + continue; + } + if (coordination.leaseHolder(CandyboxKeys.ownerResource(bp.box(), bp.partition())) + .isPresent()) { + continue; // previous owner has not released/expired yet; retry next round + } + try { + node.openPartition(BoxName.of(bp.box()), bp.partition()); + } catch (RuntimeException ex) { + // Lost an acquire race, or the Box vanished concurrently; converge next round. + LOG.info("Node {} could not take box {} partition {} yet: {}", node.nodeId(), + bp.box(), bp.partition(), ex.getMessage()); + } + } + } +} diff --git a/candybox-server/src/main/java/me/predatorray/candybox/server/BoxOwnership.java b/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionOwnership.java similarity index 57% rename from candybox-server/src/main/java/me/predatorray/candybox/server/BoxOwnership.java rename to candybox-server/src/main/java/me/predatorray/candybox/server/PartitionOwnership.java index 0536b72..aa7ca47 100644 --- a/candybox-server/src/main/java/me/predatorray/candybox/server/BoxOwnership.java +++ b/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionOwnership.java @@ -33,13 +33,17 @@ import org.slf4j.LoggerFactory; /** - * One node's fenced ownership of one Box: the ZK lease plus the {@link BoxEngine} it guards, tied - * together by the per-Box manifest pointer in coordination. + * One node's fenced ownership of one partition of a Box: the ZK lease plus the + * {@link BoxEngine} it guards, tied together by the per-partition manifest pointer in coordination. + * Each partition is an independent LSM engine, so the partitions of one Box can be owned and served + * by different nodes concurrently. * - *

Acquisition is the Phase 2 handover sequence: + *

Acquisition is the handover sequence, unchanged from the per-Box days but at partition + * granularity: *

    - *
  1. acquire the {@code boxes/<box>/owner} lease (the single serialization point);
    2. - *
  2. read the {@code boxes/<box>/manifest} pointer;
    3. + *
  3. acquire the {@code boxes/<box>/partitions/<p>/owner} lease (the single + * serialization point for this partition);
    4. + *
  4. read the {@code boxes/<box>/partitions/<p>/manifest} pointer;
    5. *
  5. create a brand-new engine (pointer absent) or recover the prior one (pointer present), stamping * every manifest edit with the lease's fencing token;
    6. *
  6. publish the new manifest ledger id by creating / compare-and-setting the pointer — aborting and @@ -50,89 +54,102 @@ *

    The lease is the real serialization point; the pointer CAS is a backstop. Engine access is gated * on the lease still being valid, so a lost/expired owner stops serving. */ -final class BoxOwnership implements AutoCloseable { +final class PartitionOwnership implements AutoCloseable { - private static final Logger LOG = LoggerFactory.getLogger(BoxOwnership.class); + private static final Logger LOG = LoggerFactory.getLogger(PartitionOwnership.class); private final BoxName box; + private final int partition; private final CoordinationService coordination; private final Lease lease; private final BoxEngine engine; - private BoxOwnership(BoxName box, CoordinationService coordination, Lease lease, BoxEngine engine) { + private PartitionOwnership(BoxName box, int partition, CoordinationService coordination, + Lease lease, BoxEngine engine) { this.box = box; + this.partition = partition; this.coordination = coordination; this.lease = lease; this.engine = engine; } - static String ownerResource(BoxName box) { - return CandyboxKeys.ownerResource(box.value()); + static String ownerResource(BoxName box, int partition) { + return CandyboxKeys.ownerResource(box.value(), partition); } - static String manifestKey(BoxName box) { - return CandyboxKeys.manifestKey(box.value()); + static String manifestKey(BoxName box, int partition) { + return CandyboxKeys.manifestKey(box.value(), partition); } - /** Acquires ownership of a brand-new Box (the manifest pointer must not already exist). */ - static BoxOwnership createNew(BoxName box, CandyboxConfig config, LedgerStore store, - CoordinationService coordination, int nodeId, Clock clock) { - Lease lease = acquireLease(box, coordination, nodeId, config); + /** Acquires ownership of a brand-new partition (the manifest pointer must not already exist). */ + static PartitionOwnership createNew(BoxName box, int partition, CandyboxConfig config, + LedgerStore store, CoordinationService coordination, + int nodeId, Clock clock) { + Lease lease = acquireLease(box, partition, coordination, nodeId, config); try { - if (coordination.get(manifestKey(box)).isPresent()) { + if (coordination.get(manifestKey(box, partition)).isPresent()) { throw new BoxAlreadyExistsException(box.value()); } BoxEngine engine = BoxEngine.createNew(box, config, store, nodeId, clock, lease.fencingToken()); try { - coordination.create(manifestKey(box), + coordination.create(manifestKey(box, partition), new ManifestPointer(engine.manifestLedgerId(), lease.fencingToken()).encode()); } catch (CasConflictException raced) { engine.close(); // a concurrent creator won; abandon our ledgers (GC: phase-3) throw new BoxAlreadyExistsException(box.value()); } - return new BoxOwnership(box, coordination, lease, engine); + return new PartitionOwnership(box, partition, coordination, lease, engine); } catch (RuntimeException e) { lease.release(); throw e; } } - /** Acquires ownership of an existing Box by recovering its manifest (the pointer must exist). */ - static BoxOwnership recover(BoxName box, CandyboxConfig config, LedgerStore store, - CoordinationService coordination, int nodeId, Clock clock) { - Lease lease = acquireLease(box, coordination, nodeId, config); + /** Acquires ownership of an existing partition by recovering its manifest (pointer must exist). */ + static PartitionOwnership recover(BoxName box, int partition, CandyboxConfig config, + LedgerStore store, CoordinationService coordination, + int nodeId, Clock clock) { + Lease lease = acquireLease(box, partition, coordination, nodeId, config); try { - VersionedValue pointer = coordination.get(manifestKey(box)) + VersionedValue pointer = coordination.get(manifestKey(box, partition)) .orElseThrow(() -> new BoxNotFoundException(box.value())); long priorManifestLedgerId = ManifestPointer.decode(pointer.value()).ledgerId(); BoxEngine engine = BoxEngine.recover(box, config, store, nodeId, clock, priorManifestLedgerId, lease.fencingToken()); try { - coordination.compareAndSet(manifestKey(box), + coordination.compareAndSet(manifestKey(box, partition), new ManifestPointer(engine.manifestLedgerId(), lease.fencingToken()).encode(), pointer.version()); } catch (CasConflictException raced) { engine.close(); throw new NotOwnerException(box.value()); // someone advanced the pointer; we lost } - LOG.info("Node {} recovered ownership of box {} (token {})", nodeId, box, - lease.fencingToken()); - return new BoxOwnership(box, coordination, lease, engine); + LOG.info("Node {} recovered ownership of box {} partition {} (token {})", nodeId, box, + partition, lease.fencingToken()); + return new PartitionOwnership(box, partition, coordination, lease, engine); } catch (RuntimeException e) { lease.release(); throw e; } } - private static Lease acquireLease(BoxName box, CoordinationService coordination, int nodeId, - CandyboxConfig config) { - Optional lease = coordination.tryAcquireLease(ownerResource(box), nodeId, + private static Lease acquireLease(BoxName box, int partition, CoordinationService coordination, + int nodeId, CandyboxConfig config) { + Optional lease = coordination.tryAcquireLease(ownerResource(box, partition), nodeId, config.ownershipLeaseTtlMillis()); return lease.orElseThrow(() -> new NotOwnerException(box.value())); } + BoxName box() { + return box; + } + + int partition() { + return partition; + } + /** The engine, if this node still holds the lease; otherwise throws {@link NotOwnerException}. */ BoxEngine engine() { if (!lease.isValid()) { @@ -155,14 +172,30 @@ boolean renew() { lease.renew(); return true; } catch (RuntimeException e) { - LOG.warn("Lost lease on box {}: {}", box, e.getMessage()); + LOG.warn("Lost lease on box {} partition {}: {}", box, partition, e.getMessage()); return false; } } /** The coordination key + version are needed to delete the pointer when the Box is dropped. */ Optional currentPointer() { - return coordination.get(manifestKey(box)); + return coordination.get(manifestKey(box, partition)); + } + + /** + * Closes for a planned handover: flushes the memtable first (best effort) so the next owner's WAL + * replay window is small, then releases the lease. + */ + void closeForHandover() { + try { + if (lease.isValid()) { + engine.flush(); + } + } catch (RuntimeException e) { + LOG.warn("Pre-handover flush failed for box {} partition {}: {}", box, partition, + e.getMessage()); + } + close(); } @Override diff --git a/candybox-server/src/main/java/me/predatorray/candybox/server/ServerConfig.java b/candybox-server/src/main/java/me/predatorray/candybox/server/ServerConfig.java index 7753e43..d2c5b17 100644 --- a/candybox-server/src/main/java/me/predatorray/candybox/server/ServerConfig.java +++ b/candybox-server/src/main/java/me/predatorray/candybox/server/ServerConfig.java @@ -264,6 +264,9 @@ CandyboxConfig buildTuning() { applyLong("multipart.upload.ttl.millis", b::multipartUploadTtlMillis); applyInt("multipart.max.concurrent.uploads.per.box", b::multipartMaxConcurrentUploadsPerBox); + applyInt("partitions.per.box.default", b::partitionsPerBoxDefault); + applyLong("balancer.interval.millis", b::balancerIntervalMillis); + applyInt("balancer.max.moves.per.round", b::balancerMaxMovesPerRound); // Per-role BookKeeper quorum overrides, "E/Qw/Qa" (e.g. 1/1/1 for a single-bookie dev box). applyQuorum("quorum.wal", LedgerRole.WAL, b); applyQuorum("quorum.manifest", LedgerRole.MANIFEST, b); diff --git a/candybox-server/src/test/java/me/predatorray/candybox/server/BoxHandoverTest.java b/candybox-server/src/test/java/me/predatorray/candybox/server/BoxHandoverTest.java index 86f8754..a5b35a8 100644 --- a/candybox-server/src/test/java/me/predatorray/candybox/server/BoxHandoverTest.java +++ b/candybox-server/src/test/java/me/predatorray/candybox/server/BoxHandoverTest.java @@ -56,9 +56,9 @@ void secondNodeTakesOverAfterLeaseExpiresAndOldOwnerIsFenced() { CandyboxNode nodeA = new CandyboxNode(1, config, store, coordination, clock); CandyboxNode nodeB = new CandyboxNode(2, config, store, coordination, clock); try { - nodeA.createBox(box); + nodeA.createBox(box, 1); // Unflushed write: lives only in A's WAL, exercising WAL recovery on handover. - nodeA.engine(box).putCandy(CandyKey.of("k"), bytes("v1"), null, Map.of(), null); + nodeA.enginePartition(box, 0).putCandy(CandyKey.of("k"), bytes("v1"), null, Map.of(), null); // While A holds a valid lease, B cannot take over. assertThatThrownBy(() -> nodeB.openBox(box)).isInstanceOf(NotOwnerException.class); @@ -67,15 +67,15 @@ void secondNodeTakesOverAfterLeaseExpiresAndOldOwnerIsFenced() { clock.advance(11_000); nodeB.openBox(box); - assertThat(nodeB.engine(box).getCandy(CandyKey.of("k"))).isEqualTo(bytes("v1")); + assertThat(nodeB.enginePartition(box, 0).getCandy(CandyKey.of("k"))).isEqualTo(bytes("v1")); // The old owner is fenced: it no longer owns the Box. - assertThatThrownBy(() -> nodeA.engine(box)).isInstanceOf(NotOwnerException.class); + assertThatThrownBy(() -> nodeA.enginePartition(box, 0)).isInstanceOf(NotOwnerException.class); // The manifest pointer now names B's manifest ledger. ManifestPointer pointer = ManifestPointer.decode( - coordination.get(BoxOwnership.manifestKey(box)).orElseThrow().value()); - assertThat(pointer.ledgerId()).isEqualTo(nodeB.engine(box).manifestLedgerId()); + coordination.get(PartitionOwnership.manifestKey(box, 0)).orElseThrow().value()); + assertThat(pointer.ledgerId()).isEqualTo(nodeB.enginePartition(box, 0).manifestLedgerId()); assertThat(pointer.ownerToken()).isGreaterThan(1L); } finally { nodeA.close(); @@ -95,13 +95,13 @@ void newWriterAfterHandoverWinsLwwDespiteAOlderWrite() { CandyboxNode nodeA = new CandyboxNode(1, config, store, coordination, clock); CandyboxNode nodeB = new CandyboxNode(2, config, store, coordination, clock); try { - nodeA.createBox(box); - nodeA.engine(box).putCandy(CandyKey.of("k"), bytes("v1"), null, Map.of(), null); + nodeA.createBox(box, 1); + nodeA.enginePartition(box, 0).putCandy(CandyKey.of("k"), bytes("v1"), null, Map.of(), null); clock.advance(11_000); nodeB.openBox(box); - nodeB.engine(box).putCandy(CandyKey.of("k"), bytes("v2"), null, Map.of(), null); - assertThat(nodeB.engine(box).getCandy(CandyKey.of("k"))).isEqualTo(bytes("v2")); + nodeB.enginePartition(box, 0).putCandy(CandyKey.of("k"), bytes("v2"), null, Map.of(), null); + assertThat(nodeB.enginePartition(box, 0).getCandy(CandyKey.of("k"))).isEqualTo(bytes("v2")); } finally { nodeA.close(); nodeB.close(); @@ -121,7 +121,7 @@ void nonOwnerHandlerReturnsMovedToTheOwner() { CandyboxNode owner = new CandyboxNode(1, config, store, coordination, clock); CandyboxNode other = new CandyboxNode(2, config, store, coordination, clock); try { - owner.createBox(box); + owner.createBox(box, 1); // A request for the Box that lands on the non-owner is redirected to the owner (node 1). Message moved = codec.decode(other.requestHandler() diff --git a/candybox-server/src/test/java/me/predatorray/candybox/server/CandyboxNodeTest.java b/candybox-server/src/test/java/me/predatorray/candybox/server/CandyboxNodeTest.java index bf4bd56..37aa0cd 100644 --- a/candybox-server/src/test/java/me/predatorray/candybox/server/CandyboxNodeTest.java +++ b/candybox-server/src/test/java/me/predatorray/candybox/server/CandyboxNodeTest.java @@ -44,7 +44,7 @@ void handlesPutGetDeleteThroughTheProtocol() { InMemoryLedgerStore store = new InMemoryLedgerStore(); try (CandyboxNode node = new CandyboxNode(1, CandyboxConfig.defaults(), store, new InMemoryCoordinationService(), new ManualClock(1000))) { - node.createBox(BoxName.of("my-box")); + node.createBox(BoxName.of("my-box"), 1); RequestHandler handler = node.requestHandler(); assertThat(roundTrip(handler, new Message.PutCandyRequest("my-box", "k", @@ -73,7 +73,7 @@ void multipartTtlSweeperAbortsStaleUploadsOlderThanTtl() { .build(); try (CandyboxNode node = new CandyboxNode(1, cfg, store, new InMemoryCoordinationService(), clock)) { - node.createBox(BoxName.of("my-box")); + node.createBox(BoxName.of("my-box"), 1); RequestHandler handler = node.requestHandler(); Message create = roundTrip(handler, new Message.CreateMultipartUploadRequest("my-box", @@ -103,7 +103,7 @@ void handlesHeadCandyListBoxesAndHeadBox() { InMemoryLedgerStore store = new InMemoryLedgerStore(); try (CandyboxNode node = new CandyboxNode(1, CandyboxConfig.defaults(), store, new InMemoryCoordinationService(), new ManualClock(1000))) { - node.createBox(BoxName.of("my-box")); + node.createBox(BoxName.of("my-box"), 1); RequestHandler handler = node.requestHandler(); roundTrip(handler, new Message.PutCandyRequest("my-box", "k", "text/plain", Map.of("m", "v"), null, "candy".getBytes(StandardCharsets.UTF_8))); @@ -136,7 +136,7 @@ void busyResponseSurfacedUnderWriteStall() { InMemoryLedgerStore store = new InMemoryLedgerStore(); try (CandyboxNode node = new CandyboxNode(1, stall, store, new InMemoryCoordinationService(), new ManualClock(1000))) { - node.createBox(BoxName.of("my-box")); + node.createBox(BoxName.of("my-box"), 1); RequestHandler handler = node.requestHandler(); roundTrip(handler, put("my-box", "k1")); @@ -156,12 +156,12 @@ void compactionServiceMergesL0Tables() { InMemoryLedgerStore store = new InMemoryLedgerStore(); ManualClock clock = new ManualClock(1000); try (CandyboxNode node = new CandyboxNode(1, cfg, store, new InMemoryCoordinationService(), clock)) { - node.createBox(BoxName.of("my-box")); + node.createBox(BoxName.of("my-box"), 1); RequestHandler handler = node.requestHandler(); for (int i = 0; i < 4; i++) { roundTrip(handler, put("my-box", "key-" + i)); } - var engine = node.engine(BoxName.of("my-box")); + var engine = node.enginePartition(BoxName.of("my-box"), 0); assertThat(engine.manifestState().level0().size()).isGreaterThanOrEqualTo(3); CompactionService compaction = new CompactionService(store, cfg, clock); @@ -188,17 +188,17 @@ void backgroundCompactionWorkerMergesOwnedBoxes() { InMemoryLedgerStore store = new InMemoryLedgerStore(); try (CandyboxNode node = new CandyboxNode(1, cfg, store, new InMemoryCoordinationService(), new ManualClock(1000))) { - node.createBox(BoxName.of("my-box")); + node.createBox(BoxName.of("my-box"), 1); RequestHandler handler = node.requestHandler(); for (int i = 0; i < 5; i++) { roundTrip(handler, put("my-box", "key-" + i)); } - assertThat(node.engine(BoxName.of("my-box")).manifestState().level0().size()) + assertThat(node.enginePartition(BoxName.of("my-box"), 0).manifestState().level0().size()) .isGreaterThanOrEqualTo(3); int performed = node.compactOwnedBoxesOnce(); assertThat(performed).isGreaterThanOrEqualTo(1); - assertThat(node.engine(BoxName.of("my-box")).manifestState().level0()).isEmpty(); + assertThat(node.enginePartition(BoxName.of("my-box"), 0).manifestState().level0()).isEmpty(); // Data survives the background compaction. for (int i = 0; i < 5; i++) { Message resp = roundTrip(handler, new Message.GetCandyRequest("my-box", "key-" + i)); @@ -219,14 +219,14 @@ void gcDeletesLedgersOfCompactedInputs() { InMemoryLedgerStore store = new InMemoryLedgerStore(); try (CandyboxNode node = new CandyboxNode(1, cfg, store, new InMemoryCoordinationService(), new ManualClock(1000))) { - node.createBox(BoxName.of("my-box")); + node.createBox(BoxName.of("my-box"), 1); RequestHandler handler = node.requestHandler(); for (int i = 0; i < 5; i++) { roundTrip(handler, put("my-box", "key-" + i)); } java.util.Set inputLedgerIds = new java.util.HashSet<>(); - for (var table : node.engine(BoxName.of("my-box")).manifestState().level0()) { + for (var table : node.enginePartition(BoxName.of("my-box"), 0).manifestState().level0()) { inputLedgerIds.add(table.ledgerId()); } assertThat(inputLedgerIds.size()).isGreaterThanOrEqualTo(3); @@ -234,7 +234,7 @@ void gcDeletesLedgersOfCompactedInputs() { // Compaction merges L0 into L1 (inputs leave the committed manifest)... node.compactOwnedBoxesOnce(); - assertThat(node.engine(BoxName.of("my-box")).manifestState().level0()).isEmpty(); + assertThat(node.enginePartition(BoxName.of("my-box"), 0).manifestState().level0()).isEmpty(); assertThat(store.listLedgers()).containsAll(inputLedgerIds); // not deleted yet // ...then GC physically deletes the obsolete input ledgers (plus rotated WALs). @@ -263,7 +263,7 @@ void gcReclaimsOrphanedSyrupsAfterOverwrite() { InMemoryLedgerStore store = new InMemoryLedgerStore(); try (CandyboxNode node = new CandyboxNode(1, cfg, store, new InMemoryCoordinationService(), new ManualClock(1000))) { - node.createBox(BoxName.of("my-box")); + node.createBox(BoxName.of("my-box"), 1); RequestHandler handler = node.requestHandler(); // Two versions of the same key => two Syrups, two L0 tables. @@ -271,13 +271,13 @@ void gcReclaimsOrphanedSyrupsAfterOverwrite() { roundTrip(handler, putValue("my-box", "k", "v2")); java.util.Set liveBefore = - new java.util.HashSet<>(node.engine(BoxName.of("my-box")).manifestState().liveSyrups()); + new java.util.HashSet<>(node.enginePartition(BoxName.of("my-box"), 0).manifestState().liveSyrups()); assertThat(liveBefore).hasSize(2); // Compaction keeps only v2 (S2); the v1 Syrup (S1) becomes unreferenced. node.compactOwnedBoxesOnce(); java.util.Set referenced = - node.engine(BoxName.of("my-box")).manifestState().referencedSyrups(); + node.enginePartition(BoxName.of("my-box"), 0).manifestState().referencedSyrups(); assertThat(referenced).hasSize(1); java.util.Set orphan = new java.util.HashSet<>(liveBefore); orphan.removeAll(referenced); @@ -308,7 +308,7 @@ void gcDeletesRotatedWalLedgers() { InMemoryLedgerStore store = new InMemoryLedgerStore(); try (CandyboxNode node = new CandyboxNode(1, cfg, store, new InMemoryCoordinationService(), new ManualClock(1000))) { - node.createBox(BoxName.of("my-box")); + node.createBox(BoxName.of("my-box"), 1); RequestHandler handler = node.requestHandler(); roundTrip(handler, put("my-box", "a")); // flush => WAL_0 rotated out roundTrip(handler, put("my-box", "b")); // flush => WAL_1 rotated out diff --git a/candybox-server/src/test/java/me/predatorray/candybox/server/NodeRequestHandlerTest.java b/candybox-server/src/test/java/me/predatorray/candybox/server/NodeRequestHandlerTest.java index 1f14d41..713e622 100644 --- a/candybox-server/src/test/java/me/predatorray/candybox/server/NodeRequestHandlerTest.java +++ b/candybox-server/src/test/java/me/predatorray/candybox/server/NodeRequestHandlerTest.java @@ -108,10 +108,10 @@ void duplicateCreateBoxAndNonEmptyDeleteMapToErrorResponses() { try (CandyboxNode node = new CandyboxNode(1, config(), new InMemoryLedgerStore(), new InMemoryCoordinationService(), new ManualClock(1000))) { RequestHandler handler = node.requestHandler(); - assertThat(roundTrip(handler, new Message.CreateBoxRequest("dup-box"))) + assertThat(roundTrip(handler, new Message.CreateBoxRequest("dup-box", 1))) .isInstanceOf(Message.OkResponse.class); // Re-create is a typed CandyboxException -> ErrorResponse. - assertThat(roundTrip(handler, new Message.CreateBoxRequest("dup-box"))) + assertThat(roundTrip(handler, new Message.CreateBoxRequest("dup-box", 1))) .isInstanceOf(Message.ErrorResponse.class); roundTrip(handler, new Message.PutCandyRequest("dup-box", "k", null, Map.of(), null, @@ -130,7 +130,7 @@ void invalidRangeGetMapsToErrorResponse() { try (CandyboxNode node = new CandyboxNode(1, config(), new InMemoryLedgerStore(), new InMemoryCoordinationService(), new ManualClock(1000))) { RequestHandler handler = node.requestHandler(); - roundTrip(handler, new Message.CreateBoxRequest("range-box")); + roundTrip(handler, new Message.CreateBoxRequest("range-box", 1)); roundTrip(handler, new Message.PutCandyRequest("range-box", "k", null, Map.of(), null, bytes("0123456789"))); @@ -145,7 +145,7 @@ void listingAndMaintenanceCandyOperationsDispatch() { try (CandyboxNode node = new CandyboxNode(1, config(), new InMemoryLedgerStore(), new InMemoryCoordinationService(), new ManualClock(1000))) { RequestHandler handler = node.requestHandler(); - roundTrip(handler, new Message.CreateBoxRequest("ops-box")); + roundTrip(handler, new Message.CreateBoxRequest("ops-box", 1)); for (String k : List.of("a", "b", "c")) { roundTrip(handler, new Message.PutCandyRequest("ops-box", k, null, Map.of(), null, bytes(k))); @@ -156,7 +156,7 @@ void listingAndMaintenanceCandyOperationsDispatch() { assertThat(((Message.ListBoxesResponse) boxes).boxes()).contains("ops-box"); // Reverse, ranged listing exercises the toScanQuery direction/bounds mapping. - Message listed = roundTrip(handler, new Message.ListCandiesRequest("ops-box", null, null, + Message listed = roundTrip(handler, new Message.ListCandiesRequest("ops-box", 0, null, null, 100, "a", "c", true)); assertThat(listed).isInstanceOf(Message.ListCandiesResponse.class); @@ -167,7 +167,7 @@ void listingAndMaintenanceCandyOperationsDispatch() { .isInstanceOf(Message.HeadCandyResponse.class); // Range delete (window form) returns OK. - assertThat(roundTrip(handler, new Message.DeleteRangeRequest("ops-box", null, "a", "c"))) + assertThat(roundTrip(handler, new Message.DeleteRangeRequest("ops-box", 0, null, "a", "c"))) .isInstanceOf(Message.OkResponse.class); } } @@ -177,7 +177,7 @@ void multipartDispatchIncludingListings() { try (CandyboxNode node = new CandyboxNode(1, config(), new InMemoryLedgerStore(), new InMemoryCoordinationService(), new ManualClock(1000))) { RequestHandler handler = node.requestHandler(); - roundTrip(handler, new Message.CreateBoxRequest("mp-box")); + roundTrip(handler, new Message.CreateBoxRequest("mp-box", 1)); Message created = roundTrip(handler, new Message.CreateMultipartUploadRequest( "mp-box", "obj", "text/plain", Map.of())); @@ -196,7 +196,7 @@ void multipartDispatchIncludingListings() { assertThat(((Message.ListPartsResponse) parts).parts()).hasSize(2); Message uploads = roundTrip(handler, new Message.ListMultipartUploadsRequest( - "mp-box", null, null, null, 100)); + "mp-box", 0, null, null, null, 100)); assertThat(((Message.ListMultipartUploadsResponse) uploads).uploads()).hasSize(1); // Complete and read back the assembled object. diff --git a/candybox-server/src/test/java/me/predatorray/candybox/server/PartitionBalancerTest.java b/candybox-server/src/test/java/me/predatorray/candybox/server/PartitionBalancerTest.java new file mode 100644 index 0000000..ce3e919 --- /dev/null +++ b/candybox-server/src/test/java/me/predatorray/candybox/server/PartitionBalancerTest.java @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package me.predatorray.candybox.server; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.util.Map; +import me.predatorray.candybox.bookkeeper.fake.InMemoryLedgerStore; +import me.predatorray.candybox.common.BoxName; +import me.predatorray.candybox.common.CandyKey; +import me.predatorray.candybox.common.ManualClock; +import me.predatorray.candybox.common.Partitioning; +import me.predatorray.candybox.common.config.CandyboxConfig; +import me.predatorray.candybox.coordination.CandyboxKeys; +import me.predatorray.candybox.coordination.LeaseInfo; +import me.predatorray.candybox.coordination.fake.InMemoryCoordinationService; +import org.junit.jupiter.api.Test; + +/** + * Drives the {@link PartitionBalancer} deterministically (the scheduler is disabled; rounds run via + * {@link CandyboxNode#runBalancerOnce()}): elected-coordinator assignment, even spread, the + * per-round move rate limit, stickiness once balanced, dead-node failover, and the deleted-Box + * sweep. + */ +class PartitionBalancerTest { + + private static CandyboxConfig config(int maxMovesPerRound) { + return CandyboxConfig.builder() + .leaseRenewIntervalMillis(0) + .balancerMaxMovesPerRound(maxMovesPerRound) + .build(); + } + + private static int ownerOf(InMemoryCoordinationService coordination, String box, int partition) { + return coordination.leaseHolder(CandyboxKeys.ownerResource(box, partition)) + .map(LeaseInfo::ownerNodeId) + .orElse(-1); + } + + private static long countOwnedBy(InMemoryCoordinationService coordination, String box, + int partitions, int nodeId) { + long owned = 0; + for (int p = 0; p < partitions; p++) { + if (ownerOf(coordination, box, p) == nodeId) { + owned++; + } + } + return owned; + } + + @Test + void spreadsPartitionsEvenlyAcrossNodesAndStaysSticky() { + ManualClock clock = new ManualClock(1_000); + InMemoryLedgerStore store = new InMemoryLedgerStore(); + InMemoryCoordinationService coordination = new InMemoryCoordinationService(clock); + BoxName box = BoxName.of("spread-box"); + + CandyboxNode nodeA = new CandyboxNode(1, config(8), store, coordination, clock); + CandyboxNode nodeB = new CandyboxNode(2, config(8), store, coordination, clock); + try { + nodeA.createBox(box, 4); // all 4 partitions start on node 1 + assertThat(countOwnedBy(coordination, box.value(), 4, 1)).isEqualTo(4); + + // One round on each node: the coordinator (node 1) publishes a 2/2 split and releases + // its overflow; node 2 then acquires the freed partitions. + nodeA.runBalancerOnce(); + nodeB.runBalancerOnce(); + + assertThat(countOwnedBy(coordination, box.value(), 4, 1)).isEqualTo(2); + assertThat(countOwnedBy(coordination, box.value(), 4, 2)).isEqualTo(2); + + // Balanced state is sticky: further rounds change nothing. + int[] before = new int[4]; + for (int p = 0; p < 4; p++) { + before[p] = ownerOf(coordination, box.value(), p); + } + nodeA.runBalancerOnce(); + nodeB.runBalancerOnce(); + for (int p = 0; p < 4; p++) { + assertThat(ownerOf(coordination, box.value(), p)).isEqualTo(before[p]); + } + + // The moved partitions still serve their data from the new owner. + for (int p = 0; p < 4; p++) { + CandyboxNode owner = ownerOf(coordination, box.value(), p) == 1 ? nodeA : nodeB; + String key = keyInPartition(p, 4); + owner.engine(box, key).putCandy(CandyKey.of(key), + "v".getBytes(StandardCharsets.UTF_8), null, Map.of(), null); + assertThat(owner.engine(box, key).getCandy(CandyKey.of(key))).isNotEmpty(); + } + } finally { + nodeA.close(); + nodeB.close(); + store.close(); + } + } + + @Test + void migrationAwayFromLiveOwnersIsRateLimitedPerRound() { + ManualClock clock = new ManualClock(1_000); + InMemoryLedgerStore store = new InMemoryLedgerStore(); + InMemoryCoordinationService coordination = new InMemoryCoordinationService(clock); + BoxName box = BoxName.of("rate-box"); + + CandyboxNode nodeA = new CandyboxNode(1, config(1), store, coordination, clock); + CandyboxNode nodeB = new CandyboxNode(2, config(1), store, coordination, clock); + try { + nodeA.createBox(box, 6); + + // Round 1: capacity is 3, node 1 is 3 over, but only 1 move is allowed per round. + nodeA.runBalancerOnce(); + nodeB.runBalancerOnce(); + assertThat(countOwnedBy(coordination, box.value(), 6, 2)).isEqualTo(1); + + // Convergence proceeds one partition per round. + nodeA.runBalancerOnce(); + nodeB.runBalancerOnce(); + assertThat(countOwnedBy(coordination, box.value(), 6, 2)).isEqualTo(2); + + nodeA.runBalancerOnce(); + nodeB.runBalancerOnce(); + assertThat(countOwnedBy(coordination, box.value(), 6, 2)).isEqualTo(3); + assertThat(countOwnedBy(coordination, box.value(), 6, 1)).isEqualTo(3); + } finally { + nodeA.close(); + nodeB.close(); + store.close(); + } + } + + @Test + void deadNodesPartitionsFailOverWithoutRateLimit() { + ManualClock clock = new ManualClock(1_000); + InMemoryLedgerStore store = new InMemoryLedgerStore(); + InMemoryCoordinationService coordination = new InMemoryCoordinationService(clock); + BoxName box = BoxName.of("failover-box"); + + CandyboxNode nodeA = new CandyboxNode(1, config(1), store, coordination, clock); + nodeA.createBox(box, 4); + nodeA.engine(box, "k").putCandy(CandyKey.of("k"), "v".getBytes(StandardCharsets.UTF_8), + null, Map.of(), null); + nodeA.close(); // releases all leases and unregisters from membership + + CandyboxNode nodeB = new CandyboxNode(2, config(1), store, coordination, clock); + try { + // Failover is not a "move": all 4 unowned partitions reassign in a single round even + // though the move limit is 1. + nodeB.runBalancerOnce(); + assertThat(countOwnedBy(coordination, box.value(), 4, 2)).isEqualTo(4); + // The takeover recovered the prior owner's data. + assertThat(nodeB.engine(box, "k").getCandy(CandyKey.of("k"))) + .isEqualTo("v".getBytes(StandardCharsets.UTF_8)); + } finally { + nodeB.close(); + store.close(); + } + } + + @Test + void ownersDropPartitionsOfDeletedBoxes() { + ManualClock clock = new ManualClock(1_000); + InMemoryLedgerStore store = new InMemoryLedgerStore(); + InMemoryCoordinationService coordination = new InMemoryCoordinationService(clock); + BoxName box = BoxName.of("doomed-box"); + + CandyboxNode nodeA = new CandyboxNode(1, config(8), store, coordination, clock); + CandyboxNode nodeB = new CandyboxNode(2, config(8), store, coordination, clock); + try { + nodeA.createBox(box, 2); + // A force-delete from a node that owns nothing: the descriptor goes away; node 1's + // partitions are cleaned up by its own next balancer round (the sweep). + nodeB.deleteBox(box, true); + assertThat(nodeB.boxExists(box)).isFalse(); + + nodeA.runBalancerOnce(); + assertThat(nodeA.ownedBoxStats()).isEmpty(); + assertThat(coordination.get(CandyboxKeys.manifestKey(box.value(), 0))).isEmpty(); + assertThat(coordination.get(CandyboxKeys.manifestKey(box.value(), 1))).isEmpty(); + } finally { + nodeA.close(); + nodeB.close(); + store.close(); + } + } + + /** Finds a key that hashes to {@code partition} under {@code count} partitions. */ + private static String keyInPartition(int partition, int count) { + for (int i = 0; i < 10_000; i++) { + String candidate = "key-" + i; + if (Partitioning.partitionOf(candidate, count) == partition) { + return candidate; + } + } + throw new AssertionError("no key found for partition " + partition); + } +} From fe084be8888a8570568f526bdd61ee7c348be75f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 15:51:01 +0000 Subject: [PATCH 2/2] Cover the partition routing, lifecycle, and fan-out paths left cold by codecov The codecov patch gate flagged 138 uncovered lines in the partitioning change. Adds targeted unit tests for the cold branches: - NodeRequestHandler: MOVED redirection for every keyed/partitioned request type, BoxInfo dispatch, the NotOwner deleteBox error path, and the cross-partition zero-copy rejection. - CandyboxNode: createBox rollback on a half-created Box, non-force deleteBox against a live foreign owner, releaseBox/openBox replay. - CandyboxClient: scatter-gather merge order/truncation/reverse, descriptor caching + invalidation on deleteBox, delete-range fan-out, merged multipart listings, and the cross-partition copy/rename/ uploadPartCopy byte-copy fallbacks (against a recording stub node). - CandyboxConfig partitioning/balancer knobs + validation; PartitionAssignment round-trip (and drops its unused targetNode); dashboard owner summary for spread partition ownership. https://claude.ai/code/session_015ZRRX63Hie7PDKxsH4Zosn --- .../candybox/admin/LiveDashboardDataTest.java | 21 ++ .../client/PartitionedRoutingClientTest.java | 276 ++++++++++++++++++ .../common/config/CandyboxConfigTest.java | 27 ++ .../candybox/server/PartitionAssignment.java | 5 - .../candybox/server/CandyboxNodeTest.java | 65 +++++ .../server/NodeRequestHandlerTest.java | 88 ++++++ .../server/PartitionAssignmentTest.java | 54 ++++ 7 files changed, 531 insertions(+), 5 deletions(-) create mode 100644 candybox-client/src/test/java/me/predatorray/candybox/client/PartitionedRoutingClientTest.java create mode 100644 candybox-server/src/test/java/me/predatorray/candybox/server/PartitionAssignmentTest.java diff --git a/candybox-admin-api/src/test/java/me/predatorray/candybox/admin/LiveDashboardDataTest.java b/candybox-admin-api/src/test/java/me/predatorray/candybox/admin/LiveDashboardDataTest.java index 1ff93f7..b8da662 100644 --- a/candybox-admin-api/src/test/java/me/predatorray/candybox/admin/LiveDashboardDataTest.java +++ b/candybox-admin-api/src/test/java/me/predatorray/candybox/admin/LiveDashboardDataTest.java @@ -150,6 +150,27 @@ void boxesAndLsmReadOwnerAndManifestVersion() throws Exception { assertThat(orphan.fencingToken()).isEqualTo(-1); } + @Test + void ownerSummarizesSpreadPartitionOwnership() { + // A Box whose two partitions are owned by different nodes reports both, sorted and joined. + InMemoryCoordinationService coord = new InMemoryCoordinationService(); + coord.create(CandyboxKeys.boxMetaKey("spread"), new BoxDescriptor(2).encode()); + coord.tryAcquireLease(CandyboxKeys.ownerResource("spread", 0), 2, 60_000); + coord.tryAcquireLease(CandyboxKeys.ownerResource("spread", 1), 1, 60_000); + + FakeBoxClient client = new FakeBoxClient(); + client.boxes.add("spread"); + LiveDashboardData data = new LiveDashboardData(coord, client); + + assertThat(data.boxes()).singleElement() + .satisfies(row -> assertThat(row.owner()).isEqualTo("1,2")); + // One LSM row per partition, each naming its own holder. + assertThat(data.lsm()).extracting(DashboardData.LsmRow::box) + .containsExactly("spread/0", "spread/1"); + assertThat(data.lsm()).extracting(DashboardData.LsmRow::owner) + .containsExactly("2", "1"); + } + @Test void mutatingOpsDelegateToClient() { InMemoryCoordinationService coord = new InMemoryCoordinationService(); diff --git a/candybox-client/src/test/java/me/predatorray/candybox/client/PartitionedRoutingClientTest.java b/candybox-client/src/test/java/me/predatorray/candybox/client/PartitionedRoutingClientTest.java new file mode 100644 index 0000000..3227989 --- /dev/null +++ b/candybox-client/src/test/java/me/predatorray/candybox/client/PartitionedRoutingClientTest.java @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package me.predatorray.candybox.client; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import me.predatorray.candybox.common.Partitioning; +import me.predatorray.candybox.protocol.Message; +import me.predatorray.candybox.protocol.MessageCodec; +import me.predatorray.candybox.protocol.transport.LoopbackTransport; +import me.predatorray.candybox.protocol.transport.RequestHandler; +import org.junit.jupiter.api.Test; + +/** + * Pins {@link CandyboxClient}'s partition-aware behaviour against a recording two-partition stub + * node: descriptor caching, scatter-gather list merging (order, truncation, reverse), fanned-out + * range deletes, merged multipart-upload listings, and the cross-partition byte-copy fallbacks for + * copy/rename/uploadPartCopy. + */ +class PartitionedRoutingClientTest { + + private static final MessageCodec CODEC = new MessageCodec(); + private static final int PARTITIONS = 2; + + private static byte[] bytes(String s) { + return s.getBytes(StandardCharsets.UTF_8); + } + + /** A key hashing to the given partition under {@link #PARTITIONS}. */ + private static String keyIn(int partition, String tag) { + for (int i = 0; i < 10_000; i++) { + String candidate = tag + "-" + i; + if (Partitioning.partitionOf(candidate, PARTITIONS) == partition) { + return candidate; + } + } + throw new AssertionError("no key found for partition " + partition); + } + + /** Records requests; answers BoxInfo with two partitions and per-partition canned pages. */ + private static final class StubNode implements RequestHandler { + final AtomicInteger boxInfoCalls = new AtomicInteger(); + final List requests = new ArrayList<>(); + Map listPages = Map.of(); + Map uploadPages = Map.of(); + + @Override + public synchronized me.predatorray.candybox.protocol.Frame handle( + me.predatorray.candybox.protocol.Frame request) { + Message message = CODEC.decode(request); + requests.add(message); + return CODEC.encode(dispatch(message)); + } + + private Message dispatch(Message message) { + if (message instanceof Message.BoxInfoRequest) { + boxInfoCalls.incrementAndGet(); + return new Message.BoxInfoResponse(PARTITIONS); + } else if (message instanceof Message.ListCandiesRequest m) { + return listPages.get(m.partition()); + } else if (message instanceof Message.ListMultipartUploadsRequest m) { + return uploadPages.get(m.partition()); + } else if (message instanceof Message.GetCandyRequest) { + return new Message.CandyDataResponse(7, "text/plain", Map.of("m", "x"), 9, + bytes("payload")); + } else if (message instanceof Message.RangeGetCandyRequest) { + return new Message.CandyDataResponse(4, 10, "text/plain", Map.of(), 9, bytes("2345")); + } else if (message instanceof Message.HeadCandyRequest) { + return new Message.HeadCandyResponse(7, "text/plain", Map.of("m", "x"), 9, 1); + } else if (message instanceof Message.UploadPartRequest m) { + return new Message.UploadPartResponse(5, m.data().length); + } else if (message instanceof Message.UploadPartCopyRequest) { + return new Message.UploadPartResponse(6, 10); + } else if (message instanceof Message.CopyCandyRequest + || message instanceof Message.RenameCandyRequest) { + return new Message.HeadCandyResponse(7, "text/plain", Map.of(), 9, 1); + } + return new Message.OkResponse(); + } + + synchronized List recorded(Class type) { + List out = new ArrayList<>(); + for (Message m : requests) { + if (type.isInstance(m)) { + out.add(type.cast(m)); + } + } + return out; + } + } + + private static Message.ListedCandy row(String key) { + return new Message.ListedCandy(key, 1, 2); + } + + @Test + void listMergesPartitionsInKeyOrderAndCachesTheDescriptor() { + StubNode node = new StubNode(); + node.listPages = Map.of( + 0, new Message.ListCandiesResponse(List.of(row("a"), row("c")), null), + 1, new Message.ListCandiesResponse(List.of(row("b"), row("d")), null)); + try (CandyboxClient client = new CandyboxClient(new LoopbackTransport(node), "x", 0)) { + CandyboxClient.Listing page = client.listCandies("box", null, null, 3); + assertThat(page.entries()).extracting(CandyboxClient.Listing.Entry::key) + .containsExactly("a", "b", "c"); // global order; "d" overflows the page + assertThat(page.nextStartAfter()).isEqualTo("c"); + + // Under the page size and no partition truncated: no continuation. + CandyboxClient.Listing all = client.listCandies("box", null, null, 10); + assertThat(all.entries()).hasSize(4); + assertThat(all.isTruncated()).isFalse(); + + // The descriptor was fetched exactly once across both listings (it is immutable). + assertThat(node.boxInfoCalls.get()).isEqualTo(1); + } + } + + @Test + void listHonorsPartitionTruncationAndReverseOrder() { + StubNode node = new StubNode(); + node.listPages = Map.of( + 0, new Message.ListCandiesResponse(List.of(row("a")), "a"), // truncated partition + 1, new Message.ListCandiesResponse(List.of(row("b")), null)); + try (CandyboxClient client = new CandyboxClient(new LoopbackTransport(node), "x", 0)) { + CandyboxClient.Listing page = client.listCandies("box", null, null, 10); + // Fewer rows than maxKeys, but partition 0 had more: the page must carry a cursor. + assertThat(page.entries()).hasSize(2); + assertThat(page.nextStartAfter()).isEqualTo("b"); + } + + StubNode reverseNode = new StubNode(); + reverseNode.listPages = Map.of( + 0, new Message.ListCandiesResponse(List.of(row("c"), row("a")), null), + 1, new Message.ListCandiesResponse(List.of(row("d"), row("b")), null)); + try (CandyboxClient client = new CandyboxClient(new LoopbackTransport(reverseNode), "x", 0)) { + CandyboxClient.Listing page = client.listCandies("box", null, null, null, null, true, 10); + assertThat(page.entries()).extracting(CandyboxClient.Listing.Entry::key) + .containsExactly("d", "c", "b", "a"); + } + } + + @Test + void deleteRangeFansOutToEveryPartition() { + StubNode node = new StubNode(); + try (CandyboxClient client = new CandyboxClient(new LoopbackTransport(node), "x", 0)) { + client.deleteRangeByPrefix("box", "logs/"); + client.deleteRange("box", "a", "m"); + } + List sent = node.recorded(Message.DeleteRangeRequest.class); + assertThat(sent).extracting(Message.DeleteRangeRequest::partition) + .containsExactly(0, 1, 0, 1); + assertThat(sent.get(0).prefix()).isEqualTo("logs/"); + assertThat(sent.get(2).startKey()).isEqualTo("a"); + } + + @Test + void listMultipartUploadsMergesAndPaginatesAcrossPartitions() { + StubNode node = new StubNode(); + node.uploadPages = Map.of( + 0, new Message.ListMultipartUploadsResponse( + List.of(new Message.InProgressUpload("u2", "k1", 1)), null, null), + 1, new Message.ListMultipartUploadsResponse( + List.of(new Message.InProgressUpload("u1", "k0", 1)), null, null)); + try (CandyboxClient client = new CandyboxClient(new LoopbackTransport(node), "x", 0)) { + CandyboxClient.MultipartListing merged = + client.listMultipartUploads("box", null, null, null, 10); + assertThat(merged.uploads()).extracting(CandyboxClient.UploadEntry::key) + .containsExactly("k0", "k1"); // merged in key order + assertThat(merged.isTruncated()).isFalse(); + + // A page smaller than the merged rows reports markers from the last returned row. + CandyboxClient.MultipartListing paged = + client.listMultipartUploads("box", null, null, null, 1); + assertThat(paged.uploads()).hasSize(1); + assertThat(paged.nextKeyMarker()).isEqualTo("k0"); + assertThat(paged.nextUploadIdMarker()).isEqualTo("u1"); + } + } + + @Test + void crossPartitionCopyAndRenameFallBackToGetPutDelete() { + String src = keyIn(0, "src"); + String dst = keyIn(1, "dst"); + StubNode node = new StubNode(); + try (CandyboxClient client = new CandyboxClient(new LoopbackTransport(node), "x", 0)) { + CandyboxClient.CandyInfo copied = client.copyCandy("box", src, dst, "tok"); + assertThat(copied.contentLength()).isEqualTo(7); + // No server-side copy was attempted; the bytes and metadata were re-put at dst. + assertThat(node.recorded(Message.CopyCandyRequest.class)).isEmpty(); + Message.PutCandyRequest put = node.recorded(Message.PutCandyRequest.class).get(0); + assertThat(put.key()).isEqualTo(dst); + assertThat(put.contentType()).isEqualTo("text/plain"); + assertThat(put.userMetadata()).containsEntry("m", "x"); + assertThat(put.idempotencyToken()).isEqualTo("tok"); + assertThat(node.recorded(Message.DeleteCandyRequest.class)).isEmpty(); + + client.renameCandy("box", src, dst, null); + assertThat(node.recorded(Message.RenameCandyRequest.class)).isEmpty(); + Message.DeleteCandyRequest deleted = node.recorded(Message.DeleteCandyRequest.class).get(0); + assertThat(deleted.key()).isEqualTo(src); // rename = copy + delete source + } + } + + @Test + void samePartitionCopyAndRenameStayServerSide() { + String src = keyIn(0, "zsrc"); + String dst = keyIn(0, "zdst"); + StubNode node = new StubNode(); + try (CandyboxClient client = new CandyboxClient(new LoopbackTransport(node), "x", 0)) { + client.copyCandy("box", src, dst, null); + client.renameCandy("box", src, dst, null); + } + assertThat(node.recorded(Message.CopyCandyRequest.class)).hasSize(1); + assertThat(node.recorded(Message.RenameCandyRequest.class)).hasSize(1); + assertThat(node.recorded(Message.PutCandyRequest.class)).isEmpty(); + } + + @Test + void uploadPartCopyDegradesToClientSideCopyOnlyAcrossPartitions() { + String target = keyIn(0, "upc"); + String srcSame = keyIn(0, "same"); + String srcOther = keyIn(1, "other"); + StubNode node = new StubNode(); + try (CandyboxClient client = new CandyboxClient(new LoopbackTransport(node), "x", 0)) { + // Same partition: the zero-copy server op goes through. + client.uploadPartCopy("box", target, "u1", 1, srcSame, -1, -1); + assertThat(node.recorded(Message.UploadPartCopyRequest.class)).hasSize(1); + + // Cross-partition, whole source: GET + UploadPart. + CandyboxClient.PartUploadInfo whole = + client.uploadPartCopy("box", target, "u1", 2, srcOther, -1, -1); + assertThat(whole.partLength()).isEqualTo(7); + assertThat(node.recorded(Message.GetCandyRequest.class)).hasSize(1); + + // Cross-partition, ranged: Range GET + UploadPart. + CandyboxClient.PartUploadInfo ranged = + client.uploadPartCopy("box", target, "u1", 3, srcOther, 2, 5); + assertThat(ranged.partLength()).isEqualTo(4); + assertThat(node.recorded(Message.RangeGetCandyRequest.class)).hasSize(1); + assertThat(node.recorded(Message.UploadPartCopyRequest.class)).hasSize(1); // unchanged + } + } + + @Test + void deleteBoxInvalidatesTheCachedDescriptor() { + StubNode node = new StubNode(); + node.listPages = Map.of( + 0, new Message.ListCandiesResponse(List.of(), null), + 1, new Message.ListCandiesResponse(List.of(), null)); + try (CandyboxClient client = new CandyboxClient(new LoopbackTransport(node), "x", 0)) { + client.listCandies("box", null, null, 5); + assertThat(node.boxInfoCalls.get()).isEqualTo(1); + client.deleteBox("box", true); + client.listCandies("box", null, null, 5); // re-resolves the descriptor + assertThat(node.boxInfoCalls.get()).isEqualTo(2); + } + } +} diff --git a/candybox-common/src/test/java/me/predatorray/candybox/common/config/CandyboxConfigTest.java b/candybox-common/src/test/java/me/predatorray/candybox/common/config/CandyboxConfigTest.java index 7552257..e64acb9 100644 --- a/candybox-common/src/test/java/me/predatorray/candybox/common/config/CandyboxConfigTest.java +++ b/candybox-common/src/test/java/me/predatorray/candybox/common/config/CandyboxConfigTest.java @@ -154,4 +154,31 @@ void buildRejectsNonPositiveMultipartMaxParts() { .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("multipartMaxParts"); } + + @Test + void partitioningAndBalancerDefaultsAndOverrides() { + CandyboxConfig defaults = CandyboxConfig.defaults(); + assertThat(defaults.partitionsPerBoxDefault()).isEqualTo(8); + assertThat(defaults.balancerIntervalMillis()).isZero(); // off by default (tests drive rounds) + assertThat(defaults.balancerMaxMovesPerRound()).isEqualTo(4); + + CandyboxConfig cfg = CandyboxConfig.builder() + .partitionsPerBoxDefault(2) + .balancerIntervalMillis(5_000) + .balancerMaxMovesPerRound(1) + .build(); + assertThat(cfg.partitionsPerBoxDefault()).isEqualTo(2); + assertThat(cfg.balancerIntervalMillis()).isEqualTo(5_000); + assertThat(cfg.balancerMaxMovesPerRound()).isEqualTo(1); + } + + @Test + void buildRejectsNonPositivePartitionCountAndMoveBudget() { + assertThatThrownBy(() -> CandyboxConfig.builder().partitionsPerBoxDefault(0).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("partitionsPerBoxDefault"); + assertThatThrownBy(() -> CandyboxConfig.builder().balancerMaxMovesPerRound(0).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("balancerMaxMovesPerRound"); + } } diff --git a/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionAssignment.java b/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionAssignment.java index a8cfb13..5d51ef6 100644 --- a/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionAssignment.java +++ b/candybox-server/src/main/java/me/predatorray/candybox/server/PartitionAssignment.java @@ -52,11 +52,6 @@ Map targets() { return java.util.Collections.unmodifiableMap(targets); } - /** The node assigned to a partition, or {@code null} if the table does not mention it. */ - Integer targetNode(String box, int partition) { - return targets.get(new BoxPartition(box, partition)); - } - byte[] encode() { BinaryWriter w = new BinaryWriter(64); w.writeByte(FORMAT_VERSION); diff --git a/candybox-server/src/test/java/me/predatorray/candybox/server/CandyboxNodeTest.java b/candybox-server/src/test/java/me/predatorray/candybox/server/CandyboxNodeTest.java index 37aa0cd..e4a59c0 100644 --- a/candybox-server/src/test/java/me/predatorray/candybox/server/CandyboxNodeTest.java +++ b/candybox-server/src/test/java/me/predatorray/candybox/server/CandyboxNodeTest.java @@ -19,10 +19,16 @@ import java.nio.charset.StandardCharsets; import java.util.Map; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + import me.predatorray.candybox.bookkeeper.fake.InMemoryLedgerStore; import me.predatorray.candybox.common.BoxName; +import me.predatorray.candybox.common.CandyKey; import me.predatorray.candybox.common.ManualClock; import me.predatorray.candybox.common.config.CandyboxConfig; +import me.predatorray.candybox.common.exception.BoxAlreadyExistsException; +import me.predatorray.candybox.common.exception.NotOwnerException; +import me.predatorray.candybox.coordination.CandyboxKeys; import me.predatorray.candybox.coordination.fake.InMemoryCoordinationService; import me.predatorray.candybox.protocol.Frame; import me.predatorray.candybox.protocol.Message; @@ -322,6 +328,65 @@ void gcDeletesRotatedWalLedgers() { store.close(); } + @Test + void createBoxRollsBackWhenAPartitionCannotBeCreated() { + InMemoryLedgerStore store = new InMemoryLedgerStore(); + InMemoryCoordinationService coordination = new InMemoryCoordinationService(); + // A leftover manifest pointer for partition 1 makes the second partition's creation fail. + coordination.create(CandyboxKeys.manifestKey("half-box", 1), new byte[] {0}); + try (CandyboxNode node = new CandyboxNode(1, CandyboxConfig.defaults(), store, coordination, + new ManualClock(1000))) { + assertThatThrownBy(() -> node.createBox(BoxName.of("half-box"), 2)) + .isInstanceOf(BoxAlreadyExistsException.class); + // The descriptor and the partition that *was* created are rolled back. + assertThat(node.boxExists(BoxName.of("half-box"))).isFalse(); + assertThat(coordination.get(CandyboxKeys.boxMetaKey("half-box"))).isEmpty(); + assertThat(coordination.get(CandyboxKeys.manifestKey("half-box", 0))).isEmpty(); + assertThat(node.ownedBoxStats()).isEmpty(); + } + store.close(); + } + + @Test + void nonForceDeleteBoxFailsWhileAnotherLiveNodeOwnsAPartition() { + InMemoryLedgerStore store = new InMemoryLedgerStore(); + InMemoryCoordinationService coordination = new InMemoryCoordinationService(); + try (CandyboxNode owner = new CandyboxNode(1, CandyboxConfig.defaults(), store, coordination, + new ManualClock(1000)); + CandyboxNode other = new CandyboxNode(2, CandyboxConfig.defaults(), store, coordination, + new ManualClock(1000))) { + owner.createBox(BoxName.of("held-box"), 2); + // Node 2 cannot take over partitions whose leases node 1 still holds. + assertThatThrownBy(() -> other.deleteBox(BoxName.of("held-box"), false)) + .isInstanceOf(NotOwnerException.class); + assertThat(owner.boxExists(BoxName.of("held-box"))).isTrue(); + } + store.close(); + } + + @Test + void releaseBoxThenOpenBoxReplaysEveryPartition() { + InMemoryLedgerStore store = new InMemoryLedgerStore(); + InMemoryCoordinationService coordination = new InMemoryCoordinationService(); + try (CandyboxNode node = new CandyboxNode(1, CandyboxConfig.defaults(), store, coordination, + new ManualClock(1000))) { + BoxName box = BoxName.of("reopen-box"); + node.createBox(box, 2); + node.engine(box, "k").putCandy(CandyKey.of("k"), + "v".getBytes(StandardCharsets.UTF_8), null, Map.of(), null); + + node.releaseBox(box); + assertThat(node.ownedBoxStats()).isEmpty(); + assertThat(node.currentOwner(box, 0)).isEmpty(); // released, not just expired + + node.openBox(box); + assertThat(node.ownedBoxStats()).containsKeys("reopen-box/0", "reopen-box/1"); + assertThat(node.engine(box, "k").getCandy(CandyKey.of("k"))) + .isEqualTo("v".getBytes(StandardCharsets.UTF_8)); + } + store.close(); + } + private static Message put(String box, String key) { return putValue(box, key, "v"); } diff --git a/candybox-server/src/test/java/me/predatorray/candybox/server/NodeRequestHandlerTest.java b/candybox-server/src/test/java/me/predatorray/candybox/server/NodeRequestHandlerTest.java index 713e622..3bba95a 100644 --- a/candybox-server/src/test/java/me/predatorray/candybox/server/NodeRequestHandlerTest.java +++ b/candybox-server/src/test/java/me/predatorray/candybox/server/NodeRequestHandlerTest.java @@ -20,6 +20,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; +import me.predatorray.candybox.common.Partitioning; import me.predatorray.candybox.bookkeeper.fake.InMemoryLedgerStore; import me.predatorray.candybox.common.BoxName; import me.predatorray.candybox.common.ManualClock; @@ -103,6 +104,93 @@ void requestForBoxOwnedElsewhereIsRedirectedWithMoved() { store.close(); } + @Test + void everyKeyedAndPartitionedRequestRedirectsWithMovedFromANonOwner() { + // Node 1 owns the (single-partition) Box; every Box-routed request sent to node 2 must come + // back as MOVED(1) — this pins the per-message routing-key/partition extraction. + InMemoryLedgerStore store = new InMemoryLedgerStore(); + InMemoryCoordinationService coordination = new InMemoryCoordinationService(); + try (CandyboxNode owner = new CandyboxNode(1, config(), store, coordination, new ManualClock(1000)); + CandyboxNode other = new CandyboxNode(2, config(), store, coordination, new ManualClock(1000))) { + owner.createBox(BoxName.of("routed-box"), 1); + RequestHandler handler = other.requestHandler(); + + List boxRouted = List.of( + new Message.PutCandyRequest("routed-box", "k", null, Map.of(), null, bytes("v")), + new Message.GetCandyRequest("routed-box", "k"), + new Message.RangeGetCandyRequest("routed-box", "k", 0, 1), + new Message.HeadCandyRequest("routed-box", "k"), + new Message.DeleteCandyRequest("routed-box", "k"), + new Message.CopyCandyRequest("routed-box", "k", "k2", null), + new Message.RenameCandyRequest("routed-box", "k", "k2", null), + new Message.DeleteRangeRequest("routed-box", 0, "p/", null, null), + new Message.ListCandiesRequest("routed-box", 0, null, null, 10), + new Message.CreateMultipartUploadRequest("routed-box", "k", null, Map.of()), + new Message.UploadPartRequest("routed-box", "k", "u1", 1, bytes("v")), + new Message.CompleteMultipartUploadRequest("routed-box", "k", "u1", List.of(), null), + new Message.AbortMultipartUploadRequest("routed-box", "k", "u1"), + new Message.ListMultipartUploadsRequest("routed-box", 0, null, null, null, 10), + new Message.ListPartsRequest("routed-box", "k", "u1", 0, 10), + new Message.UploadPartCopyRequest("routed-box", "k", "u1", 1, "src", -1, -1)); + for (Message request : boxRouted) { + Message response = roundTrip(handler, request); + assertThat(response).as(request.opcode().toString()) + .isInstanceOf(Message.MovedResponse.class); + assertThat(((Message.MovedResponse) response).ownerNodeId()).isEqualTo(1); + } + + // Box-level requests are answered from coordination, not redirected. + assertThat(roundTrip(handler, new Message.HeadBoxRequest("routed-box"))) + .isInstanceOf(Message.OkResponse.class); + Message info = roundTrip(handler, new Message.BoxInfoRequest("routed-box")); + assertThat(((Message.BoxInfoResponse) info).partitionCount()).isEqualTo(1); + assertThat(roundTrip(handler, new Message.BoxInfoRequest("ghost-box"))) + .isInstanceOf(Message.NotFoundResponse.class); + + // A non-force deleteBox needs partitions node 1 still owns: a NotOwner error, not MOVED. + Message denied = roundTrip(handler, new Message.DeleteBoxRequest("routed-box", false)); + assertThat(denied).isInstanceOf(Message.ErrorResponse.class); + assertThat(((Message.ErrorResponse) denied).errorType()).isEqualTo("NotOwnerException"); + } + store.close(); + } + + @Test + void crossPartitionServerSideCopyIsRejectedAsValidationError() { + try (CandyboxNode node = new CandyboxNode(1, config(), new InMemoryLedgerStore(), + new InMemoryCoordinationService(), new ManualClock(1000))) { + RequestHandler handler = node.requestHandler(); + roundTrip(handler, new Message.CreateBoxRequest("split-box", 4)); + + // Find two keys in different partitions: the zero-copy path must refuse them (the client + // is responsible for the byte-copy fallback). + String src = keyInPartition(0, 4); + String dst = keyInPartition(3, 4); + roundTrip(handler, new Message.PutCandyRequest("split-box", src, null, Map.of(), null, + bytes("v"))); + for (Message request : List.of( + new Message.CopyCandyRequest("split-box", src, dst, null), + new Message.RenameCandyRequest("split-box", src, dst, null), + new Message.UploadPartCopyRequest("split-box", dst, "u1", 1, src, -1, -1))) { + Message response = roundTrip(handler, request); + assertThat(response).as(request.opcode().toString()) + .isInstanceOf(Message.ErrorResponse.class); + assertThat(((Message.ErrorResponse) response).message()) + .contains("Cross-partition"); + } + } + } + + private static String keyInPartition(int partition, int count) { + for (int i = 0; i < 10_000; i++) { + String candidate = "key-" + i; + if (Partitioning.partitionOf(candidate, count) == partition) { + return candidate; + } + } + throw new AssertionError("no key found for partition " + partition); + } + @Test void duplicateCreateBoxAndNonEmptyDeleteMapToErrorResponses() { try (CandyboxNode node = new CandyboxNode(1, config(), new InMemoryLedgerStore(), diff --git a/candybox-server/src/test/java/me/predatorray/candybox/server/PartitionAssignmentTest.java b/candybox-server/src/test/java/me/predatorray/candybox/server/PartitionAssignmentTest.java new file mode 100644 index 0000000..3905c7c --- /dev/null +++ b/candybox-server/src/test/java/me/predatorray/candybox/server/PartitionAssignmentTest.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package me.predatorray.candybox.server; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Map; +import me.predatorray.candybox.server.PartitionAssignment.BoxPartition; +import org.junit.jupiter.api.Test; + +class PartitionAssignmentTest { + + @Test + void encodeDecodeRoundTripsTheTargets() { + PartitionAssignment assignment = new PartitionAssignment(Map.of( + new BoxPartition("alpha", 0), 1, + new BoxPartition("alpha", 1), 2, + new BoxPartition("beta", 0), 1)); + PartitionAssignment decoded = PartitionAssignment.decode(assignment.encode()); + assertThat(decoded.targets()).isEqualTo(assignment.targets()); + + assertThat(PartitionAssignment.decode(PartitionAssignment.empty().encode()).targets()) + .isEmpty(); + } + + @Test + void rejectsUnknownFormatVersion() { + byte[] encoded = PartitionAssignment.empty().encode(); + encoded[0] = 9; + assertThatThrownBy(() -> PartitionAssignment.decode(encoded)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void boxPartitionsOrderByBoxThenPartition() { + assertThat(new BoxPartition("a", 1)).isLessThan(new BoxPartition("b", 0)); + assertThat(new BoxPartition("a", 1)).isGreaterThan(new BoxPartition("a", 0)); + assertThat(new BoxPartition("a", 1)).isEqualByComparingTo(new BoxPartition("a", 1)); + } +}