diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..43facfd --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,72 @@ +name: Docs + +# Build the Hugo documentation site and publish it to GitHub Pages. +# +# Pages must be enabled for the repository with "Source: GitHub Actions" +# (Settings → Pages). The site is served at https://predatorray.github.io/candybox/. +on: + push: + branches: [main] + paths: + - 'docs/**' + - '.github/workflows/docs.yml' + # Allow manual runs from the Actions tab. + workflow_dispatch: + +# Cancel superseded runs on the same branch to save CI minutes. +concurrency: + group: pages-${{ github.ref }} + cancel-in-progress: true + +# Least-privilege token scoped to what the Pages deploy needs. +permissions: + contents: read + pages: write + id-token: write + +env: + # Keep in sync with docs/README.md; the theme requires >= 0.158.0 extended. + HUGO_VERSION: '0.158.0' + +jobs: + build: + name: Build site + runs-on: ubuntu-latest + steps: + - name: Checkout (with theme submodule) + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 # enableGitInfo needs full history for last-modified dates + + - name: Install Hugo + uses: peaceiris/actions-hugo@v3 + with: + hugo-version: ${{ env.HUGO_VERSION }} + extended: true + + - name: Configure Pages + id: pages + uses: actions/configure-pages@v5 + + - name: Build + working-directory: docs + # baseURL comes from configure-pages so the site works regardless of repo/user-page URL. + run: hugo --gc --minify --baseURL "${{ steps.pages.outputs.base_url }}/" + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/public + + deploy: + name: Deploy to GitHub Pages + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index b86181c..b1c3893 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,8 @@ target/ # OS / editor cruft .DS_Store + +# Hugo documentation build output (docs/) +docs/public/ +docs/resources/ +.hugo_build.lock diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..47029a0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "docs/themes/hugo-book"] + path = docs/themes/hugo-book + url = https://github.com/alex-shpak/hugo-book diff --git a/README.md b/README.md index d4570fb..c2384bd 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@

+ Documentation Docker Hub License
diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..0b20ee9 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,57 @@ +# Candybox documentation site + +The Candybox user manual / project website. It is built with [Hugo](https://gohugo.io) and the +[hugo-book](https://github.com/alex-shpak/hugo-book) theme, and published to GitHub Pages at +**https://predatorray.github.io/candybox/** by the +[`docs.yml`](../.github/workflows/docs.yml) GitHub Actions workflow on every push to `main` that +touches `docs/`. + +## Layout + +``` +docs/ +├── config.toml # Hugo site configuration +├── content/ +│ ├── _index.md # landing page (https://predatorray.github.io/candybox/) +│ └── docs/ # the manual; this section becomes the left sidebar +│ ├── getting-started/ +│ ├── installation/ +│ ├── architecture/ +│ ├── client/ +│ ├── s3-gateway/ +│ ├── operations/ +│ └── reference/ +└── themes/ + └── hugo-book/ # theme, vendored as a git submodule +``` + +To add a page, drop a markdown file (with a `title` and `weight` front matter) into the right folder +under `content/docs/`. The sidebar order follows `weight`. + +## Prerequisites + +- **Hugo extended**, version **0.158.0 or newer** (the theme requires it). Install from + . +- The theme submodule. After cloning the repo: + + ```bash + git submodule update --init docs/themes/hugo-book + ``` + + (A fresh clone can also use `git clone --recurse-submodules`.) + +## Preview locally + +```bash +cd docs +hugo server # live-reloading preview at http://localhost:1313/candybox/ +``` + +## Build + +```bash +cd docs +hugo --gc --minify # output written to docs/public/ +``` + +`docs/public/` is generated output and is git-ignored. diff --git a/docs/config.toml b/docs/config.toml new file mode 100644 index 0000000..10448db --- /dev/null +++ b/docs/config.toml @@ -0,0 +1,52 @@ +# Hugo configuration for the Candybox documentation site. +# +# The site is built with Hugo (https://gohugo.io) and the hugo-book theme +# (https://github.com/alex-shpak/hugo-book), vendored as a git submodule under +# themes/hugo-book. See README.md in this directory for how to build and preview. +# +# baseURL is the GitHub Pages project URL. The publish workflow overrides it with +# --baseURL when needed, but keep it correct here so local `hugo` produces working +# links too. +baseURL = 'https://predatorray.github.io/candybox/' +title = 'Candybox' +languageCode = 'en-us' +theme = 'hugo-book' + +# hugo-book renders the section tree as the left navigation, so a deep, ordered +# content/docs tree is all that is needed for the sidebar. +enableGitInfo = true + +# Use a clean URL layout (/docs/getting-started/ rather than .../index.html). +[markup] + [markup.goldmark.renderer] + unsafe = true # allow raw HTML in markdown (badges, the landing hero) + [markup.highlight] + noClasses = false + style = 'github' + [markup.tableOfContents] + startLevel = 2 + endLevel = 3 + +[params] + # hugo-book options — see https://github.com/alex-shpak/hugo-book#configuration + BookTheme = 'auto' # light / dark / auto + BookToC = true # per-page table of contents + BookRepo = 'https://github.com/predatorray/candybox' + BookEditPath = 'edit/main/docs' # "Edit this page" points back at the source + BookSearch = true # client-side search box + BookComments = false + # Only pages under content/docs become part of the manual's menu; the landing + # page (content/_index.md) stays out of the sidebar tree. + BookSection = 'docs' + +[menu] + # A couple of fixed links pinned under the sidebar nav. + [[menu.before]] + [[menu.after]] + name = "GitHub" + url = "https://github.com/predatorray/candybox" + weight = 10 + [[menu.after]] + name = "Docker Hub" + url = "https://hub.docker.com/r/zetaplusae/candybox" + weight = 20 diff --git a/docs/content/_index.md b/docs/content/_index.md new file mode 100644 index 0000000..6b46c97 --- /dev/null +++ b/docs/content/_index.md @@ -0,0 +1,48 @@ +--- +title: Candybox +type: docs +bookToc: false +--- + +# Candybox + +> [!TIP] +> **A self-hosted, distributed object store with a sorted LSM-tree index on Apache BookKeeper.** + +Candybox is a **distributed, S3-like object store** written in Java. You create *buckets* and store +*objects* in them through a small TCP API, a command-line client, or an S3-compatible HTTP gateway; +Candybox keeps those objects durable and replicated across a cluster. + +Under the hood it is a **distributed LSM tree built on [Apache BookKeeper](https://bookkeeper.apache.org/)**: +object data and index live in BookKeeper's replicated, append-only ledgers, and a single fenced owner +per bucket partition keeps reads and writes consistent during failover, with partitions spread evenly +across the cluster. + +Because keys are stored sorted and object bytes live behind small pointers, Candybox offers a few +operations an S3-style store cannot do cheaply: bounded / reverse-order range scans, zero-copy +`copy` / `rename`, and `delete-range` over a whole prefix or key window in constant work. + +## Where to start + +- 🚀 **[Get started]({{< relref "/docs/getting-started" >}})** — spin up a full cluster with Docker + Compose, store your first object, and learn the vocabulary. +- 🧩 **[Concepts & architecture]({{< relref "/docs/architecture" >}})** — how the LSM engine, + BookKeeper ledgers, and fenced ownership fit together. +- ☁️ **[S3 gateway]({{< relref "/docs/s3-gateway" >}})** — use Candybox from any S3 SDK or tool, with + optional SigV4 auth and ACLs. + +## Vocabulary + +| Term | Meaning | +|---|---| +| **Box** | a bucket | +| **Candy** | an object | +| **CandyKey** | an object key | +| **Syrup** | a data ledger that holds object bytes | + +*(Candy in a box — that's the whole theme.)* + +--- + +Candybox is open source under the [Apache License 2.0](https://github.com/predatorray/candybox/blob/main/LICENSE). +The source lives on [GitHub](https://github.com/predatorray/candybox). diff --git a/docs/content/docs/_index.md b/docs/content/docs/_index.md new file mode 100644 index 0000000..d428de9 --- /dev/null +++ b/docs/content/docs/_index.md @@ -0,0 +1,25 @@ +--- +title: Documentation +type: docs +weight: 1 +bookFlatSection: false +--- + +# Candybox documentation + +Welcome to the Candybox manual. Candybox is a distributed, S3-like object store backed by a +distributed LSM tree on Apache BookKeeper. + +Use the navigation on the left to browse, or jump to a common starting point: + +- **[Getting started]({{< relref "/docs/getting-started" >}})** — run a cluster and store your first object. +- **[Installation & downloads]({{< relref "/docs/installation" >}})** — Docker image, distribution tarball, and client library. +- **[Concepts & architecture]({{< relref "/docs/architecture" >}})** — the LSM engine, ledgers, and fenced ownership. +- **[Command-line client]({{< relref "/docs/client" >}})** — the `candybox` CLI and the Java client library. +- **[S3 gateway]({{< relref "/docs/s3-gateway" >}})** — the S3-compatible HTTP endpoint. +- **[Operations]({{< relref "/docs/operations" >}})** — running, monitoring, and Kubernetes. +- **[Configuration reference]({{< relref "/docs/reference/configuration" >}})** — every tunable key. + +> [!WARNING] +> Candybox is pre-`1.0` (current version `0.1.0-SNAPSHOT`). APIs, wire protocol, and on-disk formats +> may still change between releases. diff --git a/docs/content/docs/architecture/_index.md b/docs/content/docs/architecture/_index.md new file mode 100644 index 0000000..62bdea7 --- /dev/null +++ b/docs/content/docs/architecture/_index.md @@ -0,0 +1,102 @@ +--- +title: Concepts & architecture +weight: 30 +bookCollapseSection: false +--- + +# Concepts & architecture + +Candybox is layered top to bottom: an S3-like object API sits on a per-Box LSM engine, which talks to +two narrow SPIs, which in turn run on Apache BookKeeper (durable ledgers) and ZooKeeper +(coordination/metadata). + +``` +┌────────────────────────────────────────────────────────────┐ +│ Client API — S3-like object store: Boxes of Candy │ +├────────────────────────────────────────────────────────────┤ +│ LSM engine (candybox-lsm) │ +│ WAL → Memtable → SSTables → Manifest │ +│ Compaction · GC · HLC · single fenced owner │ +├──────────────────────────────┬─────────────────────────────┤ +│ LedgerStore SPI │ Coordination SPI │ +│ (candybox-bookkeeper) │ (candybox-coordination) │ +│ ledger roles: WAL, │ fencing tokens, │ +│ SSTable, Syrup, manifest │ manifest pointer CAS │ +├──────────────────────────────┼─────────────────────────────┤ +│ Apache BookKeeper │ ZooKeeper │ +│ (durable ledgers) │ (metadata / CAS) │ +└──────────────────────────────┴─────────────────────────────┘ +``` + +`candybox-common` (shared records, `BinaryWriter`/`BinaryReader` serialization, HLC, config) underpins +every layer. Object bytes never enter the LSM tree: candy lives in Syrups and the tree holds only +`CandyLocator` pointers. + +## How it works + +Candybox blends three well-known designs. + +- **A LevelDB-style LSM tree** for the index. Writes land in an in-memory *memtable* fronted by a + write-ahead log; when it fills, it is flushed to an immutable, sorted **SSTable** and later merged + into larger ones by background **compaction**. Reads merge the memtable and SSTables, newest wins. + +- **Object data kept out of the tree.** Object bytes are written to dedicated data ledgers + (*Syrups*); the LSM tree stores only a small **pointer** to where each object lives. This keeps the + index tiny and compaction cheap no matter how large the objects are. + +- **BookKeeper ledgers as the durable medium.** Every SSTable, WAL, manifest, and Syrup is a + BookKeeper ledger — append-only, replicated, and self-fencing. Candybox never mutates data in + place; updates and deletes are new appends (with tombstones), Apache-Pulsar-style. + +## Fenced ownership + +Consistency rests on **single, fenced ownership per partition**: every Box is split into a fixed +number of hash partitions, and at any moment exactly one node owns a partition, holding a ZooKeeper +lease with a *fencing token*. Every state-changing operation carries that token, so if ownership moves +during a failure, a stale former owner can no longer corrupt the partition. An elected balancer +spreads partition ownership evenly across the cluster, so one Box's writes are served by many nodes. +Each write is stamped with a hybrid logical clock for last-writer-wins ordering across nodes. + +## Operations the sorted LSM tree makes cheap + +Because keys are stored sorted and object bytes live behind small pointers, Candybox offers a few +operations an S3-style store cannot do cheaply: + +- **Bounded / reverse range scans** walk a `[start, end)` window in either direction, paging with + `--start-after`. +- **Zero-copy `copy` / `rename`** point a new key at the *same* stored bytes — no data is moved — and + `rename` removes the source atomically (within a Box). +- **`delete-range`** deletes a whole prefix or key window with a single range tombstone (constant work + regardless of how many keys it covers); the bytes are reclaimed lazily by compaction. + +See the [command-line client]({{< relref "/docs/client" >}}) for how to invoke them. + +## Deeper design docs + +The authoritative design write-ups live in the repository and go well beyond this overview: + +- [`DESIGN.md`](https://github.com/predatorray/candybox/blob/main/DESIGN.md) — record formats and the + fencing / handover protocol. +- [`BOX_PARTITIONING_PLAN.md`](https://github.com/predatorray/candybox/blob/main/BOX_PARTITIONING_PLAN.md) + — partitioning. +- [`MULTIPART_RANGE_PLAN.md`](https://github.com/predatorray/candybox/blob/main/MULTIPART_RANGE_PLAN.md) + — range GET and multipart upload. +- [`S3_GATEWAY_PLAN.md`](https://github.com/predatorray/candybox/blob/main/S3_GATEWAY_PLAN.md) and + [`AUTH_PLAN.md`](https://github.com/predatorray/candybox/blob/main/AUTH_PLAN.md) — the S3 gateway and auth. + +## Module layout + +| Module | Responsibility | +|---|---| +| `candybox-common` | Domain types, versioned serialization, configuration, CRC32C, bloom filter. | +| `candybox-bookkeeper` | The `LedgerStore` abstraction over BookKeeper — the only module that touches the raw BookKeeper client — with an in-memory fake. | +| `candybox-coordination` | Membership, fenced leases, and CAS key-value over ZooKeeper, with an in-memory fake. | +| `candybox-lsm` | The LSM engine: memtable, WAL, SSTables, Syrup chunking, manifest, merge/read path, compaction. | +| `candybox-protocol` | The framed TCP wire protocol and transport. | +| `candybox-server` | The storage node: wires the engine behind the protocol, plus the runnable entrypoint, health/metrics, and ownership. | +| `candybox-client` | The thin client library and the `candybox` command-line tool. | +| `candybox-s3-gateway` | A path-style, S3-compatible HTTP gateway (Netty) with optional SigV4 auth + S3 ACL enforcement. Stateless. | +| `candybox-admin-api` | A stateless HTTP service exposing cluster / boxes / LSM / metrics as JSON, plus the static SPA mount. | +| `candybox-web` | React + TypeScript + MUI dashboard, packaged into a jar so the admin API serves it from the classpath. | +| `candybox-dist` | Packages the runnable distribution (`bin/ lib/ conf/`) and the Docker/Kubernetes assets. | +| `candybox-integration-tests` | End-to-end tests on embedded BookKeeper + ZooKeeper. | diff --git a/docs/content/docs/client/_index.md b/docs/content/docs/client/_index.md new file mode 100644 index 0000000..1114eea --- /dev/null +++ b/docs/content/docs/client/_index.md @@ -0,0 +1,51 @@ +--- +title: Command-line client +weight: 40 +--- + +# Command-line client + +The `candybox` CLI is bundled in the same image as the storage node (the image is dual-mode). Point it +at a node with the `CANDYBOX_SERVER` environment variable or the `-s host:port` flag. + +The same operations are available programmatically through the `CandyboxClient` class in the +`candybox-client` module — see [installation]({{< relref "/docs/installation" >}}). + +## Basic object operations + +```bash +candybox create-box photos +candybox put photos cat.jpg cat.jpg --content-type image/jpeg +candybox get photos cat.jpg out.jpg +candybox head photos cat.jpg # size, content-type, checksum, metadata +candybox list photos # keys in the box +candybox list-boxes +candybox help # full command list +``` + +`put` reads from a file or, if you omit the path, from standard input; `get` writes to a file or to +standard output. + +## Operations the sorted LSM tree makes cheap + +```bash +candybox list photos --start a --end m --reverse # bounded, reverse-order range scan +candybox copy photos cat.jpg cat-copy.jpg # zero-copy: shares the stored bytes +candybox rename photos cat.jpg pets/cat.jpg # zero-copy move (same Box) +candybox delete-range photos thumbnails/ # one O(1) range tombstone, not N deletes +candybox delete-range photos --start a --end m # delete a half-open [start, end) key window +``` + +- **Bounded / reverse range scans** walk a `[start, end)` window in either direction (`list --start K + --end K --reverse`), paging with `--start-after`. +- **Zero-copy `copy` / `rename`** point a new key at the *same* stored bytes — no data is moved — and + `rename` removes the source atomically (same Box; when source and destination land in different hash + partitions the client transparently falls back to a byte copy, and the rename is no longer atomic). +- **`delete-range`** deletes a whole prefix or key window with a single range tombstone (constant work + regardless of how many keys it covers); the bytes are reclaimed lazily by compaction. + +## Range GET + +Object reads accept HTTP-style ranges through the gateway and client: `bytes=A-B`, `bytes=A-`, and +`bytes=-N`, returning the requested window. Multi-range requests are rejected. See the +[S3 gateway]({{< relref "/docs/s3-gateway" >}}) for the HTTP surface. diff --git a/docs/content/docs/getting-started/_index.md b/docs/content/docs/getting-started/_index.md new file mode 100644 index 0000000..6823ed8 --- /dev/null +++ b/docs/content/docs/getting-started/_index.md @@ -0,0 +1,12 @@ +--- +title: Getting started +weight: 10 +bookCollapseSection: true +--- + +# Getting started + +This section gets you from nothing to a running cluster that stores and serves objects. + +- **[Quick start]({{< relref "quickstart" >}})** — bring up the full stack with Docker Compose and store an object. +- **[Concepts]({{< relref "concepts" >}})** — the handful of terms and ideas the rest of the manual assumes. diff --git a/docs/content/docs/getting-started/concepts.md b/docs/content/docs/getting-started/concepts.md new file mode 100644 index 0000000..e951815 --- /dev/null +++ b/docs/content/docs/getting-started/concepts.md @@ -0,0 +1,41 @@ +--- +title: Concepts +weight: 2 +--- + +# Concepts + +Candybox borrows the bucket/object model from S3 but gives the pieces its own names. + +## Vocabulary + +| Term | Meaning | +|---|---| +| **Box** | a bucket — a named, flat namespace of keys | +| **Candy** | an object — the bytes plus metadata stored under a key | +| **CandyKey** | an object key | +| **Syrup** | a data ledger that holds object bytes | +| **CandyLocator** | the small pointer the index stores to find a Candy's bytes inside a Syrup | + +*(Candy in a box — that's the whole theme.)* + +## The data model + +A **Box** is a flat, sorted key space. Keys are stored in sorted order, which is what makes range +scans, prefix listing, and directory-walk workloads cheap sequential reads rather than scatter-gather. + +Each Box is split into a fixed number of **hash partitions**. At any moment exactly one node *owns* a +partition and serves its reads and writes; ownership is spread evenly across the cluster by an elected +balancer, so a single Box's traffic is served by many nodes. + +## Durability and consistency in one paragraph + +Object data and index both live in **BookKeeper ledgers** — append-only, replicated, and +self-fencing. Candybox never mutates data in place: updates and deletes are new appends (with +tombstones). Consistency rests on **single, fenced ownership per partition**: the owner holds a +ZooKeeper lease with a *fencing token*, and every state-changing operation carries that token, so a +stale former owner cannot corrupt a partition after ownership moves during a failure. Each write is +stamped with a hybrid logical clock for last-writer-wins ordering across nodes. + +For the full picture — record formats, the read/merge path, and the handover protocol — see +[Concepts & architecture]({{< relref "/docs/architecture" >}}). diff --git a/docs/content/docs/getting-started/quickstart.md b/docs/content/docs/getting-started/quickstart.md new file mode 100644 index 0000000..9f0aec7 --- /dev/null +++ b/docs/content/docs/getting-started/quickstart.md @@ -0,0 +1,61 @@ +--- +title: Quick start +weight: 1 +--- + +# Quick start + +The fastest way to try Candybox is the bundled Docker Compose stack. It starts the full system — +ZooKeeper, 3 BookKeeper bookies, 3 Candybox nodes, and the S3 gateway (on `:9711`) — using the +published [`zetaplusae/candybox`](https://hub.docker.com/r/zetaplusae/candybox) image. + +## With Docker Compose (recommended) + +From a checkout of the [repository](https://github.com/predatorray/candybox): + +```bash +docker compose up -d +docker compose run --rm cli create-box photos +echo 'hello candybox' | docker compose run --rm -T cli put photos hello.txt +docker compose run --rm cli get photos hello.txt # -> hello candybox +``` + +Tear it down with `docker compose down` (add `-v` to also wipe the data volumes). + +### Web dashboard + +The compose stack also brings up a stateless **admin / dashboard service** at +[`http://localhost:9713/ui/`](http://localhost:9713/ui/) — a React + TypeScript + MUI single-page app +that shows cluster topology, the box browser, LSM internals (manifest version + fencing token), and a +small set of time-series charts polled from each node's `/metrics`. The same process exposes a JSON +API at `/api/*` (see [Operations]({{< relref "/docs/operations" >}})). + +## Storing and retrieving objects + +The `zetaplusae/candybox` image is dual-mode: passing `candybox ` runs the command-line client +instead of a storage node. Point it at a node with `CANDYBOX_SERVER` (or `-s host:port`); to reach the +cluster started above, join its Compose network (`candybox_default` by default) and mount a directory +to exchange files. An alias keeps the commands readable: + +```bash +alias candybox='docker run --rm -i --network candybox_default \ + -e CANDYBOX_SERVER=candybox-1:9709 -v "$PWD:/data" -w /data zetaplusae/candybox candybox' + +candybox create-box photos +candybox put photos cat.jpg cat.jpg --content-type image/jpeg +candybox get photos cat.jpg out.jpg +candybox head photos cat.jpg # size, content-type, checksum, metadata +candybox list photos # keys in the box +candybox list-boxes +candybox help # full command list +``` + +See the [command-line client]({{< relref "/docs/client" >}}) for the full command set, including the +range scans, zero-copy `copy` / `rename`, and `delete-range` that the sorted LSM tree makes cheap. + +## Next steps + +- Read the [concepts]({{< relref "concepts" >}}) to understand Boxes, Candy, and Syrups. +- Talk to Candybox over S3 SDKs via the [S3 gateway]({{< relref "/docs/s3-gateway" >}}). +- Run it for real: see [installation]({{< relref "/docs/installation" >}}) and + [operations]({{< relref "/docs/operations" >}}). diff --git a/docs/content/docs/installation/_index.md b/docs/content/docs/installation/_index.md new file mode 100644 index 0000000..9033be3 --- /dev/null +++ b/docs/content/docs/installation/_index.md @@ -0,0 +1,82 @@ +--- +title: Installation & downloads +weight: 20 +--- + +# Installation & downloads + +There are three ways to get Candybox: the Docker image (available today), a distribution tarball +built from source, and the Java client library. + +## Docker image + +The container image is published to Docker Hub and is the recommended way to run Candybox: + +```bash +docker pull zetaplusae/candybox:latest +``` + +The image is **dual-mode** — it runs a storage node by default, and `docker run … zetaplusae/candybox +candybox ` runs the bundled client CLI instead. See the +[quick start]({{< relref "/docs/getting-started/quickstart" >}}) for a full Compose stack. + +## Distribution tarball + +> [!WARNING] +> **Not yet published.** Tagged releases and pre-built distribution archives are not available on +> [GitHub Releases](https://github.com/predatorray/candybox/releases) yet. Until the first release is +> cut, build the distribution from source (below). This page will be updated with download links when +> releases are published. + +The intended download, once releases are published, will look like: + +```bash +# Placeholder — not yet published +curl -LO https://github.com/predatorray/candybox/releases/download/v0.1.0/candybox-0.1.0-bin.tar.gz +tar xzf candybox-0.1.0-bin.tar.gz +``` + +### Build it from source + +Requirements: **Java 17+** and **Maven 3.9+**. No external services are needed to build — the +integration tests run an in-JVM BookKeeper (which bundles an in-process ZooKeeper). + +```bash +git clone https://github.com/predatorray/candybox.git +cd candybox +mvn -q -DskipTests package # builds the distribution archive under candybox-dist/target/ +``` + +This produces a `bin/ lib/ conf/` distribution (and the Docker/Kubernetes assets) from the +`candybox-dist` module. + +## Java client library + +To talk to Candybox programmatically, use the `CandyboxClient` class from the `candybox-client` +module. + +> [!WARNING] +> **Not yet published to Maven Central.** The coordinates below are a placeholder for the upcoming +> first release. Until then, build and `mvn install` the project locally (`mvn -q -DskipTests install`) +> to resolve the artifact from your local repository. + +{{< tabs "client-dep" >}} +{{< tab "Maven" >}} +```xml + + + me.predatorray.candybox + candybox-client + 0.1.0 + +``` +{{< /tab >}} +{{< tab "Gradle" >}} +```groovy +// Placeholder — not yet published to Maven Central +implementation 'me.predatorray.candybox:candybox-client:0.1.0' +``` +{{< /tab >}} +{{< /tabs >}} + +See the [command-line client]({{< relref "/docs/client" >}}) page for the CLI built on the same library. diff --git a/docs/content/docs/operations/_index.md b/docs/content/docs/operations/_index.md new file mode 100644 index 0000000..35f674d --- /dev/null +++ b/docs/content/docs/operations/_index.md @@ -0,0 +1,56 @@ +--- +title: Operations +weight: 60 +--- + +# Operations + +This page is a quick orientation for running Candybox. The authoritative operations guide is +[`OPERATIONS.md`](https://github.com/predatorray/candybox/blob/main/OPERATIONS.md) in the repository, +which covers topology, consistency & backpressure, failure modes & recovery, garbage collection, +observability, and security in depth. + +## Topology + +A Candybox deployment is a set of identical storage nodes plus their dependencies: + +- **ZooKeeper** — coordination and metadata (shared with BookKeeper). +- **BookKeeper bookies** — the durable, replicated ledger store. +- **Candybox nodes** — stateless-on-restart storage processes; ownership of Box partitions is leased + and balanced across them. +- Optionally the **S3 gateway** and the **admin / dashboard API**, both stateless and horizontally + scalable behind a load balancer. + +## Observability + +Each node exposes an HTTP endpoint (default port `9710`) with `/healthz`, `/readyz`, and `/metrics`. +Wire `/healthz` and `/readyz` to your liveness/readiness probes and scrape `/metrics`. + +### Admin / dashboard API + +The `candybox-admin-api` service exposes cluster topology, boxes, LSM internals, and metrics as JSON +under `/api/*`, and serves the [web dashboard]({{< relref "/docs/getting-started/quickstart" >}}) at +`/ui/`. It is stateless — run as many replicas as you like. + +## Security + +Candybox supports SASL + TLS between clients / gateway / admin API and the storage nodes, ACL-based +authorization, and authenticated connections to ZooKeeper and BookKeeper. See the +[Security section of `OPERATIONS.md`](https://github.com/predatorray/candybox/blob/main/OPERATIONS.md#security) +for the full configuration. + +## Running on Kubernetes + +A multi-stage `Dockerfile` at the repo root builds a node image straight from source +(`docker build -t candybox:latest .`). The image is **dual-mode**: it defaults to the storage node, +but `docker run … candybox ` runs the bundled client CLI instead. + +A `StatefulSet` + headless `Service` manifest lives under +[`examples/kubernetes/`](https://github.com/predatorray/candybox/tree/main/examples/kubernetes) (also +bundled into the distribution tarball under `examples/`). The StatefulSet gives each pod a stable +identity, so `node.id` and the advertised address derive automatically from the pod name, and +liveness/readiness probes hit the health endpoint. + +For a self-contained local cluster (ZooKeeper + bookies + nodes), use the +[`docker-compose.yml`](https://github.com/predatorray/candybox/blob/main/docker-compose.yml) described +in the [quick start]({{< relref "/docs/getting-started/quickstart" >}}). diff --git a/docs/content/docs/reference/_index.md b/docs/content/docs/reference/_index.md new file mode 100644 index 0000000..23e69ee --- /dev/null +++ b/docs/content/docs/reference/_index.md @@ -0,0 +1,10 @@ +--- +title: Reference +weight: 70 +bookCollapseSection: true +--- + +# Reference + +- **[Configuration]({{< relref "configuration" >}})** — the configuration keys a node reads and how to + override them. diff --git a/docs/content/docs/reference/configuration.md b/docs/content/docs/reference/configuration.md new file mode 100644 index 0000000..b5e64e1 --- /dev/null +++ b/docs/content/docs/reference/configuration.md @@ -0,0 +1,32 @@ +--- +title: Configuration +weight: 1 +--- + +# Configuration + +A node reads `conf/candybox.properties`. **Every key can be overridden by an environment variable** +named `CANDYBOX_` (dots become underscores, upper-cased) — for example +`CANDYBOX_ZOOKEEPER_CONNECT` — and the environment value wins. This makes it easy to ship one image +and configure each instance through the environment. + +## Common keys + +| Key | Meaning | Default | +|---|---|---| +| `node.id` | Cluster-unique node id. Falls back to the trailing number in `$HOSTNAME` (so a Kubernetes pod `candybox-2` becomes node `2`). | — | +| `zookeeper.connect` | ZooKeeper connect string, shared by BookKeeper and Candybox coordination. | `127.0.0.1:2181` | +| `server.bind` | Address clients connect to. | `0.0.0.0:9709` | +| `server.advertised` | Address published to the cluster for routing (set to a reachable hostname). | bind address | +| `health.port` | HTTP port for `/healthz`, `/readyz`, `/metrics`. | `9710` | +| `quorum.*` | BookKeeper replication per ledger role (`E/Qw/Qa`). | `3/3/2` (WAL, manifest), `3/2/2` (data) | +| `multipart.upload.ttl.millis` | How long an abandoned multipart upload is kept before a background sweep aborts it. | 7 days | + +## Full reference + +The complete, commented list of keys lives in +[`candybox.properties.example`](https://github.com/predatorray/candybox/blob/main/candybox-dist/src/conf/candybox.properties.example), +and the `CandyboxConfig` section of +[`OPERATIONS.md`](https://github.com/predatorray/candybox/blob/main/OPERATIONS.md#configuration-reference-candyboxconfig) +documents the operational meaning of each tunable, including the security, consistency, and garbage +collection settings. diff --git a/docs/content/docs/s3-gateway/_index.md b/docs/content/docs/s3-gateway/_index.md new file mode 100644 index 0000000..2723a40 --- /dev/null +++ b/docs/content/docs/s3-gateway/_index.md @@ -0,0 +1,49 @@ +--- +title: S3 gateway +weight: 50 +--- + +# S3 gateway + +The `candybox-s3-gateway` module is a **path-style, S3-compatible HTTP gateway** built on Netty that +translates the S3 REST/XML API onto the Candybox client. It is stateless and runs behind an HTTP(S) +load balancer. In the bundled Compose stack it listens on `:9711`. + +It supports optional **SigV4 authentication** and **S3 ACL enforcement**, so multi-user and +cross-account access patterns work the way S3 SDKs expect. + +## Using it from an S3 SDK + +Point any S3 client at the gateway endpoint with path-style addressing. For example, with the AWS CLI: + +```bash +aws --endpoint-url http://localhost:9711 s3 mb s3://photos +echo 'hello candybox' | aws --endpoint-url http://localhost:9711 s3 cp - s3://photos/hello.txt +aws --endpoint-url http://localhost:9711 s3 ls s3://photos/ +``` + +## Range GET and multipart upload + +Object reads accept HTTP `Range: bytes=A-B` (also `bytes=A-` and `bytes=-N`) and return `206 Partial +Content` with the right `Content-Range`; multi-range requests are rejected. + +Multipart upload is fully wired through the gateway: `CreateMultipartUpload` / `UploadPart` / +`CompleteMultipart` / `AbortMultipartUpload`, plus `UploadPartCopy` and `ListMultipartUploads` / +`ListParts`. Background TTL sweeps abandon stale uploads after `multipart.upload.ttl.millis` (7 days by +default). + +## Compatibility + +The gateway's S3 compatibility is verified against the industry-standard +[`ceph/s3-tests`](https://github.com/ceph/s3-tests) suite. The latest calibration runs the gateway +with **SigV4 auth + S3 ACLs enabled**: **192 / 838** boto3 functional tests pass, zero suite errors. +The extra passes over an unauthenticated gateway are the multi-user / ACL / cross-account-access tests +that real authentication unlocks (`bucket_acl_*`, `object_acl_*`, `access_bucket_*`, anonymous-access +and bad-auth checks). + +The remaining gaps the v1 gateway does not yet implement are **versioning, SSE, POST object, +lifecycle, bucket policy, CORS, and conditional GET**. See +[`compat/s3-tests/README.md`](https://github.com/predatorray/candybox/blob/main/compat/s3-tests/README.md) +for the family-by-family breakdown, and +[`S3_GATEWAY_PLAN.md`](https://github.com/predatorray/candybox/blob/main/S3_GATEWAY_PLAN.md) / +[`AUTH_PLAN.md`](https://github.com/predatorray/candybox/blob/main/AUTH_PLAN.md) for the design. diff --git a/docs/themes/hugo-book b/docs/themes/hugo-book new file mode 160000 index 0000000..4ae5c8a --- /dev/null +++ b/docs/themes/hugo-book @@ -0,0 +1 @@ +Subproject commit 4ae5c8a6bdcaae3a89ce384c9d5dc92ff1cee005