diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index a7519a9..d638130 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -12,6 +12,16 @@ name: CI # and `pull_request`, so neither had ever run. They are defined once, in the # workflow that actually has a schedule, rather than maintained in two places # and firing in neither. +# +# The two actions below are pinned to commit SHAs for the same reason the +# GitHub workflows are: a `@v4` tag can be repointed at other code by whoever +# controls the action's repository, and a SHA cannot. Read those SHAs from +# **code.forgejo.org**, not github.com — a Forgejo runner resolves a bare +# `actions/` against its DEFAULT_ACTIONS_URL, which is code.forgejo.org +# on Codeberg. The comment after each SHA is the release tag that carried it +# there. An instance configured to pull actions from somewhere else needs the +# SHAs re-resolved against that forge; the tags alone would silently work and +# silently mean something different. on: push: @@ -29,15 +39,19 @@ jobs: verify: runs-on: docker container: - image: docker.io/eclipse-temurin:21-jdk + # Must not be older than the `-java-output-version:25` in build.mill. + # Mill provisions its own JDK from .mill-jvm-version, so the image's JDK + # is what scala-cli and any non-Mill step run on; an older image would + # let those two disagree without failing anything. + image: docker.io/eclipse-temurin:25-jdk steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Cache Coursier - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: .cache/coursier - key: coursier-${{ hashFiles('build.mill', '.mill-version') }} + key: coursier-${{ hashFiles('build.mill', '.mill-version', '.mill-jvm-version') }} restore-keys: coursier- # verify.sh runs scripts/coverage-gate.sc through scala-cli, so the image diff --git a/.github/actions/scala-toolchain/action.yml b/.github/actions/scala-toolchain/action.yml index 774156a..4d3b0e0 100644 --- a/.github/actions/scala-toolchain/action.yml +++ b/.github/actions/scala-toolchain/action.yml @@ -1,22 +1,40 @@ name: Scala toolchain description: > - JDK 21 (Temurin), a workspace-local Coursier cache and scala-cli — everything - ./verify.sh needs and nothing it does not. + JDK 25 (Temurin), a workspace-local Coursier cache, scala-cli and the Coursier + CLI — everything ./verify.sh and scripts/site.sh need, and nothing they do not. # Every workflow in this repository sets its toolchain up through this one # action. Four copies of "install a JDK, warm the cache, fetch scala-cli" is # four places for CI to drift away from itself, and the first symptom of that # drift is a release built on a different JDK from the one the tests ran on. # -# scala-cli is not optional decoration: verify.sh runs scripts/coverage-gate.sc -# on every invocation and scripts/crap.sc under --with-slow, and both are -# scala-cli scripts. A runner without scala-cli fails at the coverage step. +# It is also the single place where the release job's toolchain comes from, so +# the two `uses:` below are pinned to commit SHAs rather than to `@v5` / `@v6`. +# A major-version tag is a mutable pointer the action's maintainers — or anyone +# who compromises them — can repoint at other code, and this action runs inside +# the job that holds the artifact signing key. The trailing comment names the +# release each SHA belongs to, so the line stays readable, and +# .github/dependabot.yml lists this directory explicitly so the two pins are +# proposed for update alongside the workflow-level ones. +# +# Neither tool is optional decoration. verify.sh runs scripts/coverage-gate.sc on +# every invocation and scripts/crap.sc under --with-slow, and both are scala-cli +# scripts, so a runner without scala-cli fails at the coverage step. scripts/site.sh +# resolves its mdoc and scaladoc classpaths with `cs fetch`, so a runner without the +# Coursier CLI fails the site build before it compiles a single page. inputs: java-version: - description: JDK feature release to install. Change it in one place, here. + description: > + JDK feature release to install. Change it in one place, here. Keep it in + step with `.mill-jvm-version`: that file decides which JDK Mill compiles + with, and `-java-output-version:25` in build.mill fails on anything older + than 25. This input governs what the *rest* of the job sees — scala-cli + scripts, `javap`, anything invoked outside Mill — so letting the two drift + apart means CI silently checks a different toolchain from the one that + produces the jars. required: false - default: "21" + default: "25" cache-prefix: description: > Cache key prefix. Give a job its own prefix when its Coursier footprint @@ -38,19 +56,20 @@ runs: echo "MILL_JVM_OPTS=-Xmx3g" >> "$GITHUB_ENV" - name: Set up JDK ${{ inputs.java-version }} (Temurin) - uses: actions/setup-java@v5 + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 with: distribution: temurin java-version: ${{ inputs.java-version }} - name: Cache Coursier - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .cache/coursier - # build.mill names every dependency; .mill-version names Mill's own. - # Nothing else changes what Coursier downloads, so nothing else belongs - # in the key. - key: ${{ inputs.cache-prefix }}-${{ runner.os }}-${{ hashFiles('build.mill', '.mill-version') }} + # build.mill names every dependency; .mill-version names Mill's own; + # .mill-jvm-version names the JDK Coursier provisions and unpacks into + # this same cache. Nothing else changes what Coursier downloads, so + # nothing else belongs in the key. + key: ${{ inputs.cache-prefix }}-${{ runner.os }}-${{ hashFiles('build.mill', '.mill-version', '.mill-jvm-version') }} restore-keys: | ${{ inputs.cache-prefix }}-${{ runner.os }}- @@ -76,3 +95,25 @@ runs: rm -f "$dir/scala-cli.gz" echo "$dir" >> "$GITHUB_PATH" "$dir/scala-cli" version --cli-version + + # scripts/site.sh calls `cs fetch` to resolve the mdoc and scaladoc + # classpaths, so the site build needs the Coursier CLI as a binary on PATH — + # having a Coursier *cache* is not the same thing. Pinned by version and + # checksum for the reason the block above gives. + - name: Install the Coursier CLI + shell: bash + env: + COURSIER_CLI_VERSION: "2.1.24" + COURSIER_CLI_SHA256: "d2c0572a17fb6146ea65349b59dd216b38beff60ae22bce6e549867c6ed2eda6" + run: | + set -euo pipefail + dir="$RUNNER_TEMP/coursier" + mkdir -p "$dir" + url="https://github.com/coursier/coursier/releases/download/v${COURSIER_CLI_VERSION}/cs-x86_64-pc-linux.gz" + curl -fsSL -o "$dir/cs.gz" "$url" + echo "${COURSIER_CLI_SHA256} $dir/cs.gz" | sha256sum --check --strict + gunzip -c "$dir/cs.gz" > "$dir/cs" + chmod +x "$dir/cs" + rm -f "$dir/cs.gz" + echo "$dir" >> "$GITHUB_PATH" + "$dir/cs" version diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8d970ff --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,81 @@ +# Keeps the commit-SHA action pins from rotting. +# +# Every `uses:` in .github/ names a 40-character commit SHA rather than a tag, +# so nobody can repoint the code a workflow runs (see the header of +# .github/workflows/release.yml for why that matters in the job holding the +# signing key). The cost of that safety is that a pin is frozen: a fix released +# upstream never arrives on its own, and an unattended pin drifts from +# "deliberate" to "nobody looked at this in two years", which is its own kind of +# stale. Dependabot closes that gap — it opens a pull request when a pinned +# action publishes a new release, rewriting both the SHA and the trailing +# version comment, and a human reviews and merges it like any other change. +# +# SCOPE — this file covers GitHub Actions only. +# +# Two things it deliberately does not cover: +# +# * Scala and Mill dependencies. Dependabot has no Mill support, and +# build.mill is where those versions live. `mill mill.scalalib.Dependency/ +# showUpdates` reports them today, and docs/READINESS.md still names +# Renovate as the eventual automation for them — Renovate also runs on +# Codeberg, which Dependabot does not. +# +# * .forgejo/workflows/ci.yml. Dependabot reads .github/workflows and the +# directories listed below; it does not see the Forgejo workflow, and it +# should not touch it anyway. Those SHAs come from code.forgejo.org, and +# Dependabot would resolve versions against github.com, where the same tag +# is different code. That file is updated by hand; its own header says so. + +version: 2 + +updates: + # `directory: "/"` means ".github/workflows" for this ecosystem — the four + # workflow files. It does not reach into .github/actions/, which is why the + # composite action gets its own entry below. + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + # Monday morning, so an update is waiting at the start of the week + # rather than landing on a Friday afternoon. + day: monday + time: "07:00" + timezone: Europe/Warsaw + # One pull request for all of them. Six separate pull requests to move six + # `actions/*` pins forward is six reviews of the same decision, and the + # reliable outcome of that is that none of them get reviewed. + groups: + github-actions: + patterns: + - "*" + open-pull-requests-limit: 5 + # Produces subjects like "ci(deps): bump actions/checkout from 7.0.1 to + # 7.0.2", which is the Conventional Commits form CLAUDE.md requires. + commit-message: + prefix: ci + include: scope + labels: + - dependencies + - github-actions + + # The composite action every workflow builds its toolchain through. Its two + # pins are the ones the release job runs with, so they matter at least as + # much as the workflow-level ones. + - package-ecosystem: github-actions + directory: "/.github/actions/scala-toolchain" + schedule: + interval: weekly + day: monday + time: "07:00" + timezone: Europe/Warsaw + groups: + github-actions: + patterns: + - "*" + open-pull-requests-limit: 5 + commit-message: + prefix: ci + include: scope + labels: + - dependencies + - github-actions diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0ad216..477a1a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: # longer has hung, not slowed down. timeout-minutes: 45 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # verify.sh's scalafmt step asks Git which Scala sources are tracked # (.scalafmt.conf sets project.git = true). A shallow checkout is @@ -49,13 +49,26 @@ jobs: - name: Verify run: ./verify.sh - # Coverage HTML is the one artifact worth keeping from a pull request: - # the gate reports a percentage, and the report says which lines. + # The coverage report is the one artifact worth keeping from a pull + # request: the gate prints a percentage, the report says which lines. + # + # It is XML, not HTML. verify.sh's coverage step runs + # `mill .scoverage.xmlReport` and nothing else, because XML is + # what scripts/coverage-gate.sc and scripts/crap.sc read. Mill can also + # render `scoverage.htmlReport`, but generating it here would mean a + # `mill` invocation in this workflow, which the header above rules out, + # and adding it to verify.sh would charge every local run for a page + # nothing reads. To read a downloaded report as a web page, run + # `./mill modules.__.scoverage.htmlReport` locally against the same + # commit. + # + # The path is the report directory itself rather than the whole + # `scoverage/` tree, which is mostly Mill's own task cache. - name: Upload coverage reports if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: scoverage-${{ github.run_id }} - path: out/modules/*/scoverage/ + name: scoverage-xml-${{ github.run_id }} + path: out/modules/*/scoverage/xmlReport.dest/scoverage.xml if-no-files-found: ignore retention-days: 14 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7be6f55..0df0156 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -37,7 +37,7 @@ jobs: # produced for this repository it will be hours, not minutes. timeout-minutes: 360 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/scala-toolchain with: @@ -65,7 +65,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # The pinned spec is a snapshot of a moving target. This does not fail # the build when Codeberg deploys a new Forgejo — it reports, so that a @@ -113,7 +113,7 @@ jobs: - name: Keep the live spec for comparison if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: live-swagger-${{ github.run_id }} path: ${{ runner.temp }}/live.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 42bdf3f..2cdbf26 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,19 @@ name: Release # The full procedure — how to set the version, how the key and credentials are # produced, how to verify the release landed, what to do about a bad one — is # in RELEASING.md. This file is that document made executable. +# +# Every third-party action below is pinned to a full commit SHA, with the +# release it belongs to in a trailing comment so the line stays readable. A tag +# like `v7` is a mutable pointer: whoever controls the action's repository can +# move it to different code at any time, and this job would fetch and run that +# code with MILL_PGP_SECRET_BASE64 — the signing key — in its environment. +# A commit SHA cannot be moved. `.github/dependabot.yml` opens a pull request +# when a pinned action publishes a new release, so the pins are reviewed and +# moved forward deliberately instead of quietly ageing. +# +# `./.github/actions/scala-toolchain` is exempt because it is not fetched from +# anywhere: a `uses:` beginning with `./` runs the copy in this repository at +# the commit being built, so it is already pinned by the checkout above. on: push: @@ -34,7 +47,7 @@ jobs: # artifact reaches Central. name: maven-central steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/scala-toolchain diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml index 95ac40c..de5e5be 100644 --- a/.github/workflows/site.yml +++ b/.github/workflows/site.yml @@ -34,7 +34,7 @@ jobs: group: site-build-${{ github.ref }} cancel-in-progress: true steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/scala-toolchain with: @@ -64,7 +64,7 @@ jobs: fi - name: Package for Pages - uses: actions/upload-pages-artifact@v5 + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: out/site/html @@ -90,4 +90,4 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: - id: deployment - uses: actions/deploy-pages@v5 + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 diff --git a/.gitignore b/.gitignore index b4a7368..4f94e5d 100644 --- a/.gitignore +++ b/.gitignore @@ -265,9 +265,13 @@ docs/ai/ # AI-generated plan / scratch documents # # These are agent working notes, not project docs. If you ever want to commit -# one deliberately, use: git add -f PLAN.md +# one deliberately, use: git add -f .md +# +# PLAN.md is the exception and is deliberately absent from the list below. It +# is tracked, because two dozen tracked files cite it by section number and a +# citation to an ignored file is a dead reference for everyone who clones this +# repository. Its own header says what it is and where it has been superseded. # --------------------------------------------------------------------------- -PLAN.md PLANS.md PLANNING.md IMPLEMENTATION_PLAN.md diff --git a/.mill-checksums b/.mill-checksums new file mode 100644 index 0000000..1bd82d2 --- /dev/null +++ b/.mill-checksums @@ -0,0 +1,50 @@ +# SHA-256 digests of the Mill distributions this repository is allowed to run. +# +# WHY THIS FILE EXISTS +# +# `./mill` downloads a ~60 MB executable over the network and then runs it. +# Every other command in this project — compile, test, publish, sign — goes +# through that executable. If the download were ever swapped for something +# else, nothing downstream would notice. `scripts/cpd.sh` pins PMD by digest +# and `.github/actions/scala-toolchain/action.yml` pins scala-cli by digest for +# exactly the same reason; the launcher was the one fetch left unpinned. +# +# FORMAT +# +# +# +# The distribution id is what `./mill` names the cached file: the Mill version, +# followed by the platform suffix when a native launcher is used (an empty +# suffix means the portable JVM launcher). Blank lines and lines whose first +# word starts with `#` are ignored. +# +# HOW TO ADD A VERSION +# +# All five entries below are the artifacts published to Maven Central at +# https://repo1.maven.org/maven2/com/lihaoyi/mill-dist// . +# To add a new version, download each artifact and hash it: +# +# v=1.1.8 +# for s in "" -native-linux-amd64 -native-linux-aarch64 \ +# -native-mac-amd64 -native-mac-aarch64 ; do +# url="https://repo1.maven.org/maven2/com/lihaoyi/mill-dist$s/$v/mill-dist$s-$v.exe" +# curl -fsSL "$url" -o /tmp/mill-dist +# # Maven Central publishes a .sha1 sidecar; check the download against it +# # before trusting the sha256 you are about to record. +# [ "$(curl -fsSL "$url.sha1")" = "$(sha1sum /tmp/mill-dist | cut -d' ' -f1)" ] \ +# || { echo "sha1 mismatch for $s" ; continue ; } +# printf '%s%s %s\n' "$v" "$s" "$(sha256sum /tmp/mill-dist | cut -d' ' -f1)" +# done +# +# Only versions listed here can be launched. Pointing `MILL_VERSION` or +# `.mill-version` at a version with no entry fails with an explicit message +# rather than running an unverified binary. +# +# Recorded 2026-08-09: every digest below was produced by the loop above, and +# each download was checked against its Maven Central `.sha1` sidecar first. + +1.1.7 49c1c575ad44efba1b9103ca1d6042624ef341aad9d8f4cb58fbfc56cf930e0d +1.1.7-native-linux-amd64 d8648b12f453947f503f3906339dac622261e8ea974339ff574b0f43e480dfdb +1.1.7-native-linux-aarch64 96ad26474945865ec4498f8133a915463c88ddba59392c23f9b4a8dfb74091a2 +1.1.7-native-mac-amd64 b003acea0aa81fa4d65391ade3c1113bb99b2c7c43571c39b7bc1a5657341875 +1.1.7-native-mac-aarch64 b50a55d06a1ef4eab5eab9b5e266af4b910b97bcf806cbbb24155b6c581ab8f1 diff --git a/.mill-jvm-version b/.mill-jvm-version new file mode 100644 index 0000000..db8427c --- /dev/null +++ b/.mill-jvm-version @@ -0,0 +1 @@ +temurin:25 diff --git a/.scalafmt.conf b/.scalafmt.conf index 599d622..e9ac6e1 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,4 +1,4 @@ -version = "3.11.4" +version = "3.11.5" runner.dialect = scala3 maxColumn = 120 # hard limit — SCALA_CODE_STYLE.md @@ -115,7 +115,7 @@ rewrite.imports.expand = false # # Putting `.*` before the scala/java groups is safe because scalafmt assigns an # import to its LONGEST matching pattern, not its first, so `scala.util.Try` -# still lands in the `scala\..*` group. Verified against scalafmt 3.11.4. +# still lands in the `scala\..*` group. Verified against scalafmt 3.11.5. rewrite.imports.groups = [ ["com\\.worxbend\\..*"] ["ox\\..*", "sttp\\..*", "com\\.github\\.plokhotnyuk\\..*"] diff --git a/CHANGELOG.md b/CHANGELOG.md index a51e772..312116b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,28 +17,38 @@ Nothing yet. First release. `build.mill` publishes `0.1.0-SNAPSHOT` until the tag is cut; this entry is the release note that tag will carry. +Nothing has been published to Maven Central yet, so "Changed" and "Fixed" below +are not a migration path from an earlier release — there is no earlier release. +They are there because the surface freezes at this tag: everything listed was +changed deliberately *before* the freeze, and anyone who built against a +`0.1.0-SNAPSHOT` jar in the meantime is the one audience that has to read them. + ### Added -- **Public `Future` API.** `CodebergClient` exposes seven endpoint groups — - `repos`, `users`, `issues`, `pulls`, `organizations`, `notifications` and - `misc` — plus `version`, covering 61 REST operations against Codeberg, - Forgejo or any Gitea-compatible instance. The base URI is configuration, not - a constant. +- **Public `Future` API.** `CodebergClient` exposes nine accessors — `repos`, + `users`, `issues`, `pulls`, `organizations`, `notifications`, `misc`, + `downloads` and `version` — which between them reach 38 API classes and 439 + REST operations against Codeberg, Forgejo or any Gitea-compatible instance. + The base URI is configuration, not a constant. That is 439 of the 439 + in-scope operations, 100 % (`docs/API_INVENTORY.md` §0), and 86.8 % of the + 506 the pinned spec declares — the remaining 67 are `admin`, `activitypub` + and `package`, which `PLAN.md` §0 puts out of scope for v1. - **Two error rails over one code path.** Every operation exists twice: the convenience rail fails the `Future` with `CodebergException`, and `.attempt` returns `Future[Either[CodebergError, A]]` and never fails. Both are projections of the same `Exec[F]` pipeline, so they cannot drift. -- **A closed error ADT with call context.** `CodebergError` has five cases — - `Transport`, `Api`, `DecodingFailed`, `Validation`, `RetriesExhausted`. Every - remote case carries a `CallContext` (a stable operation id, the HTTP method, - the redacted URI, the server's `x-request-id`, the attempt duration), so a - caller can tell *which* call failed without correlating logs. Forgejo's error - payloads are parsed into `ApiErrorBody` against captured samples. +- **A closed error ADT with call context.** `CodebergError` has six cases — + `Transport`, `Api`, `DecodingFailed`, `Validation`, `RetriesExhausted`, + `WalkTruncated`. Every remote case carries a `CallContext` (a stable operation + id, the HTTP method, the redacted URI, the server's `x-request-id`, the + attempt duration), so a caller can tell *which* call failed without + correlating logs. Forgejo's error payloads are parsed into `ApiErrorBody` + against captured samples. - **Link-header pagination.** List operations return `Page[A]` with the items, the total count and the next page parsed from the RFC 8288 `Link` header - rather than guessed from a page counter. `core.Pagination` provides the - sequential `listAll` and `foldPages` drivers, so walking every page is opt-in - and never materialises the whole collection by accident. + rather than guessed from a page counter. `paging.PageWalk` provides the + sequential `all`, `fold` and `foreach` drivers, so walking every page is + opt-in and never materialises the whole collection by accident. - **Retry that honours the server.** `RetryEngine` retries `429` and `5xx` on idempotent methods only, with jittered exponential backoff, and prefers the server's `Retry-After` over its own schedule when the policy allows it. @@ -48,33 +58,218 @@ this entry is the release note that tag will carry. captured in `CallContext` — can carry one. There are tests that assert it. - **A `Telemetry` port** for request/response visibility. The library has no logging dependency and writes nothing to stdout. +- **A bounded response body.** `CodebergConfig` carries + `maxResponseBodyBytes` (16 MiB, every textual response) and + `maxDownloadBodyBytes` (50 MiB, the two ZIP-fetching operations under + `client.downloads`). Exceeding either is `TransportCause.ResponseTooLarge`, + which is deliberately *not* retryable — a retryable oversize failure would + have downloaded the same oversized body once per attempt. - **Hexagonal module layout,** published as five artifacts under `com.worxbend`: `codeberg4s-domain` (no dependencies at all), `codeberg4s-core`, `codeberg4s-codec` (jsoniter-scala), `codeberg4s-transport` (sttp client4) and `codeberg4s-client`. Naming `codeberg4s-client` pulls in - the other four transitively. -- **Verification.** 1,072 unit tests, 54 golden fixtures captured from the live - API, scoverage thresholds, and a `verify.sh` gate that also enforces the - architecture boundaries (no sttp, jsoniter-scala or `Future` below `client`; no - `Await`; no bare exceptions). + the other four transitively. The jars are Java 25 bytecode (class-file major + version 69); an older JVM cannot load them. +- **Verification.** 3,658 unit tests, 54 golden fixtures captured from the live + API, scoverage thresholds enforced by `scripts/coverage-gate.sc`, and a + `verify.sh` gate that also enforces the architecture boundaries (no `sttp`, + `upickle`, `ujson`, `Future`, `ExecutionContext`, `Await`, `Promise` or + `blocking` imported below `client`; no `sttp` in `codec`; no bare + exceptions). Measured on the commit this entry describes: `domain` 100.00 % + statement and 100.00 % branch coverage, `core` 96.59 % / 92.48 %, `codec` + 95.20 % / 91.47 %, and the CRAP gate reports a worst method of 28.0 over + 2,217 methods against a limit of 30. + +### Changed + +Every item here is a breaking change against the `0.1.0-SNAPSHOT` builds, taken +now because the tag is what freezes the surface. + +- **Response models cannot be constructed from outside the library.** All 131 + response types — `Repository`, `Issue`, `PullRequest`, `User`, + `Organization`, `NotificationThread`, `ServerVersion`, `ApiErrorBody` and the + rest — have a `private[codeberg4s]` constructor, so `apply` and `copy` are + unavailable to callers. Reading fields and pattern matching are unaffected. + This is the deliberate price of *not* owing a major version every time + Forgejo adds a field to a response. A test fixture that used to build one + directly now has to obtain it from a client call or decode a recorded + payload. +- **`CodebergError` has a sixth case, `WalkTruncated(pagesVisited, + resumeFrom)`,** and `PageWalk.all` / `fold` / `foreach` now fail with it when + they hit the page cap. They previously returned the pages gathered so far, + which a caller could not tell apart from a genuinely short collection. An + exhaustive `match` on `CodebergError` needs the new clause. +- **`core.Pagination` is removed,** along with its `listAll` and `foldPages` + methods. `com.worxbend.codeberg4s.paging.PageWalk` replaced it: + `PageWalk.all(start)(fetch)` and `PageWalk.fold(start, zero)(fetch)(step)`. +- **A response body is bytes, not a `String`.** `CodebergResponse.body` is a + `ResponseBody` and `Decode[A].apply` takes one. Ask it for `bytes`, `text` or + `isBlank`. A test fake building a response writes + `ResponseBody.utf8("[]")`, or `ResponseBody.Empty` for a `204`. +- **A JSON number is `JsonValue.Int64(Long)` or + `JsonValue.Decimal(BigDecimal)`.** `JsonValue.Num` survives as an object + holding the constructors and an extractor, so `Num(7)` and + `case Num(value)` still compile, but it is no longer a type and no longer a + case of the ADT. +- **`JsonFields` holds the parser's `Vector[(String, JsonValue)]`** rather than + a `Map`. `fields.underlying` becomes `fields.entries` or `fields.toMap`. No + accessor changed, so a DTO that only calls `text`, `number`, `nested` and + friends needs no edit. +- **`UploadAsset` and `UploadAttachment` are built through `of` / `named` / + `as`,** not through their constructors, and `as` validates the media type, so + it answers `Either[ValidationError, …]`. Both also compare their content by + its bytes now, as do `BinaryResponse`, `RequestBody.Binary` and + `RequestBody.Multipart`; code that relied on two byte-identical values + staying distinct has to say `eq`. +- **`CodebergConfig` gains `maxResponseBodyBytes` and + `maxDownloadBodyBytes`,** so a call to the full constructor needs two more + arguments. `CodebergConfig.DefaultMaxResponseBodyBytes` and + `DefaultMaxDownloadBodyBytes` reproduce what `CodebergConfig(auth)` uses. +- **`UserTokenApi.RedactedBody` is removed.** Every credential-bearing response + is now redacted the same way by the pipeline; see Security below. +- **The jars require Java 25.** They were Java 17 bytecode before. A Java 17 or + Java 21 JVM fails with `UnsupportedClassVersionError`. + +### Security + +- **A base URI carrying credentials is rejected.** + `https://user:password@forge.example/api/v1` used to be stored verbatim and + concatenated into the redacted URI that every `CallContext` carries, so the + password reached every log line written about a failed call. `BaseUri.from` + now rejects user information, a query string and a fragment — without + echoing the offending value — and `Redaction.uri` strips the same three parts + independently, because test fakes call it with a plain `String` it has not + vetted. Embedded credentials were never sent as an `Authorization` header by + the JDK HTTP client underneath, so code relying on them was making anonymous + requests and leaking the password at the same time. Use + `Auth.Basic(username, password)`. +- **Dot segments are rejected in every single-segment identifier.** `Owner`, + `RepoName`, `Username`, `OrgName` and eleven more promised in their Scaladoc + that an accepted value could not forge a path, but accepted the exact values + `"."` and `".."`. No working traversal against a real deployment is claimed + here; the narrower claim stands on its own. Git itself forbids both as path + components, so no legitimate name is lost. +- **A credential cannot reach a decode-failure snippet.** `DecodingFailed` + carries an excerpt of the body that did not match, which is what makes it + diagnosable — except for the four responses whose success body *is* a live + secret (a created access token, an OAuth2 client secret, and the Actions + runner registration token in its several forms). Those now report + `*** (N bytes withheld)`. The decision moved into `ApiPipeline` because + `Telemetry.onError` fires while the attempt is being settled, before any + endpoint could rewrite the failure. +- **A configured credential is applied after the caller's headers,** which is + what its Scaladoc always claimed and the opposite of what the code did. No + endpoint in this library sets an `Authorization` header today, so this was + latent rather than live. `Authorization`, `Proxy-Authorization` and + `User-Agent` are also dropped from caller headers first, so a request cannot + carry two credentials under two spellings of one name. +- **A response body is bounded** — see `maxResponseBodyBytes` above. Before + this, the only thing standing between the client and its heap was the read + timeout multiplied by the peer's bandwidth. + +### Fixed + +- **The HTTP client the library created is now actually shut down.** + `CodebergClient.close()` called `backend.close()`, and that call released + nothing: sttp only ends a client it built itself if the `ExecutionContext` it + was handed is *not* also a `java.util.concurrent.Executor`, and every + ordinary `ExecutionContext` is one. An application building a client per + instance leaked a connection pool and a selector thread per instance. The + JDK `HttpClient` is now built here and ended with `shutdown()` — not + `close()`, which blocks until in-flight requests finish, whereas `close()` on + this library's client is documented to return promptly. +- **A retry waiting in backoff when the client closes now fails.** It used to + be left with no outcome at all: not fulfilled, not failed, so an application + shutting down cleanly waited on that `Future` for as long as the process + lived. Such a call now fails with `java.util.concurrent.CancellationException` + — not a `CodebergError`, because closing a client while it is in use is a + defect in the calling program and must not be laundered into something a + caller would retry. +- **`RetriesExhausted` is reported only when the policy actually gave up.** A + call that met a retryable `503` and then a terminal `404` reported + `RetriesExhausted(ctx, 2, Api(404))`, so the same `404` produced two + different error shapes depending on what preceded it. It now reports the bare + `Api(404)`. +- **A failing telemetry sink no longer fails the call it was observing.** +- **A duplicate JSON key has a settled meaning** — the first occurrence wins, + at every object width, with tests at both sides of the width threshold. +- **A colour with a trailing `U+0085`, `U+2028` or `U+2029` is rejected.** Java + regular expressions let `$` match before those line terminators and `trim` + does not remove them, so `LabelColor` used to accept the control character + and quietly discard it. + +### Performance + +Every figure below is from `scripts/alloc-bench.sh` on OpenJDK 64-Bit Server VM +25.0.4+7-LTS. `B/op` is heap bytes allocated per operation, the median of the +measured rounds. That harness is deliberately not part of `verify.sh`: it is a +measurement tool, not a gate. Read the header of `scripts/alloc-bench.sc` +before quoting any of this — in particular, wall-clock times on a working +machine vary by tens of percent between rounds and are a direction of travel, +not a figure. + +- **Decoding a 170,251-byte page of 50 repositories allocates 1,133,632 B/op, + down from 1,944,816 — 41.7 % less.** Two changes account for it. `JsonFields` + used to copy the parser's `Vector[(String, JsonValue)]` into a `Map` once per + object at every nesting level, which was 819,600 bytes of the old total; it + now reads the vector directly, scanning names for a narrow object and probing + a hash index for one of eight fields or more. (The index is not decoration: a + plain scan was measured first and made assembling one `Repository` 24 % + *slower* than the `Map` it replaced.) Separately, a whole JSON number is a + `Long` rather than a `BigDecimal`, worth 32.0 bytes per number — 33.7 % off a + thousand-element array of nine-digit identifiers. +- **The end-to-end response path allocates 963,360 B/op against 1,303,928 for + the same decode done via a `String` — 26.1 % less.** A body used to be + decoded from the socket's bytes into a `String` by sttp and then encoded + straight back into a `byte[]` by the parser. The saving is 340,568 bytes and + the page is 170,251 bytes, so it is those two copies and essentially nothing + else. The harness still measures both paths side by side + (`decode.page-50-bytes` against `decode.page-50-viastring`) so the claim can + be re-checked. +- **A Forgejo timestamp parses in about 25.7 ns and 40 B/op, from about 680 ns + and roughly 1.5 KB.** `OffsetDateTime.parse` is a general RFC-3339 reader; + Forgejo emits exactly one layout. The fast path reads that layout by index + and answers `None` for anything else, so the JDK stays the authority on what + is valid. The two paths were checked against each other over a million + generated inputs. +- **Smaller allocations removed from paths every call walks:** the redacted URI + is built once per call rather than once per attempt, and with one + `StringBuilder` rather than one `String` per percent-encoded octet; the + `Link` header is parsed once per response rather than up to three times, and + in linear rather than quadratic time; `Telemetry.noOp` no longer builds a + varargs `Seq` per callback; `FutureExec.attempt` uses one `transform` rather + than a `map` and a `recover`, which is one `Future` and one executor dispatch + instead of two; four hand-written element-decoding folds became one + tail-recursive helper that stops at the first failure instead of walking the + rest of the array. ### Known limitations -- Endpoint coverage is 61 of 439 in-scope operations. The rest is the long tail - of `repository` (actions, hooks, deploy keys, wikis, attachments) and `user` - (settings, stars, blocks, GPG keys, tokens), plus most write operations - outside issues, pulls and labels. See `docs/API_INVENTORY.md`. -- `listAll` and `foldPages` live on `core.Pagination`; they are not yet surfaced - as convenience methods on the client resource groups. -- `GET /repos/issues/search` is deferred — it returns a bare array rather than - the `{ok, data}` envelope the other search endpoints use. +- The mutation score is **unproven**. `scripts/mutate.sh` exists and the + Stryker4s runner is proven against this build, but no run with the real test + command has ever produced a score for this repository, so the ≥ 80 % target + in `docs/ROADMAP.md` is a target and not a result. +- Duplication is tracked, not eliminated. `scripts/cpd.sh` reports **363 + duplication groups** at 40+ tokens (PMD 7.26.0), and `verify.sh --with-slow` + passes because it fails on an *increase* over that recorded number rather + than on the existence of duplication. `docs/LEDGER.md` § "Helpers awaiting + promotion" names the ones with owners. +- Walking every page goes through `paging.PageWalk`, which takes the listing + operation as an argument; there is no `listAll` convenience method on the + client resource groups themselves. +- The library reads a whole response into memory and never streams, so a large + artifact is bounded rather than chunked — see `maxDownloadBodyBytes` above. - ScalaCheck property suites carry the `Property` tag and are excluded from the - default gate; `docs/ROADMAP.md` tracks how much of the planned surface they - reach. Runners exist for mutation testing (Stryker4s), duplication (PMD CPD) - and CRAP, but `docs/CONSTITUTION_MAPPING.md` is the authority on which of - them are actually proven against Mill and Scala 3. -- No binary-compatibility baseline. 0.1.0 is that baseline; MIMA gets wired - against it for 0.1.1. + default gate, and one of the three areas `PLAN.md` §6.3 names still has none: + the page walker has example-based tests only. `docs/ROADMAP.md` tracks the + rest, and `docs/CONSTITUTION_MAPPING.md` is the authority on which quality + runners are actually proven against Mill and Scala 3. +- No binary-compatibility baseline — 0.1.0 *is* that baseline. MIMA is wired + in `build.mill` and covers all five artifacts, but it has nothing to compare + against until 0.1.0 is on Maven Central, so it reports nothing until 0.1.1. + `RELEASING.md` § "Binary compatibility is checked by MIMA" is the procedure, + and measures what MIMA does and does not see through the response models' + `private[codeberg4s]` constructors. - Out of scope by design: OAuth2 token acquisition, ActivityPub federation, admin endpoints, attachment streaming above 50 MB, and Scala.js / Native. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0901c8c..3f10fc3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,10 +13,19 @@ is the source of truth for style; this document does not repeat it. ## Getting set up -You need a JDK — CI uses **Temurin 21** — and nothing else. Mill bootstraps +You need a JDK — CI uses **Temurin 25** — and nothing else. Mill bootstraps itself from the committed `./mill` script and `.mill-version`, and downloads everything into a Coursier cache. +You do not have to install Java 25 yourself to compile: `.mill-jvm-version` +pins **`temurin:25`**, and Mill downloads that JDK into the Coursier cache and +runs on it whatever your `java -version` says. That pin is load-bearing rather +than tidiness — `build.mill` sets `-java-output-version:25`, which is Scala 3's +name for `javac --release`, and a compiler running on an older JDK rejects it +outright with `25 is not a valid choice for -java-output-version`. A JDK 25 on +your PATH is still worth having, because `./verify.sh` shells out to scala-cli +scripts that Mill does not run. + ```bash git clone https://codeberg.org/worxbend/codeberg4s cd codeberg4s diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..f37c11f --- /dev/null +++ b/PLAN.md @@ -0,0 +1,313 @@ +# codeberg4s — Agent Implementation Plan + +Ground-up Scala 3 HTTP client library for the Codeberg (Forgejo) REST API v1. Future-based public API, sttp client4 transport, SoftwareMill ecosystem utilities, hexagonal architecture, full swarm-forge constitution compliance. + +--- + +## Status of this document — read first + +This is the **plan of record as written before implementation began**, kept +verbatim so that the twenty-odd files across this repository that cite it by +section number (`PLAN.md §3.1`, `PLAN.md ADR-3`, `PLAN.md §6.6`) point at +something a reader can actually open. It is a historical document, not a +description of the code as it stands. + +Where the code and this plan disagree, an ADR in [`docs/adr/`](docs/adr/) says +which won and why, and the ADR is authoritative. The divergences that matter: + +| This plan says | What shipped | Recorded in | +| --- | --- | --- | +| upickle for JSON (§3.3 ADR-3, §3.1 `codec`) | jsoniter-scala, with a hand-written codec over a document model rather than derived per-DTO codecs | [ADR-0003](docs/adr/0003-jsoniter-for-json.md) | +| package prefix `codeberg4s.*` (§3) | `com.worxbend.codeberg4s.*` | [ADR-0005](docs/adr/0005-future-public-api.md) | +| module `sttp-transport` (§3) | module `transport`, under the Mill `modules//src` layout | [docs/VERSIONS.md §2](docs/VERSIONS.md) | +| softwaremill/retry evaluated first (§3.3 ADR-4) | `RetryPolicy` implemented in `core`, no new dependency | [ADR-0004](docs/adr/0004-retry-implemented-in-core.md) | +| Gherkin acceptance pipeline (§2, §6.2), `acceptance-runner` module | not built; no Gherkin tooling exists in this repository | [CLAUDE.md](CLAUDE.md) § Acceptance tests, [docs/CONSTITUTION_MAPPING.md](docs/CONSTITUTION_MAPPING.md) | +| quicklens as a candidate utility (§0) | not a dependency | [docs/VERSIONS.md §6](docs/VERSIONS.md) | + +For what is actually built and what is left, read +[`docs/ROADMAP.md`](docs/ROADMAP.md). For the pinned versions, read +[`docs/VERSIONS.md`](docs/VERSIONS.md). For how the constitution's rules were +translated to this toolchain, read +[`docs/CONSTITUTION_MAPPING.md`](docs/CONSTITUTION_MAPPING.md). + +--- + +## 0. Mission Statement + +Build **codeberg4s**: a publishable, well-tested, hexagonally structured Scala 3 client library for the Codeberg API (`https://codeberg.org/api/v1`), which is the Forgejo v1 API. The library must: + +- Expose a **convenient `Future`-based interface** — no effect-system dependency leaks into the public API. +- Use **sttp client4** as the transport layer and other SoftwareMill utilities where they fit (sttp-model, quicklens, softwaremill/retry). +- Work against **any Forgejo/Gitea-compatible instance** via configurable base URL (Codeberg is the default, not a hardcode). +- Follow clean code, clean architecture, parse-don't-validate, rich error context propagation, and the swarm-forge engineering constitution (adapted for Scala — see §2). + +Non-goals (v1): OAuth2 token *acquisition* flows (accept pre-obtained tokens only), ActivityPub federation endpoints, admin endpoints, streaming attachments over 50 MB, Scala.js / Native cross-builds (design for it, don't ship it). + +--- + +## 1. Research Phase (Agent Task R1 — must complete before any code) + +### R1.1 Spec acquisition +1. Download the spec: `https://codeberg.org/swagger.v1.json` (Swagger 2.0). Mirror fallback: `https://code.forgejo.org/swagger.v1.json`. +2. Record the Forgejo version it corresponds to (`GET /api/v1/version`) in `docs/SPEC_PROVENANCE.md`. Vendor the spec file into `spec/swagger.v1.json` with a checksum — all model work references this pinned copy, never a live fetch. +3. Convert to OpenAPI 3 for tooling convenience if needed (`converter.swagger.io`), but treat the Swagger 2.0 original as source of truth. + +### R1.2 Spec inventory +Produce `docs/API_INVENTORY.md`: +- Full endpoint list grouped by tag (repository, issue, user, organization, notification, package, miscellaneous, settings, activitypub, admin). +- For each group: endpoint count, auth requirements, pagination behavior, known spec quirks. +- Mark each group **in-scope-v1 / deferred / out-of-scope** per §0. + +### R1.3 Known spec hazards to verify and document +These are documented pain points from other Forgejo client generators — verify each against the pinned spec: +- **Multiple security definitions** (BasicAuth, Token, AccessToken, AuthorizationHeaderToken, SudoParam, TOTPHeader). We support: `AuthorizationHeaderToken` (`Authorization: token `), Basic auth, and anonymous. Document the rest as unsupported. +- **Endpoints with union response types** — e.g. `/repos/{owner}/{repo}/contents/{filepath}` returns either a file object or a list. Model these as explicit Scala 3 union/enum ADTs, never `ujson.Value` passthrough. +- **Optionality lies**: Swagger 2.0 has no `nullable`; fields marked required may be absent in practice and vice versa. Every model must be validated against **live golden responses** (R1.4), not just the spec. +- **Pagination contract**: `page`/`limit` query params, `x-total-count` response header, RFC 5988 `Link` header. Verify which endpoints honor it. +- **Rate limiting**: capture actual headers Codeberg returns (Forgejo does not emit GitHub-style `X-RateLimit-*` on all deployments — verify empirically). +- **Error body shape**: Forgejo errors are typically `{"message": ..., "url": ...}` with occasional `errors: [...]`. Capture real samples for 401/403/404/409/422. + +### R1.4 Golden fixture harvest +Using anonymous read-only calls against codeberg.org (respect rate limits; a scratch account + token for authed shapes if available): +- Capture real JSON responses for every in-scope model into `modules/codec/test/resources/golden//.json`. +- These fixtures drive codec round-trip tests and are the ground truth over the spec. + +### R1.5 Ecosystem version resolution +Resolve **latest stable** versions at build start (constitution: no stale caches): Scala 3 LTS line, Mill, sttp client4, upickle, softwaremill/retry, munit, scalacheck, Stryker4s, scoverage, scalafmt, scalafix. Record in `docs/VERSIONS.md`. + +**Gate G-R:** API_INVENTORY.md + SPEC_PROVENANCE.md + at least 30 golden fixtures + hazard verification notes exist and are committed. No production code before this gate. + +--- + +## 2. Constitution Compliance (swarm-forge `engineering.prompt` → Scala) + +The constitution's startup tool table covers only Go, Clojure, and Java. Scala is unlisted, so we adopt the closest faithful equivalents and record the mapping in `docs/CONSTITUTION_MAPPING.md`: + +| Constitution requirement | Scala adaptation | +|---|---| +| Mutation tool (`mutate4*`) | **Stryker4s** (Mill via command-runner or sbt-shim module if needed; verify current Mill support at bootstrap, fall back to running Stryker4s CLI) | +| CRAP tool (`crap4*`) | **scoverage per-method coverage × cyclomatic complexity** via a small project-local `crap4scala` script (`scripts/crap.sc`, scala-cli) computing CRAP = comp² × (1 − cov)³ + comp from scoverage XML + scalameta complexity walk | +| DRY tool (`dry4*`) | **PMD CPD** with Scala language support; threshold config committed | +| Acceptance Pipeline (APS) | **Use as-is** — `gherkin-parser` / `gherkin-mutator` from `github.com/unclebob/Acceptance-Pipeline-Specification`, **Babashka variants preferred**, Go variants only as fallback. Install fresh from upstream at startup, never vendored/stale | +| Speclj / Clojure defaults | N/A (Clojure-only rules) | +| "Avoid Maven for Java tests; dedicated runners" | Analog: acceptance tests get a **dedicated runner module** (`modules/acceptance-runner`), not run through the general `mill __.test` sweep | + +Constitution rules adopted verbatim: +- **Small, reviewable increments** — every task in §7 must land as an independently green, conventional-commit PR ≤ ~400 changed lines. +- **Testable vs environmentally-unsuitable module separation** (§3 architecture enforces this): anything touching the live network (integration tests against a real Forgejo) lives in `modules/it` and is **excluded** from unit coverage, mutation, CRAP, and DRY-with-tests runs. +- **Property tests separated**: ScalaCheck suites live under a `Property` munit tag in dedicated `*Props.scala` files; excluded from normal unit coverage/mutation/CRAP unless explicitly requested. +- **Acceptance generation and acceptance tests run sequentially**, never concurrently with the whole-suite unit test command. +- **Gherkin mutation runs must emit periodic progress output** (wrap `gherkin-mutator` invocation in a script printing heartbeats). +- **Project-local caches**: `COURSIER_CACHE`, Mill `out/`, and tool caches pinned inside the worktree (`.cache/`); CI and agent sandboxes must not write outside the project. +- **Never hand-edit mutation/acceptance-mutation manifests**; only the tools update them. +- **Inspect `--help`/docs before relying on any unfamiliar command.** +- **Run local verification (`./verify.sh`, §6.6) before every handoff.** + +**Gate G-C:** All tools install fresh from upstream and run green on the empty skeleton before Phase 1 begins. + +--- + +## 3. Architecture + +Hexagonal / ports-and-adapters, enforced at Mill module boundaries. Domain never imports sttp, upickle, or `scala.concurrent`. + +``` +codeberg4s/ +├── build.mill +├── spec/swagger.v1.json # pinned, checksummed +├── modules/ +│ ├── domain/ # pure: models, error ADT, pagination types, ids +│ │ └── src/codeberg4s/domain/... +│ ├── core/ # ports + use-case logic, tagless over F[_] internally +│ │ └── src/codeberg4s/core/... +│ ├── codec/ # upickle ReadWriters, isolated from domain +│ ├── sttp-transport/ # adapter: sttp client4 request building/execution +│ ├── client/ # public Future façade — THE published artifact surface +│ ├── acceptance-runner/ # dedicated APS runtime + step handlers (constitution) +│ └── it/ # live-network integration tests (unsuitable boundary) +├── scripts/ # crap.sc, cpd.sh, gherkin-mutate.sh (heartbeat wrapper) +├── acceptance/ # .feature files + generated entrypoints +└── docs/ # inventory, provenance, versions, ADRs, ROADMAP.md +``` + +### 3.1 Module rules (enforced; adversarial reviewer checks every PR) +- `domain`: zero dependencies beyond stdlib. Opaque types for identifiers (`RepoName`, `Owner`, `IssueNumber`, `Sha`, `Token`), enums for states (`IssueState`, `MergeStyle`, …). **Parse, don't validate**: smart constructors return `Either[ValidationError, A]`; no `String`-typed domain fields where a refined type is meaningful. +- `core`: defines ports as traits over an abstract `F[_]` with a minimal internal capability typeclass (`Exec[F]` — pure/flatMap/raise/attempt; hand-rolled ~40 lines, **no cats dependency** to keep the published dependency footprint tiny). Contains cross-cutting logic: pagination driving, retry orchestration, error mapping. All unit-testable with `F = Either[CodebergError, *]` — synchronous, deterministic, mutation-testable. +- `codec`: upickle `ReadWriter`s only (sttp's upickle integration is the JSON path; fits the lihaoyi-stack preference and keeps transitive deps minimal). Golden-fixture round-trip tested. Codec failures never throw raw `upickle.core.Abort` outward — they map to `CodebergError.DecodingFailed` with context (§4). +- `sttp-transport`: the only module importing sttp. Implements the `HttpPort` from core: builds `Request`, executes on an injected `Backend[Future]`, translates responses to core's transport-level result type. `HttpClientFutureBackend` is the default, but the backend is constructor-injected → `SttpBackendStub` in tests, and users can bring OkHttp/Pekko backends. +- `client`: the public API. Instantiates core logic with `F = Future`. This is what users see; everything else is implementation detail (package-private where Mill allows, documented as internal otherwise). + +### 3.2 Public API shape + +```scala +val client: CodebergClient = CodebergClient( + CodebergConfig( + baseUri = uri"https://codeberg.org/api/v1", // default + auth = Auth.Token(sys.env("CODEBERG_TOKEN")), + retry = RetryPolicy.default, // backoff on 429/502/503/504 + userAgent = "codeberg4s/x.y.z", + ) +)(using ExecutionContext, backend: Backend[Future] = HttpClientFutureBackend()) + +// Resource-grouped, mirroring API tags: +client.repos.get(Owner("forgejo"), RepoName("forgejo")): Future[Repository] +client.issues.list(owner, repo, IssueQuery(state = IssueState.Open)): Future[Page[Issue]] +client.pulls.merge(owner, repo, PrNumber(42), MergeStyle.Squash): Future[Unit] +client.users.current(): Future[User] +``` + +Dual-rail error contract: +- **Convenience rail** (shown above): `Future[A]`, failing with `CodebergException` (carries the full `CodebergError` ADT — idiomatic for Future users). +- **Typed rail**: every op also available as `client.repos.attempt.get(...): Future[Either[CodebergError, Repository]]` for callers who refuse exceptions. Implemented once in core; the two rails are mechanical projections, not duplicated logic. + +Pagination conveniences: `Page[A]` carries items + `totalCount` (from `x-total-count`) + next-page handle; `client.issues.listAll(...): Future[Vector[Issue]]` drives pages sequentially with the retry policy applied per page, plus `foldPages` for bounded-memory processing. + +### 3.3 Key trade-offs (record as ADRs in `docs/adr/`) +1. **Hand-written curated models over codegen.** The Swagger 2.0 spec lies about optionality and has union types codegen handles badly (§1.3). Curated models + golden fixtures give correctness and clean-code naming; the cost (manual endpoint coverage) is mitigated by the wave plan (§7) and spec-diff checks in CI. ADR must document the rejected alternative (guardrail/openapi-generator/sttp-openapi). +2. **Hand-rolled `Exec[F]` over cats-effect/ZIO.** Future-first public API + minimal transitive deps for a publishable library (same reasoning that favored zio-config ergonomics in gitea4s — dependency footprint matters for libraries). Cost: ~40 lines of well-tested boilerplate. +3. **upickle over circe/jsoniter.** Aligns with lihaoyi-stack preference, tiny footprint, first-class sttp integration. Cost: fewer derivation knobs — mitigated by explicit `ReadWriter`s in codec (which we want anyway for golden-fixture discipline). +4. **softwaremill/retry (Future-native) evaluated first** for the retry port implementation; if its odelay dependency or maintenance status disqualifies it at version-resolution time, implement `RetryPolicy` in core (jittered exponential backoff, `Retry-After` header respected) — the port makes this swappable without API change. + +--- + +## 4. Error Context Propagation (design contract, not an afterthought) + +```scala +enum CodebergError: + case Transport(ctx: CallContext, cause: TransportCause) // DNS, TLS, timeout, connection + case Api(ctx: CallContext, status: StatusCode, body: ApiErrorBody) // 4xx/5xx with parsed Forgejo message + case DecodingFailed(ctx: CallContext, snippet: String, path: JsonPath, cause: String) + case Validation(field: String, message: String) // pre-flight, smart constructors + case RetriesExhausted(ctx: CallContext, attempts: Int, last: CodebergError) + +final case class CallContext( + operation: String, // "repos.get" — stable, greppable + method: Method, uri: Uri, // uri with token REDACTED + requestId: Option[String], // X-Request-Id if present + durationMs: Long, +) +``` + +Rules the reviewer enforces: +- Every failure path attaches `CallContext`. No bare `new Exception(msg)` anywhere in the codebase (scalafix custom rule or grep-based CI check). +- Decoding failures include a **bounded** body snippet (≤ 512 chars) and JSON path — enough to debug, no unbounded payloads in logs. +- Secrets never appear in errors, `toString`, or logs: `Token` is an opaque type with a redacting `toString`; property test asserts no configured token substring ever occurs in any rendered error (this one earns its place as a normal unit test, not just a property tag). +- `RetriesExhausted` preserves the last underlying error — no context loss through the retry loop. +- Public scaladoc on every operation documents which errors it can produce and when. + +--- + +## 5. Cross-Cutting Behaviors + +- **Auth**: `Auth.Anonymous | Auth.Token | Auth.Basic`. Applied in transport as a request transformation; core is auth-agnostic. +- **Retry**: policy on idempotent methods (GET/HEAD) for 429/5xx + transport failures; POST/PATCH/DELETE never auto-retried unless the caller opts in per-call. `Retry-After` honored when present. +- **Pagination**: as §3.2; `x-total-count` parsed defensively (absent header ≠ error). +- **Sudo / conditional requests / ETag**: out of scope v1, but `RequestCustomizer` hook (`Request => Request`) in config keeps the escape hatch open. +- **Logging**: no logging dependency in the library. A `Telemetry` port (callback trait: `onRequest/onResponse/onError`, no-op default) lets applications wire their own — keeps the published artifact silent and dependency-free. + +--- + +## 6. Testing & Verification Strategy + +### 6.1 Unit tests (munit) +- `core` logic tested with `F = Either` — fully synchronous, no `Future.await` flakiness, mutation-testing friendly. +- `sttp-transport` tested with `SttpBackendStub` (request-shape assertions: path, query encoding, headers, auth redaction). +- `codec` tested by golden-fixture round-trips (decode → encode → decode == identity where the API contract allows; decode-only otherwise). +- Coverage target: ≥ 90% line / ≥ 85% branch on `domain`+`core`+`codec`; transport measured but gated at ≥ 80% (stub-reachable paths). + +### 6.2 Acceptance tests (constitution APS pipeline) +- Gherkin `.feature` files per resource group in `acceptance/` (e.g. `issues.feature`: "listing open issues returns the first page with total count"). +- `gherkin-parser` + `gherkin-mutator` installed fresh from `unclebob/Acceptance-Pipeline-Specification` (Babashka preferred, Go fallback). +- Project components we own: acceptance entrypoint generator (scala-cli script emitting munit suites from parsed Gherkin), acceptance runtime, step handlers (backed by `SttpBackendStub` with golden fixtures — **testable module**, no live network), runner adapter, convenience scripts. +- Acceptance runs are **sequential** and via the **dedicated runner module**, never mixed into `mill __.test`. +- `scripts/gherkin-mutate.sh` wraps mutator runs with heartbeat output every 15s. + +### 6.3 Property tests (ScalaCheck, `Property` munit tag, `*Props.scala`) +- Codec laws over generated model instances; pagination driver invariants (no page fetched twice, ordering preserved); retry policy bounds (attempt count, monotone backoff, jitter within envelope). Excluded from normal coverage/mutation/CRAP runs per constitution. + +### 6.4 Mutation, CRAP, DRY +- **Stryker4s** on `domain`, `core`, `codec`; threshold ≥ 80% mutation score, manifest tool-managed only. +- **`scripts/crap.sc`**: fails build on any method with CRAP > 30; report committed to CI artifacts. +- **CPD**: fails on duplicated blocks > 40 tokens across production sources (test fixtures exempt). + +### 6.5 Integration tests (`modules/it` — the unsuitable boundary) +- **Testcontainers with a Forgejo image** (deterministic, seedable, CI-friendly) as primary target; optional live-Codeberg smoke suite behind `CODEBERG_IT=1` + token, read-only ops only, never in default CI. +- Excluded from coverage, mutation, CRAP, DRY-with-tests, and acceptance mutation. + +### 6.6 `./verify.sh` (pre-handoff, constitution-mandated, in order) +1. `scalafmt --check` + scalafix check +2. `mill __.compile` (fatal warnings on) +3. Unit tests (excluding `Property` tag and `it`) +4. Coverage report + threshold check +5. Acceptance generation → acceptance tests (sequential) +6. CPD, CRAP script +7. (nightly/pre-release only) Stryker4s + gherkin-mutator + +--- + +## 7. Phased Roadmap (ROADMAP.md-driven; each phase = milestone gate) + +### Phase 0 — Bootstrap (sequential, one agent) +Repo skeleton, Mill build with all modules, CI (Forgejo Actions or Woodpecker — decide by where the repo lives; mirror to GitHub if tooling needs it), constitution tooling installed fresh (§2), empty-walking-skeleton `verify.sh` green, `.editorconfig`, scalafmt/scalafix configs, conventional-commit lint hook, ADR template. +**Gate G0** = G-C + CI green on skeleton. + +### Phase 1 — Vertical slice (sequential — this de-risks everything) +One endpoint end-to-end: `GET /version` + `GET /repos/{owner}/{repo}` through all layers: domain model, codec + golden fixture, port, transport, Future façade, both rails, one acceptance feature, unit + property + IT coverage, full error paths (404, decode failure, timeout, retry-then-succeed). +**Gate G1**: vertical slice passes full `verify.sh` including mutation ≥ 80% on touched code; adversarial review sign-off on the architecture seams. **Everything after G1 is pattern replication.** + +### Phase 2 — Cross-cutting hardening (2 parallel tracks) +- **Track A**: retry engine, pagination driver + `Page`/`listAll`/`foldPages`, Telemetry port. +- **Track B**: full error ADT, error-body parsing against captured samples, redaction guarantees, `CodebergException` bridging. +**Gate G2**: both tracks merged, property suites green, no CRAP regressions. + +### Phase 3 — Endpoint waves (parallel tracks with beat rotation; each wave = one agent lane) +Priority order (value-weighted): +1. **users** (current user, keys, by-name lookups) +2. **repos** (CRUD, contents [union type!], branches, tags, releases, topics) +3. **issues** (CRUD, comments, labels, milestones) +4. **pulls** (CRUD, merge, reviews, diff/patch) +5. **orgs** (org, teams, membership) +6. **notifications** +7. **misc** (markdown render, settings, search) + +Per-wave definition of done: models from golden fixtures, codec round-trips, both rails, acceptance feature(s), scaladoc with error contracts, inventory checkbox flipped, coverage/CRAP/CPD thresholds hold. Deduplication ledger (`docs/LEDGER.md`) tracks shared models (User appears in issues, pulls, repos — first wave to need it owns it; later waves consume). +**Gate G3** per wave; **Gate G3-final** when in-scope-v1 inventory is 100% checked. + +### Phase 4 — Release engineering +- mdoc-checked README (every snippet compiles), scaladoc site, CHANGELOG (conventional-commit generated). +- Publishing: Maven Central via Sonatype Central portal, Mill publish setup, MIMA binary-compat baseline from 0.1.0 onward. +- Nightly job: spec drift detector (fetch live swagger, diff against pinned, open issue on divergence) + full mutation/acceptance-mutation run. +**Gate G4**: `0.1.0` published, README quickstart verified against live Codeberg by a human. + +--- + +## 8. Agent Orchestration + +- **Roles**: research agent (R1), implementer lanes (one per wave/track), **adversarial reviewer** (every PR: architecture-boundary violations, error-context gaps, constitution breaches, test-theater detection — asserts that tests would actually fail if the behavior broke), release agent (Phase 4). +- **Workflow**: ROADMAP.md is the single source of truth; agents claim tasks by PR referencing roadmap IDs; small increments (≤ ~400 lines); conventional commits; no unrelated changes or generated artifacts committed (constitution guardrail). +- **Reviewer veto list** (auto-reject): sttp/upickle import in domain or core; `Await.result` in production code; bare exceptions; unredacted token in any string rendering; new endpoint without golden fixture; acceptance manifest hand-edits; coverage/mutation threshold lowered without ADR. +- **Handoff protocol**: `verify.sh` green + roadmap checkbox + ledger update, or the handoff is invalid. + +--- + +## 9. Risk Register + +| Risk | Mitigation | +|---|---| +| Spec optionality lies → runtime decode failures | Golden fixtures as ground truth; `DecodingFailed` carries path+snippet; nightly drift detector | +| Forgejo API changes between Codeberg deploys | Pinned spec + provenance doc; drift detector opens issues, doesn't auto-break | +| Stryker4s/Mill integration friction | Verify at G0; CLI fallback documented; worst case run via thin sbt shim confined to CI | +| `Future` eagerness makes retry/pagination subtle | All logic in core over `Exec[F]`, tested synchronously with `Either`; Future is only the outermost projection | +| softwaremill/retry maintenance status | Port-based design; drop-in internal implementation ready (ADR-4) | +| APS Babashka tools fail in sandbox | Constitution-sanctioned Go fallback; verified at G-C, not discovered mid-flight | +| Rate limits during fixture harvest / IT | Testcontainers-Forgejo as primary IT target; live smoke opt-in only | + +--- + +## 10. Definition of Done (v1 / 0.1.0) + +- All in-scope inventory endpoints implemented on both rails with documented error contracts. +- `verify.sh` green; mutation ≥ 80%, coverage per §6.1, zero CRAP > 30, CPD clean. +- Acceptance features + Gherkin mutation run clean. +- Published to Maven Central; README quickstart works against live codeberg.org. +- Constitution mapping doc, ADRs 1–4, API inventory, and spec provenance all current. diff --git a/README.md b/README.md index 3d19ce2..18b9c8c 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,22 @@ def mvnDeps = Seq(mvn"com.worxbend::codeberg4s-client:0.1.0") libraryDependencies += "com.worxbend" %% "codeberg4s-client" % "0.1.0" ``` +### Requires a Java 25 runtime + +**The jars are compiled for Java 25** (class-file major version 69), the current +long-term-support release. A Java 21 or Java 17 JVM cannot load them: it fails +at class-load time with an `UnsupportedClassVersionError` naming "class file +version 69.0", which says nothing about which library caused it. Check what you +are on with `java -version` before adding the dependency. + +This is deliberate, and it does narrow who can adopt the library — see +[`CONTRIBUTING.md`](CONTRIBUTING.md#getting-set-up) for the same requirement on +the build side. Java 25 is a policy floor, not a technical one: the lowest +release the source actually compiles against is Java 21, because +`SttpHttpPort` calls `java.net.http.HttpClient.shutdown()` and that method was +added in Java 21. If a Java 21 baseline would unblock you, open an issue and +say so — moving the floor down is a one-line change to `build.mill`. + `codeberg4s-client` pulls in `-transport`, `-codec`, `-core` and `-domain` transitively. Depend on a narrower one if you want less: `codeberg4s-domain` is the models and the error ADT with no dependencies at all, which is enough to @@ -412,7 +428,7 @@ val attempted: Future[Either[CodebergError, Repository]] = Pick one per call site. `.attempt` is the convenience rail with its failure channel materialised, so the two cannot drift. -`CodebergError` is a closed family of five: +`CodebergError` is a closed family of six: | Case | Means | Reaction | | ------------------- | ---------------------------------------------------------------- | -------- | @@ -421,6 +437,7 @@ channel materialised, so the two cannot drift. | `DecodingFailed` | a 2xx payload did not match the model | retrying will not help; `path` and `snippet` are what a bug report needs | | `Validation` | a smart constructor rejected an argument | fix the argument | | `RetriesExhausted` | the retry engine gave up; `last` is preserved | surface `last` | +| `WalkTruncated` | a `PageWalk` hit its page cap with pages still to come | walk again from `resumeFrom`, or narrow the query | There is **no** `RateLimited` case. Forgejo reports rate limiting as an ordinary `429`, so it arrives as `Api(ctx, 429, body)` — and the retry engine @@ -430,6 +447,8 @@ which arrives as `RetriesExhausted` wrapping that `Api`. Every remote case carries a `CallContext` — operation id, method, redacted URI, optional request id, elapsed milliseconds — so you can tell *which* call failed without correlating logs. `error.describe` renders it, bounded and secret-free. +`Validation` and `WalkTruncated` carry none, because neither of them is a +request that reached a server. ## Pagination @@ -513,6 +532,16 @@ PageWalk.all(PageParams.First): params => `PageWalk.fold` and `PageWalk.foreach` are the bounded-memory forms — reach for those on a repository with tens of thousands of issues. +A walk visits at most `PageWalk.MaxPages` (10 000) pages, so an instance that +offers a next page forever cannot hang your process. Reaching that cap with the +server still offering another page **fails** the `Future` with +`WalkTruncated(pagesVisited, resumeFrom)` rather than handing back what it had +gathered: a short answer shaped exactly like a complete one is the failure mode +this whole section exists to prevent. `resumeFrom` is the window the walk was +about to request, page size included, so continuing is `PageWalk.all(resumeFrom)`. +A listing whose last page happens to be the ten-thousandth and offers nothing +further has ended naturally and succeeds. + ## Configuration ```scala @@ -532,22 +561,46 @@ val selfHosted: Either[ValidationError, CodebergConfig] = agent <- UserAgent.from("my-app/1.0") size <- PageSize.from(50) yield CodebergConfig( - baseUri = base, - auth = Auth.Anonymous, - retry = RetryPolicy.Default, - userAgent = agent, - defaultPageSize = size, - connectTimeout = 10.seconds, - readTimeout = 30.seconds, + baseUri = base, + auth = Auth.Anonymous, + retry = RetryPolicy.Default, + userAgent = agent, + defaultPageSize = size, + connectTimeout = 10.seconds, + readTimeout = 30.seconds, + maxResponseBodyBytes = CodebergConfig.DefaultMaxResponseBodyBytes, + maxDownloadBodyBytes = CodebergConfig.DefaultMaxDownloadBodyBytes, ) ``` -Every field is a validated type, so a misconfigured client fails at -construction rather than on its first call. `CodebergConfig.toString` is safe to -log: the credential types redact themselves. +Every field naming a domain concept is a validated type, so a misconfigured +client fails at construction rather than on its first call. The timeouts and the +two byte bounds are plain quantities and are taken as given. +`CodebergConfig.toString` is safe to log: the credential types redact +themselves. `Auth` is `Anonymous`, `Token(ApiToken)` or `Basic(username, Password)`. +### Response size + +This library reads a whole response into memory; it does not stream. So every +request carries a byte bound, and a body that passes it is abandoned part-read +as `CodebergError.Transport(ctx, TransportCause.ResponseTooLarge(detail))`. + +- `maxResponseBodyBytes` — 16 MiB, applied to every textual response. The + largest JSON body Forgejo produces is a file's contents, a blob capped by the + instance's `default_max_blob_size` (10 MiB on codeberg.org) and then + base64-encoded, which costs four bytes per three; 16 MiB clears that. +- `maxDownloadBodyBytes` — 50 MiB, applied only to `client.downloads`, which + fetches ZIP archives. An artifact is whatever a workflow uploaded, so nothing + about `default_max_blob_size` bounds it, and one shared number would have had + to be either too small for ordinary artifacts or too large to bound JSON + usefully. + +Exceeding either bound is **not** retried. Repeating the call would download the +oversized body once per attempt, which turns one oversized response into +`maxAttempts` of them. + ### Retries `RetryPolicy.Default` is 3 attempts, 250 ms base delay, 8 s ceiling, full diff --git a/RELEASING.md b/RELEASING.md index c18604c..c432923 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -78,15 +78,50 @@ is actually in the jar. - Add a new value to an *open* enum such as `NotificationSubjectType`, which already carries an `Other(raw)` case precisely so that upstream additions are not breaking. +- **Add a field to a response model** — a model decoded from a Forgejo payload + and never built by a caller, such as `Repository`, `Issue`, `PullRequest`, + `User` or `ServerApiSettings`. Their constructors are `private[codeberg4s]`, + so no caller outside this library can be calling `apply` or `copy`, and a + new field cannot break source compatibility for anyone. See the note below + on why this exemption exists and what it does not cover. ### These are breaking, and are never a patch -- **Adding, removing or reordering a field on a public `final case class`.** +- **Adding, removing or reordering a field on a *command* `final case class`.** The generated `apply`, `copy` and `unapply` change signature, so previously - compiled callers fail to link. Every model in this library is a case class, - so this is the most likely way to break the world by accident. Adding a - field with a default does not help: default arguments are banned here - anyway, and they would not make it binary compatible. + compiled callers fail to link. A command model — `CreateIssue`, + `EditRepository`, `MergePullRequest`, every `*Query` — has a public + constructor because a caller has to build one to make a request, so its + shape is part of the API. Adding a field with a default does not help: + default arguments are banned here anyway, and they would not make it binary + compatible. + +#### Why response models are exempt + +Forgejo adds fields to its response payloads routinely. When every model was a +public case class, each of those additions changed a generated ``, +`apply`, `copy` and `copy$default$N` in this library, so tracking upstream +meant a breaking release — a major version, once `0.1.0` is the baseline. That +would have been a major version of this library for a field nobody asked for. + +The constructors of response models are therefore `private[codeberg4s]`. Every +source tree in this repository lives under `com.worxbend.codeberg4s`, so each +DTO's `toDomain` and every test fixture still builds them; only code outside +the library loses the constructor. Reading fields and pattern matching are +untouched, so a caller can still destructure a `Repository` in a `match`. + +Two things this exemption does **not** cover: + +- **Removing, renaming or retyping an existing field is still breaking**, on + both source and binary compatibility. The exemption is for growth only. +- **A qualified-private constructor is still public in the bytecode**, so a + binary-compatibility checker can still complain about one. It complains far + less than expected, and only about one member — see + ["What MIMA reports for a response model"](#what-mima-reports-for-a-response-model) + below, which measures it. The short version: put a new field **last**, and + the whole cost is one filter line for that model. Those filters are lines to + write, not releases to renumber, because no external caller can have + compiled against the member being filtered. - **Adding a case to a closed `enum`.** `CodebergError` has exactly five cases — `Transport`, `Api`, `DecodingFailed`, `Validation`, `RetriesExhausted` — and every consumer that matches on it exhaustively @@ -110,37 +145,210 @@ is actually in the jar. now a `ValidationError`. Compatible at link time, breaking at run time, which is worse. -### Binary compatibility is not currently enforced +### Binary compatibility is checked by MIMA -**MIMA is not wired, because there is no baseline to check against.** No -version of this library has ever been published, so there is nothing for a -compatibility checker to compare a build to; adding it now would be a plugin -that runs and reports on the empty set. +MIMA — the Migration Manager — is the tool that turns the policy above from a +promise into a check. It reads the class files this build produces, reads the +class files of an already-released version, and reports every difference that +would stop a program compiled against the old jar from linking against the new +one. It works on bytecode, so it catches the changes that are invisible from +the source side: a `copy` overload that quietly changed arity, an opaque type +whose representation moved. -**This is the first task after `0.1.0` ships.** The work is: +It is wired. `build.mill`'s header declares the plugin -```scala -//| mvnDeps: +``` //| - com.github.lolgab::mill-mima::0.2.2 +``` + +and `Codeberg4sPublishModule` mixes in `com.github.lolgab.mill.mima.Mima`. +The two colons before the version are Mill's "add the Mill platform suffix" +spelling: Mill resolves that coordinate to `mill-mima_mill1_3`, the build for +the Mill 1.x line that `.mill-version` pins at `1.1.7`. `0.2.2` was the latest +stable version in +`repo1.maven.org/maven2/com/github/lolgab/mill-mima_mill1_3/maven-metadata.xml` +when this was written. + +Mixing it into the shared trait is what makes one declaration cover all five +artifacts. The plugin builds each coordinate to download out of +`pomSettings().organization`, `artifactId()` and `mimaPreviousVersions`, so +`modules.codec` is checked against `com.worxbend:codeberg4s-codec_3` without +that string appearing anywhere. -import com.github.lolgab.mill.mima.Mima +Run it across all five, or one module at a time: -trait Codeberg4sPublishModule extends Codeberg4sModule with PublishModule with Mima: - def mimaPreviousVersions = Seq("0.1.0") +```bash +./mill modules.__.mimaReportBinaryIssues +./mill modules.domain.mimaReportBinaryIssues ``` -on the shared publish trait, so all five artifacts are covered by one -declaration, plus a `./mill modules.__.mimaReportBinaryIssues` step in -`verify.sh`. `com.github.lolgab::mill-mima::0.2.2` is published for Mill 1.x -(`mill-mima_mill1_3`), which is the Mill in `.mill-version`; the trait is -`com.github.lolgab.mill.mima.Mima` and the task is `mimaPreviousVersions`. -That coordinate and those names were read off the published artifact, not -recalled — but the plugin has not been run against this build, so treat the -snippet as the starting point of that task and not as a verified -configuration. +#### It cannot pass yet, and that is the intended state + +Nothing has been published, so there is no jar to compare against. +`Publish.binaryCompatibleWith` in `build.mill` is therefore `Seq.empty`, and +the command stops with the plugin's own message: + +``` +[error] modules.domain.mimaPreviousArtifacts No previous artifacts configured. +Please override mimaPreviousVersions or mimaPreviousArtifacts. +``` -Until then, the policy above is enforced by review alone. Say so in the pull -request when a change touches a public signature. +That is the honest answer to "is this build compatible with nothing?", and it +costs nothing, because no other task depends on that command. `compile`, +`test` and `verify.sh` do not reach it, so the empty list cannot fail the +gate. + +**It is deliberately not in `verify.sh`.** The check downloads the previous +artifacts from Maven Central, and the fast gate has to run offline and in +seconds. It belongs in the release procedure instead, which is where the +checklist at the bottom of this document puts it. + +#### Turning it on, after `0.1.0` is published + +In the follow-up commit that moves `main` on to the next `-SNAPSHOT`, change +one line in `build.mill`: + +```scala +val binaryCompatibleWith: Seq[String] = Seq("0.1.0") +``` + +From then on the list holds every release inside the current compatibility +window. Under Early SemVer that is every `0.1.x` while the minor is still +`1`; when a deliberate break bumps the minor to `0.2.0`, the list resets to +just `0.2.0` and grows again from there. + +One trap is worth naming before somebody hits it. Because +`mimaPreviousVersions` lives on the shared trait, **every module that will +ever extend that trait inherits the claim that each listed version of it +exists on Central**. A sixth artifact first published in, say, `0.3.0` would +send MIMA looking for a `codeberg4s-newthing_3:0.1.0` that was never uploaded, +and the run would fail on a download error that says nothing at all about +compatibility. Such a module overrides the list with the releases that really +exist for it; the scaladoc on `Codeberg4sPublishModule.mimaPreviousVersions` +carries the snippet. + +#### What MIMA reports for a response model + +The policy above lets a `0.x.0` add a field to a response model, on the +grounds that its constructor is `private[codeberg4s]` and no outside caller +can be calling it. Scala erases qualified private to plain `public` bytecode, +so the obvious worry is that MIMA sees ``, `apply` and `copy` on all +131 of those classes and objects to a field being added to any of them. + +**MEASURED, NOT RECALLED.** On 2026-08-09, against mill-mima `0.2.2` and +Scala `3.8.4`, the worry turns out to be mostly unfounded, and the part that +survives is one line per model: + +| Change | Problems reported | +| --- | --- | +| Field appended to `HeatmapEntry` (2 fields, `private[codeberg4s]`) | 1 | +| Field appended to `User` (21 fields, `private[codeberg4s]`) | 1 | +| Field inserted at position 1 of `User` | 6 | +| Field inserted at position 1 of `CreateIssue` (public constructor) | 14 | + +The single problem in the first two rows is always the same shape: + +``` +* static method apply(...)com.worxbend.codeberg4s.users.User + in class com.worxbend.codeberg4s.users.User + does not have a correspondent in current version + filter with: ProblemFilter.exclude[DirectMissingMethodProblem]( + "com.worxbend.codeberg4s.users.User.apply") +``` + +Read the words `static method … in class`. That is not the companion object's +`apply`; it is the **static forwarder** Scala 3 emits on the class so Java +callers can reach the companion's method. MIMA does read Scala 3's own +signature and does honour `private[codeberg4s]` — the constructor (`this`), +`copy`, every `copy$default$N` and the companion object's real `apply` are all +correctly treated as inaccessible and never reported. The forwarder is the one +member that carries no Scala-side access information, so it leaks, and it +leaks exactly once per model. + +Two consequences, both practical: + +- **Append new fields; never insert them.** The `_1`, `_2`, … accessors that + `Product` requires are genuinely public and genuinely change result type + when a field is inserted ahead of them. Row three above is the same + one-field change as row two, moved to the front, and it costs five extra + reports that are not synthetic noise. Appending keeps the bill at one line. +- **The exemption really is about response models.** Row four is the same + insertion into a command model, whose constructor is public: `this`, `copy`, + every `copy$default$N`, both `apply`s and the `_N` accessors are all + reported. That is the check doing its job, and it is why the filter written + below names one class at a time. + +##### The filter, and how to write it + +`mill-mima` accepts filters through `mimaBinaryIssueFilters`. The name is +matched against the fully qualified member name and `*` is a wildcard that +spans package dots — both spellings below were confirmed to clear the report +in the measurement above. **Use the exact one.** A wildcard such as +`"com.worxbend.codeberg4s.users.*"` also works, and that is the problem: it +would silence a genuinely breaking change to a command model in the same +package just as effectively. + +So the filter is written **when a field is actually added**, one line for the +model that gained it, in the same commit — not as a standing 131-line blanket +that hides nothing today and something real tomorrow. There is no +`mimaBinaryIssueFilters` in `build.mill` right now for exactly that reason. + +When the day comes, widen the import in `build.mill` and add the override to +`Codeberg4sPublishModule`: + +```scala +import com.github.lolgab.mill.mima.{DirectMissingMethodProblem, Mima, ProblemFilter} + +// …inside trait Codeberg4sPublishModule… + + /** One line per response model that gained a field since the versions in + * `Publish.binaryCompatibleWith`. Each entry filters the static `apply` + * forwarder Scala 3 emits for a `private[codeberg4s]` constructor, which no + * caller outside this library can have compiled against. + */ + def mimaBinaryIssueFilters = Task { + Seq( + // 0.2.0: Forgejo added `pronouns` to the user payload. + ProblemFilter.exclude[DirectMissingMethodProblem]("com.worxbend.codeberg4s.users.User.apply") + ) + } +``` + +Note it is `ProblemFilter.exclude`, singular — sbt-mima spells the same thing +`ProblemFilters.exclude`, and the plural does not compile here. + +Every entry carries a comment naming the release and the field, so the list +can be pruned when the compatibility window resets at the next minor bump. +Anything MIMA reports that is *not* that one forwarder shape is a real +finding: read it against the policy above and renumber the release rather than +filtering it. + +##### Reproducing the measurement + +The numbers above are worth re-taking whenever Scala or mill-mima moves, +because they are a fact about a compiler's code generation, not about this +library. The procedure, which touches nothing outside the worktree except a +local Ivy directory it then deletes: + +```bash +# 1. Give MIMA something to compare against. `publishLocal` writes to +# ~/.ivy2/local, which Coursier searches by default, so the plugin can +# resolve it with no network and no Central involved. +sed -i 's/0.1.0-SNAPSHOT/0.1.0/' build.mill # temporarily +./mill modules.domain.publishLocal + +# 2. Point the check at it, and make the change being measured. +# Set `binaryCompatibleWith` to Seq("0.1.0") and edit a model. +./mill modules.domain.mimaReportBinaryIssues + +# 3. Put everything back. Leaving a fake 0.1.0 in the local Ivy cache would +# make a later run check against a jar nobody released. +git checkout -- build.mill modules/domain/src +rm -rf ~/.ivy2/local/com.worxbend/codeberg4s-domain_3 +``` + +Until `0.1.0` exists on Central, the policy in this section is enforced by +review. Say so in the pull request when a change touches a public signature. --- @@ -235,7 +443,21 @@ an artifact becomes immutable. ./verify.sh --with-slow # duplication against its baseline, and CRAP ``` -5. **Prove the artifacts assemble** before asking a public repository to +5. **Check binary compatibility**, which the gate deliberately leaves out + because it needs the network: + + ```bash + ./mill modules.__.mimaReportBinaryIssues + ``` + + Skip this one for `0.1.0` only — `Publish.binaryCompatibleWith` is empty + until `0.1.0` is on Central, so there is nothing to compare against and the + command says so. From `0.1.1` onwards it must be green, or every report it + makes must be either a filter written per + ["The filter, and how to write it"](#the-filter-and-how-to-write-it) or a + reason to renumber the release. + +6. **Prove the artifacts assemble** before asking a public repository to accept them: ```bash @@ -247,13 +469,13 @@ an artifact becomes immutable. try. This catches a broken POM, a missing transitive dependency and a `docJar` that failed, none of which the test suite can see. -6. **Commit, on a branch, with the version bump as its own change.** +7. **Commit, on a branch, with the version bump as its own change.** ``` chore(build): release 0.1.0 ``` -7. **Tag the merge commit** and push the tag. +8. **Tag the merge commit** and push the tag. ```bash git tag -a v0.1.0 -m "codeberg4s 0.1.0" @@ -264,11 +486,11 @@ an artifact becomes immutable. The workflow refuses to publish when the two disagree, and refuses a tag whose version still ends in `-SNAPSHOT`. -8. **Watch the workflow.** It re-runs `./verify.sh` from a clean checkout +9. **Watch the workflow.** It re-runs `./verify.sh` from a clean checkout before publishing anything. A release is not exempt from the gate; it is the build that most needs it. -9. **Verify it landed** — see below — and only then announce it. +10. **Verify it landed** — see below — and only then announce it. ### What `publishAll` actually does @@ -334,7 +556,10 @@ Publication to Central is asynchronous. Do all four: Then push a follow-up commit setting `Publish.version` to the next `-SNAPSHOT`, so `main` is never sitting on a version that has already been -published. +published. The same commit adds the version just released to +`Publish.binaryCompatibleWith`, which is what arms MIMA for the next release — +the artifacts are on Central by now, so there is finally something to compare +against. --- @@ -376,6 +601,8 @@ Copy this into the release pull request. - [ ] `CHANGELOG.md` entry written, breaking changes first - [ ] `Publish.version` set, `-SNAPSHOT` dropped - [ ] `./verify.sh` and `./verify.sh --with-slow` green +- [ ] `./mill modules.__.mimaReportBinaryIssues` green, or every report + filtered with a reason (not applicable to `0.1.0`) - [ ] `./mill modules.__.publishLocal`, then something compiled against it - [ ] Public signature changes reviewed against the versioning policy above - [ ] Version bump committed on its own, `chore(build): release X.Y.Z` @@ -383,3 +610,5 @@ Copy this into the release pull request. - [ ] Release workflow green - [ ] All four verification steps done - [ ] `main` moved on to the next `-SNAPSHOT` +- [ ] `Publish.binaryCompatibleWith` extended with the version just released, + in that same follow-up commit diff --git a/SCALA_CODE_STYLE.md b/SCALA_CODE_STYLE.md index a221112..22bc3a8 100644 --- a/SCALA_CODE_STYLE.md +++ b/SCALA_CODE_STYLE.md @@ -68,7 +68,7 @@ Pinned versions for this repository: | --------- | --------- | --------------------------------------------------------- | | Scala | 3.8.4 | Latest stable; pin in `build.mill`, do not float. | | Mill | 0.12.x | Pinned in `.mill-version`, which is committed. | -| Scalafmt | 3.11.4 | Pinned in `.scalafmt.conf`; matches the installed binary. | +| Scalafmt | 3.11.5 | Pinned in `.scalafmt.conf`; matches the installed binary. | | Ox | 1.0.6 | Direct-style concurrency. | Resolve versions from the canonical resolver — diff --git a/SECURITY.md b/SECURITY.md index 2b47bb2..f9d5bad 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -105,7 +105,13 @@ a smuggled query parameter, a CRLF injected into a header — is in scope. ### Parsing and resource use - A crafted 2xx payload that causes unbounded memory growth, non-terminating - decoding, or an exception that escapes the `CodebergError` channel. + decoding, or an exception that escapes the `CodebergError` channel. Note the + documented guard: every response is read under a byte bound — + `CodebergConfig.maxResponseBodyBytes`, 16 MiB by default, and + `maxDownloadBodyBytes`, 50 MiB, for the two archive downloads — and a body + that passes it is abandoned as `TransportCause.ResponseTooLarge` rather than + read to the end. A payload that exhausts memory while staying *inside* that + bound is a report. - A `Link` header that drives a pagination walk into an infinite loop. Note the documented guard: a walk stops when the server stops offering a next page, and callers are told to guard on an empty page as well. @@ -132,8 +138,9 @@ none at all. `DownloadActionArtifact` and `repoGetActionRunLogs`, reachable as `client.downloads` — hold the whole archive in memory; the library does not stream, and attachment streaming above 50 MB is explicitly out of scope for - v1. A large artifact exhausting the heap is documented behaviour. If you can - make it happen with a *small* request, that is a report. + v1. Those two are bounded by `CodebergConfig.maxDownloadBodyBytes`, which + defaults to 50 MiB for exactly that reason; raising the setting yourself and + then running out of heap is your configuration, not a vulnerability. - **Missing hardening with no exploit path**, such as the absence of certificate pinning. @@ -148,6 +155,9 @@ Stated so you know what to test against, not as a claim of safety: `new Exception` in production code fails `./verify.sh`. - Redaction is tested directly, including property suites asserting that no rendering path emits a credential. +- Every request carries a response-body byte bound, so an instance cannot + answer a call with as many bytes as it likes. Exceeding it is not retried: + repeating the call would download the oversized body once per attempt. - Two dependencies, both widely used, both pinned in `build.mill`. - Released artifacts are PGP-signed and built by [`.github/workflows/release.yml`](.github/workflows/release.yml) from a diff --git a/build.mill b/build.mill index 64d697a..3ab5feb 100644 --- a/build.mill +++ b/build.mill @@ -1,9 +1,11 @@ //| mvnDeps: //| - com.lihaoyi::mill-contrib-scoverage:1.1.7 //| - com.goyeau::mill-scalafix::0.6.2 +//| - com.github.lolgab::mill-mima::0.2.2 package build +import com.github.lolgab.mill.mima.Mima import com.goyeau.mill.scalafix.ScalafixModule import mill.* import mill.contrib.scoverage.ScoverageModule @@ -15,12 +17,12 @@ object Versions: val scala = "3.8.4" val sttp = "4.0.26" val sttpModel = "1.7.18" - val jsoniter = "2.39.1" - val munit = "1.3.4" + val jsoniter = "2.40.1" + val munit = "1.3.5" val munitScalcheck = "1.3.0" val scalacheck = "1.19.0" val testcontainers = "0.44.1" - val scoverage = "2.3.0" + val scoverage = "2.5.2" /** Everything the published POMs say about this project. * @@ -54,6 +56,33 @@ object Publish: developers = Seq(Developer("w0rxbend", "w0rxbend", "https://codeberg.org/worxbend")), ) + /** The already-released versions that every artifact in this build must stay binary compatible with. + * + * MIMA — the Migration Manager, `com.typesafe:mima-core` driven here by the `mill-mima` plugin — answers one + * question: can a program that was compiled against version X of a jar be run against this build without relinking? + * It answers it by comparing the class files, so it catches the changes a compiler cannot see from this side, such + * as a removed `copy` overload or a widened opaque type. + * + * '''This list is empty on purpose, and emptying it was not an oversight.''' Nothing has ever been published under + * `com.worxbend`, so there is no jar on Maven Central to compare against; a non-empty list here would make + * `mimaReportBinaryIssues` fail with a resolution error rather than a compatibility report. The moment `v0.1.0` is + * on Central, this becomes: + * + * {{{ + * val binaryCompatibleWith: Seq[String] = Seq("0.1.0") + * }}} + * + * and stays a list of every release in the current compatibility window — under Early SemVer, every `0.1.x` while + * the minor is `1`, reset to just the new minor when a deliberate break bumps it. `RELEASING.md` § "Binary + * compatibility is checked by MIMA" owns that procedure, measures what MIMA does and does not see through a + * `private[codeberg4s]` constructor, and the release checklist carries the step. + * + * While the list is empty the check is wired but inert: `./mill modules.__.mimaReportBinaryIssues` fails with the + * plugin's own "No previous artifacts configured" message, which is the honest answer to "is this build compatible + * with nothing?". No other task depends on it, so an empty list cannot fail a compile, a test or `verify.sh`. + */ + val binaryCompatibleWith: Seq[String] = Seq.empty + /** Common settings for every module in the build, published or not. * * Warnings are errors here on purpose — see CLAUDE.md "Compiler settings". @@ -62,7 +91,14 @@ trait Codeberg4sModule extends ScalaModule with ScalafmtModule with ScoverageMod def scalaVersion = Versions.scala def scoverageVersion = Versions.scoverage + /** `-java-output-version` is Scala 3's spelling of `javac --release`: it both stamps the class-file version and hides + * any JDK API newer than the named release, so a call that would not exist on a Java 25 runtime fails here rather + * than at a consumer's link time. Without it the compiler emits its own default — Java 17 bytecode for Scala 3.8.4 — + * no matter which JDK ran the build, which is a promise nobody made on purpose. `.mill-jvm-version` pins the JDK + * that has to understand `25`. + */ def scalacOptions = Seq( + "-java-output-version:25", "-deprecation", "-feature", "-explain", @@ -92,12 +128,39 @@ trait Codeberg4sModule extends ScalaModule with ScalafmtModule with ScoverageMod * Artifact ids are written out per module rather than derived from the module name, because `moduleSegments` would * publish `modules.domain` as `modules-domain` and because an artifact id is a permanent promise that must not move * when a module is renamed. + * + * Mixing in `Mima` here rather than per module is what makes one line cover all five artifacts: the plugin derives the + * coordinate to download from `pomSettings().organization`, `artifactId()` and each entry of `mimaPreviousVersions`, + * so `modules.codec` is checked against `com.worxbend:codeberg4s-codec_3:` without naming it anywhere. */ -trait Codeberg4sPublishModule extends Codeberg4sModule with PublishModule: +trait Codeberg4sPublishModule extends Codeberg4sModule with PublishModule with Mima: def pomSettings = Publish.pom def publishVersion = Publish.version def versionScheme = Some(VersionScheme.EarlySemVer) + /** The releases this artifact is checked against by `./mill .mimaReportBinaryIssues`. + * + * '''Every module that inherits this trait inherits the claim that each listed version of it exists on Maven + * Central.''' That claim is true for the five artifacts released together as `0.1.0`, and it is false for any + * artifact added later: a sixth module first published in, say, `0.3.0` would send MIMA looking for a + * `codeberg4s-newthing_3:0.1.0` that was never uploaded, and the check would fail on a download error that says + * nothing about compatibility. + * + * A module first published after the versions in [[Publish.binaryCompatibleWith]] therefore overrides this with the + * releases that really exist for it — and only those: + * + * {{{ + * object newthing extends Codeberg4sPublishModule: + * def artifactName = "codeberg4s-newthing" + * // First shipped in 0.3.0; there is no 0.1.0 or 0.2.0 of this artifact to compare against. + * override def mimaPreviousVersions = Task { Seq("0.3.0") } + * }}} + * + * For the release in which the new module is itself brand new, that override is `Seq.empty` until the release lands, + * exactly as the shared list is empty today. + */ + def mimaPreviousVersions = Task(Publish.binaryCompatibleWith) + /** Mill feeds `scalacOptions` to Scaladoc as well as to the compiler, so `-Werror` would let one unresolved * `[[link]]` fail `docJar` — and Maven Central rejects a bundle that has no Javadoc jar. Doc warnings still print; * they just do not block a release. The compiler's `-Werror` is untouched. @@ -123,14 +186,20 @@ object modules extends Module: * * Depends on `core` because it is the adapter that implements core's `Decode` port — an adapter depending on the * port it satisfies is the expected direction. It still must not import sttp. + * + * Only `jsoniter-scala-core` is declared. The companion `jsoniter-scala-macros` artifact exists to '''derive''' a + * `JsonValueCodec[A]` from a case class at compile time, and this module derives nothing: `JsonValue.scala` + * hand-writes the single `JsonValueCodec` over a document model, because the Forgejo API needs "every field + * optional, `null` and absent identical, unknown kinds tolerated" — a shape derivation cannot express (see + * `docs/HAZARDS.md` §1). Declaring the macros artifact anyway put roughly a megabyte on every consumer's classpath + * for a `JsonCodecMaker` nothing calls. */ object codec extends Codeberg4sPublishModule: def artifactName = "codeberg4s-codec" def moduleDeps = Seq(domain, core) def mvnDeps = Seq( - mvn"com.github.plokhotnyuk.jsoniter-scala::jsoniter-scala-core:${Versions.jsoniter}", - mvn"com.github.plokhotnyuk.jsoniter-scala::jsoniter-scala-macros:${Versions.jsoniter}", + mvn"com.github.plokhotnyuk.jsoniter-scala::jsoniter-scala-core:${Versions.jsoniter}" ) object test extends Codeberg4sTests diff --git a/docs/CONSTITUTION_MAPPING.md b/docs/CONSTITUTION_MAPPING.md index 5cd2111..ecfabe0 100644 --- a/docs/CONSTITUTION_MAPPING.md +++ b/docs/CONSTITUTION_MAPPING.md @@ -21,7 +21,7 @@ distinguishes four things: | Constitution requirement | Scala / Mill equivalent | Status | | ----------------------------------- | ------------------------------------------------------------------------------ | ------ | | Compile with warnings fatal | `scalacOptions` with `-Werror -Wunused:all -Wvalue-discard -Wnonunit-statement` | Wired | -| Formatter | Scalafmt 3.11.4, `mill mill.scalalib.scalafmt/` | Wired | +| Formatter | Scalafmt 3.11.5, `mill mill.scalalib.scalafmt/` | Wired | | Linter / semantic rules | Scalafix, `mill modules.__.fix`, rules in `.scalafix.conf` | Wired | | Coverage | scoverage via `mill-contrib-scoverage`; thresholds in `scripts/coverage-gate.sc`, called by `verify.sh` | Wired (report + gate script); thresholds never yet asserted on a real report | | Mutation tool (`mutate4*`) | **Stryker4s 1.1.1** command runner over `domain` + `core` + `codec`, wrapped by `scripts/mutate.sh` | **Runner proven, result not** | @@ -49,10 +49,14 @@ repository it tokenises Scala 3 indentation syntax, `given`/`using`, `enum`, `extension` and end markers without a single lexical error, and the duplications it reports are genuine. It is a real gate, not a stub. -It is also **red**. `scripts/cpd.sh --report` finds **323 duplication groups at -the 40-token threshold** — 128 source locations in `codec`, 40 in `client`, 15 -in `domain`, 2 in `core` — so `scripts/cpd.sh` in gate mode exits 1 today and -`./verify.sh --with-slow` fails at that step. The findings corroborate +It is also **red**. Measured on 2026-08-09, `scripts/cpd.sh --report` finds +**378 duplication groups at the 40-token threshold**, spread over 1360 source +locations — 764 in `codec`, 541 in `client`, 45 in `domain`, 8 in `core`, 2 in +`transport`. `scripts/cpd.sh` in gate mode therefore exits 1 today. +`./verify.sh --with-slow` does not fail on that count alone: it compares +against `CPD_BASELINE_GROUPS` in `verify.sh`, which records the same 378, and +fails only when the count rises above it. The recorded number is debt written +down, not debt forgiven. The findings corroborate `docs/LEDGER.md` §"Helpers awaiting promotion" (`FilterToken.from` against `PathSegment.from`, the repeated `Wire.required`/`Wire.validated` blocks in the DTOs, the `page`/`limit` pair). Some groups are import blocks, which is CPD diff --git a/docs/LEDGER.md b/docs/LEDGER.md index 881aeb7..dac7e9b 100644 --- a/docs/LEDGER.md +++ b/docs/LEDGER.md @@ -9,10 +9,13 @@ The rule from `PLAN.md` §7: **the first wave that needs a shared model owns it. Later waves import it and must not redefine, fork, or "temporarily" copy it. A duplicated model is a review-blocking defect — and it is what PMD CPD catches. The duplication gate is now real: `scripts/cpd.sh` runs PMD 7.26.0's Scala -tokenizer over the production sources and, at the 40-token threshold, currently -reports **62 duplication groups**. Several of them are exactly the helpers this -file lists below. `./verify.sh --with-slow` fails at that step until they are -fixed. +tokenizer over the production sources and, at the 40-token threshold, reports +**378 duplication groups** as of 2026-08-09. Several of them are exactly the +helpers this file lists below. `./verify.sh --with-slow` does not fail on that +number by itself — it compares it against `CPD_BASELINE_GROUPS` in `verify.sh`, +which records today's count, and fails on any increase. Bringing the count down +is what closes these entries; the baseline is then lowered in the same commit so +the ground gained is held. This file records who owns what. A wave updates it as part of its definition of done, in the same commit that introduces the model. diff --git a/docs/READINESS.md b/docs/READINESS.md index f301d23..3faab1d 100644 --- a/docs/READINESS.md +++ b/docs/READINESS.md @@ -1,41 +1,45 @@ # Readiness review — what stands between this repo and a third party using it -The library itself is done: 439 of 439 in-scope operations, `./verify.sh` green, -100 % domain coverage, 3524 unit tests and 1338 property tests. What follows is +The library itself is done: 439 of 439 in-scope operations, `./verify.sh +--with-slow` green, 100 % domain coverage, 3658 tests. What follows is everything *else* a consumer needs, assessed honestly against the state of the repo rather than against intentions. +Every claim below was checked against the repository on 2026-08-09. Where +something could not be checked from inside a clone — whether a workflow has +actually run on the forge, whether a page is actually served — it says so +rather than guessing. + ## Blocking — a stranger cannot use the library without these | Gap | Why it blocks | Where it lands | | --- | --- | --- | -| **Nothing is published.** `build.mill` has full `PublishModule` config at `0.1.0-SNAPSHOT`, but no artifact exists on Maven Central, so the README's coordinates resolve to nothing. | A dependency line that does not resolve is the first and last thing a new user tries. | `RELEASING.md`, release workflow | -| **No release process.** No documented steps, no signing key handling, no tag convention, no way to reproduce a release. | Publishing by hand from a laptop is how a supply chain gets compromised and how versions get skipped. | `RELEASING.md`, `.github/workflows/release.yml` | -| **No public API documentation.** Scaladoc exists on every public member — it is thorough — but it is never rendered or hosted. | A published library whose docs live only in source is a library people read on the train, not one they adopt. | site pipeline, Pages workflow | -| **No worked examples that compile.** README snippets were verified once by hand; nothing in the build stops them rotting. | Examples that no longer compile are worse than no examples: they teach the wrong API. | `modules/examples`, mdoc | +| **Nothing is published.** `build.mill` has full `PublishModule` config, but `Publish.version` is still `0.1.0-SNAPSHOT`, `git tag` lists nothing, and no artifact exists on Maven Central — so the README's coordinates resolve to nothing. | A dependency line that does not resolve is the first and last thing a new user tries. | cut the `v0.1.0` tag; `.github/workflows/release.yml` | +| **No mutation score.** `scripts/mutate.sh` exists and the Stryker4s runner is proven against these sources, but a scored run has never completed — it needs roughly 1400 `mill test` invocations. `PLAN.md` §10 asks for ≥ 80 %. | The number that would justify trusting the test suite does not exist. Coverage says lines were executed, not that a broken line would be caught. | a nightly run long enough to finish | ## Important — these decide whether adoption survives contact | Gap | Why it matters | | --- | --- | -| **No CI that a contributor can see.** `.forgejo/workflows/ci.yml` exists but has never run — the repo has no remote. There is no GitHub Actions equivalent, and the nightly and drift jobs were unreachable until recently. | -| **No binary-compatibility policy.** Five artifacts, no MIMA, no stated versioning scheme. Consumers pin versions; they need to know what a minor bump may break. | -| **No contribution path.** No `CONTRIBUTING.md`, no issue or pull-request templates, no code of conduct, no security policy. A drive-by bug report has nowhere to go, and a vulnerability has no private channel. | -| **No task-oriented documentation.** `README.md` is a tour and the ADRs are rationale. Neither answers "how do I paginate every issue in a repository without running out of memory" — which is what a working developer actually asks. | -| **`--with-slow` is red.** PMD CPD reports 323 duplication groups, mostly the helper duplication `docs/LEDGER.md` already tracks. Honest, but a contributor running the documented gate hits a failure that is not theirs. | +| **The site has never been observed deployed.** `.github/workflows/site.yml` builds with `scripts/site.sh` and publishes `out/site/html`, and the guides' snippets are compiled against the real sources on every pull request. But the workflow depends on a repository setting (Settings → Pages → Source must be "GitHub Actions") that cannot be verified from a clone. Until someone confirms the page is served, treat hosted documentation as unproven rather than done. | +| **Duplication is tracked debt, not a clean result.** `./verify.sh --with-slow` passes, but it passes against a *recorded baseline* of 363 groups, of which 762 of the 1350 reported locations are in `modules/codec`. The gate fails on any increase, which is the useful property; it does not mean the code is free of duplication. `docs/LEDGER.md` names the helpers involved. | +| **Property coverage is narrow.** ScalaCheck suites carry the `Property` tag and are excluded from the default gate. They reach the domain module's identifiers, secrets, pagination and retry bounds; the codec round-trip laws `PLAN.md` §6.3 asks for still have no property suite. | + +## Resolved since this document was first written + +Checked, not assumed: + +- **Release process** — `RELEASING.md` documents the steps, the signing key handling and the tag convention. +- **Contribution path** — `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`, `SECURITY.md`, `.github/pull_request_template.md` and four issue templates all exist. +- **CI a contributor can see** — GitHub workflows (`ci`, `nightly`, `release`, `site`) alongside the Forgejo one, and the repository now has a remote. Every `uses:` is pinned to a commit SHA, and `.github/dependabot.yml` keeps those pins current. +- **Binary-compatibility policy** — MIMA is wired on the shared publish trait via `mill-mima` 0.2.2, armed by setting `Publish.binaryCompatibleWith` once 0.1.0 exists. `RELEASING.md` carries the measured filter needed for the static forwarders on response models. +- **Worked examples that compile** — `modules/examples` is inside the gate, so an example that stops compiling fails the build. `verify.sh` step 3 exists specifically to stop a source tree drifting outside the build. +- **Task-oriented documentation** — ten guides under `site/src/guides/`, with mdoc compiling their snippets. +- **`--with-slow` is green** — it was red at 378 groups against a stale 323 baseline; the baseline is now measured, and two refactors paid 15 groups off rather than absorbing them. ## Nice to have — not blocking 0.1.0 -- Dependency update automation (Renovate; Codeberg supports it). +- Dependency update automation for the Scala and Mill side (Renovate; Codeberg supports it, and Dependabot has no Mill support). The GitHub Actions pins are already covered by `.github/dependabot.yml`. - A spec-drift issue opener rather than a warning in a log. -- Stryker4s producing an actual mutation score (the engine is proven on Scala 3.8.4; a real run has never completed). - Scala.js / Native cross-builds, which `PLAN.md` §0 explicitly defers. - -## What is being built in response - -1. **`modules/examples`** — runnable programs compiled by the build under `-Werror`, so an example cannot rot silently. -2. **mdoc** over every documentation snippet, so the guides compile against the real API on every site build. -3. **A microsite** (Laika) with a landing page, task-oriented guides and the full Scaladoc, deployable to GitHub Pages and Codeberg Pages. -4. **Guides written for a developer who is new to this library** — and, in places, new to Scala — covering the first request, authentication, errors, pagination, retries, testing, and a recipe cookbook. -5. **CI and release workflows** for GitHub Actions alongside the existing Forgejo ones, including a tag-driven publish and a Pages deploy. -6. **Community health files** — contributing, code of conduct, security policy, release process, issue and pull-request templates. +- A JMH harness. `scripts/alloc-bench.sh` measures allocation and wall-clock off the real golden fixtures and is what the codec work was judged against, but it is a single-threaded counter that does not fork a JVM per benchmark — its own header documents the two ways that misleads. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index b66a4b0..cc2562c 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -9,16 +9,20 @@ Gate names match `PLAN.md` §7. ## Phase 0 — Bootstrap · **Gate G0** - [x] Mill 1.1.7 pinned in `.mill-version`; bootstrap `./mill` committed (ADR-0006) -- [x] Six hexagonal modules: `domain`, `core`, `codec`, `transport`, `client`, `it` +- [x] Seven modules under `object modules`: the five published ones — `domain`, + `core`, `codec`, `transport`, `client` — plus `it` (integration suites) + and `examples` (compiled by the build so a stale example breaks it) - [x] `scalacOptions` with warnings fatal - [x] Scalafmt + Scalafix configs, `mill modules.__.reformat` green - [x] scoverage wired via `mill-contrib-scoverage` - [x] ADRs 0001–0006, constitution mapping - [x] Scalafix wired via `mill-scalafix` (Mill 1.x has no built-in `fix`; ADR-0006) - [x] `verify.sh` with the ordered gate and an architecture-boundary check -- [x] CI pipeline (`.forgejo/workflows/ci.yml`) — the `verify` job runs on push - and pull request. The `nightly` and `spec-drift` jobs are defined but - unreachable; see Phase 4. +- [x] CI pipeline — `.forgejo/workflows/ci.yml` runs the `verify` job on push + and pull request, and `.github/workflows/` carries the same gate plus + `nightly.yml`, which holds the slow analysis and the spec-drift detector + behind a real `schedule:` trigger. Every action is pinned to a commit + SHA rather than a tag, and Dependabot keeps those pins moving. ## Phase 1 — Recon and foundation · **Gate G-R**, **Gate G1** @@ -28,14 +32,14 @@ Gate names match `PLAN.md` §7. PLAN.md's assumptions turned out to be wrong - [x] 54 golden fixtures captured (`modules/codec/test/resources/golden/`) - [x] `domain`: error ADT, `CallContext`, opaque identifiers, `Auth`, config, paging value types -- [x] `core`: `Exec[F]`, ports, `RetryEngine`, `Pagination`, `LinkHeader`, `Pages`, +- [x] `core`: `Exec[F]`, ports, `RetryEngine`, `LinkHeader`, `Pages`, `ApiPipeline`, `StatusMapping`, `Redaction` - [x] Vertical slice: `GET /version` and `GET /repos/{owner}/{repo}` through every layer, both rails - [x] Full error paths on the slice: 404, decode failure, transport failure, retry-then-succeed ## Phase 2 — Cross-cutting hardening · **Gate G2** -- [x] Track A — retry engine, pagination driver, `Page` / `foldPages`, `Telemetry` port +- [x] Track A — retry engine, pagination driver, `Page` / `PageWalk`, `Telemetry` port wired through both `CodebergClient` factories - [x] Track B — error ADT, Forgejo error-body parsing against captured samples, redaction guarantees, `CodebergException` bridging @@ -45,13 +49,16 @@ Gate names match `PLAN.md` §7. `listAll` on each of the thirty-eight API classes — the termination rule is the subtle part of pagination and belongs in one place. - [ ] Property suites (`*Props.scala`, `Property` tag) for codec laws, pagination - invariants, retry bounds. **Partially done:** `modules/domain` now has + invariants, retry bounds. **Partially done:** `modules/domain` has `PropertyBase` (pinned ScalaCheck seed, `Property` tag), `IdentifierProps` - (13 properties over the opaque identifiers and `BaseUri`) and - `SecretProps` (9 properties asserting no rendering path emits a - credential). The three areas `PLAN.md` §6.3 actually names — codec - round-trip laws in `codec`, pagination-driver invariants and retry bounds - in `core` — have no property suite yet. + (13 properties over the opaque identifiers and `BaseUri`), `SecretProps` + (9 properties asserting no rendering path emits a credential), + `CodebergErrorProps` and `PageProps`. Of the three areas `PLAN.md` §6.3 + names, two are covered: codec round-trip and totality laws by `JsonProps` + and `ApiErrorBodyCodecProps`, retry bounds by `RetryEngineProps`. The + pagination driver is the one still open — `PaginationProps` was deleted + along with `core.Pagination`, and the walker that replaced it, + `paging.PageWalk`, has example-based tests only. ## Phase 3 — Endpoint waves · **Gate G3** per wave @@ -92,23 +99,28 @@ on paper only until those tools are proven — `docs/CONSTITUTION_MAPPING.md`.) - [ ] Stryker4s wired, ≥ 80 % mutation score on `domain` + `core` + `codec` — `scripts/mutate.sh` exists and the **runner is proven**: the 1.1.1 command - runner generates mutants from these sources, scalameta parses all 196 - production files under the Scala 3 dialect, the instrumented output - recompiles under `-Werror`, and the break threshold demonstrably fails the - run. **No score exists for this repository** — that needs a run with the - real test command, roughly 1400 `mill test` invocations. The ≥ 80 % figure - is still a target. `docs/CONSTITUTION_MAPPING.md` has the detail. -- [ ] PMD CPD wired, fails above 40 duplicated tokens in production sources — - **wired and verified; the gate is red.** PMD 7.26.0's scalameta Scala - module tokenises Scala 3 here without a lexical error, and - `scripts/cpd.sh --report` currently finds **323 duplication groups** (128 - locations in `codec`, 40 in `client`, 15 in `domain`, 2 in `core`), so - `./verify.sh --with-slow` fails at that step. Unticked because the - codebase does not pass, not because the tool does not work. The fix is - `docs/LEDGER.md` §"Helpers awaiting promotion", not a lower threshold. + runner generates mutants from these sources, scalameta parsed every one of + the 196 production files those modules held at the time under the Scala 3 + dialect, the instrumented output recompiles under `-Werror`, and the break + threshold demonstrably fails the run. Those modules hold 503 production + files today, so even the parse half of that proof predates the current + sources. **No score exists for this repository** — that needs a run with + the real test command, roughly 1400 `mill test` invocations. The ≥ 80 % + figure is still a target and is still **unproven**. + `docs/CONSTITUTION_MAPPING.md` has the detail. +- [x] PMD CPD wired, fails above 40 duplicated tokens in production sources — + **wired, verified, and green.** PMD 7.26.0's scalameta Scala module + tokenises Scala 3 here without a lexical error, and `scripts/cpd.sh + --report` finds **363 duplication groups** at 40+ tokens. Ticked because + the tool runs and the gate holds, not because the code is + duplication-free: `verify.sh` compares that count against + `CPD_BASELINE_GROUPS`, so it fails on an increase and tells you to bank a + decrease. `CPD_MIN_TOKENS` is still 40 and every group is still reported. + Paying the debt down is `docs/LEDGER.md` §"Helpers awaiting promotion". - [x] `scripts/crap.sc`, fails on any method with CRAP > 30 — implemented over - the scoverage XML. Note the complexity input is a documented proxy - (`branch="true"` statement count), not a control-flow analysis; the + the scoverage XML, and **run against a fresh report**: 2,217 methods + measured, worst 28.0, limit 30. Note the complexity input is a documented + proxy (`branch="true"` statement count), not a control-flow analysis; the script's header lists the three directions it is known to be wrong in. Read it as a ranking, not a certified metric. - [x] `modules/it` — Testcontainers-Forgejo suite (`ForgejoContainerSuite`) plus @@ -117,24 +129,35 @@ on paper only until those tools are proven — `docs/CONSTITUTION_MAPPING.md`.) - [x] `scripts/coverage-gate.sc` — reads scoverage's own `statement-rate` and `branch-rate`, floors at 90/85 for `domain`+`core`+`codec` and 80/80 for `transport`+`client`, and treats a missing report as a failure rather than - a skip. Called by `verify.sh`; the thresholds have not yet been asserted - against a freshly generated report. -- [ ] README with compiling examples, `CHANGELOG.md` — `CHANGELOG.md` written; - README rewritten and every Scala block compiled against the current - sources under the project's flags. Unticked because that check is manual: - `PLAN.md` §"Phase 4" asks for mdoc so the *build* enforces it. + a skip. Called by `verify.sh`, and **asserted against a freshly generated + report**: `domain` 100.00 % statement / 100.00 % branch, `core` 96.59 % / + 92.48 %, `codec` 95.20 % / 91.47 %. +- [ ] README with compiling examples, `CHANGELOG.md` — `CHANGELOG.md` written + and corrected against the current sources; README rewritten and every + Scala block compiled against those sources under the project's flags. + Unticked because that check is manual: `PLAN.md` §"Phase 4" asks for mdoc + so the *build* enforces it. - [ ] Maven Central publishing config, MIMA baseline from 0.1.0 — publishing is **configured**: `build.mill` publishes five artifacts (`codeberg4s-domain`, `-core`, `-codec`, `-transport`, `-client`) under `com.worxbend` at `0.1.0-SNAPSHOT`, MIT, `EarlySemVer`, with `modules.it` - deliberately excluded. Nothing has been published, and there is **no MIMA - setup at all** — no plugin, no baseline. Both remain. -- [ ] Nightly spec-drift detector — `.forgejo/workflows/ci.yml` defines both a - `nightly` job and a `spec-drift` job, and **both are gated on - `github.event_name == 'schedule'` while the workflow declares only `push` - and `pull_request` triggers.** Neither has ever run. Adding a `schedule:` - trigger is the whole fix, and the drift job currently only warns on a - sha mismatch — it does not open an issue, as `PLAN.md` §7 asks. + deliberately excluded. **MIMA is now wired** through + `com.github.lolgab::mill-mima::0.2.2`, mixed into + `Codeberg4sPublishModule` so one declaration covers all five artifacts. + What remains is the publication itself: `binaryCompatibleWith` is + `Seq.empty` because there is nothing on Central to compare against, so + `mimaReportBinaryIssues` reports nothing until 0.1.0 is released. + `RELEASING.md` § "Binary compatibility is checked by MIMA" carries the + measured detail of what MIMA does and does not see through the response + models' `private[codeberg4s]` constructors. +- [x] Nightly spec-drift detector — it lives in `.github/workflows/nightly.yml` + behind a real `schedule:` trigger (03:00 UTC) plus `workflow_dispatch`, + alongside the `verify.sh --nightly` job. It was previously declared in + `.forgejo/workflows/ci.yml` gated on a `schedule` event that workflow + never emitted, so it had never run. It compares the sha256 of the live + `swagger.v1.json` against the pinned copy and, on a mismatch, emits a + warning and a truncated diff in the job summary. It still does **not** + open an issue, as `PLAN.md` §7 asks — that is the piece left. ## Distance to 0.1.0 @@ -142,21 +165,23 @@ on paper only until those tools are proven — `docs/CONSTITUTION_MAPPING.md`.) | Definition-of-done clause | State | | ------------------------- | ----- | -| All in-scope endpoints on both rails with documented error contracts | 61 / 439 — every one of the 61 is on both rails with a Scaladoc error contract, so the shape is right and the surface is 14 % of the way there | -| `verify.sh` green | Default run yes: format, lint, zero-warning compile, 1072 unit tests, boundary check, coverage. **`--with-slow` is red** at the CPD step | -| Coverage per §6.1 (≥ 90 % line / ≥ 85 % branch on `domain`+`core`+`codec`) | Report is produced and `scripts/coverage-gate.sc` now enforces the floors; the assertion has not yet been run against a fresh report | -| Mutation ≥ 80 % | Runner proven, **no score produced** — see Phase 4 | -| Zero CRAP > 30 | Gate implemented; not yet run against a fresh coverage report, and its complexity input is a proxy | -| CPD clean | **No — 323 duplication groups at 40 tokens.** The tool works; the codebase does not pass it yet | +| All in-scope endpoints on both rails with documented error contracts | **439 / 439, 100 %** (`docs/API_INVENTORY.md` §0) — every one on both rails with a Scaladoc error contract | +| `verify.sh` green | **Yes, both modes.** Default run: format, lint, zero-warning compile, **3,658 unit tests**, boundary check, coverage. `--with-slow` adds duplication and CRAP and also passes | +| Coverage per §6.1 (≥ 90 % line / ≥ 85 % branch on `domain`+`core`+`codec`) | **Enforced and asserted against a fresh report** by `scripts/coverage-gate.sc`: `domain` 100.00 % / 100.00 %, `core` 96.59 % / 92.48 %, `codec` 95.20 % / 91.47 % | +| Mutation ≥ 80 % | Runner proven, **no score produced — the figure is unproven** — see Phase 4 | +| Zero CRAP > 30 | **Yes:** 2,217 methods measured, worst 28.0. Its complexity input is a documented proxy, so read it as a ranking | +| CPD clean | **No — 363 duplication groups at 40 tokens.** The gate passes because it fails on an increase over that recorded number, not because the duplication is gone. A real finding about the code; `docs/LEDGER.md` names most of them | | Acceptance features + Gherkin mutation clean | Dormant by decision — `docs/CONSTITUTION_MAPPING.md` | -| Published to Maven Central | Configured but not published; no MIMA baseline | +| Published to Maven Central | Configured and MIMA wired, but nothing published, so there is still no baseline to compare against | | README quickstart works against live codeberg.org | Samples are checked against the source signatures by hand. `CodebergLiveSmokeSuite` exercises the same calls against codeberg.org under `CODEBERG_IT=1`, but nobody has run the README itself | | Mapping doc, ADRs, inventory, provenance current | Yes, as of this revision | -The single largest remaining item is the endpoint surface. The most urgent one -is the CPD result: the duplication gate now works, and it says the codebase has -323 duplication groups. That is a real finding about the code, not a tooling -problem, and `docs/LEDGER.md` already names most of them. +The endpoint surface is done. What is left is evidence, not code. Two clauses +are genuinely unmet — the mutation score does not exist, and the codebase +carries 363 duplication groups that the gate records rather than forgives — and +two more are met only by hand: the README's examples are checked by a person +rather than by mdoc, and nothing has been published, so MIMA has nothing to +compare 0.1.1 against until the 0.1.0 tag is on Central. ## Out of scope for 0.1.0 @@ -164,3 +189,8 @@ Per `PLAN.md` §0: OAuth2 token *acquisition* flows (pre-obtained tokens only), ActivityPub federation, admin endpoints, attachment streaming above 50 MB, and Scala.js / Native cross-builds. The Gherkin acceptance pipeline is dormant — see `docs/CONSTITUTION_MAPPING.md`. + +The 50 MB line is now enforced rather than merely written down: +`CodebergConfig.maxDownloadBodyBytes` defaults to 50 MiB and a body past it is +`TransportCause.ResponseTooLarge`, non-retryable. Textual responses have their +own, smaller bound at `maxResponseBodyBytes`. diff --git a/docs/VERSIONS.md b/docs/VERSIONS.md index 7ff89cf..64e4665 100644 --- a/docs/VERSIONS.md +++ b/docs/VERSIONS.md @@ -6,7 +6,7 @@ can tell a deliberate pin from a stale one. | | | | --- | --- | -| Resolved on | `2026-08-01` | +| Resolved on | `2026-08-09` | | Resolver | `https://repo1.maven.org/maven2` (`maven-metadata.xml`) | | Authority for build pins | [`build.mill`](../build.mill) — this document mirrors it and must never contradict it | @@ -50,9 +50,19 @@ against the pin in `build.mill`. | Mill | `1.1.7` | `.mill-version` (committed) | `1.1.7` (`com.lihaoyi:mill-dist`) | current | | `mill-contrib-scoverage` | `1.1.7` | `build.mill` header `//| mvnDeps:` | `1.1.7` | current — must track the Mill version exactly | -The `./mill` launcher script carries `DEFAULT_MILL_VERSION="1.1.6-104-5bbe1e"`, -but `.mill-version` is present and overrides it, so the effective version is -`1.1.7`. The launcher default is a bootstrap fallback only. +The `./mill` launcher script carries `DEFAULT_MILL_VERSION="1.1.7"`, matching +`.mill-version`, which overrides it anyway. The launcher default only applies to +a checkout with no `.mill-version` at all; it used to name `1.1.6-104-5bbe1e`, +an untagged snapshot 104 commits past the `1.1.6` tag, which is not a version +anyone can reason about. + +`.mill-checksums` records the SHA-256 of every Mill distribution the launcher is +allowed to run — one line per platform, because the launcher picks a different +native binary per OS and architecture. `./mill` checks the file it is about to +execute against that digest on every run, whether it was downloaded a moment ago +or cached months ago, and refuses to run anything unlisted. Bumping Mill +therefore means editing two files: `.mill-version` and `.mill-checksums`. The +header of `.mill-checksums` carries the exact commands for regenerating it. **Deviation from SCALA_CODE_STYLE.md.** That guide's pinned-versions table names Mill `0.12.x` and Ox `1.0.6`. Both are superseded here by decisions recorded in @@ -72,30 +82,56 @@ deliberately short (PLAN.md ADR-2, ADR-3). | --- | --- | --- | --- | --- | --- | | sttp client4 core | `com.softwaremill.sttp.client4::core` | `4.0.26` | `4.0.26` | current | `transport` | | sttp-model core | `com.softwaremill.sttp.model::core` | `1.7.18` | `1.7.18` | current | `transport` | -| jsoniter-scala core | `com.github.plokhotnyuk.jsoniter-scala::jsoniter-scala-core` | `2.39.1` | `2.39.1` | current | `codec` | -| jsoniter-scala macros | `com.github.plokhotnyuk.jsoniter-scala::jsoniter-scala-macros` | `2.39.1` | `2.39.1` | current | `codec` | +| jsoniter-scala core | `com.github.plokhotnyuk.jsoniter-scala::jsoniter-scala-core` | `2.40.1` | `2.40.1` | current | `codec` | + +Three artifacts, which is what README.md's "sttp client4 and jsoniter-scala, +that is the list" claims. `jsoniter-scala-macros` used to be a fourth. It is the +artifact that *derives* a codec from a case class at compile time, and this +build derives none — `modules/codec` hand-writes one `JsonValueCodec` over a +document model instead, for the reasons in `docs/HAZARDS.md` §1. Nothing under +`modules/` imports anything outside `jsoniter_scala.core`, so the macros jar was +about a megabyte of dead weight on every consumer's classpath. `modules/domain` and `modules/core` declare **no** `mvnDeps` at all — the hexagonal boundary is enforced by the build graph, not by convention (PLAN.md §3.1). -**JSON library:** PLAN.md §3.3 originally selected upickle; the project now uses **jsoniter-scala**, and -`build.mill` pins it. SCALA_CODE_STYLE.md's "JSON Codecs" section shows -jsoniter-scala examples; those examples do not apply to this repository. The rule -they illustrate — *derive the codec on the DTO, next to the DTO, and provide a -codec for the list type as well as the element type* — does apply, translated to -jsoniter-scala `ReadWriter`s. +**JSON library:** PLAN.md ADR-3 originally selected upickle. The project now +uses **jsoniter-scala**, and `build.mill` pins it; upickle and its `ujson` +document model are gone from the build and from the sources. Nothing in this +repository has a `ReadWriter` — that is upickle's codec type, and the +jsoniter-scala equivalent is `JsonValueCodec[A]`, from +`com.github.plokhotnyuk.jsoniter_scala.core`. + +SCALA_CODE_STYLE.md's "JSON Codecs" section is written for the same library, so +its vocabulary is this repository's vocabulary. Its *mechanism* is not: the +examples there call `JsonCodecMaker.make` to derive a `JsonValueCodec` per DTO +at compile time, and that macro lives in `jsoniter-scala-macros`, which this +build deliberately does not depend on (see the paragraph above). The reason is +in `docs/HAZARDS.md` §1 — no response definition in the pinned spec declares +`required`, `nullable` never appears, and live payloads send JSON `null` where +the spec promises an array or an object. A derived codec answers a payload like +that by failing. + +What `modules/codec` does instead is parse once into a document model, +`JsonValue`, whose single hand-written `JsonValueCodec[JsonValue]` is the only +codec in the build, and then assemble each DTO from that document. So the rule +SCALA_CODE_STYLE.md's example illustrates — *a top-level JSON array needs a +codec too, not only its element type* — is satisfied structurally rather than +per type: `JsonValue.Arr` is a case of the same model, so a list body and an +object body are decoded by the one codec and there is no per-DTO codec that +could be forgotten. ## 4. Test dependencies (not published) | Dependency | Coordinate | Pinned | Latest stable | Status | | --- | --- | --- | --- | --- | -| munit | `org.scalameta::munit` | `1.3.4` | `1.3.4` | current | +| munit | `org.scalameta::munit` | `1.3.5` | `1.3.5` | current | | munit-scalacheck | `org.scalameta::munit-scalacheck` | `1.3.0` | `1.3.0` | current | | ScalaCheck | `org.scalacheck::scalacheck` | `1.19.0` | `1.19.0` | current | | testcontainers-scala-munit | `com.dimafeng::testcontainers-scala-munit` | `0.44.1` | `0.44.1` | current | -`munit` and `munit-scalacheck` version independently — `1.3.4` and `1.3.0` are +`munit` and `munit-scalacheck` version independently — `1.3.5` and `1.3.0` are both the newest stable of their own artifact, not a mismatch. testcontainers-scala is confined to `modules/it`, the environmentally unsuitable @@ -106,37 +142,56 @@ test sweep. | Tool | Pinned | Pinned in | Latest stable | Status | | --- | --- | --- | --- | --- | -| scoverage | `2.3.0` | `build.mill` → `Versions.scoverage` | `2.5.2` | **behind — see below** | -| Scalafmt | `3.11.4` | `.scalafmt.conf` → `version` | `3.11.5` | **one patch behind — see below** | +| scoverage | `2.5.2` | `build.mill` → `Versions.scoverage` | `2.5.2` | current | +| Scalafmt | `3.11.5` | `.scalafmt.conf` → `version` | `3.11.5` | current | | Scalafix | via Mill's `__.fix` | `.scalafix.conf` (rules only, no version) | `scalafix-core` `0.14.7` | resolved transitively by Mill | | Stryker4s | not yet wired | — | — | scaffold only when the task calls for it (CLAUDE.md § Quality analysis) | -### scoverage `2.3.0` vs `2.5.2` +### scoverage `2.5.2` For Scala 3.4+ the coverage instrumentation lives in the compiler itself; Mill's -`ScoverageModule` resolves `org.scoverage::scalac-scoverage-serializer` at -`scoverageVersion`. `2.3.0` is a real published version of that artifact -(confirmed against `maven-metadata.xml`), so the build is valid — it is simply -not the newest. Bumping to `2.5.2` is a one-line `build.mill` change and belongs -in its own `build(deps):` commit per CLAUDE.md's granularity rule. **Not changed -by this lane**, which owns only `spec/` and `docs/`. - -### Scalafmt `3.11.4` vs `3.11.5` - -`.scalafmt.conf` pins `3.11.4`. The newest stable `org.scalameta:scalafmt-core` -is `3.11.5`. SCALA_CODE_STYLE.md requires the pin to match the installed binary, -and `align.preset = most` means a version bump can produce a repository-wide -realignment diff. If bumped, it must land as a standalone `style:` commit with -`mill mill.scalalib.scalafmt/` run over the whole tree, never mixed with logic. -**Not changed by this lane.** +`ScoverageModule` resolves `org.scoverage::scalac-scoverage-reporter` (and its +`-serializer` / `-domain` siblings) at `scoverageVersion`. Those artifacts read +the compiler's output and turn it into the XML and HTML reports, so a bump here +changes reporting, not instrumentation. + +Taken from `2.3.0` on 2026-08-09. `2.4.0` is the only release in that range with +a breaking change, and it does not touch this build: it drops support for Scala +2.13.15-and-earlier and 2.12.16, and this project is Scala 3 only. `2.4.1` fixes +instrumentation of pattern-matching assignments, `2.4.2` and `2.5.1` add Scala 2 +versions, `2.5.0` is dependency updates, and `2.5.2` adds incremental coverage. +The measured line and branch percentages in `verify.sh`'s coverage gate were +identical before and after the bump. + +### Scalafmt `3.11.5` + +Taken from `3.11.4` on 2026-08-09, in a standalone `style:` commit as the rule +below requires: `align.preset = most` means a formatter bump *can* realign the +whole repository, and that churn must never share a commit with a logic or +dependency change. + +In the event it realigned nothing. `mill mill.scalalib.scalafmt/` under `3.11.5` +rewrote 0 of 850 files, so the only line in that commit's diff outside the +documentation is the `version` key itself. The 3.11.5 changes are a website +migration and four fixes — inverted offsets on empty trees, a CLI error that +could mask a real one, the runner reporting which failure it exited on, and a +`RemoveScala3OptionalBraces` brace/colon oscillation — none of which this +configuration triggers. + +That zero-file result is also what re-verifies the longest-match claim in +`.scalafmt.conf`'s `rewrite.imports.groups` comment: if 3.11.5 had changed how +an import is assigned to a group, `scala.*` imports across the tree would have +moved and the reformat would not have been a no-op. ## 6. Not adopted | Candidate | Decision | Reason | | --- | --- | --- | | Ox | **not a dependency** | Public API is `Future`-based (PLAN.md §3.2). Overrides SCALA_CODE_STYLE.md's Ox chapter for this repo. | -| cats-effect / ZIO | rejected | PLAN.md ADR-2 — hand-rolled `Exec[F]` keeps the published dependency footprint at four artifacts. | -| circe / jsoniter-scala | rejected | PLAN.md ADR-3 — jsoniter-scala, first-class sttp integration, tiny footprint. | +| cats-effect / ZIO | rejected | PLAN.md ADR-2 (docs/adr/0002) — hand-rolled `Exec[F]` keeps the published dependency footprint at the three artifacts in §3. | +| circe | rejected | docs/adr/0003 — a larger dependency, and its optics would not change the shape of the problem the document model in `modules/codec` solves. | +| upickle / ujson | **removed** | PLAN.md ADR-3 chose it and the first docs/adr/0003 confirmed it; the current docs/adr/0003 supersedes both and the code moved to jsoniter-scala. No `upickle` or `ujson` import remains anywhere under `modules/`. | +| jsoniter-scala-macros | rejected | docs/adr/0003 — `JsonCodecMaker` derives a codec per DTO, and this build derives none. Mill's `mvnDeps` is runtime scope too, so declaring it would put roughly a megabyte of derivation machinery on every consumer's classpath. | | softwaremill/retry | **undecided** | PLAN.md ADR-4 evaluates it at Phase 2; not pinned in `build.mill` yet. If its `odelay` dependency or maintenance status disqualifies it, `RetryPolicy` is implemented in `core` with no new dependency. Decide before Phase 2 Track A, and record the outcome here. | | quicklens | not pinned | PLAN.md §0 mentions it as a candidate SoftwareMill utility; no module needs it yet. | diff --git a/docs/adr/0003-jsoniter-for-json.md b/docs/adr/0003-jsoniter-for-json.md index 297d957..128a701 100644 --- a/docs/adr/0003-jsoniter-for-json.md +++ b/docs/adr/0003-jsoniter-for-json.md @@ -2,13 +2,13 @@ - Status: accepted - Date: 2026-08-02 -- Supersedes: the original ADR-0003, which chose jsoniter-scala +- Supersedes: the original ADR-0003, which chose upickle ## Context `SCALA_CODE_STYLE.md` §"JSON Codecs" specifies jsoniter-scala. `PLAN.md` §3.3 -specified jsoniter-scala. The first version of this ADR chose jsoniter-scala, on two grounds: -that sttp client4 ships a first-party jsoniter-scala integration, and that jsoniter-scala's +specified upickle. The first version of this ADR chose upickle, on two grounds: +that sttp client4 ships a first-party upickle integration, and that upickle's transitive footprint is small. Both grounds turned out to be weaker than they looked. @@ -21,12 +21,21 @@ build declared and no code imported. It has been removed. The footprint argument was a wash: jsoniter-scala's core is comparable, and its macros module is compile-time only. +> **Later correction.** That last clause was wrong twice over. `mvnDeps` in Mill +> is compile *and* runtime scope, so a declared `jsoniter-scala-macros` reaches +> every consumer's classpath through the published POM; and this build never +> needed it in the first place, because the section below hand-writes its one +> codec instead of deriving any. The dependency has since been dropped — see +> "Consequences". + Meanwhile the style guide — which `CLAUDE.md` names the single source of truth for the HTTP/JSON boundary — said jsoniter all along. ## Decision -Use **jsoniter-scala** (`com.github.plokhotnyuk.jsoniter-scala`), 2.39.1. +Use **jsoniter-scala** (`com.github.plokhotnyuk.jsoniter-scala`), 2.39.1 at the +time this was decided. Only the `jsoniter-scala-core` artifact; the running pin +lives in `build.mill` → `Versions.jsoniter` and moves with routine bumps. ## How it is used, and why not the obvious way @@ -52,18 +61,30 @@ swapping the engine changed the two files underneath and left the DTOs alone. Good: - The style guide and the code agree again. -- One fewer dependency: the unused sttp-jsoniter-scala integration is gone. +- One fewer dependency: the unused sttp-upickle integration is gone. +- **And one fewer again: `jsoniter-scala-macros` is not declared.** Its whole + purpose is `JsonCodecMaker.make`, which derives a `JsonValueCodec[A]` from a + case class at compile time. The two-step boundary above derives nothing, so + every `com.github.plokhotnyuk` import under `modules/` is from + `jsoniter_scala.core`. `modules/codec` declares only that artifact, which + keeps roughly a megabyte of derivation machinery off a consumer's classpath + and makes README.md's "sttp client4 and jsoniter-scala, that is the list" + literally true. - **Numbers are exact.** The previous document model parsed every JSON number as - a `Double`, which silently loses precision above 2^53. `JsonValue.Num` holds a - `BigDecimal`, and a test round-trips 2^53 + 1 to prove it. + a `Double`, which silently loses precision above 2^53. `JsonValue` holds a + whole number as a `Long` (`JsonValue.Int64`) and everything else as a + `BigDecimal` (`JsonValue.Decimal`), and tests round-trip both 2^53 + 1 and a + value past `Long.MaxValue` to prove it. That split replaced an earlier + all-`BigDecimal` model, which was equally exact and cost about thirty bytes + more per number; `scripts/alloc-bench.sh` has the measurement. - **No hex dump in a failure message.** jsoniter appends one to parse errors by default; that is response payload, and this library's failures are logged, so it is switched off and a test asserts the body does not leak into the message. - **Depth is bounded.** The document reader is recursive, so a deeply nested body is remote input that could exhaust a caller's stack. `JsonValue.MaxDepth` rejects it, with a test. -- Decoders are stricter where jsoniter-scala was lenient: `JsonDecoder[String]` requires - a JSON string, where jsoniter-scala coerced `{"name": 7}` into `"7"`. Nothing wanted +- Decoders are stricter where upickle was lenient: `JsonDecoder[String]` requires + a JSON string, where upickle coerced `{"name": 7}` into `"7"`. Nothing wanted that coercion, and a silent one at the boundary is how a wrong field reaches the domain looking right. @@ -71,7 +92,7 @@ Bad: - The document model and its codec are ours to maintain: about 190 lines, covered by `JsonSuite` and the codec property suites. -- Per-field JSON paths on a *parse* failure are gone. jsoniter-scala's tracing visitor +- Per-field JSON paths on a *parse* failure are gone. upickle's tracing visitor could say `$.owner.login`; jsoniter reports an offset. In practice this costs nothing: parse failures are now always document-level (the model is total), and the field-level paths callers actually see come from each DTO's `toDomain`, @@ -81,6 +102,6 @@ Bad: | Alternative | Why rejected | | --- | --- | -| Keep jsoniter-scala | The style guide says jsoniter, the sttp integration that justified it was unused, and the `Double` numeric model was a latent precision defect. | +| Keep upickle | The style guide says jsoniter, the sttp integration that justified it was unused, and `ujson`'s `Double` numeric model was a latent precision defect. | | jsoniter with derived codecs per DTO | Cannot express "every field optional, `null` and absent identical, unknown kinds tolerated" without a per-field knob; `docs/HAZARDS.md` §1 shows the API requires exactly that. | | circe | A larger dependency, and its optics would not change the shape of the problem above. | diff --git a/docs/adr/0005-future-public-api.md b/docs/adr/0005-future-public-api.md index 23e9d00..0b90ccd 100644 --- a/docs/adr/0005-future-public-api.md +++ b/docs/adr/0005-future-public-api.md @@ -69,6 +69,10 @@ all such logic lives in `core` over `Exec[F]` and is tested with `F = Either`; `Future` appears only at the outermost projection, where each attempt is a fresh thunk (`Exec.suspend`). -Bad: the style guide's Ox and jsoniter examples no longer match the code. -`SCALA_CODE_STYLE.md` is left unedited — it is upstream-derived — and this ADR is -the pointer that explains the divergence. +Bad: the style guide's examples no longer match the code in two places. Its Ox +examples describe a dependency this build does not have at all. Its jsoniter +examples now name the right library — that part was settled by ADR-0003 — but +they derive a codec per DTO with `JsonCodecMaker`, and this build derives none; +`modules/codec` hand-writes a single codec for a document model instead, for the +reasons ADR-0003 gives. `SCALA_CODE_STYLE.md` is left unedited — it is +upstream-derived — and this ADR is the pointer that explains the divergence. diff --git a/mill b/mill index bd4b8fd..6920cb2 100755 --- a/mill +++ b/mill @@ -2,7 +2,18 @@ set -e -if [ -z "${DEFAULT_MILL_VERSION}" ] ; then DEFAULT_MILL_VERSION="1.1.6-104-5bbe1e"; fi +# This is the upstream Mill bootstrap script with two local additions, both +# marked "LOCAL CHANGE" below: the default version is a released one, and the +# downloaded distribution is checked against `.mill-checksums` before it runs. +# Re-generating this file from upstream drops both; re-apply them. + +# LOCAL CHANGE: upstream ships an untagged development build here +# ("1.1.6-104-5bbe1e" — a snapshot 104 commits past the 1.1.6 tag). It is only +# a fallback for a checkout without `.mill-version`, but a fallback that lands +# on an unreleased snapshot is still a build nobody can reproduce. `1.1.7` is +# the release `.mill-version` already pins, so the fallback and the pin now +# agree. +if [ -z "${DEFAULT_MILL_VERSION}" ] ; then DEFAULT_MILL_VERSION="1.1.7"; fi if [ -z "${GITHUB_RELEASE_CDN}" ] ; then GITHUB_RELEASE_CDN=""; fi @@ -119,6 +130,73 @@ esac MILL="${MILL_FINAL_DOWNLOAD_FOLDER}/$MILL_VERSION$ARTIFACT_SUFFIX" +# LOCAL CHANGE: verify the distribution before it is ever executed. +# +# Everything this project builds, tests, signs and publishes runs through the +# executable named by ${MILL}, so a swapped download is a swapped build. The +# three helpers below check it against a digest committed in `.mill-checksums`. +# +# `.mill-checksums` is read relative to the working directory, the same way the +# script already reads `.mill-version` a few lines up: `./mill` is always run +# from the repository root. +MILL_CHECKSUMS_FILE=".mill-checksums" +MILL_DIST_ID="$MILL_VERSION$ARTIFACT_SUFFIX" + +# Print the SHA-256 of "$1" as lowercase hex. sha256sum is coreutils (Linux), +# shasum ships with macOS, openssl is the last resort. No fallback to "skip the +# check": a machine that cannot hash the file cannot vouch for it either. +mill_sha256() { + if command -v sha256sum > /dev/null 2>&1 ; then + sha256sum "$1" | cut -d ' ' -f 1 + elif command -v shasum > /dev/null 2>&1 ; then + shasum -a 256 "$1" | cut -d ' ' -f 1 + elif command -v openssl > /dev/null 2>&1 ; then + openssl dgst -sha256 "$1" | sed 's/.*= *//' + else + echo "mill: cannot verify the download: no sha256sum, shasum or openssl on PATH" 1>&2 + exit 1 + fi +} + +# Print the recorded digest for distribution id "$1", or fail if there is none. +mill_expected_sha256() { + if [ ! -f "${MILL_CHECKSUMS_FILE}" ] ; then return 1 ; fi + awk -v id="$1" ' + $1 ~ /^#/ { next } + $1 == id { print $2 ; found = 1 ; exit } + END { exit !found } + ' "${MILL_CHECKSUMS_FILE}" +} + +# Set MILL_WANT_SHA256 to the recorded digest, or abort saying how to add one. +# Called before the download too, so an unlisted version fails in a second +# instead of after pulling 60 MB it was never going to be allowed to run. +mill_require_sha256() { + if ! MILL_WANT_SHA256="$(mill_expected_sha256 "${MILL_DIST_ID}")" ; then + echo "mill: no recorded SHA-256 for Mill ${MILL_DIST_ID}" 1>&2 + echo "mill: ${MILL_CHECKSUMS_FILE} decides which Mill distributions this" 1>&2 + echo "mill: repository will run. Add an entry for ${MILL_DIST_ID} (that" 1>&2 + echo "mill: file explains how) or pin a version already listed there." 1>&2 + exit 1 + fi +} + +# Abort unless the file at "$1" is the distribution we recorded. +mill_verify_sha256() { + mill_require_sha256 + MILL_GOT_SHA256="$(mill_sha256 "$1")" + if [ "${MILL_GOT_SHA256}" != "${MILL_WANT_SHA256}" ] ; then + echo "mill: SHA-256 mismatch for Mill ${MILL_DIST_ID} at $1" 1>&2 + echo "mill: expected ${MILL_WANT_SHA256}" 1>&2 + echo "mill: actual ${MILL_GOT_SHA256}" 1>&2 + echo "mill: refusing to run it. If a retry does not fix this, the file is" 1>&2 + echo "mill: not the published distribution — do not execute it." 1>&2 + exit 1 + fi + unset MILL_WANT_SHA256 + unset MILL_GOT_SHA256 +} + # If not already downloaded, download it if [ ! -s "${MILL}" ] || [ "$MILL_TEST_DRY_RUN_LAUNCHER_SCRIPT" = "1" ] ; then case $MILL_VERSION in @@ -168,9 +246,18 @@ if [ ! -s "${MILL}" ] || [ "$MILL_TEST_DRY_RUN_LAUNCHER_SCRIPT" = "1" ] ; then exit 0 fi + # LOCAL CHANGE: fail now if this version has no recorded digest, rather than + # after spending the bandwidth. + mill_require_sha256 + echo "Downloading mill ${MILL_VERSION} from ${MILL_DOWNLOAD_URL} ..." 1>&2 curl -f -L -o "${MILL_TEMP_DOWNLOAD_FILE}" "${MILL_DOWNLOAD_URL}" + # LOCAL CHANGE: check it while it is still an inert file in out/, before it + # is made executable and before it is moved into the shared cache. A download + # that fails the check never becomes something a later run would trust. + mill_verify_sha256 "${MILL_TEMP_DOWNLOAD_FILE}" + chmod +x "${MILL_TEMP_DOWNLOAD_FILE}" mkdir -p "${MILL_FINAL_DOWNLOAD_FOLDER}" @@ -178,6 +265,14 @@ if [ ! -s "${MILL}" ] || [ "$MILL_TEST_DRY_RUN_LAUNCHER_SCRIPT" = "1" ] ; then unset MILL_TEMP_DOWNLOAD_FILE unset MILL_DOWNLOAD_SUFFIX +else + # LOCAL CHANGE: a cached distribution is not a verified one. The cache lives + # outside the repository, is shared by every project on the machine, and may + # predate this check entirely — as it did on every machine that ran Mill + # before this commit. Hashing 60 MB costs tens of milliseconds; re-checking + # on each run is what makes "verified" mean the binary about to be executed + # rather than a download that happened once, months ago. + mill_verify_sha256 "${MILL}" fi MILL_FIRST_ARG="" @@ -192,6 +287,8 @@ unset MILL_OLD_DOWNLOAD_PATH unset OLD_MILL unset MILL_VERSION unset MILL_REPO_URL +unset MILL_CHECKSUMS_FILE +unset MILL_DIST_ID # -D mill.main.cli is for compatibility with Mill 0.10.9 - 0.13.0-M2 # We don't quote MILL_FIRST_ARG on purpose, so we can expand the empty value without quotes diff --git a/modules/client/src/com/worxbend/codeberg4s/CodebergClient.scala b/modules/client/src/com/worxbend/codeberg4s/CodebergClient.scala index 885b9cd..ccc872e 100644 --- a/modules/client/src/com/worxbend/codeberg4s/CodebergClient.scala +++ b/modules/client/src/com/worxbend/codeberg4s/CodebergClient.scala @@ -2,6 +2,7 @@ package com.worxbend.codeberg4s import com.worxbend.codeberg4s.client.FutureExec import com.worxbend.codeberg4s.client.FutureTimer +import com.worxbend.codeberg4s.client.GuardedTelemetry import com.worxbend.codeberg4s.codec.ApiErrorBodyCodec import com.worxbend.codeberg4s.core.ApiPipeline import com.worxbend.codeberg4s.core.BinaryHttpPort @@ -101,13 +102,24 @@ final class CodebergClient private ( * [[CodebergClient.usingBackend]] belongs to the caller, who may well be sharing it with the rest of their * application, and closing it here would break them. * - * Idempotent and safe to call from any thread: the second and later calls do nothing. Backend shutdown is - * asynchronous in sttp, so this method returns before the backend's own connections are gone; nothing in this - * library observes that, and waiting for it would make an ordinary `finally` block block. + * Closing an owned backend ends the JDK `java.net.http.HttpClient` under it: this library calls that client's + * `shutdown()`, which refuses new requests, lets the ones already sent run to completion, and returns without + * waiting for them. The JDK also offers a `close()` that waits, and this method deliberately does not use it — see + * "Idempotent" below. The `ExecutionContext` the client was built with is untouched, because that one belongs to the + * caller. * - * Using a client after closing it is a defect. Calls will fail with a rejected-execution failure from the scheduler - * rather than with a [[CodebergError]], because a closed client is a programming mistake and not a remote failure to + * Idempotent and safe to call from any thread: the second and later calls do nothing. Nothing here blocks, so this + * method returns before the last in-flight response has been delivered; nothing in this library observes that, and + * waiting for it would make an ordinary `finally` block block. + * + * Using a client after closing it is a defect. A call started afterwards fails with a rejected-execution failure + * from the scheduler, and a call that was already sitting in retry backoff fails with a + * `java.util.concurrent.CancellationException` — see [[com.worxbend.codeberg4s.client.FutureTimer.close]]. Neither + * is reported as a [[CodebergError]], because a closed client is a programming mistake and not a remote failure to * be retried. + * + * Both are failures, which is the guarantee that matters at shutdown: no `Future` this client handed out is left + * without an outcome, so an application closing down never waits on one that cannot finish. */ def close(): Unit = if closed.compareAndSet(false, true) then @@ -120,7 +132,8 @@ object CodebergClient: /** Builds a client that creates and owns its own HTTP backend. * * The backend is a JDK-HTTP-client sttp backend configured with [[CodebergConfig.connectTimeout]], and [[close]] - * shuts it down. This is the right constructor unless the application already has an sttp backend it wants reused. + * shuts it down — the connection pool and the JDK client's own selector thread go with it. This is the right + * constructor unless the application already has an sttp backend it wants reused. * * @param config * the instance to talk to, the credentials, the retry policy and the timeouts @@ -132,7 +145,10 @@ object CodebergClient: * * This library has no logging dependency and writes nothing anywhere, so this is the only way to see requests. A * [[com.worxbend.codeberg4s.core.Telemetry]] failure never fails the call it was observing — instrumentation that - * breaks must not break the application it instruments. + * breaks must not break the application it instruments. That covers both ways a callback can go wrong: throwing + * where it stands, and returning a `Future` that fails afterwards. Either way the observation is lost and the + * request's own outcome is what the caller receives. A fatal error — an `OutOfMemoryError`, say — is not swallowed, + * because it says the process is no longer sound. * * The callbacks receive a [[CallContext]] whose URI is already redacted, so an implementation cannot leak a token by * logging what it is handed. @@ -192,7 +208,13 @@ object CodebergClient: // Resolved here rather than at the call site because Telemetry.noOp needs // the Exec[Future] that only exists once this method has built it. - val observer = telemetry.getOrElse(Telemetry.noOp[Future]) + // + // A caller's sink is wrapped so that a callback which throws, or which + // returns a failed Future, cannot fail the request it was watching; see + // GuardedTelemetry for why the guard belongs here and not in Exec.attempt. + // Telemetry.noOp is this library's own code and cannot fail, so it is left + // unwrapped rather than paying for a guard on every unconfigured request. + val observer = telemetry.fold(Telemetry.noOp[Future])(GuardedTelemetry(_)) val port = SttpHttpPort(backend, config) diff --git a/modules/client/src/com/worxbend/codeberg4s/client/FutureExec.scala b/modules/client/src/com/worxbend/codeberg4s/client/FutureExec.scala index 87a8ec2..a1b92d2 100644 --- a/modules/client/src/com/worxbend/codeberg4s/client/FutureExec.scala +++ b/modules/client/src/com/worxbend/codeberg4s/client/FutureExec.scala @@ -6,6 +6,8 @@ import com.worxbend.codeberg4s.core.Exec import scala.concurrent.ExecutionContext import scala.concurrent.Future +import scala.util.Failure +import scala.util.Success /** The [[com.worxbend.codeberg4s.core.Exec]] instance the published client runs on. * @@ -45,10 +47,16 @@ final class FutureExec(using executionContext: ExecutionContext) extends Exec[Fu * * This is what the typed rail is built from: `client.repos.attempt.get(…)` is `attempt(client.repos.get(…))`, so the * two rails cannot drift apart. + * + * Written as one `transform` rather than `map(…).recover(…)`: the pair would build an intermediate `Future` and + * dispatch to the execution context twice for every call, and this sits on the path of every request the typed rail + * makes. */ override def attempt[A](fa: Future[A]): Future[Either[CodebergError, A]] = - fa.map(Right.apply).recover: - case CodebergException(error) => Left(error) + fa.transform: + case Success(value) => Success(Right(value)) + case Failure(CodebergException(error)) => Success(Left(error)) + case Failure(other) => Failure(other) /** Defers `thunk` until the returned effect is composed, so each retry issues a fresh request. See the class note. */ override def suspend[A](thunk: () => Future[A]): Future[A] = Future.delegate(thunk()) diff --git a/modules/client/src/com/worxbend/codeberg4s/client/FutureTimer.scala b/modules/client/src/com/worxbend/codeberg4s/client/FutureTimer.scala index ff87956..255db25 100644 --- a/modules/client/src/com/worxbend/codeberg4s/client/FutureTimer.scala +++ b/modules/client/src/com/worxbend/codeberg4s/client/FutureTimer.scala @@ -11,8 +11,10 @@ import scala.util.Failure import scala.util.Success import scala.util.Try -import java.util.concurrent.Executors -import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.CancellationException +import java.util.concurrent.Delayed +import java.util.concurrent.RunnableScheduledFuture +import java.util.concurrent.ScheduledThreadPoolExecutor import java.util.concurrent.ThreadFactory import java.util.concurrent.TimeUnit @@ -28,11 +30,15 @@ import java.util.concurrent.TimeUnit * even if an application forgets to [[close]] its client. Forgetting is still a leak — the thread lives as long as the * process — which is why [[com.worxbend.codeberg4s.CodebergClient.close]] releases it. * + * '''Every sleep ends.''' A `Future` handed out by [[sleep]] always reaches an outcome: it succeeds when the delay + * elapses, and it fails when [[close]] is called first. Nothing this type returns is left without one, so a caller can + * always await it. + * * '''Ownership.''' Whoever calls [[FutureTimer.apply]] owns the result and must [[close]] it. A * [[com.worxbend.codeberg4s.CodebergClient]] creates its own and closes it, so an application that only uses the * client never touches this type. */ -final class FutureTimer private (scheduler: ScheduledExecutorService) extends Timer[Future]: +final class FutureTimer private (scheduler: ScheduledThreadPoolExecutor) extends Timer[Future]: /** The wall clock, read when this method is called rather than when the returned effect completes. * @@ -47,21 +53,31 @@ final class FutureTimer private (scheduler: ScheduledExecutorService) extends Ti * rejects new work; that is reported as a failed `Future` rather than as a thrown exception, because using a closed * client is a defect and a defect must not be laundered into a [[com.worxbend.codeberg4s.CodebergError]] the caller * would then retry. + * + * A sleep that is already waiting when [[close]] arrives is failed rather than dropped — see there. */ override def sleep(duration: FiniteDuration): Future[Unit] = if duration <= Duration.Zero then Future.unit else - val promise = Promise[Unit]() - Try(scheduler.schedule(FutureTimer.completing(promise), duration.toMillis, TimeUnit.MILLISECONDS)) match - case Success(_) => promise.future + val completing = FutureTimer.Completing(Promise[Unit]()) + Try(scheduler.schedule(completing, duration.toMillis, TimeUnit.MILLISECONDS)) match + case Success(_) => completing.future case Failure(reason) => Future.failed(reason) - /** Releases the scheduler thread. Idempotent, and safe to call from any thread. + /** Releases the scheduler thread and fails every sleep that was still waiting. Idempotent, and safe to call from any + * thread. + * + * A sleep whose delay has not elapsed yet is abandoned: its `Future` is completed with a + * [[java.util.concurrent.CancellationException]]. Completing it is the whole point. A `Future` that is neither + * fulfilled nor failed has no outcome at all, so an application that closed its client while a call sat in retry + * backoff would wait on that `Future` for as long as the process lived. Failing it means "the client was closed + * under you", which a caller can see, log and shut down on. * - * Work already scheduled is abandoned, so a `Future` returned by an in-flight [[sleep]] never completes. That is the - * intended reading of "the client is closed": a retry that was waiting is not resumed. + * The failure is a `CancellationException` rather than a [[com.worxbend.codeberg4s.CodebergError]] for the same + * reason [[sleep]] reports a closed scheduler that way: closing a client that is still in use is a defect in the + * calling program, not a remote failure worth retrying. */ - def close(): Unit = scheduler.shutdownNow().discard + def close(): Unit = scheduler.shutdownNow().forEach(task => FutureTimer.abandon(task)) object FutureTimer: @@ -69,12 +85,68 @@ object FutureTimer: val ThreadName: String = "codeberg4s-timer" /** Creates a timer with its own daemon scheduler thread. The caller owns the result and must close it. */ - def apply(): FutureTimer = new FutureTimer(Executors.newSingleThreadScheduledExecutor(DaemonThreads)) + def apply(): FutureTimer = new FutureTimer(QueueKeepingPromises(DaemonThreads)) private val DaemonThreads: ThreadFactory = (runnable: Runnable) => val thread = Thread(runnable, ThreadName) thread.setDaemon(true) thread - private def completing(promise: Promise[Unit]): Runnable = - () => promise.success(()).discard + /** Fails the promise behind an abandoned queue entry, ignoring anything else the queue happened to hold. */ + private def abandon(task: Runnable): Unit = task match + case waiting: Waiting[?] => waiting.origin.abandon() + case _ => () + + /** The scheduler, subclassed so that its queue remembers which promise each waiting entry would have completed. + * + * A `ScheduledThreadPoolExecutor` does not put the `Runnable` it was handed onto its queue; it puts an internal task + * object that wraps it, and that wrapper — not the original — is what `shutdownNow()` gives back. Walking the + * returned list would therefore find nothing recognisable. `decorateTask` is the supported hook for choosing what + * goes onto the queue, so overriding it is how [[FutureTimer.close]] gets to see the waiting promises at all. + * + * `Executors.newSingleThreadScheduledExecutor` builds exactly this executor with a core pool size of one; the only + * thing given up by constructing it directly is the wrapper that stops callers reconfiguring it, and the instance + * never leaves [[FutureTimer]]. + */ + private final class QueueKeepingPromises(threads: ThreadFactory) extends ScheduledThreadPoolExecutor(1, threads): + + override protected def decorateTask[V]( + runnable: Runnable, + task: RunnableScheduledFuture[V], + ): RunnableScheduledFuture[V] = + runnable match + case completing: Completing => Waiting(completing, task) + case _ => task + + /** The scheduled work itself: complete the promise a [[FutureTimer.sleep]] handed out. + * + * It is a named class rather than a lambda so that [[QueueKeepingPromises.decorateTask]] can recognise it and carry + * its promise onto the queue. + */ + private final class Completing(promise: Promise[Unit]) extends Runnable: + + /** The effect [[FutureTimer.sleep]] returns; it ends in exactly one of [[run]] or [[abandon]]. */ + def future: Future[Unit] = promise.future + + override def run(): Unit = promise.trySuccess(()).discard + + /** Ends the sleep as a failure because the timer was closed before the delay elapsed. */ + def abandon(): Unit = + promise.tryFailure(CancellationException("codeberg4s timer closed while a retry was still waiting")).discard + + /** A queue entry that behaves exactly like the scheduler's own but also names the [[Completing]] it will run. + * + * Every method delegates; the single added member is [[origin]], which is what makes the list returned by + * `shutdownNow()` worth walking. + */ + private final class Waiting[V](val origin: Completing, delegate: RunnableScheduledFuture[V]) + extends RunnableScheduledFuture[V]: + override def run(): Unit = delegate.run() + override def isPeriodic: Boolean = delegate.isPeriodic + override def getDelay(unit: TimeUnit): Long = delegate.getDelay(unit) + override def compareTo(other: Delayed): Int = delegate.compareTo(other) + override def cancel(mayInterrupt: Boolean): Boolean = delegate.cancel(mayInterrupt) + override def isCancelled: Boolean = delegate.isCancelled + override def isDone: Boolean = delegate.isDone + override def get(): V = delegate.get() + override def get(timeout: Long, unit: TimeUnit): V = delegate.get(timeout, unit) diff --git a/modules/client/src/com/worxbend/codeberg4s/client/GuardedTelemetry.scala b/modules/client/src/com/worxbend/codeberg4s/client/GuardedTelemetry.scala new file mode 100644 index 0000000..70a5698 --- /dev/null +++ b/modules/client/src/com/worxbend/codeberg4s/client/GuardedTelemetry.scala @@ -0,0 +1,89 @@ +package com.worxbend.codeberg4s.client + +import com.worxbend.codeberg4s.CallContext +import com.worxbend.codeberg4s.CodebergError +import com.worxbend.codeberg4s.core.Telemetry + +import scala.concurrent.ExecutionContext +import scala.concurrent.Future +import scala.util.Failure +import scala.util.Success +import scala.util.Try +import scala.util.control.NonFatal + +import java.util.concurrent.ExecutionException + +/** Wraps a caller's [[com.worxbend.codeberg4s.core.Telemetry]] so that a broken sink cannot break the call it is + * watching. + * + * A sink is application code that this library agreed to run on its own request path, and there are exactly two ways + * for that code to go wrong: it throws where it stands, before any `Future` exists to carry the failure, or it returns + * a `Future` that fails later. Neither is a [[com.worxbend.codeberg4s.CodebergError]], so neither can be materialised + * by [[com.worxbend.codeberg4s.core.Exec.attempt]] — which catches this library's own failures and deliberately lets + * every other throwable stay failed, so that a defect in caller code is never laundered into an error the caller is + * told to act on. Correct as that is, it left an unobserved gap: a `NullPointerException` from a logging callback + * reached the caller of `repos.get` as the outcome of a request the server had already answered successfully. + * + * This class closes the gap at the only place where it can be closed without weakening `attempt`: the boundary where + * the caller's sink enters the library. Both failure modes end as a successful `Future[Unit]`, and the observation is + * lost — which is the intended trade, because instrumentation that breaks must not break the application it + * instruments. + * + * '''Fatal errors are not swallowed.''' A `VirtualMachineError` such as `OutOfMemoryError`, a `LinkageError`, a + * `ControlThrowable` or an `InterruptedException` passes straight through, on either rail. Those say the process is no + * longer sound, or that someone asked for cancellation; hiding one to protect a single API call would trade a visible + * crash for a silent corruption, or a cancellation for a hang. + * + * Only a sink the caller supplied is wrapped. [[com.worxbend.codeberg4s.core.Telemetry.noOp]] is this library's own + * code, it cannot fail, and it is on the path of every request an unconfigured client makes, so it is left alone + * rather than paying for a guard it does not need. + * + * @param sink + * the caller's observer, called exactly once per callback + * @param executionContext + * where the guard's continuation runs; the caller's, as everywhere else in this library + */ +private[codeberg4s] final class GuardedTelemetry(sink: Telemetry[Future])(using executionContext: ExecutionContext) + extends Telemetry[Future]: + + override def onRequest(ctx: CallContext): Future[Unit] = guarded(sink.onRequest(ctx)) + + override def onResponse(ctx: CallContext, status: Int): Future[Unit] = guarded(sink.onResponse(ctx, status)) + + override def onError(ctx: CallContext, error: CodebergError): Future[Unit] = guarded(sink.onError(ctx, error)) + + /** Runs one callback and reports success whatever it does, short of a fatal error. + * + * `callback` is by-name because evaluating it is itself one of the two failure modes: the `try` has to be around the + * call to the sink, not only around the `Future` the call produced. + */ + private def guarded(callback: => Future[Unit]): Future[Unit] = + try callback.transform(GuardedTelemetry.swallowed) + catch case NonFatal(_) => Future.unit + +private[codeberg4s] object GuardedTelemetry: + + /** The one outcome a guarded callback reports. Held as a value so that no callback allocates it. */ + private val Observed: Try[Unit] = Success(()) + + /** Turns any hideable outcome into success, and leaves a fatal one exactly as it was. */ + private val swallowed: Try[Unit] => Try[Unit] = + case failure @ Failure(error) if isFatal(error) => failure + case _ => Observed + + /** Whether `error` says the process is no longer sound — seeing through the box a `Future` puts one in. + * + * `scala.util.control.NonFatal` answers this directly for a throwable that arrives unaltered, which is what happens + * when a callback throws where it stands. It does not answer it for one that arrives through a `Future`: completing + * a promise with a `VirtualMachineError`, a `LinkageError`, an `InterruptedException` or a `ControlThrowable` + * replaces it with a `java.util.concurrent.ExecutionException` wrapping the original, and that wrapper is an + * ordinary exception. Asking `NonFatal` alone would therefore hide the one class of failure that must never be + * hidden — an interrupt above all, because a swallowed `InterruptedException` is a cancellation the caller asked for + * and did not get. + * + * The recursion is for a cause that is itself boxed, which costs one line and removes the question. + */ + private def isFatal(error: Throwable): Boolean = + error match + case boxed: ExecutionException => Option(boxed.getCause).exists(isFatal) + case other => !NonFatal(other) diff --git a/modules/client/src/com/worxbend/codeberg4s/client/WireDecode.scala b/modules/client/src/com/worxbend/codeberg4s/client/WireDecode.scala index 54700c0..73d9ed5 100644 --- a/modules/client/src/com/worxbend/codeberg4s/client/WireDecode.scala +++ b/modules/client/src/com/worxbend/codeberg4s/client/WireDecode.scala @@ -2,6 +2,7 @@ package com.worxbend.codeberg4s.client import com.worxbend.codeberg4s.core.Decode import com.worxbend.codeberg4s.core.DecodeFailure +import com.worxbend.codeberg4s.core.ResponseBody /** Joins the two halves of reading a response: parse the wire DTO, then project it into the domain. * @@ -24,4 +25,4 @@ private[codeberg4s] object WireDecode: * the DTO's own projection, which reports the JSON path of whatever the domain required and did not get */ def of[D, A](wire: Decode[D])(toDomain: D => Either[DecodeFailure, A]): Decode[A] = - (body: String) => wire(body).flatMap(toDomain) + (body: ResponseBody) => wire(body).flatMap(toDomain) diff --git a/modules/client/src/com/worxbend/codeberg4s/issues/IssueAttachmentApi.scala b/modules/client/src/com/worxbend/codeberg4s/issues/IssueAttachmentApi.scala index 0233577..fcc6446 100644 --- a/modules/client/src/com/worxbend/codeberg4s/issues/IssueAttachmentApi.scala +++ b/modules/client/src/com/worxbend/codeberg4s/issues/IssueAttachmentApi.scala @@ -37,10 +37,14 @@ import scala.concurrent.Future * ==Nothing here downloads== * * Every method returns metadata. [[IssueAttachment.browserDownloadUrl]] is the supported route to the content: hand it - * to an HTTP client that can stream bytes. [[com.worxbend.codeberg4s.core.CodebergResponse]] carries a body as - * `String` and the transport reads every response with sttp's `asStringAlways`, so an arbitrary file that has been - * through a UTF-8 decoder is no longer that file — which is the same reason - * [[com.worxbend.codeberg4s.repositories.actions.RepositoryActionApi]] does not offer its two ZIP endpoints. + * to an HTTP client that can stream bytes. + * + * That used to be forced by the library: a response body was a `String`, and an arbitrary file that has been through a + * UTF-8 decoder is no longer that file. It is no longer forced — [[com.worxbend.codeberg4s.core.CodebergResponse]] + * carries a [[com.worxbend.codeberg4s.core.ResponseBody]], which is bytes — so an attachment-download operation is now + * something this group '''could''' offer. It does not yet, because adding one is a new endpoint with its own tests + * rather than a rider on the change that made it possible, and because an attachment can be arbitrarily large and + * nothing here streams. * * ==Not paged, and that is the endpoints' decision== * diff --git a/modules/client/src/com/worxbend/codeberg4s/issues/IssueDecoders.scala b/modules/client/src/com/worxbend/codeberg4s/issues/IssueDecoders.scala index 581a0f8..18a5282 100644 --- a/modules/client/src/com/worxbend/codeberg4s/issues/IssueDecoders.scala +++ b/modules/client/src/com/worxbend/codeberg4s/issues/IssueDecoders.scala @@ -4,6 +4,7 @@ import com.worxbend.codeberg4s.JsonPath import com.worxbend.codeberg4s.client.WireDecode import com.worxbend.codeberg4s.codec.Json import com.worxbend.codeberg4s.core.Decode +import com.worxbend.codeberg4s.core.ResponseBody import com.worxbend.codeberg4s.issues.wire.AttachmentDto import com.worxbend.codeberg4s.issues.wire.CommentDto import com.worxbend.codeberg4s.issues.wire.IssueDeadlineDto @@ -55,7 +56,7 @@ private[issues] object IssueDecoders: val comment: Decode[Option[Comment]] = val present = WireDecode.of(Json.decoder[CommentDto])(_.toDomain) - (body: String) => if body.isBlank then Right(None) else present(body).map(Some.apply) + (body: ResponseBody) => if body.isBlank then Right(None) else present(body).map(Some.apply) /** A bare array of comment objects, as the repository-wide comment listing returns it. */ val comments: Decode[Vector[Comment]] = diff --git a/modules/client/src/com/worxbend/codeberg4s/miscellaneous/PlainText.scala b/modules/client/src/com/worxbend/codeberg4s/miscellaneous/PlainText.scala index 68fac7d..d388dd0 100644 --- a/modules/client/src/com/worxbend/codeberg4s/miscellaneous/PlainText.scala +++ b/modules/client/src/com/worxbend/codeberg4s/miscellaneous/PlainText.scala @@ -1,6 +1,7 @@ package com.worxbend.codeberg4s.miscellaneous import com.worxbend.codeberg4s.core.Decode +import com.worxbend.codeberg4s.core.ResponseBody /** The identity [[com.worxbend.codeberg4s.core.Decode]]: a response body, unchanged. * @@ -23,13 +24,18 @@ import com.worxbend.codeberg4s.core.Decode */ private[codeberg4s] object PlainText: - /** The body exactly as received: no trimming, no charset guessing, no parsing. + /** The body exactly as received: no trimming, no parsing. * - * The transport has already decoded the bytes into a `String`, so the only thing left to get wrong would be changing - * them. + * '''This is one of the few places that genuinely turns bytes into text.''' The transport no longer does it — it + * carries the bytes and the charset the response declared, so that the JSON endpoints, which are nearly all of them, + * never pay for a decoding they do not want. Here the decoding is exactly what was asked for, and it uses the + * charset the response declared rather than one picked here: see [[com.worxbend.codeberg4s.core.ResponseBody.text]]. + * In practice that charset is UTF-8, because Forgejo answers `text/plain; charset=utf-8` on every endpoint that + * reaches this decoder, but the header is read rather than assumed and an instance behind a proxy that rewrote it is + * still understood. */ val decoder: Decode[String] = - (body: String) => Right(body) + (body: ResponseBody) => Right(body.text) /** The identity decoder followed by `interpret`, for an endpoint whose body is text but whose result is not a * `String`. @@ -43,4 +49,4 @@ private[codeberg4s] object PlainText: * [[com.worxbend.codeberg4s.miscellaneous.SigningKey]] */ def decodedAs[A](interpret: String => A): Decode[A] = - (body: String) => decoder(body).map(interpret) + (body: ResponseBody) => decoder(body).map(interpret) diff --git a/modules/client/src/com/worxbend/codeberg4s/organizations/actions/OrganizationActionDecoders.scala b/modules/client/src/com/worxbend/codeberg4s/organizations/actions/OrganizationActionDecoders.scala index f4a11e7..b49683a 100644 --- a/modules/client/src/com/worxbend/codeberg4s/organizations/actions/OrganizationActionDecoders.scala +++ b/modules/client/src/com/worxbend/codeberg4s/organizations/actions/OrganizationActionDecoders.scala @@ -50,13 +50,20 @@ private[actions] object OrganizationActionDecoders: val runners: Decode[Vector[ActionRunner]] = WireDecode.of(Json.decoder[Vector[ActionRunnerDto]])(dtos => ActionRunnerDto.toDomainAll(JsonPath.Root, dtos)) - /** The `{id, uuid, token}` object a runner registration returns. */ + /** The `{id, uuid, token}` object a runner registration returns, whose `token` is a live credential. + * + * Marked [[com.worxbend.codeberg4s.core.Decode.sensitive]]: anyone holding that token can attach a runner that + * executes workflow code, so a payload that does not decode must not put an excerpt of this body into + * [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]]. + */ val registeredRunner: Decode[RegisteredRunner] = - WireDecode.of(Json.decoder[RegisteredRunnerDto])(_.toDomain) + Decode.sensitive(WireDecode.of(Json.decoder[RegisteredRunnerDto])(_.toDomain)) - /** The one-key object the registration-token endpoint returns. */ + /** The one-key object the registration-token endpoint returns — the same credential with nothing around it, and + * [[com.worxbend.codeberg4s.core.Decode.sensitive]] for the same reason as [[registeredRunner]]. + */ val registrationToken: Decode[RunnerRegistrationToken] = - WireDecode.of(Json.decoder[RegistrationTokenDto])(_.toDomain) + Decode.sensitive(WireDecode.of(Json.decoder[RegistrationTokenDto])(_.toDomain)) /** A bare array of job objects, as the runner job search returns it. */ val jobs: Decode[Vector[ActionRunJob]] = diff --git a/modules/client/src/com/worxbend/codeberg4s/paging/PageWalk.scala b/modules/client/src/com/worxbend/codeberg4s/paging/PageWalk.scala index 668aea5..8afa639 100644 --- a/modules/client/src/com/worxbend/codeberg4s/paging/PageWalk.scala +++ b/modules/client/src/com/worxbend/codeberg4s/paging/PageWalk.scala @@ -1,5 +1,8 @@ package com.worxbend.codeberg4s.paging +import com.worxbend.codeberg4s.CodebergError +import com.worxbend.codeberg4s.CodebergException + import scala.collection.immutable.VectorBuilder import scala.concurrent.ExecutionContext import scala.concurrent.Future @@ -33,6 +36,12 @@ import scala.concurrent.Future * [[com.worxbend.codeberg4s.CodebergException]] on the convenience rail. A failure part-way through a walk discards * the pages already gathered; use [[fold]] if partial progress needs to be kept somewhere. * + * '''The page cap is a failure, not a quiet stop.''' A walk visits at most [[MaxPages]] pages. Reaching that cap with + * the server still offering another page fails the `Future` with + * [[com.worxbend.codeberg4s.CodebergError.WalkTruncated]] rather than returning what was gathered so far, because a + * short answer that looks exactly like a complete one is the worse of the two outcomes. The error carries the window + * to resume from, so a caller who genuinely wants more than half a million items can continue from there. + * * '''Memory.''' [[all]] holds every item. A repository can have tens of thousands of issues, so prefer [[fold]] or * [[foreach]] when the result does not need to exist all at once — that is the whole reason no operation in this * library returns an unbounded collection by default. @@ -41,6 +50,10 @@ object PageWalk: /** The most pages any bounded walk visits before giving up, so a server that always offers a next page cannot spin * forever. Deliberately generous: 10 000 pages of 50 is half a million items. + * + * A walk that reaches this many pages and is offered another fails with + * [[com.worxbend.codeberg4s.CodebergError.WalkTruncated]]. A walk whose last page happens to be the ten-thousandth + * and offers nothing further has reached the natural end of the listing and succeeds. */ val MaxPages: Int = 10_000 @@ -69,18 +82,25 @@ object PageWalk: * the listing operation, applied once per page * @param step * combines the state so far with the page just fetched + * @return + * the folded state, or a `Future` failed with [[com.worxbend.codeberg4s.CodebergException]] wrapping + * [[com.worxbend.codeberg4s.CodebergError.WalkTruncated]] when [[MaxPages]] was reached with pages still to come */ def fold[A, B](first: PageParams, zero: B)(fetch: PageParams => Future[Page[A]])( step: (B, Page[A]) => B )(using ExecutionContext): Future[B] = def loop(params: PageParams, state: B, visited: Int): Future[B] = - if visited >= MaxPages then Future.successful(state) - else - fetch(params).flatMap: page => - val next = step(state, page) - page.nextPage match - case Some(number) => loop(params.at(number), next, visited + 1) - case None => Future.successful(next) + fetch(params).flatMap: page => + val next = step(state, page) + val fetched = visited + 1 + page.nextPage match + // The cap is tested against a page the server actually offered, so a + // listing that ends on the last page the cap allows is complete and + // succeeds. Only an offer this walk refuses to follow is truncation. + case Some(number) if fetched >= MaxPages => + Future.failed(CodebergException(CodebergError.WalkTruncated(fetched, params.at(number)))) + case Some(number) => loop(params.at(number), next, fetched) + case None => Future.successful(next) loop(first, zero, 0) diff --git a/modules/client/src/com/worxbend/codeberg4s/repositories/actions/ActionDownloadApi.scala b/modules/client/src/com/worxbend/codeberg4s/repositories/actions/ActionDownloadApi.scala index 86e5f05..15a592a 100644 --- a/modules/client/src/com/worxbend/codeberg4s/repositories/actions/ActionDownloadApi.scala +++ b/modules/client/src/com/worxbend/codeberg4s/repositories/actions/ActionDownloadApi.scala @@ -14,16 +14,23 @@ import scala.concurrent.Future /** The two Actions endpoints whose success body is a ZIP archive rather than text. * - * Reached as `client.repos.actions.downloads`. They are separate from [[RepositoryActionApi]] because they are the - * only operations in the library that need a byte-carrying transport - * ([[com.worxbend.codeberg4s.core.BinaryHttpPort]]), and folding that requirement into the class that serves the other - * twenty-six would have made every one of them depend on a capability none of them use. + * Reached as `client.downloads`. They are separate from [[RepositoryActionApi]] because they are the only operations + * in the library that need a byte-carrying transport ([[com.worxbend.codeberg4s.core.BinaryHttpPort]]), and folding + * that requirement into the class that serves the other twenty-six would have made every one of them depend on a + * capability none of them use. * * '''Memory.''' Both operations hold the whole archive in memory as an `Array[Byte]`. A CI artifact can be large, and * this library does not stream. Check [[com.worxbend.codeberg4s.repositories.actions.ActionArtifact.sizeInBytes]] * before downloading if that matters, or fetch * [[com.worxbend.codeberg4s.repositories.actions.ActionArtifact.archiveDownloadUrl]] with your own HTTP client. * + * The heap is not the only thing standing in the way: these two are the operations + * [[com.worxbend.codeberg4s.CodebergConfig.maxDownloadBodyBytes]] bounds — 50 MiB by default, rather than the 16 MiB + * every other operation gets, because an artifact is whatever a workflow uploaded and is legitimately far larger than + * a JSON document. An archive past the bound fails as [[com.worxbend.codeberg4s.TransportCause.ResponseTooLarge]] and + * is not retried, since a second attempt would download it again. Raise the setting if you need bigger archives and + * have the memory for them. + * * '''Failures.''' As everywhere else: [[com.worxbend.codeberg4s.CodebergError.Api]] for a non-2xx — `404` when the * artifact or run does not exist, has expired, or belongs to a repository the token cannot see, and `410` when Forgejo * has garbage-collected it — [[com.worxbend.codeberg4s.CodebergError.Transport]] when nothing arrived, and diff --git a/modules/client/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionApi.scala b/modules/client/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionApi.scala index 4b864b1..7866ff5 100644 --- a/modules/client/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionApi.scala +++ b/modules/client/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionApi.scala @@ -84,13 +84,10 @@ import scala.concurrent.Future * - `GET /repos/{owner}/{repo}/actions/artifacts/{artifact_id}/zip` (`DownloadActionArtifact`); * - `GET /repos/{owner}/{repo}/actions/runs/{run_id}/logs` (`repoGetActionRunLogs`). * - * Both answer a ZIP archive. [[com.worxbend.codeberg4s.core.CodebergResponse]] carries a body as `String`, and the - * transport reads every response with sttp's `asStringAlways` — a ZIP that has been through a UTF-8 decoder is no - * longer a ZIP, and no amount of re-encoding recovers it. There is no `Decode[Array[Byte]]` and no streaming response - * in the core vocabulary to model them with, so they are not offered rather than offered broken. - * - * The supported routes are [[ActionArtifact.archiveDownloadUrl]], which a caller hands to an HTTP client that can - * stream bytes, and [[jobLogs]], which is genuinely text and is implemented. + * Both answer a ZIP archive, which is not text, so neither belongs on a class whose every other operation decodes one. + * They are '''implemented''', on [[ActionDownloadApi]] — reached as `client.downloads` — which reads a body as bytes. + * [[ActionArtifact.archiveDownloadUrl]] remains available for a caller who would rather stream the archive with their + * own HTTP client, since nothing in this library streams. [[jobLogs]] is genuinely text and is implemented here. * * @param pipeline * the shared request pipeline; the only thing here that reaches the network diff --git a/modules/client/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionDecoders.scala b/modules/client/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionDecoders.scala index 5199221..acbc966 100644 --- a/modules/client/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionDecoders.scala +++ b/modules/client/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionDecoders.scala @@ -4,6 +4,7 @@ import com.worxbend.codeberg4s.JsonPath import com.worxbend.codeberg4s.client.WireDecode import com.worxbend.codeberg4s.codec.Json import com.worxbend.codeberg4s.core.Decode +import com.worxbend.codeberg4s.core.ResponseBody import com.worxbend.codeberg4s.miscellaneous.PlainText import com.worxbend.codeberg4s.repositories.actions.wire.ActionArtifactDto import com.worxbend.codeberg4s.repositories.actions.wire.ActionRunDto @@ -72,13 +73,20 @@ private[actions] object RepositoryActionDecoders: val runners: Decode[Vector[ActionRunner]] = WireDecode.of(Json.decoder[Vector[ActionRunnerDto]])(dtos => ActionRunnerDto.toDomainAll(JsonPath.Root, dtos)) - /** The `{id, uuid, token}` object a runner registration returns. */ + /** The `{id, uuid, token}` object a runner registration returns, whose `token` is a live credential. + * + * Marked [[com.worxbend.codeberg4s.core.Decode.sensitive]]: anyone holding that token can attach a runner that + * executes workflow code, so a payload that does not decode must not put an excerpt of this body into + * [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]]. + */ val registeredRunner: Decode[RegisteredRunner] = - WireDecode.of(Json.decoder[RegisteredRunnerDto])(_.toDomain) + Decode.sensitive(WireDecode.of(Json.decoder[RegisteredRunnerDto])(_.toDomain)) - /** The one-key object the registration-token endpoint returns. */ + /** The one-key object the registration-token endpoint returns — the same credential with nothing around it, and + * [[com.worxbend.codeberg4s.core.Decode.sensitive]] for the same reason as [[registeredRunner]]. + */ val registrationToken: Decode[RunnerRegistrationToken] = - WireDecode.of(Json.decoder[RegistrationTokenDto])(_.toDomain) + Decode.sensitive(WireDecode.of(Json.decoder[RegistrationTokenDto])(_.toDomain)) /** A bare array of secret objects — names and timestamps, never values. */ val secrets: Decode[Vector[ActionSecret]] = @@ -103,7 +111,7 @@ private[actions] object RepositoryActionDecoders: val dispatchedRun: Decode[Option[DispatchedWorkflowRun]] = val present = WireDecode.of(Json.decoder[DispatchedWorkflowRunDto])(_.toDomain) - (body: String) => if body.isBlank then Right(None) else present(body).map(Some.apply) + (body: ResponseBody) => if body.isBlank then Right(None) else present(body).map(Some.apply) /** A job's log, exactly as the instance sent it. * diff --git a/modules/client/src/com/worxbend/codeberg4s/repositories/admin/RepositoryAdminApi.scala b/modules/client/src/com/worxbend/codeberg4s/repositories/admin/RepositoryAdminApi.scala index defcbc8..7c23871 100644 --- a/modules/client/src/com/worxbend/codeberg4s/repositories/admin/RepositoryAdminApi.scala +++ b/modules/client/src/com/worxbend/codeberg4s/repositories/admin/RepositoryAdminApi.scala @@ -106,10 +106,9 @@ import java.time.LocalDate * - `GET /repos/{owner}/{repo}/actions/artifacts/{artifact_id}/zip` (`DownloadActionArtifact`); * - `GET /repos/{owner}/{repo}/actions/runs/{run_id}/logs` (`repoGetActionRunLogs`). * - * Both answer a ZIP archive. [[com.worxbend.codeberg4s.core.CodebergResponse]] carries a body as `String`, and the - * transport reads every response with sttp's `asStringAlways` — a ZIP that has been through a UTF-8 decoder is no - * longer a ZIP, and no amount of re-encoding recovers it. Offering them would mean handing back corrupted bytes and - * calling it success. + * Both answer a ZIP archive, so neither is served by a class whose every other operation decodes text. They are + * implemented on [[com.worxbend.codeberg4s.repositories.actions.ActionDownloadApi]], reached as `client.downloads`, + * which reads a body as bytes. * * @param pipeline * the shared request pipeline; the only thing here that reaches the network diff --git a/modules/client/src/com/worxbend/codeberg4s/repositories/gitdata/RepositoryGitApi.scala b/modules/client/src/com/worxbend/codeberg4s/repositories/gitdata/RepositoryGitApi.scala index f629bfb..8a727b9 100644 --- a/modules/client/src/com/worxbend/codeberg4s/repositories/gitdata/RepositoryGitApi.scala +++ b/modules/client/src/com/worxbend/codeberg4s/repositories/gitdata/RepositoryGitApi.scala @@ -53,11 +53,15 @@ import scala.concurrent.Future * ==Five of these endpoints do not answer JSON== * * `{sha}.diff` and `{sha}.patch` produce `text/plain`, and this library returns that text unchanged. `/raw`, `/media` - * and `/archive` produce '''bytes''' — `application/octet-stream`, a zip, a gzipped tar — and that is a shape the - * current core vocabulary cannot express: [[com.worxbend.codeberg4s.core.CodebergResponse]] carries its body as a - * `String`, so the transport has already decoded those bytes as text before any code here sees them. The three methods - * therefore return a `String` and say so plainly on each of them rather than pretending to hand back a file. A - * byte-carrying response body is a change to `modules/core` and `modules/transport`, not to this group. + * and `/archive` produce '''bytes''' — `application/octet-stream`, a zip, a gzipped tar — and the three methods that + * serve them return a `String`, which is lossy for anything that is not text. + * + * That was once a limitation of core: a response body was a `String`, so the bytes were already gone before any code + * here saw them. It no longer is. [[com.worxbend.codeberg4s.core.CodebergResponse]] carries a + * [[com.worxbend.codeberg4s.core.ResponseBody]], so the bytes survive as far as the decoder, and the `String` these + * three return is now this group's own choice rather than something forced on it. Changing their return type is a + * change to the published API of this group, with its own tests and its own migration note, so it is deliberately not + * folded into the change that removed the constraint. */ final class RepositoryGitApi private[codeberg4s] (pipeline: ApiPipeline[Future])(using exec: Exec[Future]): @@ -497,13 +501,12 @@ final class RepositoryGitApi private[codeberg4s] (pipeline: ApiPipeline[Future]) /** Reads a file's raw bytes — `GET /repos/{owner}/{repo}/raw/{filepath}`. * * '''The result is the response body decoded as text, and that is a real limitation.''' The endpoint produces - * `application/octet-stream`; the transport turns the bytes into a `String` before any code here sees them, because - * [[com.worxbend.codeberg4s.core.CodebergResponse]] has nowhere else to put them. For a text file that is exactly - * what a caller wants. '''For a binary file it is lossy''' — bytes that are not valid in the response's charset - * become replacement characters, and re-encoding the result does not give the file back. Use + * `application/octet-stream`, and this method decodes it with the charset the response declared. For a text file + * that is exactly what a caller wants. '''For a binary file it is lossy''' — bytes that are not valid in that + * charset become replacement characters, and re-encoding the result does not give the file back. Use * [[com.worxbend.codeberg4s.repositories.RepositoryApi.getContents]] for a binary blob under the instance's inline - * size limit, whose base64 payload does survive; there is no lossless path here for one above it until core grows a - * byte-carrying response. + * size limit, whose base64 payload does survive. The bytes now reach the decoder intact, so a lossless variant of + * this method has become possible; see the group note above for why it is not part of this signature yet. * * Unlike the contents endpoint this returns the file itself with no envelope, and is therefore the cheap way to read * a large text file. @@ -555,8 +558,9 @@ final class RepositoryGitApi private[codeberg4s] (pipeline: ApiPipeline[Future]) * * '''An archive is always binary, so the text limitation on [[getRawFile]] is not a caveat here but the whole * story.''' A zip or a gzipped tar decoded as text is not recoverable. This method builds and issues the request - * correctly and returns what the current transport can express; it is not a way to obtain a usable archive file, and - * making it one is a change to `modules/core` and `modules/transport`. + * correctly and returns what its own signature can express; it is not a way to obtain a usable archive file. Making + * it one is now a change to this method's return type alone — the transport and core carry the bytes intact, and + * `com.worxbend.codeberg4s.repositories.actions.ActionDownloadApi` shows the shape such an operation takes. * * '''Failures.''' The group contract above. [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]] is not * reachable: nothing is parsed. diff --git a/modules/client/src/com/worxbend/codeberg4s/users/account/UserAccountDecoders.scala b/modules/client/src/com/worxbend/codeberg4s/users/account/UserAccountDecoders.scala index 13b9828..e89b282 100644 --- a/modules/client/src/com/worxbend/codeberg4s/users/account/UserAccountDecoders.scala +++ b/modules/client/src/com/worxbend/codeberg4s/users/account/UserAccountDecoders.scala @@ -47,10 +47,27 @@ import com.worxbend.codeberg4s.users.account.wire.UserSettingsDto */ private[account] object UserAccountDecoders: - /** One OAuth2 application object, which on a creation response carries the client secret. */ + /** One OAuth2 application object as a '''read''' returns it — never with a secret in it. + * + * Forgejo stores the secret hashed, so `GET /user/applications/oauth2/{id}` cannot carry one. That is why this keeps + * the ordinary body excerpt on a decoding failure while [[issuedApplication]] does not: the two responses have the + * same shape and different consequences, and one decoder for both would have to be as cautious as the more dangerous + * of them. + */ val application: Decode[OAuth2Application] = WireDecode.of(Json.decoder[OAuth2ApplicationDto])(_.toDomain) + /** The same object as [[application]], from the responses that '''do''' carry `client_secret`. + * + * Those are the `201` of a registration and the `200` of an update — the spec does not say whether an update + * re-issues the secret, so this treats it as though it does, which is the safe direction to be wrong in. Marked + * [[com.worxbend.codeberg4s.core.Decode.sensitive]], so a payload that does not decode reports + * [[com.worxbend.codeberg4s.core.ApiPipeline.redactedSnippet]] rather than an excerpt containing the one copy of + * that credential. + */ + val issuedApplication: Decode[OAuth2Application] = + Decode.sensitive(application) + /** A bare array of OAuth2 application objects, as the listing returns it — never with a secret in it. */ val applications: Decode[Vector[OAuth2Application]] = WireDecode.of(Json.decoder[Vector[OAuth2ApplicationDto]]): dtos => @@ -118,13 +135,20 @@ private[account] object UserAccountDecoders: val jobs: Decode[Vector[ActionRunJob]] = WireDecode.of(Json.decoder[Vector[ActionRunJobDto]])(dtos => ActionRunJobDto.toDomainAll(JsonPath.Root, dtos)) - /** The `{id, uuid, token}` object a runner registration returns. */ + /** The `{id, uuid, token}` object a runner registration returns, whose `token` is a live credential. + * + * Marked [[com.worxbend.codeberg4s.core.Decode.sensitive]]: anyone holding that token can attach a runner that + * executes workflow code, so a payload that does not decode must not put an excerpt of this body into + * [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]]. + */ val registeredRunner: Decode[RegisteredRunner] = - WireDecode.of(Json.decoder[RegisteredRunnerDto])(_.toDomain) + Decode.sensitive(WireDecode.of(Json.decoder[RegisteredRunnerDto])(_.toDomain)) - /** The one-key object the registration-token endpoint returns. */ + /** The one-key object the registration-token endpoint returns — the same credential with nothing around it, and + * [[com.worxbend.codeberg4s.core.Decode.sensitive]] for the same reason as [[registeredRunner]]. + */ val registrationToken: Decode[RunnerRegistrationToken] = - WireDecode.of(Json.decoder[RegistrationTokenDto])(_.toDomain) + Decode.sensitive(WireDecode.of(Json.decoder[RegistrationTokenDto])(_.toDomain)) /** One variable object. */ val variable: Decode[ActionVariable] = diff --git a/modules/client/src/com/worxbend/codeberg4s/users/account/UserApplicationApi.scala b/modules/client/src/com/worxbend/codeberg4s/users/account/UserApplicationApi.scala index 751747d..d81dc65 100644 --- a/modules/client/src/com/worxbend/codeberg4s/users/account/UserApplicationApi.scala +++ b/modules/client/src/com/worxbend/codeberg4s/users/account/UserApplicationApi.scala @@ -33,6 +33,14 @@ import scala.concurrent.Future * of the application holding it, so it cannot leak into a log or into a [[com.worxbend.codeberg4s.CodebergError]] on * the way past. Revealing it is an explicit act; see that type. * + * The mask only covers a secret this library managed to decode. The other way out is the raw body: a `2xx` that does + * not match the model puts an excerpt of the payload into + * [[com.worxbend.codeberg4s.CodebergError.DecodingFailed.snippet]], and on these two responses that payload is the + * secret. So [[create]] and [[update]] read their response through a decoder marked + * [[com.worxbend.codeberg4s.core.Decode.sensitive]], which makes the pipeline report + * [[com.worxbend.codeberg4s.core.ApiPipeline.redactedSnippet]] instead. [[get]] and [[list]] keep the excerpt, because + * the bodies they read carry no secret to lose. + * * ==Evidence== * * '''Every model here is derived from `spec/swagger.v1.json`, not from a captured response.''' The harvest behind @@ -110,7 +118,7 @@ final class UserApplicationApi private[codeberg4s] (pipeline: ApiPipeline[Future */ def create(definition: OAuth2ApplicationDefinition): Future[OAuth2Application] = pipeline.call(UserApplicationApi.createRequest(definition), RetryEligibility.Never)(using - UserAccountDecoders.application) + UserAccountDecoders.issuedApplication) /** Replaces an application's definition — `PATCH /user/applications/oauth2/{id}`. * @@ -132,7 +140,7 @@ final class UserApplicationApi private[codeberg4s] (pipeline: ApiPipeline[Future */ def update(id: OAuth2ApplicationId, definition: OAuth2ApplicationDefinition): Future[OAuth2Application] = pipeline.call(UserApplicationApi.updateRequest(id, definition), RetryEligibility.Never)(using - UserAccountDecoders.application) + UserAccountDecoders.issuedApplication) /** Deletes one of the account's applications — `DELETE /user/applications/oauth2/{id}`. * diff --git a/modules/client/src/com/worxbend/codeberg4s/users/social/SocialDecoders.scala b/modules/client/src/com/worxbend/codeberg4s/users/social/SocialDecoders.scala index cc3702f..424825f 100644 --- a/modules/client/src/com/worxbend/codeberg4s/users/social/SocialDecoders.scala +++ b/modules/client/src/com/worxbend/codeberg4s/users/social/SocialDecoders.scala @@ -5,6 +5,7 @@ import com.worxbend.codeberg4s.client.WireDecode import com.worxbend.codeberg4s.codec.Json import com.worxbend.codeberg4s.core.Decode import com.worxbend.codeberg4s.core.DecodeFailure +import com.worxbend.codeberg4s.core.ResponseBody import com.worxbend.codeberg4s.issues.TrackedTime import com.worxbend.codeberg4s.issues.wire.TrackedTimeDto import com.worxbend.codeberg4s.miscellaneous.PlainText @@ -40,9 +41,9 @@ import com.worxbend.codeberg4s.users.wire.UserDto * `GET /user/gpg_key_token`, so the body '''is''' the token. Running it through a JSON parser would turn a good * response into a decoding failure. * - * [[createdAccessToken]] does parse JSON, and is listed here only to say where it is: it is the one decoder in the - * library that yields a usable credential, and it is deliberately not reachable from any listing — see - * [[com.worxbend.codeberg4s.users.social.wire.AccessTokenDto]]. + * [[createdAccessToken]] does parse JSON, and is listed here only to say where it is: it is the one decoder in this + * group that yields a usable credential — the others are the Actions runner registration decoders — and it is + * deliberately not reachable from any listing; see [[com.worxbend.codeberg4s.users.social.wire.AccessTokenDto]]. */ private[social] object SocialDecoders: @@ -103,9 +104,15 @@ private[social] object SocialDecoders: val accessTokens: Decode[Vector[AccessToken]] = WireDecode.of(Json.decoder[Vector[AccessTokenDto]])(dtos => AccessTokenDto.toDomainAll(JsonPath.Root, dtos)) - /** The `201` of a token creation — the one decoder in this library that yields a usable credential. */ + /** The `201` of a token creation — the one decoder in this library that yields a usable personal access token. + * + * Marked [[com.worxbend.codeberg4s.core.Decode.sensitive]], because this body '''is''' the credential: a payload + * that carries `sha1` but fails to decode for some other reason would otherwise put a live token into the excerpt on + * [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]], which is a value applications log. See + * [[com.worxbend.codeberg4s.core.ApiPipeline.redactedSnippet]] for what the excerpt becomes instead. + */ val createdAccessToken: Decode[CreatedAccessToken] = - WireDecode.of(Json.decoder[AccessTokenDto])(_.toCreated) + Decode.sensitive(WireDecode.of(Json.decoder[AccessTokenDto])(_.toCreated)) /** The plain-text challenge `GET /user/gpg_key_token` answers. * @@ -114,7 +121,7 @@ private[social] object SocialDecoders: * a caller. The failure carries [[com.worxbend.codeberg4s.ValidationError.message]] and never the rejected body. */ val verificationToken: Decode[GpgKeyToken] = - (body: String) => + (body: ResponseBody) => PlainText .decoder(body) .flatMap(text => GpgKeyToken.from(text).left.map(error => DecodeFailure(JsonPath.Root, error.message))) diff --git a/modules/client/src/com/worxbend/codeberg4s/users/social/UserTokenApi.scala b/modules/client/src/com/worxbend/codeberg4s/users/social/UserTokenApi.scala index aee9d96..132592d 100644 --- a/modules/client/src/com/worxbend/codeberg4s/users/social/UserTokenApi.scala +++ b/modules/client/src/com/worxbend/codeberg4s/users/social/UserTokenApi.scala @@ -2,7 +2,6 @@ package com.worxbend.codeberg4s.users.social import com.worxbend.codeberg4s.CodebergError import com.worxbend.codeberg4s.HttpMethod -import com.worxbend.codeberg4s.auth.ApiToken import com.worxbend.codeberg4s.core.ApiPipeline import com.worxbend.codeberg4s.core.CodebergRequest import com.worxbend.codeberg4s.core.Exec @@ -34,11 +33,12 @@ import scala.concurrent.Future * - it is returned '''once'''. `GET /users/{username}/tokens` sends the same token with its material empty, and * [[AccessToken]] has no field that could hold one. A caller who does not store the value has to revoke the token * and mint another; - * - it cannot reach a failure a caller receives. [[com.worxbend.codeberg4s.CallContext]] carries a redacted URI and - * never a body, and the one channel that '''would''' have carried the body — - * [[com.worxbend.codeberg4s.CodebergError.DecodingFailed.snippet]], which - * [[com.worxbend.codeberg4s.core.ApiPipeline]] fills with an excerpt of the response — is emptied by [[create]] - * before the failure is raised. See that method for what is emptied and what is not; + * - it cannot reach a failure a caller receives, or one a telemetry sink observes. + * [[com.worxbend.codeberg4s.CallContext]] carries a redacted URI and never a body, and the one channel that + * '''would''' have carried the body — [[com.worxbend.codeberg4s.CodebergError.DecodingFailed.snippet]], which + * [[com.worxbend.codeberg4s.core.ApiPipeline]] fills with an excerpt of the response — is replaced by a + * placeholder inside the pipeline, because the decoder this endpoint uses is marked + * [[com.worxbend.codeberg4s.core.Decode.sensitive]]. See [[create]]; * - it never travels in a request. The material is generated by the instance, so no renderer in this library holds * one and no request body can leak one. * @@ -116,29 +116,28 @@ final class UserTokenApi private[codeberg4s] (pipeline: ApiPipeline[Future])(usi * [[com.worxbend.codeberg4s.core.ApiPipeline]] puts a bounded excerpt of the response body into every * [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]], which is exactly the right default and exactly wrong * here: the `201` body of this endpoint '''is''' a credential, so a payload that did not decode — one carrying - * `sha1` but no `id`, say — would put a minted, working token into whatever the caller logs. This method therefore - * replaces that excerpt with [[UserTokenApi.RedactedBody]] before raising, on both rails. The `path` and the reason - * survive untouched, so the failure is still diagnosable; what is lost is the payload dump, which for this one - * endpoint is the part nobody may see. + * `sha1` but no `id`, say — would put a minted, working token into whatever the caller logs. * - * '''One gap remains, and it is core's rather than this group's.''' A [[com.worxbend.codeberg4s.core.Telemetry]] - * implementation observing `onError` is called by the pipeline with the un-emptied failure, because the hook fires - * inside the pipeline and before this method sees anything. A deployment that logs raw telemetry errors '''and''' - * hits a malformed `201` on this endpoint would record the token. Closing that would mean teaching `ApiPipeline` - * which endpoints carry credentials, which is a change to core. + * The decoder this call passes to the pipeline is therefore marked + * [[com.worxbend.codeberg4s.core.Decode.sensitive]], and the pipeline substitutes + * [[com.worxbend.codeberg4s.core.ApiPipeline.redactedSnippet]] for the excerpt as it builds the failure. The `path` + * and the reason survive untouched, so the failure is still diagnosable, and the placeholder still reports that a + * body arrived and how many bytes it held; what is lost is the payload dump, which for this one endpoint is the part + * nobody may see. + * + * '''The substitution happens inside the pipeline, and that is the point.''' Scrubbing the failure here, after the + * call returned, would have left [[com.worxbend.codeberg4s.core.Telemetry.onError]] observing the un-scrubbed one: + * the hook fires while the attempt is being settled, before any endpoint sees the result. A deployment that logs raw + * telemetry errors '''and''' hits a malformed `201` on this endpoint would have recorded the token. * * '''Failures.''' The group contract above; a decoding failure means the payload carried no `id`, or no usable * `sha1` — and [[com.worxbend.codeberg4s.ValidationError]] never echoes a rejected value, so the reason text does * not carry the credential either. */ def create(username: Username, command: CreateAccessToken): Future[CreatedAccessToken] = - val sent = pipeline.call(UserTokenApi.createRequest(username, command), RetryEligibility.Never)(using + pipeline.call(UserTokenApi.createRequest(username, command), RetryEligibility.Never)(using SocialDecoders.createdAccessToken) - exec.flatMap(exec.attempt(sent)): - case Right(created) => exec.pure(created) - case Left(error) => exec.raise(UserTokenApi.withoutBody(error)) - /** Revokes an access token — `DELETE /users/{username}/tokens/{token}`. * * '''Whether this is retried depends on how the token was addressed''', and the difference is a real hazard rather @@ -169,27 +168,6 @@ object UserTokenApi: /** The stable operation id of [[UserTokenApi.delete]]. */ val DeleteOperation: String = "users.tokens.delete" - /** What replaces the response excerpt on a failed [[UserTokenApi.create]]. - * - * The same constant [[com.worxbend.codeberg4s.auth.ApiToken]] renders itself as, so a reader who sees it in a log - * recognises it as "a credential was here" rather than as an empty body. - */ - val RedactedBody: String = ApiToken.Redacted - - /** `error` with any response excerpt replaced by [[RedactedBody]]. - * - * Only [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]] carries one, and only [[UserTokenApi.create]] calls - * this. [[com.worxbend.codeberg4s.CodebergError.RetriesExhausted]] is unwrapped anyway, even though a creation is - * never retried, so the guarantee does not depend on that staying true. - */ - private def withoutBody(error: CodebergError): CodebergError = - error match - case CodebergError.DecodingFailed(ctx, _, path, cause) => - CodebergError.DecodingFailed(ctx, RedactedBody, path, cause) - case CodebergError.RetriesExhausted(ctx, attempts, last) => - CodebergError.RetriesExhausted(ctx, attempts, withoutBody(last)) - case other => other - /** Whether revoking a token addressed by `token` may be attempted again. * * Exposed rather than hidden, because it is the one place in this group where a retry decision depends on a value diff --git a/modules/client/test/src/com/worxbend/codeberg4s/CodebergClientSuite.scala b/modules/client/test/src/com/worxbend/codeberg4s/CodebergClientSuite.scala index da222bf..a4b44a9 100644 --- a/modules/client/test/src/com/worxbend/codeberg4s/CodebergClientSuite.scala +++ b/modules/client/test/src/com/worxbend/codeberg4s/CodebergClientSuite.scala @@ -8,6 +8,8 @@ import com.worxbend.codeberg4s.repositories.Repository import com.worxbend.codeberg4s.repositories.RepositoryApi import com.worxbend.codeberg4s.retry.Jitter import com.worxbend.codeberg4s.retry.RetryPolicy +import com.worxbend.codeberg4s.syntax.discard +import com.worxbend.codeberg4s.transport.SttpHttpPort import sttp.client4.Backend import sttp.client4.testing.BackendStub @@ -20,8 +22,15 @@ import munit.FunSuite import scala.concurrent.ExecutionContext import scala.concurrent.Future import scala.concurrent.duration.DurationInt +import scala.concurrent.duration.FiniteDuration +import scala.jdk.DurationConverters.ScalaDurationOps + +import java.util.concurrent.Executors /** The published façade, end to end, over a `BackendStub`: nothing in this suite opens a socket. + * + * The one exception is the JDK-HTTP-client test below, which builds a real client to watch it shut down. It sends no + * request, so it opens no socket either. * * The subject here is the wiring, not the codecs — `modules/codec` already asserts decoding against the golden * captures. The payloads below are therefore small hand-written bodies chosen to exercise the seams: what a caller @@ -106,6 +115,43 @@ final class CodebergClientSuite extends FunSuite: assertEquals(backend.closes, 1) + /** The stub above proves `close` was called; this proves the call reaches something. + * + * `CountingBackend` cannot tell a `close()` that released a connection pool from one that returned an + * already-completed `Future` and did nothing, which is exactly what sttp's own backend used to do here. So this test + * runs against a real JDK HTTP client and asks the client itself. No request is sent — the JDK starts the client's + * selector thread when the client is built — so nothing here opens a socket. + * + * The execution context is this test's own rather than the suite's, because the assertion blocks the calling thread + * and the client's callbacks must have a thread of their own to run on. + */ + test("close terminates the JDK HTTP client behind a backend the client owns"): + // munit continues on whichever thread completed the previous test's Future, + // and the retry tests above are completed by the client's own scheduler + // thread — which `CodebergClient.close` then interrupts, by design, to + // abandon a retry that was waiting. So this body can start on a thread + // whose interrupt flag is already set, and the blocking wait below would + // throw InterruptedException before it waited at all. Clearing the flag + // makes the test independent of which thread it was handed; nothing in this + // suite is waiting to be interrupted. + Thread.interrupted().discard + + val executor = ExecutionContext.fromExecutorService(Executors.newSingleThreadExecutor()) + + try + val http = SttpHttpPort.defaultHttpClient(CodebergClientSuite.ConnectTimeout, executor) + val client = CodebergClient.owning(configFor(Auth.Anonymous), SttpHttpPort.owning(http, executor))(using executor) + + assert(!http.isTerminated, "a client that was never closed already reports itself terminated") + + client.close() + + assert( + http.awaitTermination(CodebergClientSuite.TerminationLimit.toJava), + "close() left the JDK HTTP client running", + ) + finally executor.shutdown() + test("close leaves a backend the caller supplied open, because the caller owns it"): val backend = CountingBackend(responding(200, CodebergClientSuite.VersionBody)) val client = CodebergClient.usingBackend(configFor(Auth.Anonymous), backend) @@ -178,6 +224,13 @@ object CodebergClientSuite: /** A token that must not turn up in any rendering of a failure. */ private val Secret: String = "cb-0123456789abcdef-secret" + private val ConnectTimeout: FiniteDuration = 3.seconds + + /** Generous on purpose: an idle JDK HTTP client terminates in single-digit milliseconds, so this is a hang detector + * rather than a race the test is trying to win. + */ + private val TerminationLimit: FiniteDuration = 10.seconds + private val ExpectedVersion: ServerVersion = ServerVersion("16.0.0-dev-668-1bdb1938+gitea-1.22.0") private val VersionBody: String = s"""{"version": "${ExpectedVersion.raw}"}""" diff --git a/modules/client/test/src/com/worxbend/codeberg4s/TelemetryFailureSuite.scala b/modules/client/test/src/com/worxbend/codeberg4s/TelemetryFailureSuite.scala new file mode 100644 index 0000000..d91045e --- /dev/null +++ b/modules/client/test/src/com/worxbend/codeberg4s/TelemetryFailureSuite.scala @@ -0,0 +1,125 @@ +package com.worxbend.codeberg4s + +import com.worxbend.codeberg4s.auth.Auth +import com.worxbend.codeberg4s.client.GuardedTelemetry +import com.worxbend.codeberg4s.core.Telemetry + +import sttp.client4.Backend +import sttp.client4.testing.BackendStub +import sttp.client4.testing.ResponseStub +import sttp.model.StatusCode + +import munit.FunSuite + +import scala.concurrent.ExecutionContext +import scala.concurrent.Future + +/** The promise on [[CodebergClient.apply]]: a telemetry sink that breaks must not break the call it was watching. + * + * A sink is application code the library agreed to run on its own request path, and there are two ways for that code + * to go wrong — it throws where it stands, or it hands back a `Future` that later fails. Neither is a + * [[CodebergError]], so neither is something the caller of `repos.get` can act on; both used to reach that caller + * anyway. Every test below asserts the same thing from a different angle: the response still arrives. + */ +final class TelemetryFailureSuite extends FunSuite: + + private given ExecutionContext = munitExecutionContext + + private val Instance: BaseUri = BaseUri.from("https://forge.example/api/v1") match + case Right(value) => value + case Left(error) => fail(s"invalid fixture: ${error.field} ${error.message}") + + test("a sink that throws inside onRequest does not fail the call"): + onClient(TelemetryFailureSuite.throwing): client => + client.version.get().map(version => assertEquals(version.raw, TelemetryFailureSuite.Version)) + + test("a sink that throws inside onRequest does not fail the typed rail either"): + onClient(TelemetryFailureSuite.throwing): client => + client.version.attempt + .get() + .map(result => assertEquals(result.map(_.raw), Right(TelemetryFailureSuite.Version))) + + test("a sink whose onRequest returns a failed Future does not fail the call"): + onClient(TelemetryFailureSuite.failing): client => + client.version.get().map(version => assertEquals(version.raw, TelemetryFailureSuite.Version)) + + test("a sink whose onRequest returns a failed Future does not fail the typed rail either"): + onClient(TelemetryFailureSuite.failing): client => + client.version.attempt + .get() + .map(result => assertEquals(result.map(_.raw), Right(TelemetryFailureSuite.Version))) + + test("a fatal error from a sink is not swallowed"): + val guarded = GuardedTelemetry(TelemetryFailureSuite.fatal) + + guarded.onRequest(TelemetryFailureSuite.SomeCall).failed.map: thrown => + assertEquals(rootCause(thrown).getClass, classOf[OutOfMemoryError]) + + test("an interrupt from a sink is not swallowed"): + val guarded = GuardedTelemetry(TelemetryFailureSuite.interrupted) + + guarded.onRequest(TelemetryFailureSuite.SomeCall).failed.map: thrown => + assertEquals(rootCause(thrown).getClass, classOf[InterruptedException]) + + // --- assertions ----------------------------------------------------------- + + /** The throwable underneath however many `ExecutionException`s `Future` wrapped it in on the way here. + * + * Completing a promise with a fatal throwable boxes it, so the failure a test observes is the wrapper and the + * identity worth asserting on is the thing inside. + */ + private def rootCause(thrown: Throwable): Throwable = + Option(thrown.getCause) match + case Some(cause) => rootCause(cause) + case None => thrown + + // --- fixtures ------------------------------------------------------------- + + /** Runs `use` against a client that answers `200` with a version payload and reports to `telemetry`. */ + private def onClient[A](telemetry: Telemetry[Future])(use: CodebergClient => Future[A]): Future[A] = + val config = CodebergConfig(Auth.Anonymous).copy(baseUri = Instance) + val client = CodebergClient.usingBackend(config, responding, telemetry) + + use(client).transform: outcome => + client.close() + outcome + + private def responding: Backend[Future] = + BackendStub.asynchronousFuture.whenAnyRequest + .thenRespond(ResponseStub.adjust(s"""{"version": "${TelemetryFailureSuite.Version}"}""", StatusCode(200))) + +object TelemetryFailureSuite: + + private val Version: String = "12.0.1+gitea-1.22.0" + + /** What an ordinarily broken sink reports. Not a [[CodebergError]], so nothing downstream can turn it into one. */ + private def boom: RuntimeException = IllegalStateException("the telemetry sink is broken") + + /** A sink that does `request` when a call is announced and nothing at all afterwards. + * + * The four sinks below break the same callback and differ only in how they break it, so they are one definition with + * four arguments rather than four near-identical anonymous classes. `request` is by-name because one of the four + * throws rather than returning. + */ + private def brokenSink(request: => Future[Unit]): Telemetry[Future] = new Telemetry[Future]: + + override def onRequest(ctx: CallContext): Future[Unit] = request + + override def onResponse(ctx: CallContext, status: Int): Future[Unit] = Future.unit + + override def onError(ctx: CallContext, error: CodebergError): Future[Unit] = Future.unit + + /** Throws where it stands, before any `Future` exists to carry the failure. */ + private def throwing: Telemetry[Future] = brokenSink(throw boom) + + /** Returns normally and fails afterwards. */ + private def failing: Telemetry[Future] = brokenSink(Future.failed(boom)) + + /** Reports that the process is no longer sound. The guard must let this through untouched. */ + private def fatal: Telemetry[Future] = brokenSink(Future.failed(OutOfMemoryError("the heap is gone"))) + + /** Reports that someone asked for cancellation. Swallowing that would turn a cancellation into a hang. */ + private def interrupted: Telemetry[Future] = brokenSink(Future.failed(InterruptedException("cancelled"))) + + /** A context to hand a callback that is being tested on its own, away from a request. */ + private val SomeCall: CallContext = CallContext("version.get", HttpMethod.Get, "https://forge.example", None, 0L) diff --git a/modules/client/test/src/com/worxbend/codeberg4s/client/FutureTimerSuite.scala b/modules/client/test/src/com/worxbend/codeberg4s/client/FutureTimerSuite.scala index d1137f0..3122db1 100644 --- a/modules/client/test/src/com/worxbend/codeberg4s/client/FutureTimerSuite.scala +++ b/modules/client/test/src/com/worxbend/codeberg4s/client/FutureTimerSuite.scala @@ -1,13 +1,19 @@ package com.worxbend.codeberg4s.client +import com.worxbend.codeberg4s.syntax.discard + import munit.FunSuite +import scala.concurrent.Await import scala.concurrent.ExecutionContext import scala.concurrent.duration.Duration import scala.concurrent.duration.DurationInt import scala.concurrent.duration.FiniteDuration +import scala.util.Failure import scala.util.Success +import scala.util.Try +import java.util.concurrent.CancellationException import java.util.concurrent.RejectedExecutionException /** The `Future` instance of [[com.worxbend.codeberg4s.core.Timer]]. @@ -19,6 +25,15 @@ final class FutureTimerSuite extends FunSuite: private given ExecutionContext = munitExecutionContext + /** Clears a left-over thread interrupt before each test. + * + * Closing a timer interrupts the scheduler's thread, and one test below deliberately continues on that thread in + * order to inspect it. munit may then start the next test on the same, now-interrupted, thread. An interrupted + * thread cannot wait: `Await` throws `InterruptedException` at once instead of honouring its timeout. Clearing the + * flag here keeps that leak from turning into a flake in whichever test happens to run next. + */ + override def beforeEach(context: BeforeEach): Unit = Thread.interrupted().discard + test("a non-positive delay completes immediately, without going near the scheduler"): val timer = FutureTimer() @@ -75,6 +90,20 @@ final class FutureTimerSuite extends FunSuite: case _: RejectedExecutionException => () case other => fail(s"expected the scheduler to reject the work, got $other") + test("closing the timer fails every sleep that was still waiting, rather than leaving it without an outcome"): + val timer = FutureTimer() + val waiting = List.fill(2)(timer.sleep(FutureTimerSuite.UnreachableDelay)) + + timer.close() + + // A bounded `Await`, deliberately, rather than returning a mapped Future. The bug this guards against left the + // promise with no outcome at all, and a continuation on a Future that never completes never runs — so a regression + // would hang the suite instead of failing it. + waiting.foreach: effect => + Try(Await.result(effect, FutureTimerSuite.CompletionBound)) match + case Failure(_: CancellationException) => () + case outcome => fail(s"closing the timer left a waiting sleep at $outcome") + /** Runs a continuation on whatever thread completed the promise — for a scheduled sleep, the scheduler's own. */ private val OnSchedulerThread: ExecutionContext = ExecutionContext.parasitic @@ -82,3 +111,9 @@ object FutureTimerSuite: /** Long enough that a scheduled completion cannot plausibly have happened before the next statement runs. */ private val ObservableDelay: FiniteDuration = 200.millis + + /** Far longer than the test can run, so a sleep given this delay is certainly still waiting when the timer closes. */ + private val UnreachableDelay: FiniteDuration = 1.hour + + /** How long a closed timer is given to fail its waiting sleeps. Closing does the work inline, so this is slack. */ + private val CompletionBound: FiniteDuration = 5.seconds diff --git a/modules/client/test/src/com/worxbend/codeberg4s/paging/PageWalkSuite.scala b/modules/client/test/src/com/worxbend/codeberg4s/paging/PageWalkSuite.scala index 192b2b0..45ac609 100644 --- a/modules/client/test/src/com/worxbend/codeberg4s/paging/PageWalkSuite.scala +++ b/modules/client/test/src/com/worxbend/codeberg4s/paging/PageWalkSuite.scala @@ -1,5 +1,8 @@ package com.worxbend.codeberg4s.paging +import com.worxbend.codeberg4s.CodebergError +import com.worxbend.codeberg4s.CodebergException + import munit.FunSuite import scala.collection.mutable.ListBuffer @@ -155,9 +158,55 @@ final class PageWalkSuite extends FunSuite: visited.append(params.page.value) Future.successful(Page(Vector(1), params, None, PageNumber.from(params.page.value + 1).toOption, None)) + val result = PageWalk.fold(PageParams.First, 0)(fetch)((count, page) => count + page.items.size) + + assertEquals(visited.size, PageWalk.MaxPages) + assert(result.value.exists(_.isFailure), "the walk should not have returned a value") + + test("hitting the page cap fails rather than returning the pages gathered so far"): + // The defect this replaced: the cap returned the accumulated state, which is + // the same shape a complete walk returns. A caller could not tell a listing + // of ten thousand pages from one of ten thousand and one. + def fetch(params: PageParams): Future[Page[Int]] = + Future.successful(Page(Vector(1), params, None, PageNumber.from(params.page.value + 1).toOption, None)) + + val result = PageWalk.fold(PageParams.First, 0)(fetch)((count, page) => count + page.items.size) + val expected: Throwable = + CodebergException(CodebergError.WalkTruncated(PageWalk.MaxPages, PageParams(cappedPage, PageSize.Default))) + + assertEquals(result.value.flatMap(_.failed.toOption), Some(expected)) + + test("the truncation error resumes at the page the walk refused, keeping the caller's page size"): + val size = PageSize.from(7).toOption.getOrElse(PageSize.Default) + val start = PageParams(PageNumber.First, size) + + def fetch(params: PageParams): Future[Page[Int]] = + Future.successful(Page(Vector.empty, params, None, PageNumber.from(params.page.value + 1).toOption, None)) + + val result = PageWalk.fold(start, 0)(fetch)((count, _) => count) + + result.value.flatMap(_.failed.toOption) match + case Some(CodebergException(CodebergError.WalkTruncated(pagesVisited, resumeFrom))) => + assertEquals(pagesVisited, PageWalk.MaxPages) + assertEquals(resumeFrom.page, cappedPage) + assertEquals(resumeFrom.size, size) + case other => + fail(s"expected a truncated walk, got $other") + + test("a listing that ends on the last page the cap allows is complete, not truncated"): + // The boundary the cap must not get wrong. Exactly MaxPages pages arrive and + // the last one offers nothing further, so the walk saw the whole collection + // and there is nothing to report. + def fetch(params: PageParams): Future[Page[Int]] = + val next = if params.page.value < PageWalk.MaxPages then PageNumber.from(params.page.value + 1).toOption else None + Future.successful(Page(Vector(1), params, None, next, None)) + val counted = await(PageWalk.fold(PageParams.First, 0)(fetch)((count, page) => count + page.items.size)) assertEquals(counted, PageWalk.MaxPages) - assertEquals(visited.size, PageWalk.MaxPages) + + /** The page a capped walk is offered and declines: one past the last it fetched. */ + private val cappedPage: PageNumber = + PageNumber.from(PageWalk.MaxPages + 1).toOption.getOrElse(PageNumber.First) extension [A](value: A) private def discard: Unit = () diff --git a/modules/client/test/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionApiSuite.scala b/modules/client/test/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionApiSuite.scala index 6bdc60f..73f07c6 100644 --- a/modules/client/test/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionApiSuite.scala +++ b/modules/client/test/src/com/worxbend/codeberg4s/repositories/actions/RepositoryActionApiSuite.scala @@ -272,6 +272,17 @@ final class RepositoryActionApiSuite extends FunSuite: .registerRunner(Handle, Name, orFail(RegisterRunner.named("build-box-3"))) .map(_ => assertEquals(backend.allInteractions.size, 1, "the POST was retried")) + test("a registration response that does not decode reports no part of the credential"): + val truncated = RepositoryActionApiSuite.RegisteredBody.dropRight(1) + + onBackend(RecordingBackend(responding(201, truncated))): api => + api.attempt.registerRunner(Handle, Name, orFail(RegisterRunner.named("build-box-3"))).map: + case Left(error @ CodebergError.DecodingFailed(_, snippet, _, _)) => + assertEquals(snippet, ApiPipeline.redactedSnippet(truncated.length)) + assert(!error.describe.contains("QWERTY123"), s"the body excerpt carried the token: ${error.describe}") + case other => + fail(s"expected a decoding failure, got $other") + test("actions.runners.delete is a DELETE on the runner"): val backend = RecordingBackend(responding(204, "")) diff --git a/modules/client/test/src/com/worxbend/codeberg4s/users/account/UserApplicationApiSuite.scala b/modules/client/test/src/com/worxbend/codeberg4s/users/account/UserApplicationApiSuite.scala index c6e10f9..45e4ac8 100644 --- a/modules/client/test/src/com/worxbend/codeberg4s/users/account/UserApplicationApiSuite.scala +++ b/modules/client/test/src/com/worxbend/codeberg4s/users/account/UserApplicationApiSuite.scala @@ -2,6 +2,7 @@ package com.worxbend.codeberg4s.users.account import com.worxbend.codeberg4s.CodebergError import com.worxbend.codeberg4s.CodebergException +import com.worxbend.codeberg4s.core.ApiPipeline import com.worxbend.codeberg4s.paging.PageParams import sttp.client4.Backend @@ -165,14 +166,26 @@ final class UserApplicationApiSuite extends AccountApiSuite: assert(!s"$application".contains("gto_"), "interpolation leaked the secret") assert(!application.clientSecret.toString.contains("gto_"), "the Option's toString leaked the secret") - test("a body that failed to decode is snippetted verbatim, credential included — the pipeline's contract, pinned"): + test("a creation body that failed to decode is withheld, not snippetted, because it carries the secret"): onApi(responding(201, UserApplicationApiSuite.SecretWithoutIdBody)): api => api.attempt.create(definition).map: outcome => assertEquals(decodingPathOf(outcome), "$.id") - assert( - describe(outcome).contains("gto_"), - "the snippet no longer carries the body; if that is deliberate, ClientSecret's Scaladoc must be updated", + assertEquals( + snippetOf(outcome), + ApiPipeline.redactedSnippet(UserApplicationApiSuite.SecretWithoutIdBody.length), ) + assert(!describe(outcome).contains("gto_"), s"the snippet leaked the secret: ${describe(outcome)}") + + test("a read that failed to decode keeps its snippet, because no read can carry a secret"): + val body = """{"name":"no id"}""" + + onApi(responding(200, body)): api => + api.attempt.get(Application).map(outcome => assertEquals(snippetOf(outcome), body)) + + private def snippetOf[A](outcome: Either[CodebergError, A]): String = + outcome match + case Left(CodebergError.DecodingFailed(_, snippet, _, _)) => snippet + case other => fail(s"expected a decoding failure, got $other") private def describe[A](outcome: Either[CodebergError, A]): String = outcome match diff --git a/modules/client/test/src/com/worxbend/codeberg4s/users/social/UserTokenApiSuite.scala b/modules/client/test/src/com/worxbend/codeberg4s/users/social/UserTokenApiSuite.scala index acaa8b1..2bcaeff 100644 --- a/modules/client/test/src/com/worxbend/codeberg4s/users/social/UserTokenApiSuite.scala +++ b/modules/client/test/src/com/worxbend/codeberg4s/users/social/UserTokenApiSuite.scala @@ -4,6 +4,7 @@ import com.worxbend.codeberg4s.CodebergError import com.worxbend.codeberg4s.CodebergException import com.worxbend.codeberg4s.HttpMethod import com.worxbend.codeberg4s.auth.ApiToken +import com.worxbend.codeberg4s.core.ApiPipeline import com.worxbend.codeberg4s.paging.PageParams import com.worxbend.codeberg4s.repositories.Owner import com.worxbend.codeberg4s.repositories.RepoName @@ -120,7 +121,7 @@ final class UserTokenApiSuite extends FunSuite with SocialApiHarness: api.attempt.create(Handle, orFail(CreateAccessToken.named("ci"))).map: case Left(error @ CodebergError.DecodingFailed(_, snippet, path, _)) => assertEquals(path.render, "$.id") - assertEquals(snippet, UserTokenApi.RedactedBody) + assertEquals(snippet, ApiPipeline.redactedSnippet(UserTokenApiSuite.CreatedWithoutIdBody.length)) assert( !error.describe.contains(UserTokenApiSuite.Material), s"the body excerpt leaked the credential into describe: ${error.describe}", @@ -128,7 +129,7 @@ final class UserTokenApiSuite extends FunSuite with SocialApiHarness: case other => fail(s"expected a decoding failure, got $other") - test("the excerpt is emptied on the convenience rail too, not only on the typed one"): + test("the placeholder reaches the convenience rail too, not only the typed one"): onStub(responding(201, UserTokenApiSuite.CreatedWithoutIdBody)): api => api.create(Handle, orFail(CreateAccessToken.named("ci"))).failed.map: case CodebergException(error) => diff --git a/modules/codec/src/com/worxbend/codeberg4s/codec/ArrayElements.scala b/modules/codec/src/com/worxbend/codeberg4s/codec/ArrayElements.scala new file mode 100644 index 0000000..caf0efb --- /dev/null +++ b/modules/codec/src/com/worxbend/codeberg4s/codec/ArrayElements.scala @@ -0,0 +1,48 @@ +package com.worxbend.codeberg4s.codec + +import com.worxbend.codeberg4s.core.DecodeFailure + +import scala.annotation.tailrec + +/** Applies a conversion that can fail to every element of an array, stopping at the first element that fails. + * + * Four places in this module wanted exactly this and each had grown its own copy: [[JsonDecoder.arrayOf]] and + * [[JsonDecoder.all]] for `JSON → DTO`, and the two `wire` helpers + * ([[com.worxbend.codeberg4s.repositories.wire.Elements]] and [[com.worxbend.codeberg4s.issues.wire.WireElements]]) + * for `DTO → domain`. The copies agreed on the contract, which is the only reason they were survivable; they are here + * once so they cannot start disagreeing. + * + * '''One bad element fails the whole array.''' A listing that silently dropped a malformed element would under-report, + * and a caller cannot tell an under-report from a short page. + * + * The conversion receives each element's zero-based position, because three of the four callers turn it into a + * [[com.worxbend.codeberg4s.JsonPath]] segment so a failure reads `$[7].sha` rather than `$`. + */ +private[codeberg4s] object ArrayElements: + + /** Converts every element in order, answering the '''first''' failure or all of the converted values. + * + * Written as a loop over a `Vector.newBuilder` rather than as a fold over `Either`, because the fold cost one tuple, + * one `Either` and one whole-vector copy per element and kept walking the rest of the array after it already knew + * the answer. The builder is local and never escapes, so the mutation is not observable. + * + * @param values + * the elements, in the order the server sent them + * @param one + * converts a single element, given the element and its zero-based position + */ + def convert[D, A](values: Vector[D])(one: (D, Int) => Either[DecodeFailure, A]): Either[DecodeFailure, Vector[A]] = + val converted = Vector.newBuilder[A] + converted.sizeHint(values.size) + + @tailrec + def loop(index: Int): Either[DecodeFailure, Vector[A]] = + if index < values.size then + one(values(index), index) match + case Left(failure) => Left(failure) + case Right(element) => + converted.addOne(element) + loop(index + 1) + else Right(converted.result()) + + loop(0) diff --git a/modules/codec/src/com/worxbend/codeberg4s/codec/Json.scala b/modules/codec/src/com/worxbend/codeberg4s/codec/Json.scala index ae56471..98a6aac 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/codec/Json.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/codec/Json.scala @@ -3,8 +3,10 @@ package com.worxbend.codeberg4s.codec import com.worxbend.codeberg4s.JsonPath import com.worxbend.codeberg4s.core.Decode import com.worxbend.codeberg4s.core.DecodeFailure +import com.worxbend.codeberg4s.core.ResponseBody import com.github.plokhotnyuk.jsoniter_scala.core.ReaderConfig +import com.github.plokhotnyuk.jsoniter_scala.core.readFromArray import com.github.plokhotnyuk.jsoniter_scala.core.readFromString import com.github.plokhotnyuk.jsoniter_scala.core.writeToString @@ -12,11 +14,15 @@ import scala.util.control.NonFatal /** The single door between a response body and a wire DTO. * - * Nothing else in this library calls jsoniter's `readFromString`. Keeping the call in one place is what lets the - * module promise that a decoding failure is always a [[com.worxbend.codeberg4s.core.DecodeFailure]] value and never an + * Nothing else in this library asks jsoniter to read anything. Keeping the call in one place is what lets the module + * promise that a decoding failure is always a [[com.worxbend.codeberg4s.core.DecodeFailure]] value and never an * escaping `JsonReaderException` — the promise ADR-0003 makes and `SCALA_CODE_STYLE.md` restates as "malformed and * unexpected JSON produces a failure value, never an exception". * + * '''Bytes are the primary input shape.''' A response arrives as bytes and jsoniter reads bytes, so the pair of entry + * points that take an `Array[Byte]` is the one the request pipeline uses; the `String` overloads exist for the callers + * that genuinely start from text and pay a UTF-8 encoding to join the same path. + * * '''Where a path comes from.''' Decoding is two steps: jsoniter parses the body into [[JsonValue]], then the DTO * assembles itself from that document. Only the first step can fail structurally, and when it does the problem is the * document as a whole, so the failure carries [[com.worxbend.codeberg4s.JsonPath.Root]]. A field the domain genuinely @@ -50,7 +56,11 @@ object Json: .withMaxBufSize(1 << 22) .withPreferredBufSize(1 << 14) - /** Decodes a body into `A`. + /** Decodes a body into `A`, from the bytes that arrived. + * + * This is the shape the request pipeline uses. jsoniter — like every JSON parser worth using — reads bytes, so + * handing it the bytes is the direct route; handing it a `String` makes it encode that `String` back into a `byte[]` + * before it can start, which is a full copy of the payload for nothing. * * '''Never throws.''' Everything jsoniter can raise — a body that is not JSON, a truncated body, an empty body, a * document nested past [[JsonValue.MaxDepth]] — is caught and returned as a @@ -58,7 +68,19 @@ object Json: * [[MaxReasonLength]]. * * @param body - * the raw response body, exactly as received + * the raw response body as UTF-8 bytes, exactly as received + */ + def decode[A](body: Array[Byte])(using decoder: JsonDecoder[A]): Either[DecodeFailure, A] = + parse(body).flatMap(decoder.decode) + + /** Decodes a body given as text. + * + * Kept for the callers that genuinely hold a `String` and not bytes — the error-payload parser, which is handed + * already-decoded text, and tests written against a literal. It encodes to UTF-8 and then parses, so prefer the + * `Array[Byte]` overload wherever the bytes are still available. + * + * @param body + * the raw response body as text, exactly as received */ def decode[A](body: String)(using decoder: JsonDecoder[A]): Either[DecodeFailure, A] = parse(body).flatMap(decoder.decode) @@ -67,24 +89,41 @@ object Json: * * Use this to hand a DTO to a use case without core learning that jsoniter exists. Instances are stateless and safe * to share between threads. + * + * Reads [[com.worxbend.codeberg4s.core.ResponseBody.utf8Bytes]] rather than the raw bytes, because JSON is UTF-8 by + * RFC 8259 §8.1 and jsoniter reads UTF-8 and nothing else. For every response Forgejo has ever sent, that is the + * array the socket produced and no work happens at all; for a server that declared something else, it is a + * transcoding, which is still right and merely slow. */ def decoder[A](using JsonDecoder[A]): Decode[A] = - (body: String) => decode[A](body) + (body: ResponseBody) => decode[A](body.utf8Bytes) - /** Parses a body into the document model, without interpreting it. + /** Parses UTF-8 bytes into the document model, without interpreting it. * * The bare literal `null` is a successful parse producing [[JsonValue.Null]], not a `null` reference — which is the * whole reason this library models JSON rather than mapping it onto Scala types at the parser. Whether `null` is an * acceptable document is the decoder's question, and [[JsonDecoder.objectOf]] answers no. */ + def parse(body: Array[Byte]): Either[DecodeFailure, JsonValue] = + read(readFromArray[JsonValue](body, Config)(using JsonValue.codec)) + + /** [[parse]] for a body already held as text; it is encoded to UTF-8 and parsed. */ def parse(body: String): Either[DecodeFailure, JsonValue] = - try Right(readFromString[JsonValue](body, Config)(using JsonValue.codec)) - catch case NonFatal(error) => Left(DecodeFailure(JsonPath.Root, reasonOf(error))) + read(readFromString[JsonValue](body, Config)(using JsonValue.codec)) /** Renders a document to its compact wire form. The only place this library serialises JSON. */ def render(value: JsonValue): String = writeToString(value)(using JsonValue.codec) + /** The promise that no `JsonReaderException` escapes, made once for both entry points. + * + * `parsed` is by-name so that the parse happens inside the `try` rather than at the call site, which is the whole + * point of routing both overloads through here. + */ + private def read(parsed: => JsonValue): Either[DecodeFailure, JsonValue] = + try Right(parsed) + catch case NonFatal(error) => Left(DecodeFailure(JsonPath.Root, reasonOf(error))) + private def reasonOf(error: Throwable): String = Option(error.getMessage).map(bound).getOrElse(error.getClass.getName) diff --git a/modules/codec/src/com/worxbend/codeberg4s/codec/JsonDecoder.scala b/modules/codec/src/com/worxbend/codeberg4s/codec/JsonDecoder.scala index d1f95da..07c343a 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/codec/JsonDecoder.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/codec/JsonDecoder.scala @@ -40,7 +40,7 @@ object JsonDecoder: * assembles the value from the object's fields */ def objectOf[A](build: JsonFields => A): JsonDecoder[A] = - case JsonValue.Obj(fields) => Right(build(JsonFields(fields.toMap))) + case JsonValue.Obj(fields) => Right(build(JsonFields(fields))) case other => Left(DecodeFailure(JsonPath.Root, s"expected an object but found ${other.kind}")) /** A decoder for a top-level array of objects, which is what most Forgejo list endpoints return. @@ -50,15 +50,13 @@ object JsonDecoder: */ def arrayOf[A](element: JsonDecoder[A]): JsonDecoder[Vector[A]] = case JsonValue.Arr(values) => - values.zipWithIndex.foldLeft(Right(Vector.empty): Either[DecodeFailure, Vector[A]]): - case (Left(failure), _) => Left(failure) - case (Right(built), (raw, at)) => - element.decode(raw).left.map(failure => failure.copy(path = JsonPath.Root.index(at))).map(built :+ _) + ArrayElements.convert(values): (raw, at) => + element.decode(raw).left.map(failure => failure.copy(path = JsonPath.Root.index(at))) case other => Left(DecodeFailure(JsonPath.Root, s"expected an array but found ${other.kind}")) /** As [[objectOf]], for a build step that can itself fail — an envelope whose elements are decoded, say. */ def objectOfEither[A](build: JsonFields => Either[DecodeFailure, A]): JsonDecoder[A] = - case JsonValue.Obj(fields) => build(JsonFields(fields.toMap)) + case JsonValue.Obj(fields) => build(JsonFields(fields)) case other => Left(DecodeFailure(JsonPath.Root, s"expected an object but found ${other.kind}")) /** Decodes every element of an already-extracted array, failing on the first element that will not decode. @@ -67,8 +65,7 @@ object JsonDecoder: * under-report, and a caller cannot tell an under-report from a short page. */ def all[A](values: Vector[JsonValue])(using element: JsonDecoder[A]): Either[DecodeFailure, Vector[A]] = - values.foldLeft(Right(Vector.empty): Either[DecodeFailure, Vector[A]]): (built, raw) => - built.flatMap(soFar => element.decode(raw).map(soFar :+ _)) + ArrayElements.convert(values)((raw, _) => element.decode(raw)) /** A decoder that hands the parsed document over untouched, for the few payloads whose shape is not fixed. */ given identity: JsonDecoder[JsonValue] = (value: JsonValue) => Right(value) @@ -80,10 +77,14 @@ object JsonDecoder: */ given vector[A](using element: JsonDecoder[A]): JsonDecoder[Vector[A]] = arrayOf(element) - /** A JSON object, as its fields. The shape [[JsonFields]] is built from, exposed for a caller decoding a payload - * whose keys are data rather than a schema — an EditorConfig, say. + /** A JSON object, as a map from field name to value, for a caller decoding a payload whose keys are data rather than + * a schema — an EditorConfig, say. + * + * This is the one place in the library that pays for a map. [[JsonFields]] reads named fields straight out of the + * vector the parser built; a caller who does not know the names has to enumerate them, and a map is the shape that + * caller wants. Anything with a fixed set of fields should use [[JsonFields.reader]] instead. */ - given fields: JsonDecoder[Map[String, JsonValue]] = objectOf(_.underlying) + given fields: JsonDecoder[Map[String, JsonValue]] = objectOf(_.toMap) /** A JSON string. * @@ -100,7 +101,13 @@ object JsonDecoder: case JsonValue.Bool(value) => Right(value) case other => Left(DecodeFailure(JsonPath.Root, s"expected a boolean but found ${other.kind}")) - /** A JSON number, truncated toward zero. */ + /** A JSON number, truncated toward zero. + * + * The two number cases are named rather than matched through `JsonValue.Num`, whose extractor would build a + * `BigDecimal` for the [[JsonValue.Int64]] case that this decoder would then throw away — which is the cost the two + * cases exist to avoid. + */ given long: JsonDecoder[Long] = - case JsonValue.Num(value) => Right(value.toLong) - case other => Left(DecodeFailure(JsonPath.Root, s"expected a number but found ${other.kind}")) + case JsonValue.Int64(value) => Right(value) + case JsonValue.Decimal(value) => Right(value.toLong) + case other => Left(DecodeFailure(JsonPath.Root, s"expected a number but found ${other.kind}")) diff --git a/modules/codec/src/com/worxbend/codeberg4s/codec/JsonFields.scala b/modules/codec/src/com/worxbend/codeberg4s/codec/JsonFields.scala index 866d843..1b304cf 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/codec/JsonFields.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/codec/JsonFields.scala @@ -1,5 +1,7 @@ package com.worxbend.codeberg4s.codec +import scala.annotation.tailrec + /** A total, read-only view over the fields of one decoded JSON object. * * Every accessor answers `None` (or an empty collection) instead of failing, which is the only workable reading of the @@ -18,16 +20,79 @@ package com.worxbend.codeberg4s.codec * all, or an array where an object was expected — are still failures, and are reported by [[JsonDecoder.objectOf]] * before this view is ever built. * + * ==Why this is the parser's own vector and not a map== + * + * [[JsonValue.Obj]] already holds an object's fields as a `Vector[(String, JsonValue)]` in document order, because a + * request body is rendered from one of these and a stable field order makes a recorded request assertable. This view + * used to copy that vector into a `Map` — once per object, at every level of every response — and then look fields up + * by hash. + * + * Building the map was the single largest allocation in the decode path. Measured with `scripts/alloc-bench.sh` on a + * page of fifty repositories: 1,944,816 bytes to decode the page, of which the `toMap` calls were 819,600 — 42% of the + * whole decode, spent copying a list the parser had already built into a structure thrown away one object later. + * + * So the vector is kept and a field is found by looking through it. How it is looked through depends on how wide the + * object is, and the split is the one [[JsonValue]] already makes for its repeated-key check, at the same width and + * for the same reason: + * + * - a narrow object has its names compared one after another. That allocates nothing and beats a hash lookup + * outright at this width, and most of the objects a Forgejo response nests are this narrow — a repository's + * `permissions` has three fields and its `internal_tracker` three. + * - a wide object has its field positions indexed by name hash once, at construction, and a lookup probes that index + * instead. This is what keeps a wide object off the quadratic path: `RepositoryDto` reads 63 fields out of a + * 64-key object, and scanning for each of them would compare about two thousand names to assemble one repository. + * The index costs one small `Array[Int]`, which on the same fifty-repository page adds 35,200 bytes back — so the + * measured saving over the whole decode is 785,200 bytes rather than the full 819,600. + * + * '''No field can be shadowed by another.''' Both strategies answer with the *first* field of a given name — a scan + * because it stops there, the index because a colliding name inserted later probes past the earlier one — and + * [[JsonValue.field]] answers the same way. The question does not arise in a parsed document at all: [[Json.parse]] + * rejects one that names a field twice, for the reasons set out on [[JsonValue.Obj]]. + * * Instances are immutable and safe to share. * - * @param underlying - * the decoded object, keyed by the wire (snake_case) field name + * @param entries + * the object's fields in document order, named as they were on the wire (snake_case) */ -final case class JsonFields(underlying: Map[String, JsonValue]): +final case class JsonFields(entries: Vector[(String, JsonValue)]): + + /** Field positions by name hash for a wide object, empty for a narrow one; see `JsonFields.index`. */ + private val slots: Array[Int] = JsonFields.index(entries) /** The raw value at `name`, absent when the key is missing or explicitly `null`. */ def value(name: String): Option[JsonValue] = - underlying.get(name).filterNot(_.isNull) + lookup(name).filterNot(_.isNull) + + /** The first field named `name`, or `None` when the object has no such field. */ + private def lookup(name: String): Option[JsonValue] = + val at = if slots.isEmpty then scanFor(name, 0) else probeFor(name, JsonFields.slotFor(name, slots.length)) + + if at < 0 then None else Some(entries(at)._2) + + /** The position of the first field named `name` at or after `index`, or `-1`. + * + * An indexed loop rather than `entries.indexWhere`, because this runs once per field of every object of every + * response and the closure `indexWhere` takes would be allocated at each of those call sites. + */ + @tailrec + private def scanFor(name: String, index: Int): Int = + if index >= entries.length then -1 + else if entries(index)._1.contentEquals(name) then index + else scanFor(name, index + 1) + + /** The position [[slots]] records for `name`, or `-1`, probing on from `slot` while the slot is taken by another + * name. + * + * Names are compared in full on a hash hit: a hash agreeing is not two names agreeing, and answering on the hash + * alone would return a neighbouring field's value whenever two names in one object collided. + */ + @tailrec + private def probeFor(name: String, slot: Int): Int = + val taken = slots(slot) - 1 + + if taken < 0 then -1 + else if entries(taken)._1.contentEquals(name) then taken + else probeFor(name, (slot + 1) & (slots.length - 1)) /** The string at `name`, verbatim — the empty string is preserved. Use [[text]] to fold Forgejo's `""`-for-absent * convention away. @@ -46,11 +111,12 @@ final case class JsonFields(underlying: Map[String, JsonValue]): /** The number at `name`, truncated to a `Long`. * - * [[JsonValue.Num]] holds a `BigDecimal`, so an identifier beyond 2^53 keeps its precision on the way through. The - * previous document model parsed every number as a `Double` and would have lost it silently. + * An identifier beyond 2^53 keeps its precision on the way through: the document model reads a whole number straight + * into a `Long` and anything else into a `BigDecimal`, neither of which rounds. The model before it parsed every + * number as a `Double` and would have lost the identifier silently. */ def number(name: String): Option[Long] = - value(name).flatMap(_.numOpt).map(_.toLong) + value(name).flatMap(_.longOpt) /** The boolean at `name`. */ def boolean(name: String): Option[Boolean] = @@ -58,7 +124,7 @@ final case class JsonFields(underlying: Map[String, JsonValue]): /** The nested object at `name`, as another view. */ def nested(name: String): Option[JsonFields] = - value(name).flatMap(_.objOpt).map(entries => JsonFields(entries.toMap)) + value(name).flatMap(_.objOpt).map(JsonFields.apply) /** The elements of the array at `name`; empty when the key is absent, `null`, or not an array. */ def values(name: String): Vector[JsonValue] = @@ -70,12 +136,76 @@ final case class JsonFields(underlying: Map[String, JsonValue]): /** The elements of the array at `name` that are objects, each as another view. */ def nestedAll(name: String): Vector[JsonFields] = - values(name).flatMap(_.objOpt).map(entries => JsonFields(entries.toMap)) + values(name).flatMap(_.objOpt).map(JsonFields.apply) + + /** The fields as a map, for the few payloads whose keys are data rather than a schema. + * + * An EditorConfig response, a hook's `config` and a language breakdown all have keys the caller has never heard of + * and must enumerate, and a map is the shape their DTO exposes. Everything else reads named fields through the + * accessors above and must not call this: it copies the vector, which is the cost this type exists to avoid. + * + * A repeated key cannot reach here — [[Json.parse]] rejects the document first — so nothing is dropped by the copy. + */ + def toMap: Map[String, JsonValue] = + entries.toMap object JsonFields: + /** How many fields an object may have before a lookup stops comparing names and starts probing a hash index. + * + * The same number [[JsonValue]] splits its repeated-key check at, for the same reason: below this width comparing + * names is cheaper than building and probing a table, and above it the comparisons grow with the square of the + * width, because a DTO reads about as many fields as the object has. It is a cost split and not a limit — an object + * of any width is readable, and none is rejected for being wide. + * + * `inline` so that it is a compile-time constant rather than a field of this object. [[Empty]] below builds a view, + * which reads this while the object is still initialising, and a plain `val` declared after it would read as zero. + */ + private inline val MaxScannedFields = 8 + /** An empty view — every accessor answers as though the object had no keys. */ - val Empty: JsonFields = JsonFields(Map.empty) + val Empty: JsonFields = JsonFields(Vector.empty) + + /** An open-addressed table of field positions, keyed by the hash of the field name. + * + * A narrow object gets `Array.emptyIntArray`, the standard library's shared zero-length array, so that being narrow + * costs no allocation at all — and so that this answer cannot depend on a field of this object being initialised. + * + * A slot holds a position plus one, so a fresh array of zeroes already reads as an empty table. Insertion runs + * forwards through `entries`, which is what makes a probe answer with the first of two fields sharing a name: the + * earlier one already occupies the slot the later one would want, and the later one is pushed past it. + */ + private def index(entries: Vector[(String, JsonValue)]): Array[Int] = + val size = entries.length + + if size <= MaxScannedFields then Array.emptyIntArray + else + // A power of two, so a probe wraps with a mask; at least twice the field + // count, so the table stays under half full and a probe stays short. + val slots = new Array[Int](Integer.highestOneBit(size) * 2) + + @tailrec def place(position: Int, slot: Int): Unit = + if slots(slot) < 1 then slots(slot) = position + 1 + else place(position, (slot + 1) & (slots.length - 1)) + + @tailrec def placeFrom(position: Int): Unit = + if position < size then + place(position, slotFor(entries(position)._1, slots.length)) + placeFrom(position + 1) + + placeFrom(0) + slots + + /** Where a name's probe starts in a table of `length` slots. + * + * The high bits of a `String` hash carry most of its entropy for the short, similar names a JSON object has, and the + * mask keeps only the low ones. Folding one half onto the other is what `java.util.HashMap` does about that, for the + * same reason. + */ + private def slotFor(name: String, length: Int): Int = + val hash = name.hashCode + + (hash ^ (hash >>> 16)) & (length - 1) /** Builds a decoder for a type assembled field by field from one JSON object. * diff --git a/modules/codec/src/com/worxbend/codeberg4s/codec/JsonValue.scala b/modules/codec/src/com/worxbend/codeberg4s/codec/JsonValue.scala index ed81191..4c8240f 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/codec/JsonValue.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/codec/JsonValue.scala @@ -18,8 +18,24 @@ import scala.annotation.tailrec * So the boundary is two steps rather than one: parse into this model, then assemble the DTO from it. The parse is * jsoniter's, through the hand-written [[JsonValue.codec]] below. * - * Numbers are `BigDecimal` rather than `Double`. The previous model parsed every number as a `Double`, which silently - * loses precision above 2^53 — fine for Forgejo's row ids today, but a latent defect rather than a decision. + * ==Numbers come in two cases, and neither of them is a `Double`== + * + * The model before this one parsed every number as a `Double`, which silently loses precision above 2^53 — fine for + * Forgejo's row ids today, but a latent defect rather than a decision. What replaced it made every number a + * `BigDecimal`, which is exact and costs: measured with `scripts/alloc-bench.sh`, parsing a thousand small integers + * allocated 88.5 bytes an element against 26.6 for the same array of booleans, so roughly sixty of those bytes were + * the `BigDecimal` and the `java.math.BigDecimal` inside it. A response is mostly numbers, and every row id, count and + * timestamp offset in it was paying that. + * + * So a number is now one of two cases: + * + * - [[JsonValue.Int64]] holds a `Long`, and is what a number written without a fractional part or an exponent + * becomes when it fits in 64 bits — which is every identifier, count and offset a Forgejo response carries; + * - [[JsonValue.Decimal]] holds a `BigDecimal`, and is what everything else becomes: a fractional value, an exponent + * form, or a whole number too large for a `Long`. Still exact, still never a `Double`. + * + * Which case a parsed number lands in follows the text that was on the wire and nothing else, so [[Json.render]] + * writes back what it read. [[JsonValue.Num]] builds and reads either one without a caller having to know which. */ sealed trait JsonValue: @@ -33,10 +49,32 @@ sealed trait JsonValue: case JsonValue.Str(value) => Some(value) case _ => None - /** The number, when this is one. */ + /** The number, when this is one, as an exact decimal. + * + * This builds a `BigDecimal` for an [[JsonValue.Int64]], which is the case almost every number a response carries + * lands in — so it undoes, at this one call, the saving the two cases exist for. Reach for [[longOpt]] instead + * whenever a `Long` is what the caller wanted anyway, which in this library it always is. + */ def numOpt: Option[BigDecimal] = this match - case JsonValue.Num(value) => Some(value) - case _ => None + case JsonValue.Int64(value) => Some(BigDecimal(value)) + case JsonValue.Decimal(value) => Some(value) + case _ => None + + /** The number, when this is one, truncated toward zero. + * + * Truncation is what a wire `int64` field wants and what the previous accessor did — `numOpt.map(_.toLong)` — so a + * fractional value still answers with its whole part rather than with `None`. Nothing is allocated for an + * [[JsonValue.Int64]] beyond the `Option` itself. + */ + def longOpt: Option[Long] = this match + case JsonValue.Int64(value) => Some(value) + case JsonValue.Decimal(value) => Some(value.toLong) + case _ => None + + /** Whether this is a number, of either case. */ + def isNum: Boolean = this match + case JsonValue.Int64(_) | JsonValue.Decimal(_) => true + case _ => false /** The boolean, when this is one. */ def boolOpt: Option[Boolean] = this match @@ -58,6 +96,9 @@ sealed trait JsonValue: * Total, like every accessor here: an absent key, a non-object receiver and an explicit `null` are all `None`. There * is deliberately no throwing `apply`; a JSON document is remote input and this library has no accessor that can * fail on it. + * + * This takes the first field named `name`, which in a parsed document is also the only one: [[JsonValue.Obj]] + * explains why a document that names a field twice is rejected instead of being read. */ def field(name: String): Option[JsonValue] = objOpt.flatMap(_.collectFirst { case (key, value) if key.contentEquals(name) => value }).filterNot(_.isNull) @@ -69,10 +110,10 @@ sealed trait JsonValue: def kind: String = this match case JsonValue.Null => "null" case JsonValue.Bool(_) => "a boolean" - case JsonValue.Num(_) => "a number" - case JsonValue.Str(_) => "a string" - case JsonValue.Arr(_) => "an array" - case JsonValue.Obj(_) => "an object" + case JsonValue.Int64(_) | JsonValue.Decimal(_) => "a number" + case JsonValue.Str(_) => "a string" + case JsonValue.Arr(_) => "an array" + case JsonValue.Obj(_) => "an object" object JsonValue: @@ -81,7 +122,30 @@ object JsonValue: final case class Bool(value: Boolean) extends JsonValue - final case class Num(value: BigDecimal) extends JsonValue + /** A whole number that fits in 64 bits — the case nearly every number in a Forgejo response lands in. + * + * A parsed document puts a number here when it was written on the wire without a fractional part and without an + * exponent, and its digits fit a `Long`. `12` is one of these; `12.0` and `1.2e1` are [[Decimal]], because that is + * what they were written as and [[Json.render]] has to be able to write them back. + * + * '''Build one through [[Num]] rather than directly.''' [[Num.apply]] is what decides which of the two cases a value + * belongs in, and the two are only ever distinguishable if that decision is made in one place: + * `Decimal(BigDecimal(5))` renders as `5`, exactly as `Int64(5)` does, yet is a different value from it. This is the + * same unpoliced invariant [[Obj]] carries about a repeated key, and it is unpoliced for the same reason — a + * constructor cannot refuse a value that is perfectly well formed on its own. + */ + final case class Int64(value: Long) extends JsonValue + + /** Every number that is not an [[Int64]]: a fractional value, an exponent form, or a whole number past 64 bits. + * + * Exact, and deliberately not a `Double` — the model this replaced parsed `9007199254740993` into a `Double` and + * gave back `9007199254740992` without saying so. + * + * The note on [[Int64]] about building through [[Num]] applies here too, and matters more: handing this constructor + * a value that is a whole number inside the `Long` range produces a document that does not compare equal to the one + * [[Json.parse]] reads back from its own rendering. + */ + final case class Decimal(value: BigDecimal) extends JsonValue final case class Str(value: String) extends JsonValue @@ -91,22 +155,75 @@ object JsonValue: * * Order is preserved because a request body is rendered from one of these, and a stable field order makes a recorded * request assertable. + * + * '''A repeated key is a rejected document, not a resolved one.''' JSON does not forbid `{"id":1,"id":2}`, and the + * three ways of reading this type had drifted into three different answers for it: [[JsonValue.field]] scans and + * finds the first, `fields.toMap` keeps the last, and [[Json.render]] writes both back out. Rather than pick a + * winner, [[Json.parse]] refuses the document and says `duplicated field "id"`, so no value parsed by this library + * ever carries a repeated key and the three readings cannot disagree. + * + * Refusing is the narrower promise, and it is the one this library can keep. Choosing a winner would mean silently + * dropping a value the sender wrote, with no way for a caller to learn that it happened — the same trade + * [[JsonDecoder.arrayOf]] already refuses when it fails a page rather than skip an element it cannot read. It is + * also not the leniency `docs/HAZARDS.md` §1 argues for: that leniency is for wire shapes measured coming out of a + * real Forgejo — `null` for an array, `""` for absent — whereas Forgejo serialises from Go structs and maps and + * cannot emit a repeated key at all. A response that has one was rewritten between the server and here, which is + * worth a failure rather than a guess. + * + * The one thing this does not police is an object built in code: the constructor takes the vector as given, and + * [[Json.render]] writes whatever it is handed. Handing it a repeated key produces a request body this library would + * refuse to read back, so do not. */ final case class Obj(fields: Vector[(String, JsonValue)]) extends JsonValue + /** Builds and reads a number of either case, so that a caller who has one does not have to know which. + * + * This is the only constructor that keeps [[Int64]] and [[Decimal]] apart correctly, and it is why they were split + * without every call site in the library having to change: `JsonValue.Num(7)` still builds a number and + * `case JsonValue.Num(value)` still matches one. + */ object Num: - def apply(value: Long): Num = Num(BigDecimal(value)) - def apply(value: Int): Num = Num(BigDecimal(value)) + /** A whole number. */ + def apply(value: Long): JsonValue = Int64(value) + + /** A whole number. */ + def apply(value: Int): JsonValue = Int64(value.toLong) /** A number from a `Double`, rendered without a fractional part when it has none. * * `BigDecimal(102.0)` keeps a scale of one and renders `102.0`, which Forgejo's integer fields reject and which - * would silently change every request body carrying an identifier. A whole value therefore becomes a whole - * `BigDecimal`. + * would silently change every request body carrying an identifier. A whole value therefore becomes an [[Int64]], + * which renders `102`. + * + * The bound is strict at the top and not at the bottom because `Long.MaxValue` has no exact `Double` — the nearest + * one is 2^63, one past the largest `Long` — whereas `Long.MinValue` is exactly -2^63 and does. Without the strict + * bound a `Double` of 2^63 would silently become `Long.MaxValue`, which is the class of quiet rounding this whole + * type exists to avoid. + */ + def apply(value: Double): JsonValue = + if value.isWhole && value >= Long.MinValue.toDouble && value < Long.MaxValue.toDouble then Int64(value.toLong) + else Decimal(BigDecimal(value)) + + /** A number from an exact decimal, put into whichever case renders the same text back. + * + * A `BigDecimal` of scale zero that fits a `Long` writes itself as plain digits — `BigDecimal(5)` renders `5` — so + * it becomes an [[Int64]] and parsing that rendering returns the same value. Everything else keeps its scale and + * stays a [[Decimal]]: `BigDecimal("1.0")` renders `1.0`, `BigDecimal("1E+3")` renders `1E+3`, and both read back + * as themselves. */ - def apply(value: Double): Num = - if value.isWhole then Num(BigDecimal(value.toLong)) else Num(BigDecimal(value)) + def apply(value: BigDecimal): JsonValue = + value.scale match + case 0 if value.isValidLong => Int64(value.toLong) + case _ => Decimal(value) + + /** The number, whichever case it is, as an exact decimal. + * + * Present so that `case JsonValue.Num(value)` keeps meaning what it meant when `Num` was a single case class + * holding a `BigDecimal`. It allocates one for an [[Int64]], so a match that only wants a `Long` should say + * [[JsonValue.longOpt]] or name the two cases instead. + */ + def unapply(value: JsonValue): Option[BigDecimal] = value.numOpt object Arr: /** An array from any collection of values. */ @@ -130,6 +247,18 @@ object JsonValue: */ val MaxDepth: Int = 128 + /** How many fields an object may have before the repeated-key check stops comparing names and starts hashing them. + * + * Both strategies give the same answer; they cost differently. Comparing every pair allocates nothing and is the + * cheaper of the two while an object is narrow, which the objects nested inside a response mostly are — a + * repository's `permissions` has three fields and its `internal_tracker` three. The comparisons grow with the square + * of the width, so a wide object gets a table instead, at the price of one array. + * + * Eight is where the pair count (28) stops being obviously smaller than the work of allocating and filling a table. + * It is a cost split, not a limit: an object of any width is checked, and none is rejected for being wide. + */ + private val MaxComparedFields: Int = 8 + /** The jsoniter codec for the document model. * * Hand-written rather than derived: [[JsonValue]] is a recursive sum type whose `Obj` case is an ordered field list, @@ -147,7 +276,7 @@ object JsonValue: private val UnreachableString: String = "" /** As [[UnreachableString]], for the number reader. */ - private val UnreachableNumber: BigDecimal = BigDecimal(0) + private val UnreachableNumber: java.lang.Number = java.lang.Long.valueOf(0L) private def read(in: JsonReader, depth: Int): JsonValue = if depth > MaxDepth then in.decodeError(s"the document nests deeper than $MaxDepth levels") @@ -157,7 +286,7 @@ object JsonValue: // below re-reads the token itself and so needs it put back. in.nextToken() match case 'n' => in.readNullOrError(Null, "expected the literal null") - // The argument to readString and readBigDecimal is what jsoniter returns + // The argument to readString and readNumber is what jsoniter returns // for a JSON null. Generated codecs pass null there; these defaults are // unreachable instead, because the 'n' branch above has already taken // every null — which keeps a null literal out of the codebase. @@ -165,7 +294,31 @@ object JsonValue: case 't' | 'f' => in.rollbackToken(); Bool(in.readBoolean()) case '[' => in.rollbackToken(); readArray(in, depth) case '{' => in.rollbackToken(); readObject(in, depth) - case _ => in.rollbackToken(); Num(in.readBigDecimal(UnreachableNumber)) + case _ => in.rollbackToken(); readNumber(in) + + /** Reads a number into whichever of the two number cases matches how it was written. + * + * `readNumber` is jsoniter's own answer to this question and does the hard part: it returns a `java.lang.Long` for a + * number written without a fractional part or an exponent that fits in 64 bits, a `java.math.BigInteger` for one + * that does not, and a `java.math.BigDecimal` for everything else. Reading a `BigDecimal` unconditionally, which is + * what this used to do, is what made every row id cost one. + * + * Those three classes are the whole of what 2.40.1 returns — checked against the library rather than recalled. The + * fourth branch is there because `java.lang.Number` is a plain abstract class that anyone may extend, so the match + * has to be total; reaching it would mean jsoniter had grown a return type this reader does not know how to keep + * exactly, and answering with an approximation is the one thing this type must not do. + * + * One allocation is left on this path and is jsoniter's rather than this reader's: the `java.lang.Long` it boxes to + * return, which dies immediately. Avoiding it needs a look-ahead the reader interface does not offer — the only way + * to it is to try `readLong` behind a mark and catch the failure, which hands a remote party a document that throws + * once per fractional number. + */ + private def readNumber(in: JsonReader): JsonValue = + in.readNumber(UnreachableNumber) match + case whole: java.lang.Long => Int64(whole.longValue) + case exact: java.math.BigDecimal => Decimal(BigDecimal(exact)) + case big: java.math.BigInteger => Decimal(BigDecimal(BigInt(big))) + case other => in.decodeError(s"read a number as an unsupported ${other.getClass.getName}") private def readArray(in: JsonReader, depth: Int): JsonValue = if !in.isNextToken('[') then in.decodeError("expected an array") @@ -196,19 +349,85 @@ object JsonValue: else if !in.isCurrentToken('}') then in.objectEndOrCommaError() loop() - Obj(fields.result()) + val entries = fields.result() + rejectRepeatedKey(in, entries) + Obj(entries) + + /** Fails the whole document when one object names the same field twice — the rule [[Obj]] documents. + * + * Two strategies, because this runs on every object of every response. Up to [[MaxComparedFields]] the check + * compares the names against each other, which allocates nothing; a wider object is indexed by hash instead, because + * comparing every pair of a 500-key object is a quarter of a million comparisons and a remote party chooses that + * width. + */ + private def rejectRepeatedKey(in: JsonReader, entries: Vector[(String, JsonValue)]): Unit = + val size = entries.size + + if size <= MaxComparedFields then compareFields(in, entries, size) else indexFields(in, entries, size) + + /** The pairwise check: every field against the ones before it, `size * (size - 1) / 2` comparisons and no allocation. */ + private def compareFields(in: JsonReader, entries: Vector[(String, JsonValue)], size: Int): Unit = + @tailrec def compare(index: Int, earlier: Int): Unit = + if index >= size then () + else if earlier >= index then compare(index + 1, 0) + else if entries(earlier)._1.contentEquals(entries(index)._1) then repeatedKeyError(in, entries(index)._1) + else compare(index, earlier + 1) + + compare(1, 0) + + /** The hashed check: one open-addressed table of field positions, probed linearly. + * + * A slot holds a field's position plus one, so that a fresh array of zeroes already reads as an empty table. Names + * are still compared in full on a hash hit — a hash agreeing is not two names agreeing, and a check that took it for + * one would reject a document over a collision. + */ + private def indexFields(in: JsonReader, entries: Vector[(String, JsonValue)], size: Int): Unit = + // A power of two, so a probe wraps with a mask; at least twice the field + // count, so that the table stays under half full and a probe stays short. + val slots = new Array[Int](Integer.highestOneBit(size) * 2) + val mask = slots.length - 1 + + @tailrec def probe(index: Int, slot: Int): Unit = + val taken = slots(slot) - 1 + + if taken < 0 then slots(slot) = index + 1 + else if entries(taken)._1.contentEquals(entries(index)._1) then repeatedKeyError(in, entries(index)._1) + else probe(index, (slot + 1) & mask) + + @tailrec def indexFrom(index: Int): Unit = + if index < size then + // The high bits of a String hash carry most of its entropy for the + // short, similar names a JSON object has, and the mask below keeps only + // the low ones. Folding one half onto the other is what java.util.HashMap + // does about that, for the same reason. + val hash = entries(index)._1.hashCode + + probe(index, (hash ^ (hash >>> 16)) & mask) + indexFrom(index + 1) + + indexFrom(0) + + /** The failure a repeated field produces, worded as jsoniter's own `duplicatedKeyError` words it. + * + * That method is not called here because it formats the name out of the reader's character buffer, which by the time + * an object is complete holds the last string the reader saw rather than the offending key. `Json.parse` bounds the + * message at `Json.MaxReasonLength`, so a pathologically long key cannot turn this into a payload dump. + */ + private def repeatedKeyError(in: JsonReader, name: String): Nothing = + in.decodeError(s"""duplicated field "$name"""") private def write(value: JsonValue, out: JsonWriter): Unit = value match - case Null => out.writeNull() - case Bool(flag) => out.writeVal(flag) - case Num(number) => out.writeVal(number) - case Str(text) => out.writeVal(text) - case Arr(values) => + case Null => out.writeNull() + case Bool(flag) => out.writeVal(flag) + case Int64(whole) => out.writeVal(whole) + case Decimal(number) => out.writeVal(number) + case Str(text) => out.writeVal(text) + case Arr(values) => out.writeArrayStart() values.foreach(element => write(element, out)) out.writeArrayEnd() - case Obj(fields) => + case Obj(fields) => out.writeObjectStart() fields.foreach: (name, field) => out.writeKey(name) diff --git a/modules/codec/src/com/worxbend/codeberg4s/codec/Timestamps.scala b/modules/codec/src/com/worxbend/codeberg4s/codec/Timestamps.scala index ca4fb48..324f934 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/codec/Timestamps.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/codec/Timestamps.scala @@ -1,9 +1,11 @@ package com.worxbend.codeberg4s.codec +import scala.annotation.tailrec import scala.util.Try import java.time.Instant import java.time.OffsetDateTime +import java.time.Year /** Turns Forgejo's timestamp strings into instants, sentinels included. * @@ -24,11 +26,164 @@ object Timestamps: * any instant at or before the Unix epoch — see the sentinels above. The epoch boundary means this cannot represent * a genuine pre-1970 timestamp; Forgejo has no field that could carry one, since every timestamp it emits describes * an event on a Git forge. + * + * Two parsers sit behind this. [[fixedLayout]] reads the one shape Forgejo actually sends, character by character, + * with no formatter and no intermediate date objects. Anything it does not recognise — a lowercase `t`, an offset + * carrying seconds, a year outside four digits, or a value that is simply not a timestamp — falls through to + * `java.time.OffsetDateTime.parse`, so the set of strings this accepts is exactly what it has always been, and only + * the speed of the common case changes. */ def parse(value: String): Option[Instant] = - Try(OffsetDateTime.parse(value.trim).toInstant).toOption + val trimmed = value.trim + fixedLayout(trimmed) + .orElse(Try(OffsetDateTime.parse(trimmed).toInstant).toOption) .filter(_.isAfter(Instant.EPOCH)) /** [[parse]] lifted over an optional wire value, for the common `dto.createdAt.flatMap(...)` shape. */ def parseOptional(value: Option[String]): Option[Instant] = value.flatMap(parse) + + /** The shortest string the fixed layout can be: `yyyy-MM-ddTHH:mm:ssZ`. */ + private val MinimumLength: Int = 20 + + private val SecondsPerDay: Long = 86400L + + /** `java.time.ZoneOffset` refuses anything beyond ±18:00, so the fixed path refuses it too rather than inventing an + * instant the JDK would have rejected. + */ + private val MaxOffsetSeconds: Int = 18 * 3600 + + /** Not a possible offset, so it can stand for "these characters are not an offset" without an `Option` wrapper on a + * path that runs once per timestamp field of every decoded object. + */ + private val OffsetMismatch: Int = Int.MinValue + + /** Reads `yyyy-MM-ddTHH:mm:ss`, optional `.fraction`, then `Z` or `±HH:mm`, and answers the instant it names. + * + * Answers `None` whenever the string departs from that layout in any way, including when it departs by being an + * impossible date such as `2023-02-29`. `None` here means "not handled", not "invalid": the caller retries with the + * JDK parser, which is the authority on what is valid. That keeps this function free to bail out early on anything + * awkward instead of having to reproduce every corner of RFC-3339. + * + * The whole parse is integer arithmetic over `charAt`. It allocates the resulting `Instant` and nothing else — no + * formatter lookup, no `LocalDate`, no `OffsetDateTime`, no exception for the failure case. + */ + private def fixedLayout(value: String): Option[Instant] = + if value.length < MinimumLength || !hasFixedSeparators(value) then None + else + val year = fourDigits(value, 0) + val month = twoDigits(value, 5) + val day = twoDigits(value, 8) + val hour = twoDigits(value, 11) + val minute = twoDigits(value, 14) + val second = twoDigits(value, 17) + if year < 0 || !isRealDate(year, month, day) || !isRealTime(hour, minute, second) then None + else + val fractionStart = MinimumLength + val hasFraction = value.startsWith(".", fractionStart - 1) + val fractionEnd = if hasFraction then digitsEnd(value, fractionStart) else fractionStart - 1 + val fractionDigits = fractionEnd - fractionStart + // A dot with no digits after it, or more than nanosecond precision: rare enough to hand to the JDK. + if hasFraction && (fractionDigits < 1 || fractionDigits > 9) then None + else + offsetSeconds(value, fractionEnd) match + case OffsetMismatch => None + case offset => + val nanos = + if hasFraction then scaleToNanos(digitsValue(value, fractionStart, fractionEnd, 0), fractionDigits) + else 0 + val secondOfDay = hour * 3600 + minute * 60 + second + val epochSecond = epochDay(year, month, day) * SecondsPerDay + secondOfDay - offset + Some(Instant.ofEpochSecond(epochSecond, nanos.toLong)) + + /** `String.startsWith(prefix, offset)` rather than a comparison against a `Char`, because `.scalafix.conf` bans + * universal equality — the same reason `LinkHeader` reaches for `equalsIgnoreCase`. + */ + private def hasFixedSeparators(value: String): Boolean = + value.startsWith("-", 4) && value.startsWith("-", 7) && value.startsWith("T", 10) && + value.startsWith(":", 13) && value.startsWith(":", 16) + + /** Reads `Z` or `±HH:mm` at `index`, and only if it runs to the end of the string — which is what rejects trailing + * garbage. Answers seconds east of UTC, or [[OffsetMismatch]]. + */ + private def offsetSeconds(value: String, index: Int): Int = (value.length - index) match + case 1 if value.startsWith("Z", index) => 0 + case 6 if value.startsWith(":", index + 3) => signedOffsetSeconds(value, index) + case _ => OffsetMismatch + + /** The `±HH:mm` at `index`, which the caller has already checked is six characters long with a colon in the middle. */ + private def signedOffsetSeconds(value: String, index: Int): Int = + val hours = twoDigits(value, index + 1) + val minutes = twoDigits(value, index + 4) + val total = hours * 3600 + minutes * 60 + if hours < 0 || minutes < 0 || minutes > 59 || total > MaxOffsetSeconds then OffsetMismatch + else + value.charAt(index) match + case '+' => total + case '-' => -total + case _ => OffsetMismatch + + private def isRealDate(year: Int, month: Int, day: Int): Boolean = + month >= 1 && month <= 12 && day >= 1 && day <= lengthOfMonth(year, month) + + private def isRealTime(hour: Int, minute: Int, second: Int): Boolean = + hour >= 0 && hour <= 23 && minute >= 0 && minute <= 59 && second >= 0 && second <= 59 + + /** `java.time.Year.isLeap` is a static arithmetic method — no object is created — so the leap-year rule stays the + * JDK's rather than a second copy of it here. + */ + private def lengthOfMonth(year: Int, month: Int): Int = + month match + case 2 => if Year.isLeap(year.toLong) then 29 else 28 + case 4 | 6 | 9 | 11 => 30 + case _ => 31 + + /** Days from 1970-01-01 to the given proleptic-Gregorian date, by Howard Hinnant's `days_from_civil`. + * + * The trick is to start the year in March, which pushes the leap day to the end of the year and makes the + * day-of-year a closed-form expression, then to count whole 400-year eras, each of which has exactly 146,097 days. + * The date must already be known to be real; nothing here range-checks it. + */ + private def epochDay(year: Int, month: Int, day: Int): Long = + val shiftedYear = if month <= 2 then year - 1 else year + val era = (if shiftedYear >= 0 then shiftedYear else shiftedYear - 399) / 400 + val yearOfEra = shiftedYear - era * 400 // 0 to 399 + val marchMonth = (month + 9) % 12 // March is 0, February is 11 + val dayOfYear = (153 * marchMonth + 2) / 5 + day - 1 // 0 to 365 + val dayOfEra = yearOfEra * 365 + yearOfEra / 4 - yearOfEra / 100 + dayOfYear + val daysToMarch1 = 719468 // 1970-01-01 measured from 0000-03-01 + era * 146097L + dayOfEra - daysToMarch1 + + /** Left-aligns a fraction of `digits` places into nanoseconds: `.123` is 123,000,000ns, `.1` is 100,000,000ns. */ + @tailrec + private def scaleToNanos(fraction: Int, digits: Int): Int = + if digits >= 9 then fraction else scaleToNanos(fraction * 10, digits + 1) + + @tailrec + private def digitsEnd(value: String, index: Int): Int = + if index < value.length && isDigit(value.charAt(index)) then digitsEnd(value, index + 1) else index + + @tailrec + private def digitsValue(value: String, index: Int, end: Int, accumulated: Int): Int = + if index >= end then accumulated + else digitsValue(value, index + 1, end, accumulated * 10 + (value.charAt(index) - '0')) + + private def isDigit(character: Char): Boolean = + character >= '0' && character <= '9' + + /** The digit at `index`, or `-1` if that character is not a digit. */ + private def digit(value: String, index: Int): Int = + val character = value.charAt(index) + if isDigit(character) then character - '0' else -1 + + /** The two-digit number at `index`, or a negative value if either character is not a digit. */ + private def twoDigits(value: String, index: Int): Int = + val tens = digit(value, index) + val units = digit(value, index + 1) + if tens < 0 || units < 0 then -1 else tens * 10 + units + + /** The four-digit number at `index`, or a negative value if any character is not a digit. */ + private def fourDigits(value: String, index: Int): Int = + val hundreds = twoDigits(value, index) + val units = twoDigits(value, index + 2) + if hundreds < 0 || units < 0 then -1 else hundreds * 100 + units diff --git a/modules/codec/src/com/worxbend/codeberg4s/issues/wire/WireElements.scala b/modules/codec/src/com/worxbend/codeberg4s/issues/wire/WireElements.scala index aa0476c..33e1fe6 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/issues/wire/WireElements.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/issues/wire/WireElements.scala @@ -1,14 +1,16 @@ package com.worxbend.codeberg4s.issues.wire import com.worxbend.codeberg4s.JsonPath +import com.worxbend.codeberg4s.codec.ArrayElements import com.worxbend.codeberg4s.core.DecodeFailure /** Converts the elements of a decoded JSON array, reporting the position of whichever one failed. * * [[com.worxbend.codeberg4s.codec.Wire]] does this for a field; there is no equivalent for an element, and this group * needs one five times over — four list endpoints plus the `labels` and `assignees` arrays nested inside every issue. - * Writing the fold once means a decoding failure says `$[2].id` or `$.labels[1].name` rather than `$`, and means the - * five call sites cannot drift into disagreeing about whether one bad element fails the page. + * Writing the path construction once means a decoding failure says `$[2].id` or `$.labels[1].name` rather than `$`, + * and means the five call sites cannot drift into disagreeing about whether one bad element fails the page. The walk + * over the elements is [[com.worxbend.codeberg4s.codec.ArrayElements]]'s. * * '''One bad element fails the whole conversion.''' That is the same contract [[com.worxbend.codeberg4s.codec.Json]] * gives a list body, and the same one [[com.worxbend.codeberg4s.repositories.wire.RepositoryDto]] gives a nested @@ -32,6 +34,4 @@ private[codeberg4s] object WireElements: def at[D, A](base: JsonPath, dtos: Vector[D])( convert: (D, JsonPath) => Either[DecodeFailure, A] ): Either[DecodeFailure, Vector[A]] = - dtos.zipWithIndex.foldLeft[Either[DecodeFailure, Vector[A]]](Right(Vector.empty)): - case (soFar, (dto, position)) => - soFar.flatMap(converted => convert(dto, base.index(position)).map(element => converted.appended(element))) + ArrayElements.convert(dtos)((dto, position) => convert(dto, base.index(position))) diff --git a/modules/codec/src/com/worxbend/codeberg4s/issues/wire/WireNumbers.scala b/modules/codec/src/com/worxbend/codeberg4s/issues/wire/WireNumbers.scala index 7a99d60..b9e5e53 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/issues/wire/WireNumbers.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/issues/wire/WireNumbers.scala @@ -4,9 +4,9 @@ import com.worxbend.codeberg4s.codec.JsonValue /** Builds the JSON scalars and arrays this group's request bodies are made of. * - * [[JsonValue.Num]] holds a `BigDecimal`, so an `int64` identifier reaches the wire exactly — the previous document - * model held a `Double` and represented integers exactly only up to 2^53. Building the scalars here rather than at - * each call site means the conversion happens once. + * [[JsonValue.Num]] puts a whole number into the document model's `Long` case, so an `int64` identifier reaches the + * wire exactly — a document model that held a `Double`, as an earlier one did, represents integers exactly only up to + * 2^53. Building the scalars here rather than at each call site means the conversion happens once. * * Internal to this group's wire package, and a candidate to move into `com.worxbend.codeberg4s.codec` once a second * endpoint group writes a request body. @@ -18,8 +18,8 @@ private[codeberg4s] object WireNumbers: JsonValue.Num(value) /** One whole number as a JSON number, for an `int64` wire field that is not an identifier — a duration in seconds, - * say. Kept distinct from [[identifier]] so a reader of a request builder can tell which is which; the `Double` - * caveat above applies to both. + * say. Kept distinct from [[identifier]] so a reader of a request builder can tell which is which; the exactness + * above applies to both. */ def whole(value: Long): JsonValue = JsonValue.Num(value) diff --git a/modules/codec/src/com/worxbend/codeberg4s/repositories/access/wire/BranchProtectionOptionDto.scala b/modules/codec/src/com/worxbend/codeberg4s/repositories/access/wire/BranchProtectionOptionDto.scala index ab8a9b8..6d011ad 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/repositories/access/wire/BranchProtectionOptionDto.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/repositories/access/wire/BranchProtectionOptionDto.scala @@ -68,7 +68,7 @@ private[wire] object BranchProtectionSettingsDto: value => key -> JsonValue.Bool(value) private def approvals(key: String): ApprovalCount => (String, JsonValue) = - count => key -> JsonValue.Num(count.value.toDouble) + count => key -> JsonValue.Num(count.value) private def text(key: String): String => (String, JsonValue) = value => key -> JsonValue.Str(value) diff --git a/modules/codec/src/com/worxbend/codeberg4s/repositories/admin/wire/LanguageStatisticsDto.scala b/modules/codec/src/com/worxbend/codeberg4s/repositories/admin/wire/LanguageStatisticsDto.scala index 614bd27..3c17e1d 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/repositories/admin/wire/LanguageStatisticsDto.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/repositories/admin/wire/LanguageStatisticsDto.scala @@ -38,8 +38,11 @@ object LanguageStatisticsDto: given JsonDecoder[LanguageStatisticsDto] = JsonFields.reader(fromFields) - /** Projects an already-decoded object by reading every key it happens to have. */ + /** Projects an already-decoded object by reading every key it happens to have. + * + * The names come straight off [[com.worxbend.codeberg4s.codec.JsonFields.entries]] rather than from a map built for + * the purpose, because the object's own key list is what has to be walked here and a map would be built only to be + * asked for its keys. + */ def fromFields(fields: JsonFields): LanguageStatisticsDto = - LanguageStatisticsDto( - fields.underlying.keys.toVector.flatMap(name => fields.number(name).map(count => name -> count)).toMap - ) + LanguageStatisticsDto(fields.entries.flatMap((name, _) => fields.number(name).map(count => name -> count)).toMap) diff --git a/modules/codec/src/com/worxbend/codeberg4s/repositories/admin/wire/MigrateRepoOptionsDto.scala b/modules/codec/src/com/worxbend/codeberg4s/repositories/admin/wire/MigrateRepoOptionsDto.scala index 72d7c49..5cef6bf 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/repositories/admin/wire/MigrateRepoOptionsDto.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/repositories/admin/wire/MigrateRepoOptionsDto.scala @@ -95,7 +95,7 @@ private[codeberg4s] object TransferRepoOptionDto: List( Some(NewOwnerKey -> JsonValue.Str(command.newOwner.value)), Option.when(command.teamIds.nonEmpty)( - TeamIdsKey -> JsonValue.Arr.from(command.teamIds.map(team => JsonValue.Num(team.value.toDouble))) + TeamIdsKey -> JsonValue.Arr.from(command.teamIds.map(team => JsonValue.Num(team.value))) ), ).flatten diff --git a/modules/codec/src/com/worxbend/codeberg4s/repositories/gitdata/wire/EditorConfigDto.scala b/modules/codec/src/com/worxbend/codeberg4s/repositories/gitdata/wire/EditorConfigDto.scala index e85b791..a8bde30 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/repositories/gitdata/wire/EditorConfigDto.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/repositories/gitdata/wire/EditorConfigDto.scala @@ -19,9 +19,10 @@ import com.worxbend.codeberg4s.repositories.gitdata.EditorConfigDefinitions * a value that is a structure — an array, an object, or `null` — is dropped, because those have no text form an * EditorConfig consumer could use. * - * A whole number renders without a fractional part: the document model parses every JSON number as a `Double`, so `4` - * arrives as `4.0` and would read as `"4.0"` if it were rendered naively. That is the one piece of arithmetic in this - * file and the reason it exists. + * A whole number renders without a fractional part. `4` arrives as a [[JsonValue.Int64]] and needs no help, but an + * instance that sends `4.0` — legal JSON for the same quantity — arrives as a [[JsonValue.Decimal]] that would read as + * `"4.0"` if it were rendered naively, and `indent_size = 4.0` is not a setting any EditorConfig consumer honours. + * Dropping the fractional part when there is nothing in it is the one piece of arithmetic in this file. * * @param values * the properties, keyed as the instance named them @@ -42,13 +43,21 @@ object EditorConfigDto: /** Projects an already-decoded object, keeping only the properties that have a text form. */ def fromFields(fields: JsonFields): EditorConfigDto = - EditorConfigDto(fields.underlying.flatMap((name, value) => rendered(value).map(text => name -> text))) + EditorConfigDto(fields.toMap.flatMap((name, value) => rendered(value).map(text => name -> text))) /** The text an EditorConfig consumer would have read, or `None` for a value that has none. */ private def rendered(value: JsonValue): Option[String] = - value.strOpt - .orElse(value.numOpt.map(number)) - .orElse(value.boolOpt.map(_.toString)) - - private def number(value: BigDecimal): String = - if value.isWhole then value.toLong.toString else value.toString + value match + case JsonValue.Str(text) => Some(text) + case JsonValue.Int64(whole) => Some(whole.toString) + case JsonValue.Decimal(number) => Some(decimal(number)) + case JsonValue.Bool(flag) => Some(flag.toString) + case _ => None + + /** A number written the way a `.editorconfig` file would have written it. + * + * `toBigInt` rather than `toLong` for the whole case: a `Long` silently wraps a value too large for it, and while no + * EditorConfig property is plausibly that large, a value arriving from a remote party decides its own size. + */ + private def decimal(value: BigDecimal): String = + if value.isWhole then value.toBigInt.toString else value.toString diff --git a/modules/codec/src/com/worxbend/codeberg4s/repositories/hooks/wire/HookWire.scala b/modules/codec/src/com/worxbend/codeberg4s/repositories/hooks/wire/HookWire.scala index 80dff40..a246de3 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/repositories/hooks/wire/HookWire.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/repositories/hooks/wire/HookWire.scala @@ -32,7 +32,7 @@ private[hooks] object HookWire: def stringMap(fields: JsonFields, name: String): Map[String, String] = fields .nested(name) - .fold(Map.empty)(nested => nested.underlying.flatMap((key, value) => value.strOpt.map(text => key -> text))) + .fold(Map.empty)(nested => nested.toMap.flatMap((key, value) => value.strOpt.map(text => key -> text))) /** The object at `name` read as a map of text, keeping values of any JSON kind. * @@ -46,4 +46,4 @@ private[hooks] object HookWire: fields .nested(name) .fold(Map.empty): nested => - nested.underlying.map((key, value) => key -> value.strOpt.getOrElse(Json.render(value))) + nested.toMap.map((key, value) => key -> value.strOpt.getOrElse(Json.render(value))) diff --git a/modules/codec/src/com/worxbend/codeberg4s/repositories/wire/Elements.scala b/modules/codec/src/com/worxbend/codeberg4s/repositories/wire/Elements.scala index 600a2b3..3d3a5f8 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/repositories/wire/Elements.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/repositories/wire/Elements.scala @@ -1,15 +1,18 @@ package com.worxbend.codeberg4s.repositories.wire import com.worxbend.codeberg4s.JsonPath +import com.worxbend.codeberg4s.codec.ArrayElements import com.worxbend.codeberg4s.core.DecodeFailure /** Converting a JSON array of DTOs into domain values, with each failure reported at its own index. * * Rule 5 of [[com.worxbend.codeberg4s.codec.WireConventions]] says a `toDomain` reports the JSON path of whatever it - * could not convert. For an array that means `$[7].sha` and not `$`, and getting there requires threading the index - * through the fold. Doing it once here is what stops every list-shaped model from growing its own copy of the same - * four lines — and a copy that quietly forgot the index would be indistinguishable from one that did not, until - * someone tried to debug a bad payload. + * could not convert. For an array that means `$[7].sha` and not `$`, and getting there requires turning each element's + * position into a path segment. Doing it once here is what stops every list-shaped model from growing its own copy — + * and a copy that quietly forgot the index would be indistinguishable from one that did not, until someone tried to + * debug a bad payload. + * + * The walk itself lives in [[com.worxbend.codeberg4s.codec.ArrayElements]]; this adds the path. * * Used from two sides: by the DTOs below for arrays nested inside a model — a commit's parents, a release's assets — * and by the client module for a response body that is an array at the top level. @@ -32,6 +35,4 @@ private[codeberg4s] object Elements: def convert[D, A](at: JsonPath, dtos: Vector[D])( one: (D, JsonPath) => Either[DecodeFailure, A] ): Either[DecodeFailure, Vector[A]] = - dtos.zipWithIndex.foldLeft[Either[DecodeFailure, Vector[A]]](Right(Vector.empty)): - case (Left(failure), _) => Left(failure) - case (Right(converted), (dto, i)) => one(dto, at.index(i)).map(converted.appended) + ArrayElements.convert(dtos)((dto, position) => one(dto, at.index(position))) diff --git a/modules/codec/src/com/worxbend/codeberg4s/repositories/wire/RepositoryContentDto.scala b/modules/codec/src/com/worxbend/codeberg4s/repositories/wire/RepositoryContentDto.scala index 8d9e7b3..6a90335 100644 --- a/modules/codec/src/com/worxbend/codeberg4s/repositories/wire/RepositoryContentDto.scala +++ b/modules/codec/src/com/worxbend/codeberg4s/repositories/wire/RepositoryContentDto.scala @@ -66,7 +66,7 @@ object RepositoryContentDto: /** Branches on the JSON kind of an already-parsed body. Total by construction — every shape maps to a case. */ def fromJson(value: JsonValue): RepositoryContentDto = value.objOpt match - case Some(entry) => Single(entryOf(entry.toMap)) + case Some(entry) => Single(entryOf(entry)) case None => value.arrOpt match case Some(elements) => listing(elements.toVector) @@ -74,15 +74,15 @@ object RepositoryContentDto: /** An array is a directory only if every element is an object; anything else is a shape this endpoint does not have. */ private def listing(elements: Vector[JsonValue]): RepositoryContentDto = - if elements.forall(_.objOpt.isDefined) then Listing(elements.flatMap(_.objOpt).map(entry => entryOf(entry.toMap))) + if elements.forall(_.objOpt.isDefined) then Listing(elements.flatMap(_.objOpt).map(entryOf)) else Unexpected("an array holding a value that is not an object") - private def entryOf(entry: scala.collection.Map[String, JsonValue]): ContentEntryDto = - ContentEntryDto.fromFields(JsonFields(entry.toMap)) + private def entryOf(entry: Vector[(String, JsonValue)]): ContentEntryDto = + ContentEntryDto.fromFields(JsonFields(entry)) private def describe(value: JsonValue): String = if value.isNull then "null" else if value.strOpt.isDefined then "a string" - else if value.numOpt.isDefined then "a number" + else if value.isNum then "a number" else if value.boolOpt.isDefined then "a boolean" else "a value of an unrecognised kind" diff --git a/modules/codec/test/src/com/worxbend/codeberg4s/codec/JsonFieldsSuite.scala b/modules/codec/test/src/com/worxbend/codeberg4s/codec/JsonFieldsSuite.scala index 35a3bb9..1bd0317 100644 --- a/modules/codec/test/src/com/worxbend/codeberg4s/codec/JsonFieldsSuite.scala +++ b/modules/codec/test/src/com/worxbend/codeberg4s/codec/JsonFieldsSuite.scala @@ -17,7 +17,7 @@ final class JsonFieldsSuite extends FunSuite: private def fieldsOf(body: String): JsonFields = Json.parse(body) match - case Right(JsonValue.Obj(fields)) => JsonFields(fields.toMap) + case Right(JsonValue.Obj(fields)) => JsonFields(fields) case other => fail(s"the fixture body is not a JSON object: $other") private val absent: JsonFields = fieldsOf("""{}""") @@ -92,6 +92,63 @@ final class JsonFieldsSuite extends FunSuite: assertEquals(JsonFields.Empty.number("anything"), None) assertEquals(JsonFields.Empty.texts("anything"), Vector.empty[String]) + test("no field can be shadowed, because a document that repeats one never becomes a view"): + // This view scans the parser's field vector and takes the first match, so a + // repeated name would read as the first of the two. That answer is + // unreachable: the parser refuses the document first, so the view is only + // ever built from names that are already distinct. See JsonValue.Obj for why + // refusing is the rule. + assert(Json.decode[Probe]("""{"a":"first","a":"second"}""").isLeft) + assertEquals(Json.decode[Probe]("""{"a":"first","b":"second"}"""), Right(Probe(Some("first")))) + + test("a wide object reads the same as a narrow one, on either side of the index threshold"): + // Above a threshold the view stops comparing names one by one and probes a + // hash index instead. Both strategies have to answer identically, so the + // same object is read at eight fields (compared) and at forty (indexed). + def objectOf(width: Int): JsonFields = + fieldsOf((0 until width).map(at => s""""f$at":$at""").mkString("{", ",", "}")) + + val narrow = objectOf(8) + val wide = objectOf(40) + + assertEquals(narrow.number("f0"), Some(0L)) + assertEquals(narrow.number("f7"), Some(7L)) + assertEquals(narrow.number("f8"), None) + + (0 until 40).foreach(at => assertEquals(wide.number(s"f$at"), Some(at.toLong), s"field f$at of a 40-field object")) + assertEquals(wide.number("f40"), None) + assertEquals(wide.number("absent"), None) + + test("two field names with the same hash are still told apart in a wide object"): + // "Aa" and "BB" have the same String hash (2112), so in an indexed object + // they want the same slot. A lookup that trusted the hash would answer one + // with the other's value; comparing the names in full is what stops it. + val colliding = + fieldsOf(s"""{"Aa":"first","BB":"second",${(0 until 8).map(at => s""""f$at":$at""").mkString(",")}}""") + + assertEquals("Aa".hashCode, "BB".hashCode, "the fixture is pointless unless these two really do collide") + assertEquals(colliding.text("Aa"), Some("first")) + assertEquals(colliding.text("BB"), Some("second")) + assertEquals(colliding.text("Ab"), None) + + test("a view built by hand from a repeated name answers with the first of them, at either width"): + // Json.parse never produces this, but the view's constructor is public and + // RepositoryContentDto builds one straight from an already-parsed object. + // First-wins is the answer JsonValue.field gives, so it is the answer here. + def repeated(padding: Int): JsonFields = + JsonFields( + Vector("a" -> JsonValue.Str("first"), "a" -> JsonValue.Str("second")) ++ + (0 until padding).map(at => s"f$at" -> JsonValue.Num(at)) + ) + + assertEquals(repeated(0).text("a"), Some("first")) + assertEquals(repeated(20).text("a"), Some("first")) + + test("toMap hands the fields over for a payload whose keys are data rather than a schema"): + assertEquals(populated.toMap("a"), JsonValue.Str("text")) + assertEquals(populated.toMap.keySet, Set("a", "n", "b", "o", "list")) + assertEquals(JsonFields.Empty.toMap, Map.empty[String, JsonValue]) + test("reader delegates the is-this-an-object question to the JSON parser"): assertEquals(Json.decode[Probe]("""{"a":"x"}"""), Right(Probe(Some("x")))) assertEquals(Json.decode[Probe]("""{}"""), Right(Probe(None))) diff --git a/modules/codec/test/src/com/worxbend/codeberg4s/codec/JsonSuite.scala b/modules/codec/test/src/com/worxbend/codeberg4s/codec/JsonSuite.scala index b78f521..558e1bb 100644 --- a/modules/codec/test/src/com/worxbend/codeberg4s/codec/JsonSuite.scala +++ b/modules/codec/test/src/com/worxbend/codeberg4s/codec/JsonSuite.scala @@ -1,9 +1,12 @@ package com.worxbend.codeberg4s.codec import com.worxbend.codeberg4s.JsonPath +import com.worxbend.codeberg4s.core.ResponseBody import munit.FunSuite +import java.nio.charset.StandardCharsets + /** `Json` is the only place in the library that calls jsoniter, so this suite is where "no codec exception ever * escapes" is proved. Every case below is a body that makes the parser throw. * @@ -85,10 +88,22 @@ final class JsonSuite extends FunSuite: assertEquals(Json.decode[Vector[Leaf]]("[]"), Right(Vector.empty)) test("decoder produces a Decode port that agrees with decode"): - assertEquals(Json.decoder[Leaf].apply(leaf), Json.decode[Leaf](leaf)) + assertEquals(Json.decoder[Leaf].apply(ResponseBody.utf8(leaf)), Json.decode[Leaf](leaf)) test("decoder never throws either"): - assert(Json.decoder[Leaf].apply("not json").isLeft) + assert(Json.decoder[Leaf].apply(ResponseBody.utf8("not json")).isLeft) + + test("the two decode overloads agree, so a caller holding text is not on a different code path"): + assertEquals(Json.decode[Leaf](leaf.getBytes(StandardCharsets.UTF_8)), Json.decode[Leaf](leaf)) + + test("decoder reads a body the response declared as something other than UTF-8"): + // Nothing Forgejo serves looks like this. The point is that the charset on + // the body is honoured rather than ignored: the same characters encoded as + // ISO-8859-1 must decode to the same value, not to mojibake. + val accented = """{"name":"café"}""" + val latin1 = ResponseBody.of(accented.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.ISO_8859_1) + + assertEquals(Json.decoder[Leaf].apply(latin1), Json.decode[Leaf](accented)) test("a document nested past the depth bound is a failure, not a stack overflow"): // Remote input must not be able to exhaust the caller's stack. @@ -111,8 +126,122 @@ final class JsonSuite extends FunSuite: assertEquals(Json.render(document), """{"z":"1","a":"2"}""") + test("a document that names a field twice is rejected rather than quietly resolved"): + // JsonValue.Obj argues the decision. In short: the three ways of reading an + // object disagreed about {"id":1,"id":2} — first, last, and both — and only + // one of the available answers loses no data, which is to refuse it. + assert(Json.parse("""{"id":1,"id":2}""").isLeft) + + test("the repeated-field failure names the field that was repeated"): + Json.parse("""{"id":1,"id":2}""") match + case Left(failure) => assert(failure.message.contains("""duplicated field "id""""), failure.message) + case Right(value) => fail(s"expected a failure, got $value") + + test("a repeated field fails at the root, like every other structural failure"): + Json.parse("""{"id":1,"id":2}""") match + case Left(failure) => assertEquals(failure.path, JsonPath.Root) + case Right(value) => fail(s"expected a failure, got $value") + + test("a repeated field is rejected whichever door the body is decoded through"): + // This is the disagreement that made the decision necessary: JsonValue.field + // answered 1, and a DTO assembled from the same object answered 2. + assert(Json.decode[Leaf]("""{"name":"a","name":"b"}""").isLeft) + assert(Json.decode[Map[String, JsonValue]]("""{"id":1,"id":2}""").isLeft) + + test("a repeated field is rejected wherever in the document it sits"): + assert(Json.parse("""{"owner":{"id":1,"id":2}}""").isLeft) + assert(Json.parse("""[{"id":1},{"id":2,"id":3}]""").isLeft) + + test("the same field name in two sibling objects is not a repeat"): + // The rule is about one object naming a field twice. Every element of a page + // carrying an "id" is what a page looks like. + assertEquals(Json.parse("""[{"id":1},{"id":2}]""").map(Json.render), Right("""[{"id":1},{"id":2}]""")) + + test("a repeated field is caught in a wide object, where the check hashes instead of comparing"): + // Objects this wide are checked by a different branch than the narrow ones + // above — a repository response has 64 keys, so both branches carry real + // traffic and both need a test. 200 keeps this clear of the width the two + // branches split at without the test having to know that width. + val distinct = (1 to 200).map(index => s""""k$index":$index""").mkString("{", ",", "}") + val repeated = distinct.replace(""""k137":137""", """"k42":137""") + + assert(Json.parse(distinct).isRight) + assert(Json.parse(repeated).isLeft) + + test("rendering stays faithful, so a document built with a repeated field is one parse refuses"): + // Json.render writes the fields it is given, and Obj says not to hand it a + // repeated key. This pins the consequence rather than hiding it: render does + // not quietly drop a field, and the parser does not quietly accept one, so + // the two never disagree about a document — they only ever both refuse. + val document = JsonValue.Obj("id" -> JsonValue.Num(1), "id" -> JsonValue.Num(2)) + + assertEquals(Json.render(document), """{"id":1,"id":2}""") + assert(Json.parse(Json.render(document)).isLeft) + test("a large integer survives the round trip, which a Double would not"): // 2^53 + 1 is the first integer a Double cannot represent. val body = """{"id":9007199254740993}""" assertEquals(Json.parse(body).map(Json.render), Right(body)) + + test("a whole number parses into the Long case and nothing else does"): + // The split is what keeps a BigDecimal off the path every row id takes. + // Which case a number lands in follows the text on the wire, so that render + // can write back what it read. + assertEquals(Json.parse("7"), Right(JsonValue.Int64(7L))) + assertEquals(Json.parse("-7"), Right(JsonValue.Int64(-7L))) + assertEquals(Json.parse("9223372036854775807"), Right(JsonValue.Int64(Long.MaxValue))) + assertEquals(Json.parse("2.5"), Right(JsonValue.Decimal(BigDecimal("2.5")))) + assertEquals(Json.parse("7.0"), Right(JsonValue.Decimal(BigDecimal("7.0")))) + assertEquals(Json.parse("7e2"), Right(JsonValue.Decimal(BigDecimal("7E+2")))) + + test("a whole number past the Long range is exact rather than rounded"): + // One past Long.MaxValue, so the Long case cannot hold it. jsoniter hands + // this back as a BigInteger; losing it to a Double or clamping it to + // Long.MaxValue are both silent corruptions of an identifier. + val body = """{"id":9223372036854775808}""" + + assertEquals(Json.parse("9223372036854775808"), Right(JsonValue.Decimal(BigDecimal("9223372036854775808")))) + assertEquals(Json.parse(body).map(Json.render), Right(body)) + + test("both number cases say they are a number when a failure has to name a kind"): + assertEquals(JsonValue.Int64(7L).kind, "a number") + assertEquals(JsonValue.Decimal(BigDecimal("2.5")).kind, "a number") + + test("Num builds whichever case renders the text back unchanged"): + // Forgejo's integer fields reject 102.0, so a whole value must never pick up + // a fractional part on its way to a request body — whichever of the four + // ways of writing it down the caller reached for. + assertEquals(Json.render(JsonValue.Num(102)), "102") + assertEquals(Json.render(JsonValue.Num(102L)), "102") + assertEquals(Json.render(JsonValue.Num(102.0)), "102") + assertEquals(Json.render(JsonValue.Num(BigDecimal(102))), "102") + assertEquals(Json.render(JsonValue.Num(102.5)), "102.5") + assertEquals(Json.render(JsonValue.Num(BigDecimal("102.0"))), "102.0") + + test("a number Num built parses back to the same value"): + // The two cases are only telling apart if one constructor decides between + // them: Decimal(BigDecimal(102)) would render 102 and then not equal the + // Int64(102) that parsing 102 produces. Num is that constructor. + val built = Vector(JsonValue.Num(102), JsonValue.Num(102.0), JsonValue.Num(BigDecimal(102))) + + built.foreach(number => assertEquals(Json.parse(Json.render(number)), Right(number))) + + test("Num matches either case and hands back an exact decimal"): + // Kept so that a call site written when Num was one case class holding a + // BigDecimal still compiles and still means the same thing. + assertEquals(JsonValue.Int64(7L).numOpt, Some(BigDecimal(7))) + assertEquals(JsonValue.Decimal(BigDecimal("2.5")).numOpt, Some(BigDecimal("2.5"))) + assertEquals(JsonValue.Str("7").numOpt, None) + + val matched = Json.parse("""[7,2.5,"7"]""").map(_.arrOpt.toVector.flatten.collect { case JsonValue.Num(n) => n }) + + assertEquals(matched, Right(Vector(BigDecimal(7), BigDecimal("2.5")))) + + test("a number read as a Long truncates toward zero, whichever case it is in"): + assertEquals(Json.decode[Long]("7"), Right(7L)) + assertEquals(Json.decode[Long]("2.9"), Right(2L)) + assertEquals(Json.decode[Long]("-2.9"), Right(-2L)) + assertEquals(JsonValue.Int64(7L).longOpt, Some(7L)) + assertEquals(JsonValue.Decimal(BigDecimal("2.9")).longOpt, Some(2L)) + assertEquals(JsonValue.Bool(true).longOpt, None) diff --git a/modules/codec/test/src/com/worxbend/codeberg4s/codec/PropertyBaseProps.scala b/modules/codec/test/src/com/worxbend/codeberg4s/codec/PropertyBaseProps.scala index 5a5749a..a0d5418 100644 --- a/modules/codec/test/src/com/worxbend/codeberg4s/codec/PropertyBaseProps.scala +++ b/modules/codec/test/src/com/worxbend/codeberg4s/codec/PropertyBaseProps.scala @@ -87,15 +87,27 @@ object PropertyBase: /** Text that survives `JsonFields.text`, which folds Forgejo's `""`-for-absent convention into `None`. */ val nonBlankText: Gen[String] = text.suchThat(_.trim.nonEmpty) - /** A JSON object key. Distinct keys are enforced where it matters, since a duplicate key is not round-trippable. */ + /** A JSON object key. The object generator below makes the names distinct, because a document that names a field + * twice does not round-trip — [[JsonValue.Obj]] explains why the parser refuses one outright. + */ val key: Gen[String] = Gen.nonEmptyListOf(Gen.oneOf(('a' to 'z') ++ ('0' to '9'))).map(_.mkString) - /** A JSON value that is not a container. */ + /** A JSON value that is not a container. + * + * Three number generators rather than one, because [[JsonValue]] represents a number two ways — a `Long` for a whole + * one that fits, a `BigDecimal` for everything else — and the round-trip property is what holds the two apart. A + * generator that only ever produced whole numbers would never build the second case and would pass whatever + * rendering did to it. The divisor is eight so that the quotient is exact in decimal, which keeps the generated + * value a fact about the model rather than about `BigDecimal`'s rounding; the multiplier is 2^70 so the value is + * past what a `Long` can hold. + */ val scalar: Gen[JsonValue] = Gen.oneOf( Gen.const[JsonValue](JsonValue.Null), Gen.oneOf(true, false).map(flag => JsonValue.Bool(flag)), Gen.choose(-100000, 100000).map(number => JsonValue.Num(number.toDouble)), + Gen.choose(-100000, 100000).map(number => JsonValue.Num(BigDecimal(number) / 8)), + Gen.choose(-100000, 100000).map(number => JsonValue.Num(BigDecimal(BigInt(number) * BigInt(2).pow(70)))), text.map(value => JsonValue.Str(value)), ) diff --git a/modules/codec/test/src/com/worxbend/codeberg4s/codec/TimestampsSuite.scala b/modules/codec/test/src/com/worxbend/codeberg4s/codec/TimestampsSuite.scala index c298b99..0ac9139 100644 --- a/modules/codec/test/src/com/worxbend/codeberg4s/codec/TimestampsSuite.scala +++ b/modules/codec/test/src/com/worxbend/codeberg4s/codec/TimestampsSuite.scala @@ -2,7 +2,10 @@ package com.worxbend.codeberg4s.codec import munit.FunSuite +import scala.util.Try + import java.time.Instant +import java.time.OffsetDateTime final class TimestampsSuite extends FunSuite: @@ -12,6 +15,17 @@ final class TimestampsSuite extends FunSuite: test("a Z-suffixed timestamp parses"): assertEquals(Timestamps.parse("2026-08-01T22:12:04Z"), Some(Instant.parse("2026-08-01T22:12:04Z"))) + test("a negative offset is subtracted, even when that moves the date"): + assertEquals(Timestamps.parse("2026-08-01T22:12:04-05:00"), Some(Instant.parse("2026-08-02T03:12:04Z"))) + + test("fractional seconds survive, and the offset still applies"): + assertEquals(Timestamps.parse("2022-11-26T18:56:24.123+01:00"), Some(Instant.parse("2022-11-26T17:56:24.123Z"))) + assertEquals(Timestamps.parse("2026-08-01T22:12:04.1Z"), Some(Instant.parse("2026-08-01T22:12:04.100Z"))) + assertEquals( + Timestamps.parse("2026-08-01T22:12:04.123456789Z"), + Some(Instant.parse("2026-08-01T22:12:04.123456789Z")), + ) + test("the Go zero-time sentinel is absence, not a year-1 timestamp"): assertEquals(Timestamps.parse("0001-01-01T00:00:00Z"), None) @@ -25,10 +39,79 @@ final class TimestampsSuite extends FunSuite: assertEquals(Timestamps.parse("notadate"), None) assertEquals(Timestamps.parse("2022-11-26"), None) + test("a timestamp with no offset at all is absence"): + assertEquals(Timestamps.parse("2026-08-01T22:12:04"), None) + assertEquals(Timestamps.parse("2026-08-01T22:12:04.123"), None) + + test("anything trailing the offset makes the whole value absence"): + assertEquals(Timestamps.parse("2026-08-01T22:12:04Zx"), None) + assertEquals(Timestamps.parse("2026-08-01T22:12:04+02:00x"), None) + assertEquals(Timestamps.parse("2026-08-01T22:12:04Z 2026-08-01T22:12:04Z"), None) + + test("a value that is not a real point on the calendar is absence"): + assertEquals(Timestamps.parse("2024-02-29T00:00:00Z"), Some(Instant.parse("2024-02-29T00:00:00Z"))) + assertEquals(Timestamps.parse("2023-02-29T00:00:00Z"), None) + assertEquals(Timestamps.parse("2026-04-31T00:00:00Z"), None) + assertEquals(Timestamps.parse("2026-13-01T00:00:00Z"), None) + assertEquals(Timestamps.parse("2026-00-01T00:00:00Z"), None) + assertEquals(Timestamps.parse("2026-08-00T00:00:00Z"), None) + assertEquals(Timestamps.parse("2026-08-01T24:00:00Z"), None) + assertEquals(Timestamps.parse("2026-08-01T22:60:04Z"), None) + assertEquals(Timestamps.parse("2026-08-01T22:12:60Z"), None) + assertEquals(Timestamps.parse("2026-08-01T22:12:04+24:00"), None) + + test("single-digit calendar fields are not padded for us"): + assertEquals(Timestamps.parse("2026-8-01T22:12:04Z"), None) + assertEquals(Timestamps.parse("2026-08-1T22:12:04Z"), None) + test("surrounding whitespace is tolerated"): assertEquals(Timestamps.parse(" 2022-11-26T18:56:24+01:00 "), Some(Instant.parse("2022-11-26T17:56:24Z"))) + /** RFC-3339 spellings Forgejo does not send, but which the library accepts today. + * + * They are pinned so that a faster parser tuned to Forgejo's exact layout cannot silently narrow what the library + * understands: whatever shape a caller was already feeding it has to keep working. + */ + test("RFC-3339 spellings outside Forgejo's own layout still parse"): + assertEquals(Timestamps.parse("2026-08-01t22:12:04z"), Some(Instant.parse("2026-08-01T22:12:04Z"))) + assertEquals(Timestamps.parse("2022-11-26T18:56+01:00"), Some(Instant.parse("2022-11-26T17:56:00Z"))) + assertEquals(Timestamps.parse("2026-08-01T22:12:04+01:00:30"), Some(Instant.parse("2026-08-01T21:11:34Z"))) + assertEquals(Timestamps.parse("2026-08-01T22:12:04-00:00"), Some(Instant.parse("2026-08-01T22:12:04Z"))) + assertEquals( + Timestamps.parse("+12026-08-01T22:12:04Z"), + Some(OffsetDateTime.parse("+12026-08-01T22:12:04Z").toInstant), + ) + test("parseOptional threads absence through"): assertEquals(Timestamps.parseOptional(None), None) assertEquals(Timestamps.parseOptional(Some("0001-01-01T00:00:00Z")), None) assertEquals(Timestamps.parseOptional(Some("2026-08-01T22:12:04Z")), Some(Instant.parse("2026-08-01T22:12:04Z"))) + + /** What [[Timestamps.parse]] has to mean, spelled out with the JDK doing the work. + * + * The sweep below compares every generated value against this, so an implementation that hand-rolls the arithmetic — + * days since the epoch, leap years, offset subtraction — is checked against `java.time` rather than against a + * handful of examples someone remembered to write down. + */ + private def viaJdk(value: String): Option[Instant] = + Try(OffsetDateTime.parse(value.trim).toInstant).toOption + .filter(_.isAfter(Instant.EPOCH)) + + test("the parser agrees with the JDK across a sweep of dates, times and offsets"): + val dates = + for + year <- Seq(1969, 1970, 1971, 1999, 2000, 2001, 2023, 2024, 2026, 2100, 2400) + month <- 1 to 12 + day <- Seq(1, 15, 28, 29, 30, 31) + yield f"$year%04d-$month%02d-$day%02d" + + val times = Seq("00:00:00", "12:34:56", "23:59:59", "22:12:04.123", "06:07:08.000000001") + val offsets = Seq("Z", "+00:00", "-00:00", "+01:00", "-05:00", "+05:30", "-09:30", "+14:00", "-12:00", "+18:00") + + for + date <- dates + time <- times + offset <- offsets + do + val value = s"${date}T$time$offset" + assertEquals(Timestamps.parse(value), viaJdk(value), value) diff --git a/modules/core/src/com/worxbend/codeberg4s/core/ApiPipeline.scala b/modules/core/src/com/worxbend/codeberg4s/core/ApiPipeline.scala index 2b09731..56bf3a5 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/ApiPipeline.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/ApiPipeline.scala @@ -13,8 +13,6 @@ import com.worxbend.codeberg4s.syntax.discard import scala.concurrent.duration.FiniteDuration import scala.util.Try -import java.nio.charset.StandardCharsets - /** The single path every API call takes: send, retry, classify, decode, observe. * * Endpoints describe *what* to call by building a [[CodebergRequest]]; this class owns *how* a call is made. Keeping @@ -25,9 +23,12 @@ import java.nio.charset.StandardCharsets * '''Order of events for one attempt.''' [[Telemetry.onRequest]], the send, then [[Telemetry.onResponse]] if a * response arrived, then [[Telemetry.onError]] if the attempt failed. The retry engine repeats that whole sequence, so * a retried call produces one triple per attempt, and [[Telemetry.onError]] is called once more with the failure the - * caller finally receives — which is [[com.worxbend.codeberg4s.CodebergError.RetriesExhausted]] when more than one - * attempt was made. A telemetry callback that fails is swallowed: observation must not decide whether a request - * succeeded. + * caller finally receives — which is [[com.worxbend.codeberg4s.CodebergError.RetriesExhausted]] when the retry policy + * ran out of attempts on a failure it was repeating, and the last failure unwrapped otherwise. A telemetry callback + * that fails in `F`'s error channel is swallowed here: observation must not decide whether a request succeeded. A + * callback that fails some other way — a raw throw, or an `F` whose failure channel is wider than + * [[com.worxbend.codeberg4s.CodebergError]] — is out of this class's reach, because [[Exec.attempt]] deliberately + * catches nothing else; whoever hands a caller's sink to this pipeline is responsible for wrapping it. * * '''Failure contract.''' `Left`/raised values are always a [[com.worxbend.codeberg4s.CodebergError]]: * - no response at all becomes [[com.worxbend.codeberg4s.CodebergError.Transport]]; @@ -36,7 +37,9 @@ import java.nio.charset.StandardCharsets * never masks the status; * - a 2xx payload that does not decode becomes [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]] with the * failing JSON path and an excerpt of the body bounded at - * [[com.worxbend.codeberg4s.CodebergError.MaxSnippetLength]]; + * [[com.worxbend.codeberg4s.CodebergError.MaxSnippetLength]] — or, when the [[Decode]] instance declared itself + * [[Decode.sensitive]] because the endpoint answers with a credential, [[ApiPipeline.redactedSnippet]] in place of + * the excerpt; * - a failure the retry engine gave up on becomes [[com.worxbend.codeberg4s.CodebergError.RetriesExhausted]], * preserving the last underlying failure. * @@ -108,21 +111,25 @@ final class ApiPipeline[F[_]]( * never need one are unaffected and every existing [[HttpPort]] fake keeps compiling. * * Retry, telemetry, status mapping and `CallContext` behave exactly as they do for a textual call. An error body is - * still JSON text even on an endpoint whose success body is binary, so a non-2xx response is decoded as UTF-8 and - * parsed the usual way; a successful body is never decoded, which is the whole point. + * still JSON text even on an endpoint whose success body is binary, so a non-2xx response is decoded with the + * charset it declared and parsed the usual way; a successful body is never decoded, which is the whole point. * * Always [[RetryEligibility.IdempotentOnly]] — every endpoint that answers bytes in this API is a `GET`. */ def callBinary(request: CodebergRequest, binary: BinaryHttpPort[F]): F[BinaryResponse] = + val uri = redactedUri(request) val attempts = engine.runWith(request.operation, request.method, RetryEligibility.IdempotentOnly)(_ => - binaryAttempt(request, binary) + binaryAttempt(request, uri, binary) ) exec.attempt(attempts).flatMap: case Right(value) => exec.pure(value) case Left(error) => reportFinal(request, error).flatMap(_ => exec.raise(error)) - private def binaryAttempt(request: CodebergRequest, binary: BinaryHttpPort[F]): F[AttemptOutcome[BinaryResponse]] = - val uri = Redaction.uri(config.baseUri.value, request.path, request.query) + private def binaryAttempt( + request: CodebergRequest, + uri: String, + binary: BinaryHttpPort[F], + ): F[AttemptOutcome[BinaryResponse]] = timer.nowMillis.flatMap: started => observe(telemetry.onRequest(contextOf(request, uri, None, 0L))).flatMap: _ => binary.sendBinary(request, uri).flatMap: sent => @@ -144,25 +151,36 @@ final class ApiPipeline[F[_]]( observe(telemetry.onResponse(ctx, response.status)).flatMap: _ => if StatusMapping.isSuccess(response.status) then exec.pure(AttemptOutcome.succeeded(response)) else - val text = String(response.bytes, StandardCharsets.UTF_8) - val error = StatusMapping.toError(ctx, response.status, parsedErrorBody(text)) + val body = ResponseBody.of(response.bytes, ResponseBody.charsetOf(response.contentType)) + val error = StatusMapping.toError(ctx, response.status, parsedErrorBody(body)) failedWith(ctx, error, response.retryAfter) private def perform[A](request: CodebergRequest, eligibility: RetryEligibility)( onSuccess: (CallContext, CodebergResponse) => Either[CodebergError, A] ): F[A] = + val uri = redactedUri(request) val attempts = engine.runWith(request.operation, request.method, eligibility)(_ => - attemptOnce(request, onSuccess) + attemptOnce(request, uri, onSuccess) ) exec.attempt(attempts).flatMap: case Right(value) => exec.pure(value) case Left(error) => reportFinal(request, error).flatMap(_ => exec.raise(error)) + /** Renders the URI every attempt of this call reports. + * + * A retried call re-sends an identical request, so the redacted URI it reports is identical too. Building it here, + * once per call rather than once per attempt, keeps the string every attempt shares — and therefore every + * `CallContext` and every telemetry event — exactly what it was, while a five-attempt call encodes its path and + * query once instead of five times. + */ + private def redactedUri(request: CodebergRequest): String = + Redaction.uri(config.baseUri.value, request.path, request.query) + private def attemptOnce[A]( request: CodebergRequest, + uri: String, onSuccess: (CallContext, CodebergResponse) => Either[CodebergError, A], ): F[AttemptOutcome[A]] = - val uri = Redaction.uri(config.baseUri.value, request.path, request.query) timer.nowMillis.flatMap: started => observe(telemetry.onRequest(contextOf(request, uri, None, 0L))).flatMap: _ => http.send(request, uri).flatMap: sent => @@ -219,22 +237,52 @@ final class ApiPipeline[F[_]]( ): CallContext = CallContext(request.operation, request.method, uri, requestId, elapsedMs) - private def decoded[A](ctx: CallContext, body: String)(using decode: Decode[A]): Either[CodebergError, A] = + private def decoded[A](ctx: CallContext, body: ResponseBody)(using decode: Decode[A]): Either[CodebergError, A] = decode(body).left.map(failure => - CodebergError.DecodingFailed(ctx, ApiPipeline.snippetOf(body), failure.path, failure.message) + CodebergError.DecodingFailed(ctx, ApiPipeline.snippetOf(body, decode.sensitive), failure.path, failure.message) ) - /** Reads a non-2xx payload, tolerating both an empty body and an injected parser that fails outright. */ - private def parsedErrorBody(body: String): ApiErrorBody = + /** Reads a non-2xx payload, tolerating both an empty body and an injected parser that fails outright. + * + * This is one of the few places that genuinely wants text: the injected parser takes a `String`, an error payload is + * a few hundred bytes, and it is only ever read on the failure path. Decoding it here rather than at the transport + * is what keeps the successful path — every listing, every read — free of the copy. + */ + private def parsedErrorBody(body: ResponseBody): ApiErrorBody = if body.isBlank then ApiErrorBody.Empty - else Try(errorBody(body)).getOrElse(ApiErrorBody.Empty) + else Try(errorBody(body.text)).getOrElse(ApiErrorBody.Empty) object ApiPipeline: - /** An excerpt of `body` no longer than [[com.worxbend.codeberg4s.CodebergError.MaxSnippetLength]] characters. + /** What a body is reported as when its [[Decode]] declared itself [[Decode.sensitive]]. + * + * A fixed string, so nothing about the payload survives into it, but not an empty one: a reader still has to be able + * to tell "the instance answered with a body this library refuses to quote" from "the instance answered with + * nothing". The size is what remains — enough to distinguish a truncated response from a complete one that did not + * match the model, and not enough to reconstruct a byte of it. + * + * @param bytes + * how many bytes the withheld body held, [[ResponseBody.size]] of the response + */ + def redactedSnippet(bytes: Int): String = + s"${Redaction.Mask} ($bytes bytes withheld)" + + /** What [[com.worxbend.codeberg4s.CodebergError.DecodingFailed.snippet]] carries for `body`. * + * Ordinarily an excerpt no longer than [[com.worxbend.codeberg4s.CodebergError.MaxSnippetLength]] '''characters'''. * Bounding happens here, once, rather than at each call site: a decoding failure on a 40 MB repository listing must - * not put 40 MB into an error value that an application is about to log. + * not put 40 MB into an error value that an application is about to log. [[ResponseBody.excerpt]] does the work, + * because bounding a body that is now bytes at a number of characters is a job with a trap in it — see its own + * documentation for why slicing the bytes and decoding the slice is not the same thing. + * + * When `sensitive` is set, the excerpt is replaced by [[redactedSnippet]] and no part of the body is quoted. That is + * decided here rather than by the endpoint that made the call, so it holds for the failure a caller receives + * '''and''' for the one [[Telemetry.onError]] observes — the hook fires inside the pipeline, so anything an endpoint + * scrubbed afterwards would already have been handed to a telemetry sink that logs what it is given. + * + * @param sensitive + * [[Decode.sensitive]] of the instance that read this body */ - private[core] def snippetOf(body: String): String = - body.take(CodebergError.MaxSnippetLength) + private[core] def snippetOf(body: ResponseBody, sensitive: Boolean): String = + if sensitive then redactedSnippet(body.size) + else body.excerpt(CodebergError.MaxSnippetLength) diff --git a/modules/core/src/com/worxbend/codeberg4s/core/BinaryHttpPort.scala b/modules/core/src/com/worxbend/codeberg4s/core/BinaryHttpPort.scala index def23d8..cf7af30 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/BinaryHttpPort.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/BinaryHttpPort.scala @@ -9,7 +9,9 @@ package com.worxbend.codeberg4s.core * this, and one that cannot simply does not. * * The failure contract matches [[HttpPort]]: any HTTP status, including `5xx`, arrives as a `Right`; a `Left` means no - * response arrived at all. + * complete response arrived. The one difference is which bound applies to the body — these operations fetch archives, + * so an adapter applies [[com.worxbend.codeberg4s.CodebergConfig.maxDownloadBodyBytes]] here rather than the smaller + * [[com.worxbend.codeberg4s.CodebergConfig.maxResponseBodyBytes]]. * * @tparam F * the effect the client runs in diff --git a/modules/core/src/com/worxbend/codeberg4s/core/BinaryResponse.scala b/modules/core/src/com/worxbend/codeberg4s/core/BinaryResponse.scala index 4ff9ba5..e703cb3 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/BinaryResponse.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/BinaryResponse.scala @@ -4,10 +4,14 @@ import scala.concurrent.duration.FiniteDuration /** A response whose body is bytes rather than text. * - * [[CodebergResponse]] carries a `String`, which is right for the JSON and `text/plain` endpoints that make up almost - * all of this API — but a few answer a ZIP: an Actions artifact and a workflow run's logs. Decoding those bytes as - * UTF-8 destroys them before any [[Decode]] could see them, so they need their own response type rather than a lossy - * reuse of the textual one. + * A few endpoints answer a ZIP rather than text: an Actions artifact and a workflow run's logs. This is what + * [[BinaryHttpPort]] hands back for those. + * + * '''It is now a near-duplicate of [[CodebergResponse]].''' It exists because [[CodebergResponse]] used to carry a + * `String`, which would have destroyed a ZIP before any [[Decode]] could see it. Now that [[CodebergResponse]] carries + * a [[ResponseBody]] the two types say the same thing, and the download endpoints could be served by the ordinary + * pipeline. Merging them changes the published signature of the download API, so it is a change of its own rather than + * a rider on the one that made it possible. * * Headers are lowercased by the transport, exactly as they are on [[CodebergResponse]]. * @@ -33,14 +37,34 @@ final case class BinaryResponse( /** The `Retry-After` delay, parsed defensively — a missing or malformed header is `None`, never a failure. */ def retryAfter: Option[FiniteDuration] = - CodebergResponse(status, headers, "").retryAfter + CodebergResponse(status, headers, ResponseBody.Empty).retryAfter /** The request id the instance echoed, when it echoed one. */ def requestId: Option[String] = - CodebergResponse(status, headers, "").requestId + CodebergResponse(status, headers, ResponseBody.Empty).requestId /** How large the body is. Cheaper to read than to render, and the thing worth logging. */ def size: Int = bytes.length + /** Structural, on the status, the headers and then the bytes. + * + * Written out because the array's own `equals` in Scala is '''identity''': the equality a case class generates would + * compare [[bytes]] by reference, so two responses carrying byte-identical archives would compare unequal and hash + * differently. That is a wrong answer with no warning attached — in an assertion, or in a `Set` — which is why the + * bytes are compared with `java.util.Arrays.equals` here. The status and the headers are compared first because they + * are the cheap half; the archive may be megabytes. + * + * The class is `final`, so no subclass can exist and the type test below is the whole of the compiler-generated + * `canEqual`; calling `canEqual` as well would add nothing. Removing `final` would change that. + */ + override def equals(other: Any): Boolean = + other match + case that: BinaryResponse => + status.equals(that.status) && headers.equals(that.headers) && java.util.Arrays.equals(bytes, that.bytes) + case _ => false + + override def hashCode(): Int = + 31 * (31 * status + headers.hashCode) + java.util.Arrays.hashCode(bytes) + /** Deliberately does not render the body: a ZIP in a log line helps nobody. */ override def toString: String = s"BinaryResponse(status=$status, size=$size)" diff --git a/modules/core/src/com/worxbend/codeberg4s/core/CodebergRequest.scala b/modules/core/src/com/worxbend/codeberg4s/core/CodebergRequest.scala index ac48c34..cc78389 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/CodebergRequest.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/CodebergRequest.scala @@ -10,7 +10,9 @@ import com.worxbend.codeberg4s.HttpMethod * * '''Security contract:''' `headers` never contains an `Authorization` header. Credentials are applied by the * transport from [[com.worxbend.codeberg4s.auth.Auth]], so no credential can reach a log line, a - * [[com.worxbend.codeberg4s.CallContext]] or an error payload through this type. + * [[com.worxbend.codeberg4s.CallContext]] or an error payload through this type. The rule is enforced and not merely + * documented: the transport adapter drops an `Authorization` or `Proxy-Authorization` entry from this list before it + * applies the configured credential, so a request built with one still goes out authenticated as `Auth` says. * * @param operation * the stable, greppable operation id, for example `"repos.get"`; it is copied into every failure diff --git a/modules/core/src/com/worxbend/codeberg4s/core/CodebergResponse.scala b/modules/core/src/com/worxbend/codeberg4s/core/CodebergResponse.scala index fb952cc..d62a76a 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/CodebergResponse.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/CodebergResponse.scala @@ -20,9 +20,10 @@ import java.util.Locale * response headers with '''already lowercased''' keys, as the transport adapter normalises them; a key maps to every * value the server sent for it, in order * @param body - * the response body as text, empty for a `204` + * the response body as the bytes that arrived, with the charset the response declared for them; empty for a `204`. + * See [[ResponseBody]] for why this is not a `String` */ -final case class CodebergResponse(status: Int, headers: Map[String, List[String]], body: String): +final case class CodebergResponse(status: Int, headers: Map[String, List[String]], body: ResponseBody): /** The first value of `name`, matched case-insensitively. * @@ -62,8 +63,11 @@ final case class CodebergResponse(status: Int, headers: Map[String, List[String] * All values of the `Link` header are considered, not just the first: a proxy is allowed to split one header into * several, and the RFC says the result is the same as if they had been joined with commas. An unreadable header * yields an empty map — see [[LinkHeader]] for why that is never an error. + * + * The header is parsed the first time this is read and the result is kept, because [[nextPage]], [[prevPage]] and + * [[lastPage]] all go through it and page one alone would otherwise parse the same string three times. */ - def links: Map[String, String] = + lazy val links: Map[String, String] = LinkHeader.parse(headers.getOrElse(LinkHeader.Name, Nil).mkString(",")) /** The page number of `rel="next"`, and nothing else. diff --git a/modules/core/src/com/worxbend/codeberg4s/core/Decode.scala b/modules/core/src/com/worxbend/codeberg4s/core/Decode.scala index 280c552..5970b11 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/Decode.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/Decode.scala @@ -3,7 +3,10 @@ package com.worxbend.codeberg4s.core /** Turns a response body into a value. * * Core never imports a JSON library; the `codec` module supplies instances of this trait, so the choice of JSON - * library is invisible above the boundary and replaceable without touching a single use case. + * library is invisible above the boundary and replaceable without touching a single use case. The argument is a + * [[ResponseBody]] — bytes and a declared charset, both from the standard library — rather than a `String`, so that a + * parser which reads bytes reads the ones that arrived instead of a re-encoding of a decoding of them. An instance + * that genuinely wants text asks the body for it; see [[ResponseBody.text]]. * * An instance must be total: a malformed, truncated or unexpected payload returns a [[DecodeFailure]], and no * implementation lets a codec exception escape. @@ -14,4 +17,37 @@ package com.worxbend.codeberg4s.core trait Decode[A]: /** Decodes `body`, or explains where and why it could not be decoded. */ - def apply(body: String): Either[DecodeFailure, A] + def apply(body: ResponseBody): Either[DecodeFailure, A] + + /** Whether the successful body this instance reads is credential material. + * + * `false` for every endpoint but a handful, and that default is deliberate: when a payload does not decode, + * [[ApiPipeline]] puts a bounded excerpt of it into [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]], + * because an excerpt is what makes such a failure diagnosable at all. + * + * A few endpoints answer `2xx` with a body that '''is''' a live secret — the `201` of a token creation carries a + * usable access token, an OAuth2 application registration carries a client secret, and a runner registration carries + * the credential the runner authenticates with. On those, the excerpt that helps everywhere else is a credential + * disclosure into whatever the application logs. Setting this to `true` tells the pipeline to substitute a fixed + * placeholder — see [[ApiPipeline.redactedSnippet]] — which reports that a body arrived and how big it was without + * reproducing any of it. + * + * Sensitivity is a property of the '''endpoint's response''', not of the decoded type: a listing that names tokens + * without carrying their material keeps its excerpt, and only the response that carries material is marked. + */ + def sensitive: Boolean = false + +/** How an instance declares that the body it reads is a credential. */ +object Decode: + + /** `decode`, marked so that a decoding failure reports a placeholder instead of an excerpt of the body. + * + * Wrapping rather than requiring an `override` at each definition site is what lets an existing instance — a SAM + * lambda, or whatever the `client` module composes out of a codec — be marked without changing how it is built. The + * decoding itself is untouched; only [[Decode.sensitive]] differs. + */ + def sensitive[A](decode: Decode[A]): Decode[A] = + new Decode[A]: + def apply(body: ResponseBody): Either[DecodeFailure, A] = decode(body) + + override def sensitive: Boolean = true diff --git a/modules/core/src/com/worxbend/codeberg4s/core/HttpPort.scala b/modules/core/src/com/worxbend/codeberg4s/core/HttpPort.scala index 5e0d3a3..0a6cdfe 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/HttpPort.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/HttpPort.scala @@ -2,7 +2,8 @@ package com.worxbend.codeberg4s.core /** The port every transport adapter implements — the single hole through which this library reaches the network. * - * '''Failure contract.''' A `Left` means no response was produced at all: DNS, TLS, connection or timeout. Every HTTP + * '''Failure contract.''' A `Left` means no complete response was produced: DNS, TLS, connection, timeout, or a body + * that passed [[com.worxbend.codeberg4s.CodebergConfig.maxResponseBodyBytes]] and was abandoned part-read. Every HTTP * status, including `4xx` and `5xx`, arrives as a `Right`; deciding what a status means belongs to [[StatusMapping]], * not to the adapter. An implementation must therefore not throw and must not translate a status into a failure. * diff --git a/modules/core/src/com/worxbend/codeberg4s/core/LinkHeader.scala b/modules/core/src/com/worxbend/codeberg4s/core/LinkHeader.scala index d332d5f..bd8ead6 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/LinkHeader.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/LinkHeader.scala @@ -3,6 +3,7 @@ package com.worxbend.codeberg4s.core import scala.annotation.tailrec import java.util.Locale +import java.util.regex.Pattern /** RFC 5988 `Link` header parsing. * @@ -37,7 +38,9 @@ object LinkHeader: private val RelParameter: String = "rel" - private val Whitespace: String = "\\s+" + // Compiled once, at class-initialisation time. Passing the pattern as a String to String#split would + // make java.util.regex recompile it on every element of every header. + private val Whitespace: Pattern = Pattern.compile("\\s+") /** Parses a raw `Link` header value into a map from relation type to target URI. * @@ -86,17 +89,23 @@ object LinkHeader: /** Splits on the commas that separate elements, ignoring any comma inside an angle-bracketed target URI. */ private def elementsOf(value: String): List[String] = - split(value, 0, 0, false, Nil).map(_.trim).filter(_.nonEmpty) + split(value, 0, 0, false, Nil).reverse.map(_.trim).filter(_.nonEmpty) + /** The elements of `value`, '''in reverse order''' — [[elementsOf]] puts them back. + * + * Prepending onto a list costs the same however long the list is, whereas appending walks it to the end, so a loop + * that appends `k` times does work proportional to `k²`. Collecting in reverse and turning the result around once at + * the end keeps the whole split proportional to the length of the header. + */ @tailrec - private def split(value: String, from: Int, at: Int, inTarget: Boolean, done: List[String]): List[String] = - if at >= value.length then done.appended(value.substring(from)) + private def split(value: String, from: Int, at: Int, inTarget: Boolean, reversed: List[String]): List[String] = + if at >= value.length then value.substring(from) :: reversed else value.charAt(at) match - case '<' => split(value, from, at + 1, true, done) - case '>' => split(value, from, at + 1, false, done) - case ',' if !inTarget => split(value, at + 1, at + 1, false, done.appended(value.substring(from, at))) - case _ => split(value, from, at + 1, inTarget, done) + case '<' => split(value, from, at + 1, true, reversed) + case '>' => split(value, from, at + 1, false, reversed) + case ',' if !inTarget => split(value, at + 1, at + 1, false, value.substring(from, at) :: reversed) + case _ => split(value, from, at + 1, inTarget, reversed) private def entriesOf(element: String): List[(String, String)] = val found = @@ -116,7 +125,7 @@ object LinkHeader: // equalsIgnoreCase rather than ==, because RFC 5988 parameter names are // case-insensitive and .scalafix.conf bans universal equality. .collectFirst { case (name, value) if name.equalsIgnoreCase(RelParameter) => unquote(value) } - .map(_.split(Whitespace).toList.map(_.toLowerCase(Locale.ROOT)).filter(_.nonEmpty)) + .map(declared => Whitespace.split(declared).toList.map(_.toLowerCase(Locale.ROOT)).filter(_.nonEmpty)) .filter(_.nonEmpty) private def parametersOf(element: String): List[(String, String)] = diff --git a/modules/core/src/com/worxbend/codeberg4s/core/Pagination.scala b/modules/core/src/com/worxbend/codeberg4s/core/Pagination.scala deleted file mode 100644 index b30d8f1..0000000 --- a/modules/core/src/com/worxbend/codeberg4s/core/Pagination.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.worxbend.codeberg4s.core - -import com.worxbend.codeberg4s.core.Exec.flatMap -import com.worxbend.codeberg4s.paging.Page -import com.worxbend.codeberg4s.paging.PageParams - -/** Walks a paginated Forgejo collection one page at a time. - * - * The driver is '''lazy''': page `n + 1` is not requested before page `n` has been handed to the fold. That is the - * whole point — a repository can hold tens of thousands of issues, and a driver that fetched everything up front would - * turn one listing into an outage. [[Exec.suspend]] is what keeps that true even when `F` is eager. - * - * A walk stops on the first of three conditions: the response offered no following page, the page came back empty, or - * the fetch failed. The empty-page guard matters against instances that advertise a next page forever; without it a - * caller would loop until the rate limit stopped them. - * - * Failures are not swallowed. The first page that fails ends the walk with that failure, and everything folded so far - * is discarded — a partial result that looks complete is worse than an error. - * - * @tparam F - * the effect the client runs in - */ -final class Pagination[F[_]](using exec: Exec[F]): - - /** Folds every page of a collection into a single value, fetching lazily. - * - * The step function sees whole pages rather than items so that a caller can use the pagination metadata — stop early - * on a total count, report progress, or write each page out before the next is requested. - * - * @param start - * where to begin, usually [[com.worxbend.codeberg4s.paging.PageParams.First]]; its size is kept for every - * following page - * @param zero - * the initial accumulator - * @param fetch - * requests one page; called once per page, never ahead of the fold - * @param step - * combines the accumulator with a page, before the next page is requested - */ - def foldPages[A, B](start: PageParams, zero: B)(fetch: PageParams => F[Page[A]])(step: (B, Page[A]) => B): F[B] = - loop(start, zero, fetch, step) - - /** Collects every item of a collection into memory. - * - * The convenient shape, and the dangerous one: the whole collection ends up in the returned vector. Use it when the - * collection is known to be small — labels, milestones, a repository's branches — and [[foldPages]] otherwise. - */ - def listAll[A](start: PageParams)(fetch: PageParams => F[Page[A]]): F[Vector[A]] = - foldPages(start, Vector.empty[A])(fetch)((collected, page) => collected ++ page.items) - - private def loop[A, B]( - params: PageParams, - accumulator: B, - fetch: PageParams => F[Page[A]], - step: (B, Page[A]) => B, - ): F[B] = - exec.suspend(() => fetch(params)).flatMap: page => - val folded = step(accumulator, page) - page.nextPage match - case Some(following) if page.items.nonEmpty => loop(params.at(following), folded, fetch, step) - case _ => exec.pure(folded) diff --git a/modules/core/src/com/worxbend/codeberg4s/core/Redaction.scala b/modules/core/src/com/worxbend/codeberg4s/core/Redaction.scala index 4c331fe..d11f0d2 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/Redaction.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/Redaction.scala @@ -1,5 +1,7 @@ package com.worxbend.codeberg4s.core +import com.worxbend.codeberg4s.syntax.discard + import java.nio.charset.StandardCharsets import java.util.Locale @@ -36,15 +38,53 @@ object Redaction: * The value of a parameter named in [[SensitiveQueryParameters]] is replaced by [[Mask]] and never encoded, so the * result shows `?token=***` rather than an encoded secret. * + * The base URI is not trusted to be clean. `com.worxbend.codeberg4s.BaseUri.from` rejects one carrying + * `user:password@`, a query or a fragment, but this method takes a plain `String` and a test fake can pass anything + * at all, so those three parts are removed here as well. Without that, a password in a hand-built base URI would be + * copied into every rendered URI, and therefore into every error and every telemetry event. + * * @param baseUri - * the API root, already normalised without a trailing slash + * the API root, expected to be normalised without a trailing slash * @param path * unencoded path segments, in order; an empty list renders just the base URI * @param query * query parameters in order, keys may repeat */ def uri(baseUri: String, path: List[String], query: List[(String, String)]): String = - s"$baseUri${renderPath(path)}${renderQuery(query)}" + s"${safeBase(baseUri)}${renderPath(path)}${renderQuery(query)}" + + /** Strips the parts of a base URI that must never be rendered: anything from the first `?` or `#`, and the + * `user:password@` prefix of the authority. + */ + private def safeBase(baseUri: String): String = + withoutUserInfo(withoutTail(baseUri)) + + /** The value up to its first `?` or `#`, dropping a query or fragment along with everything after it. */ + private def withoutTail(value: String): String = + val tail = value.indexWhere(startsTail) + if tail < 0 then value else value.take(tail) + + private def startsTail(char: Char): Boolean = + char match + case '?' | '#' => true + case _ => false + + /** The value with any `user:password@` removed from its authority. + * + * The authority runs from `://` to the next `/`, so an `@` in a path segment — `https://forge.example/api/v1/@me` — + * is left alone. A value with no `://` is returned unchanged: there is no authority to trim. + */ + private def withoutUserInfo(value: String): String = + val marker = value.indexOf(AuthorityMarker) + if marker < 0 then value + else + val start = marker + AuthorityMarker.length + val end = value.indexOf('/', start) + val at = value.lastIndexOf('@', (if end < 0 then value.length else end) - 1) + if at < start then value else value.substring(0, start) + value.substring(at + 1) + + /** What separates a scheme from an authority; the authority is where user information can hide. */ + private val AuthorityMarker: String = "://" /** Masks the value of every credential-carrying header, keeping order and every other header untouched. * @@ -70,15 +110,39 @@ object Redaction: private def renderValue(name: String, value: String): String = if isSensitiveParameter(name) then Mask else percentEncode(value) + /** The uppercase hex alphabet, indexed by nibble. A `String` rather than an `Array[Char]` so that the lookup table + * cannot be mutated by anything holding a reference to it. + */ + private val HexDigits: String = "0123456789ABCDEF" + + /** Percent-encodes `value` per RFC 3986 into one buffer. + * + * Every octet is appended to a single [[StringBuilder]] rather than turned into its own `String` first. The result + * is character-for-character what a per-octet `map(...).mkString` produces; only the allocation count differs, and + * this runs on every path segment and every query value of every attempt of every call. + */ private def percentEncode(value: String): String = - value.getBytes(StandardCharsets.UTF_8).map(encodeByte).mkString + val octets = value.getBytes(StandardCharsets.UTF_8) + val encoded = StringBuilder(octets.length) + octets.foreach(byte => appendEncoded(encoded, byte)) + encoded.toString - private def encodeByte(byte: Byte): String = + /** Appends one UTF-8 octet: literally when it is unreserved, otherwise as `%` and two uppercase hex digits. */ + private def appendEncoded(target: StringBuilder, byte: Byte): Unit = val octet = byte & 0xFF - if isUnreserved(octet.toChar) then octet.toChar.toString else f"%%$octet%02X" + if isUnreserved(octet.toChar) then target.append(octet.toChar).discard + else target.append('%').append(HexDigits(octet >> 4)).append(HexDigits(octet & 0x0F)).discard + /** Whether `char` is RFC 3986 unreserved. + * + * The four punctuation marks are matched literally rather than looked for inside a `"-._~"` string, so deciding a + * character costs a comparison instead of a scan. A `match` rather than `==` because the build's Scalafix + * configuration bans universal equality, and rather than `.equals` because that would box the `Char`. + */ private def isUnreserved(char: Char): Boolean = (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || - "-._~".contains(char) + (char match + case '-' | '.' | '_' | '~' => true + case _ => false) diff --git a/modules/core/src/com/worxbend/codeberg4s/core/RequestBody.scala b/modules/core/src/com/worxbend/codeberg4s/core/RequestBody.scala index b2f13ce..be34bcb 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/RequestBody.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/RequestBody.scala @@ -1,20 +1,34 @@ package com.worxbend.codeberg4s.core +import java.util.Arrays + /** The payload of a request, as far as core is concerned. * * Core never imports a JSON library, so a body reaches it already serialised: the `codec` module renders a wire DTO - * into a string and hands it over. The transport adapter is what turns a case of this enum into bytes and sets the + * into a string and hands it over. The transport adapter is what turns a case of this type into bytes and sets the * `Content-Type` header. * * Every case names its own media type rather than leaving the transport to guess, because Forgejo is not uniform here: * `/markdown` consumes JSON while `/markdown/raw` consumes `text/plain`, and a release asset is `multipart/form-data` * with a named file part. Encoding those differences as separate cases keeps them visible in the request builder * instead of hidden in a header override. + * + * ==Why this is a sealed trait and not an `enum`== + * + * [[RequestBody.Binary]] and [[RequestBody.Multipart]] carry an `Array[Byte]`, and an array's own `equals` in Scala is + * '''identity''' — two arrays holding the same bytes are not equal to each other. A case whose equality is to be + * decided by the bytes therefore has to write its own `equals` and `hashCode`, and a Scala 3 `enum` case cannot have a + * body to write them in. So the cases are ordinary `final case class`es under a sealed trait, which pattern matching, + * construction (`RequestBody.Json("…")`) and exhaustivity checking all see exactly as they saw the enum's cases. What + * is lost is `ordinal` and the `scala.reflect.Enum` supertype, which nothing uses. */ -enum RequestBody: +sealed trait RequestBody + +/** The cases of [[RequestBody]], and the media types they are sent under. */ +object RequestBody: /** A serialised JSON document, sent with `Content-Type: application/json`. */ - case Json(value: String) + final case class Json(value: String) extends RequestBody /** A text body sent verbatim under `mediaType`, for endpoints that consume `text/plain` rather than JSON. * @@ -23,14 +37,35 @@ enum RequestBody: * @param mediaType * the full `Content-Type` value, for example `text/plain; charset=utf-8` */ - case Text(value: String, mediaType: String) + final case class Text(value: String, mediaType: String) extends RequestBody - /** Raw bytes sent under `mediaType`, for endpoints that consume a file body directly. */ - case Binary(bytes: Array[Byte], mediaType: String) + /** Raw bytes sent under `mediaType`, for endpoints that consume a file body directly. + * + * The array is '''adopted, not copied''': a file body can be large, and copying it here to gain an immutability + * guarantee the caller can already provide is the wrong trade. Do not modify an array after handing it over. + */ + final case class Binary(bytes: Array[Byte], mediaType: String) extends RequestBody: + + /** Structural, on the media type and then on the bytes. + * + * Written out because the array's own `equals` is identity, which would make two bodies carrying byte-identical + * content compare unequal — a wrong answer with no warning attached, in an assertion or in a `Set`. The media type + * is compared first because it is the cheap half. + * + * The class is `final`, so no subclass can exist and the type test below is the whole of the compiler-generated + * `canEqual`; calling `canEqual` as well would add nothing. Removing `final` would change that. + */ + override def equals(other: Any): Boolean = + other match + case that: Binary => mediaType.equals(that.mediaType) && Arrays.equals(bytes, that.bytes) + case _ => false + + override def hashCode(): Int = 31 * mediaType.hashCode + Arrays.hashCode(bytes) /** A single-file `multipart/form-data` body, which is how Forgejo accepts release assets and avatars. * - * The transport supplies the boundary; callers only choose the part name, the file name and the bytes. + * The transport supplies the boundary; callers only choose the part name, the file name and the bytes. The array is + * adopted rather than copied, exactly as in [[RequestBody.Binary]]. * * @param fieldName * the form field name the endpoint expects, for example `attachment` @@ -41,12 +76,24 @@ enum RequestBody: * @param mediaType * the part's own content type */ - case Multipart(fieldName: String, fileName: String, bytes: Array[Byte], mediaType: String) + final case class Multipart(fieldName: String, fileName: String, bytes: Array[Byte], mediaType: String) + extends RequestBody: - /** A request with a body that is deliberately empty, as some Forgejo `PUT` endpoints require. */ - case Empty + /** Structural, on all three names and then on the bytes — see [[RequestBody.Binary.equals]] for why it is written + * out at all, and why `canEqual` does not appear. + */ + override def equals(other: Any): Boolean = + other match + case that: Multipart => + fieldName.equals(that.fieldName) && fileName.equals(that.fileName) && mediaType.equals(that.mediaType) && + Arrays.equals(bytes, that.bytes) + case _ => false -object RequestBody: + override def hashCode(): Int = + 31 * (31 * (31 * fieldName.hashCode + fileName.hashCode) + mediaType.hashCode) + Arrays.hashCode(bytes) + + /** A request with a body that is deliberately empty, as some Forgejo `PUT` endpoints require. */ + case object Empty extends RequestBody /** The media type [[RequestBody.Json]] is sent with. */ val JsonMediaType: String = "application/json" diff --git a/modules/core/src/com/worxbend/codeberg4s/core/ResponseBody.scala b/modules/core/src/com/worxbend/codeberg4s/core/ResponseBody.scala new file mode 100644 index 0000000..a36ee89 --- /dev/null +++ b/modules/core/src/com/worxbend/codeberg4s/core/ResponseBody.scala @@ -0,0 +1,194 @@ +package com.worxbend.codeberg4s.core + +import scala.annotation.tailrec +import scala.util.Try + +import java.nio.charset.Charset +import java.nio.charset.StandardCharsets +import java.util.Arrays +import java.util.Locale + +/** A response body exactly as it came off the socket — the bytes, and the charset the response declared for them. + * + * '''Why bytes and not a `String`.''' Text was the wrong currency for this library. A transport that hands over a + * `String` has already decoded the payload once, and the JSON parser then encodes that `String` straight back into a + * `byte[]` to read it, because every JSON parser worth using reads bytes. That is two full copies of a response before + * a single field is looked at — on a 170 KB listing page, a third of a megabyte of garbage per call. Carrying the + * bytes and decoding to text only where text is genuinely wanted removes both copies from the JSON path, which is + * every path but a handful. + * + * '''The charset is read, not assumed.''' [[charset]] is whatever the response's `Content-Type` declared, falling back + * to UTF-8 when it declared nothing, declared something unparseable, or declared a charset this JVM does not have — + * which is precisely what the sttp adapter used to do on this library's behalf, so nothing about text decoding changed + * when the body stopped being text. Forgejo sends `charset=utf-8` on everything, but that is an observation about one + * server rather than a licence to ignore the header, and [[utf8Bytes]] states the one place where UTF-8 is genuinely + * required rather than merely expected. + * + * '''Ownership of the array.''' [[bytes]] hands back the array this body holds, without copying it. Copying would + * reintroduce the very copy this type exists to remove, so the contract is the other way round: whoever constructs a + * `ResponseBody` gives up the array, and whoever reads [[bytes]] must not modify it. Every reader in this library + * obeys that — jsoniter reads the array and never writes to it. + * + * Instances are immutable as long as that contract is kept, and are safe to share between threads. + * + * @param charset + * the charset [[text]] decodes with, taken from the response's `Content-Type` + */ +final class ResponseBody private (private val raw: Array[Byte], val charset: Charset): + + /** The body verbatim, '''not copied'''. Read it; do not write to it. See the note on ownership above. */ + def bytes: Array[Byte] = raw + + /** How many bytes the body holds. Cheaper than decoding it, and the thing worth logging. */ + def size: Int = raw.length + + /** Whether the server sent no body at all — a `204`, or a `200` with nothing after the headers. */ + def isEmpty: Boolean = raw.isEmpty + + /** Whether the body is empty or contains nothing but ASCII whitespace. + * + * Answered on the bytes, so asking it of a 40 MB payload does not decode 40 MB of text. That makes it very slightly + * stricter than `String.isBlank`, which also counts a handful of non-ASCII Unicode separators such as `U+2028`: a + * body made only of those is reported here as non-blank. The consequence of that difference is that such a body is + * handed to the parser and fails there, rather than being silently treated as absent — the safe direction, and no + * server this library talks to has ever sent one. + */ + def isBlank: Boolean = ResponseBody.allWhitespace(raw, 0) + + /** The whole body decoded with [[charset]]. + * + * Bytes that are not valid in that charset become the replacement character rather than an error, because a body is + * only ever decoded here for something a human will read — an error payload, a `text/plain` endpoint, a failure + * excerpt — and failing to render an explanation would replace a useful message with a useless one. + * + * Decoded once and kept: the error path reads it to parse the payload and again to build the excerpt. + */ + lazy val text: String = String(raw, charset) + + /** The body as UTF-8 bytes, for a reader that requires UTF-8 rather than merely expecting it. + * + * JSON is the case. RFC 8259 §8.1 says JSON exchanged between systems that are not one closed ecosystem '''must''' + * be encoded as UTF-8, and every JSON parser this library could use reads UTF-8 bytes and nothing else. So the JSON + * path asks for this rather than for [[bytes]], and the check is written down here instead of being assumed + * anywhere: when the response really did declare UTF-8 — which is what Forgejo declares, on every endpoint — this is + * [[bytes]] and costs nothing, and when it declared something else the body is transcoded through [[text]] so that a + * non-conforming server is read correctly rather than as mojibake. + */ + def utf8Bytes: Array[Byte] = + if charset.equals(StandardCharsets.UTF_8) then raw else text.getBytes(StandardCharsets.UTF_8) + + /** An excerpt of the decoded body no longer than `maxChars` '''characters'''. + * + * Characters, not bytes, and that distinction is the whole reason this lives here. Slicing the byte array to a fixed + * length and decoding the slice would cut a multi-byte character in half and end the excerpt in a replacement + * character that the server never sent. So the slice is deliberately generous — enough bytes that `maxChars` + * characters are certainly inside it, computed from the widest encoding of one character in this charset — and the + * bound is then applied to the decoded text, where a character is a character. Any damage the generous slice did at + * its own tail sits beyond `maxChars` and is discarded with the rest. + * + * The point of the bound is that a decoding failure on a 40 MB listing must not put 40 MB into an error value an + * application is about to log. Slicing first is what keeps that promise for the decode as well as for the result. + * + * @param maxChars + * the longest excerpt wanted; zero or less yields an empty string + */ + def excerpt(maxChars: Int): String = + if maxChars <= 0 || raw.isEmpty then "" + else + val enough = math.min(raw.length.toLong, (maxChars.toLong + 1L) * widestCharInBytes).toInt + String(raw, 0, enough, charset).take(maxChars) + + /** Structural, on the bytes and the charset. Two bodies holding equal bytes decode to the same text only if they + * declare the same charset, so both halves count. Written out because the array's own `equals` is identity, which + * would make every response value compare unequal to an identical one and quietly break anybody's tests. + */ + override def equals(other: Any): Boolean = + other match + case that: ResponseBody => charset.equals(that.charset) && Arrays.equals(raw, that.raw) + case _ => false + + override def hashCode(): Int = 31 * Arrays.hashCode(raw) + charset.hashCode + + /** Deliberately does not render the body. A response payload can carry a freshly minted access token, and this + * library's failures are logged. + */ + override def toString: String = s"ResponseBody($size B, ${charset.name})" + + /** The widest one character can be in this charset, in bytes — the factor [[excerpt]] slices by. + * + * `maxBytesPerChar` is the encoder's own guarantee, so it is an upper bound for the decoding direction too: UTF-8 + * reports 3, and its 4-byte sequences produce two characters, which is 2 bytes per character. A charset that cannot + * encode at all reports nothing, and 4 is above every charset in the JDK. + */ + private def widestCharInBytes: Int = + if charset.canEncode then math.max(1, math.ceil(charset.newEncoder().maxBytesPerChar().toDouble).toInt) + else ResponseBody.WidestCharFallback + +/** How a [[ResponseBody]] is built, and how a declared charset is read. */ +object ResponseBody: + + /** No body at all: zero bytes, nominally UTF-8. What a `204` carries. */ + val Empty: ResponseBody = ResponseBody(Array.emptyByteArray, StandardCharsets.UTF_8) + + /** Bytes off the wire, with the charset the response declared for them. + * + * '''The array is adopted, not copied.''' The caller must not keep a reference it later writes through. See the + * ownership note on [[ResponseBody]] for why copying here would defeat the purpose of the type. + */ + def of(bytes: Array[Byte], charset: Charset): ResponseBody = ResponseBody(bytes, charset) + + /** A body written as text and encoded as UTF-8 — for a test fake, or for anyone standing in for a transport. */ + def utf8(text: String): ResponseBody = + ResponseBody(text.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8) + + /** The charset a `Content-Type` declared, or UTF-8. + * + * UTF-8 is the answer for a header that is absent, carries no `charset` parameter, carries one that is not a legal + * charset name, or names a charset this JVM does not provide. None of those is worth failing a whole request over, + * and UTF-8 is both the RFC 8259 requirement for JSON and what every Forgejo endpoint actually declares. The + * parameter name is matched case-insensitively and a quoted value is unquoted, as RFC 9110 §5.6.6 allows. + */ + def charsetOf(contentType: Option[String]): Charset = + contentType + .flatMap(declaredCharset) + .flatMap(supported) + .getOrElse(StandardCharsets.UTF_8) + + /** Above every `maxBytesPerChar` in the JDK, used when a charset cannot encode and so reports none. */ + private val WidestCharFallback: Int = 4 + + private val CharsetParameter: String = "charset=" + + private val Quote: String = "\"" + + private def declaredCharset(contentType: String): Option[String] = + contentType + .split(';') + .iterator + .drop(1) + .map(_.trim) + .find(_.toLowerCase(Locale.ROOT).startsWith(CharsetParameter)) + .map(parameter => unquoted(parameter.drop(CharsetParameter.length).trim)) + .filter(_.nonEmpty) + + private def unquoted(value: String): String = + if value.length >= 2 && value.startsWith(Quote) && value.endsWith(Quote) then + value.substring(1, value.length - 1) + else value + + private def supported(name: String): Option[Charset] = Try(Charset.forName(name)).toOption + + /** ASCII whitespace only, so that [[ResponseBody.isBlank]] never decodes the body. */ + @tailrec + private def allWhitespace(bytes: Array[Byte], index: Int): Boolean = + if index >= bytes.length then true + else if isAsciiWhitespace(bytes(index)) then allWhitespace(bytes, index + 1) + else false + + /** Space, tab, line feed, vertical tab, form feed and carriage return — the ASCII characters `String.isBlank` counts, + * as their byte values. + */ + private def isAsciiWhitespace(byte: Byte): Boolean = + byte match + case 32 | 9 | 10 | 11 | 12 | 13 => true + case _ => false diff --git a/modules/core/src/com/worxbend/codeberg4s/core/RetryEngine.scala b/modules/core/src/com/worxbend/codeberg4s/core/RetryEngine.scala index c725abb..f8219fb 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/RetryEngine.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/RetryEngine.scala @@ -20,11 +20,13 @@ import scala.concurrent.duration.FiniteDuration * 1. how long to wait — the server's `Retry-After` when the policy honours it, otherwise `baseDelay * 2^(n - 1)` * after the *n*-th failed attempt, clamped to `maxDelay` and then jittered. * - * Nothing is lost when it gives up: as soon as more than one attempt has been made, the failure the caller receives is + * Nothing is lost when it gives up: a call the engine was repeating and could not repeat again reports * [[com.worxbend.codeberg4s.CodebergError.RetriesExhausted]], carrying the number of attempts and the last underlying - * error verbatim. A call that fails once and is not repeated — a `404`, a `POST` under - * [[RetryEligibility.IdempotentOnly]], a policy of [[com.worxbend.codeberg4s.retry.RetryPolicy.Off]] — returns that - * failure unwrapped, because there is nothing to explain. + * error verbatim. Every other failure is reported unwrapped, because there is nothing to explain: a call that fails + * once and is not repeated — a `404`, a `POST` under [[RetryEligibility.IdempotentOnly]], a policy of + * [[com.worxbend.codeberg4s.retry.RetryPolicy.Off]] — and equally a call whose last attempt failed in a way no + * repetition could fix, such as a `503` followed by a `404`. `RetriesExhausted` therefore means the attempt budget ran + * out, never merely that more than one attempt was made. * * Waiting goes through [[Timer]] and jitter through [[JitterSource]], so the whole schedule is observable in a unit * test without a single real millisecond passing. @@ -91,7 +93,7 @@ final class RetryEngine[F[_]](policy: RetryPolicy, timer: Timer[F])(using exec: case Left(error) => if shouldRetry(method, eligibility, number, error) then waitThenRetry(operation, method, eligibility, number, attempt, outcome.retryAfter) - else exec.raise(giveUp(operation, method, number, error)) + else exec.raise(giveUp(operation, method, eligibility, number, error)) private def waitThenRetry[A]( operation: String, @@ -112,8 +114,23 @@ final class RetryEngine[F[_]](policy: RetryPolicy, timer: Timer[F])(using exec: ): Boolean = number < policy.maxAttempts && eligibility.allows(method) && RetryEngine.isRetryable(error) - private def giveUp(operation: String, method: HttpMethod, attempts: Int, error: CodebergError): CodebergError = - if attempts > 1 then + /** The failure the caller receives once the loop has stopped. + * + * Wrapping in [[com.worxbend.codeberg4s.CodebergError.RetriesExhausted]] is a claim that the attempt budget is what + * ended the call, so it is made only when every part of that claim holds: more than one attempt was made, the caller + * allowed repetition, and the failure that ended the loop is itself one the engine would have repeated had an + * attempt been left. A call that meets a `503` and then a `404` stopped because a `404` cannot be repeated, not + * because the budget ran out, so it reports that `404` unwrapped — the same failure it would have reported had the + * `404` arrived first. + */ + private def giveUp( + operation: String, + method: HttpMethod, + eligibility: RetryEligibility, + attempts: Int, + error: CodebergError, + ): CodebergError = + if attempts > 1 && eligibility.allows(method) && RetryEngine.isRetryable(error) then CodebergError.RetriesExhausted(RetryEngine.contextOf(operation, method, error), attempts, error) else error @@ -150,8 +167,15 @@ object RetryEngine: case CodebergError.DecodingFailed(_, _, _, _) => false case CodebergError.Validation(_) => false case CodebergError.RetriesExhausted(_, _, _) => false - - /** A TLS failure does not heal by itself and an interruption was asked for; everything else may be transient. */ + // Never reaches this engine — a page walk is assembled above it — and + // repeating the walk would stop at the same cap, so it is not retryable + // even in principle. + case CodebergError.WalkTruncated(_, _) => false + + /** A TLS failure does not heal by itself, an interruption was asked for, and an oversized body would arrive oversized + * again — repeating that one would download the body the bound exists to refuse once per attempt. Everything else + * may be transient. + */ private[core] def isRetryable(cause: TransportCause): Boolean = cause match case TransportCause.ConnectionFailed(_) => true @@ -160,6 +184,7 @@ object RetryEngine: case TransportCause.Unknown(_) => true case TransportCause.Tls(_) => false case TransportCause.Interrupted(_) => false + case TransportCause.ResponseTooLarge(_) => false private[core] def contextOf(operation: String, method: HttpMethod, error: CodebergError): CallContext = error match @@ -168,6 +193,7 @@ object RetryEngine: case CodebergError.DecodingFailed(ctx, _, _, _) => ctx case CodebergError.RetriesExhausted(ctx, _, _) => ctx case CodebergError.Validation(_) => CallContext(operation, method, UnknownUri, None, 0L) + case CodebergError.WalkTruncated(_, _) => CallContext(operation, method, UnknownUri, None, 0L) /** `value` doubled `times` over, stopping at `cap`. Written as a fold rather than a shift so that a large attempt * count cannot overflow the exponent. diff --git a/modules/core/src/com/worxbend/codeberg4s/core/Telemetry.scala b/modules/core/src/com/worxbend/codeberg4s/core/Telemetry.scala index cbe7d91..8e70d53 100644 --- a/modules/core/src/com/worxbend/codeberg4s/core/Telemetry.scala +++ b/modules/core/src/com/worxbend/codeberg4s/core/Telemetry.scala @@ -2,7 +2,6 @@ package com.worxbend.codeberg4s.core import com.worxbend.codeberg4s.CallContext import com.worxbend.codeberg4s.CodebergError -import com.worxbend.codeberg4s.syntax.discard /** Observation hooks for applications that want to see what the client is doing. * @@ -38,17 +37,13 @@ object Telemetry: private final class NoOp[F[_]](exec: Exec[F]) extends Telemetry[F]: + /** The one effect this sink ever returns. Built once per instance so that no callback allocates: a client that + * leaves telemetry unconfigured still reaches this class three times per attempt, on every request it makes. + */ private val done: F[Unit] = exec.pure(()) - override def onRequest(ctx: CallContext): F[Unit] = ignoring(ctx) - - override def onResponse(ctx: CallContext, status: Int): F[Unit] = ignoring(ctx, status) + override def onRequest(ctx: CallContext): F[Unit] = done - override def onError(ctx: CallContext, error: CodebergError): F[Unit] = ignoring(ctx, error) + override def onResponse(ctx: CallContext, status: Int): F[Unit] = done - /** Discards what it was told, explicitly: the build treats an unused value in statement position as an error, and a - * no-op sink is exactly the place where ignoring an argument is the intended behaviour. - */ - private def ignoring(observed: Any*): F[Unit] = - observed.discard - done + override def onError(ctx: CallContext, error: CodebergError): F[Unit] = done diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/ApiPipelineSuite.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/ApiPipelineSuite.scala index 505ce15..6f7df81 100644 --- a/modules/core/test/src/com/worxbend/codeberg4s/core/ApiPipelineSuite.scala +++ b/modules/core/test/src/com/worxbend/codeberg4s/core/ApiPipelineSuite.scala @@ -68,13 +68,13 @@ final class ApiPipelineSuite extends FunSuite: JitterSource.Deterministic) private def responseOf(status: Int, body: String, headers: (String, List[String])*): CodebergResponse = - CodebergResponse(status, headers.toMap, body) + CodebergResponse(status, headers.toMap, ResponseBody.utf8(body)) private def contextOf(operation: String, method: HttpMethod, requestId: Option[String]): CallContext = CallContext(operation, method, expectedUri, requestId, 0L) test("a 2xx body is decoded and returned"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val http = FakeHttpPort.always(responseOf(200, "payload")) val result = pipelineOf(http, FakeTimer(0L), silent, stubErrorBody) @@ -84,7 +84,7 @@ final class ApiPipelineSuite extends FunSuite: assertEquals(http.sends, 1) test("the URI handed to the transport is redacted"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val http = FakeHttpPort.always(responseOf(200, "payload")) val result = pipelineOf(http, FakeTimer(0L), silent, stubErrorBody) @@ -94,7 +94,7 @@ final class ApiPipelineSuite extends FunSuite: assertEquals(http.uris, Vector("https://codeberg.org/api/v1/repos/owner/name/issues?page=1&token=***")) test("a 404 becomes an Api failure carrying the parsed error body and the request id"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val http = FakeHttpPort.always(responseOf(404, forgejoErrorBody, "x-request-id" -> List("abc123"))) val result = pipelineOf(http, FakeTimer(0L), silent, stubErrorBody) @@ -113,7 +113,7 @@ final class ApiPipelineSuite extends FunSuite: assertEquals(http.sends, 1) test("a 500 is attempted again and the later success is returned"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val timer = FakeTimer(0L) val http = FakeHttpPort(Vector(Right(responseOf(500, "boom")), Right(responseOf(200, "payload")))) @@ -125,7 +125,7 @@ final class ApiPipelineSuite extends FunSuite: assertEquals(timer.sleeps, Vector(250.millis)) test("a 429 is retried after the delay the server asked for"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val timer = FakeTimer(0L) val http = FakeHttpPort( @@ -170,6 +170,55 @@ final class ApiPipelineSuite extends FunSuite: assertEquals(http.sends, 1) + test("a sensitive body is replaced by the placeholder, never excerpted"): + val credential = "gto_thisisarealtoken" + val payload = s"""{"sha1": "$credential"}""" + + given Decode[String] = + Decode.sensitive(_ => Left(DecodeFailure(JsonPath.of("id"), "no such field"))) + + val http = FakeHttpPort.always(responseOf(201, payload)) + + val result = pipelineOf(http, FakeTimer(0L), silent, stubErrorBody) + .call[String](creation, RetryEligibility.Never) + + result match + case Left(error @ CodebergError.DecodingFailed(_, snippet, path, cause)) => + assertEquals(snippet, ApiPipeline.redactedSnippet(payload.length)) + assertEquals(path, JsonPath.of("id")) + assertEquals(cause, "no such field") + assert(!error.describe.contains(credential), s"the body reached the rendered failure: ${error.describe}") + case other => + fail(s"expected a decoding failure, got $other") + + test("the placeholder is what a telemetry sink observes, not only what the caller receives"): + val credential = "gto_thisisarealtoken" + + given Decode[String] = Decode.sensitive(_ => Left(DecodeFailure(JsonPath.Root, "not an object"))) + + val http = FakeHttpPort.always(responseOf(201, s"""{"sha1": "$credential"}""")) + val telemetry = RecordingTelemetry(failing = false) + + pipelineOf(http, FakeTimer(0L), telemetry, stubErrorBody) + .call[String](creation, RetryEligibility.Never) + .discard + + val rendered = telemetry.observed.map(_.describe) + + assert(rendered.nonEmpty, "the sink observed no failure at all") + rendered.foreach(line => assert(!line.contains(credential), s"a credential was observed: $line")) + + test("an ordinary decoder keeps its excerpt, because that is what makes a failure diagnosable"): + given Decode[String] = _ => Left(DecodeFailure(JsonPath.Root, "not an object")) + val http = FakeHttpPort.always(responseOf(200, """{"unexpected": true}""")) + + val result = pipelineOf(http, FakeTimer(0L), silent, stubErrorBody) + .call[String](listing, RetryEligibility.Never) + + result match + case Left(CodebergError.DecodingFailed(_, snippet, _, _)) => assertEquals(snippet, """{"unexpected": true}""") + case other => fail(s"expected a decoding failure, got $other") + test("a decoding failure is never attempted again"): given Decode[String] = _ => Left(DecodeFailure(JsonPath.Root, "not an object")) val http = FakeHttpPort.always(responseOf(200, "{}")) @@ -181,7 +230,7 @@ final class ApiPipelineSuite extends FunSuite: assertEquals(http.sends, 1) test("a request that never reaches the server becomes a Transport failure"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val cause = TransportCause.Tls("certificate expired") val http = FakeHttpPort.broken(TransportFailure(cause)) @@ -192,7 +241,7 @@ final class ApiPipelineSuite extends FunSuite: assertEquals(http.sends, 1) test("a transport failure that keeps recurring ends as RetriesExhausted preserving the last failure"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val cause = TransportCause.Timeout("read timed out") val http = FakeHttpPort.broken(TransportFailure(cause)) val context = contextOf("issues.list", HttpMethod.Get, None) @@ -204,7 +253,7 @@ final class ApiPipelineSuite extends FunSuite: assertEquals(http.sends, 3) test("a successful call reports a request and a response and no error"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val telemetry = RecordingTelemetry(failing = false) val http = FakeHttpPort.always(responseOf(200, "payload")) @@ -215,7 +264,7 @@ final class ApiPipelineSuite extends FunSuite: assertEquals(telemetry.events, Vector("request issues.list", "response 200")) test("a failing call reports the attempt in order and then the failure the caller receives"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val telemetry = RecordingTelemetry(failing = false) val http = FakeHttpPort.always(responseOf(404, forgejoErrorBody)) @@ -226,7 +275,7 @@ final class ApiPipelineSuite extends FunSuite: assertEquals(telemetry.events, Vector("request issues.list", "response 404", "error Api", "error Api")) test("a retried call reports one request and one response per attempt"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val telemetry = RecordingTelemetry(failing = false) val http = FakeHttpPort(Vector(Right(responseOf(503, "down")), Right(responseOf(200, "payload")))) @@ -240,7 +289,7 @@ final class ApiPipelineSuite extends FunSuite: ) test("a telemetry sink that fails does not fail the call it is only watching"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val telemetry = RecordingTelemetry(failing = true) val http = FakeHttpPort.always(responseOf(200, "payload")) @@ -251,7 +300,7 @@ final class ApiPipelineSuite extends FunSuite: assertEquals(telemetry.events, Vector("request issues.list", "response 200")) test("an error body the parser cannot read falls back to Empty and never masks the status"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val http = FakeHttpPort.always(responseOf(422, "not json")) val result = pipelineOf(http, FakeTimer(0L), silent, failingErrorBody) @@ -263,7 +312,7 @@ final class ApiPipelineSuite extends FunSuite: ) test("an empty error body becomes Empty without consulting the parser"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val http = FakeHttpPort.always(responseOf(500, " ")) val result = pipelineOf(http, FakeTimer(0L), silent, failingErrorBody) @@ -350,7 +399,7 @@ final class ApiPipelineSuite extends FunSuite: fail(s"expected a decoding failure, got $other") test("the duration on a call context is measured with the timer around the send"): - given Decode[String] = body => Right(body) + given Decode[String] = body => Right(body.text) val http = FakeHttpPort.always(responseOf(404, forgejoErrorBody)) val result = pipelineOf(http, SteppingTimer(7L), silent, stubErrorBody) diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/BinaryPipelineSuite.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/BinaryPipelineSuite.scala index 5db161d..60758c5 100644 --- a/modules/core/test/src/com/worxbend/codeberg4s/core/BinaryPipelineSuite.scala +++ b/modules/core/test/src/com/worxbend/codeberg4s/core/BinaryPipelineSuite.scala @@ -39,7 +39,7 @@ final class BinaryPipelineSuite extends FunSuite: private def pipeline(telemetry: Telemetry[Result]): ApiPipeline[Result] = ApiPipeline[Result]( - FakeHttpPort(Vector(Right(CodebergResponse(200, Map.empty, "")))), + FakeHttpPort(Vector(Right(CodebergResponse(200, Map.empty, ResponseBody.Empty)))), config, FakeTimer(0L), telemetry, @@ -74,7 +74,7 @@ final class BinaryPipelineSuite extends FunSuite: val body = """{"message":"gone"}""".getBytes(StandardCharsets.UTF_8) val port = StubBinaryPort(List(Right(BinaryResponse(410, Map.empty, body)))) val recorded = ApiPipeline[Result]( - FakeHttpPort(Vector(Right(CodebergResponse(200, Map.empty, "")))), + FakeHttpPort(Vector(Right(CodebergResponse(200, Map.empty, ResponseBody.Empty)))), config, FakeTimer(0L), Telemetry.noOp[Result], @@ -92,7 +92,7 @@ final class BinaryPipelineSuite extends FunSuite: val port = StubBinaryPort(List(Right(BinaryResponse(404, Map.empty, body)))) val seen = ApiPipeline[Result]( - FakeHttpPort(Vector(Right(CodebergResponse(200, Map.empty, "")))), + FakeHttpPort(Vector(Right(CodebergResponse(200, Map.empty, ResponseBody.Empty)))), config, FakeTimer(0L), Telemetry.noOp[Result], diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/ByteEqualitySuite.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/ByteEqualitySuite.scala new file mode 100644 index 0000000..0451930 --- /dev/null +++ b/modules/core/test/src/com/worxbend/codeberg4s/core/ByteEqualitySuite.scala @@ -0,0 +1,118 @@ +package com.worxbend.codeberg4s.core + +import munit.FunSuite + +import java.nio.charset.StandardCharsets + +/** The three core values that carry an `Array[Byte]`, and the one rule they all have to obey: two of them holding the + * same bytes are equal. + * + * They are tested together because the defect is one defect. An array's `equals` in Scala is '''identity''', so the + * equality a case class generates for a byte field compares two archives by reference — `download(a) == download(a)` + * answers `false`, a `Set` keeps both copies, and nothing warns. Every assertion below would have passed by accident + * if the field were a `String`, which is exactly why the rule is worth pinning down for the byte case. + * + * `hashCode` is asserted alongside every equality, because equal values that hash differently break a `Set` and a + * `Map` just as thoroughly as unequal ones. + */ +final class ByteEqualitySuite extends FunSuite: + + private def bytes(text: String): Array[Byte] = text.getBytes(StandardCharsets.UTF_8) + + private val Headers: Map[String, List[String]] = Map("content-type" -> List("application/zip")) + + // --- BinaryResponse ------------------------------------------------------- + + test("two binary responses carrying equal-but-distinct arrays are equal and hash alike"): + val one = BinaryResponse(200, Headers, bytes("PK-archive")) + val two = BinaryResponse(200, Headers, bytes("PK-archive")) + + assert(!one.bytes.eq(two.bytes), "the two arrays must be distinct objects, or the test proves nothing") + assertEquals(one, two) + assertEquals(one.hashCode, two.hashCode) + + test("a set of binary responses keeps one copy of a repeated archive"): + val archive = BinaryResponse(200, Headers, bytes("PK-archive")) + val same = BinaryResponse(200, Headers, bytes("PK-archive")) + + assertEquals(Set(archive, same).size, 1) + + test("binary responses whose bytes differ are not equal"): + val one = BinaryResponse(200, Headers, bytes("PK-archive")) + val two = BinaryResponse(200, Headers, bytes("PK-archiv3")) + + assertNotEquals(one, two) + + test("a shorter body is not equal to a longer one that starts the same way"): + assertNotEquals(BinaryResponse(200, Headers, bytes("PK")), BinaryResponse(200, Headers, bytes("PK-archive"))) + + test("the status and the headers still count, so equal bytes alone are not enough"): + val archive = BinaryResponse(200, Headers, bytes("PK-archive")) + + assertNotEquals(archive, BinaryResponse(404, Headers, bytes("PK-archive"))) + assertNotEquals(archive, BinaryResponse(200, Map.empty[String, List[String]], bytes("PK-archive"))) + + test("canEqual agrees with equals: a binary response is comparable to another and to nothing else"): + // The compiler still generates canEqual for a case class that writes its own + // equals, and the two must not disagree about what is worth comparing. + val archive = BinaryResponse(200, Headers, bytes("PK-archive")) + + assert(archive.canEqual(BinaryResponse(500, Map.empty, Array.emptyByteArray))) + assert(!archive.canEqual("PK-archive")) + assertNotEquals[Any, Any](archive, "PK-archive") + + // --- RequestBody.Binary --------------------------------------------------- + + test("two binary request bodies carrying equal-but-distinct arrays are equal and hash alike"): + val one = RequestBody.Binary(bytes("avatar"), "image/png") + val two = RequestBody.Binary(bytes("avatar"), "image/png") + + assert(!one.bytes.eq(two.bytes), "the two arrays must be distinct objects, or the test proves nothing") + assertEquals(one, two) + assertEquals(one.hashCode, two.hashCode) + assertEquals(Set[RequestBody](one, two).size, 1) + + test("binary request bodies differing in their bytes or in their media type are not equal"): + val body = RequestBody.Binary(bytes("avatar"), "image/png") + + assertNotEquals(body, RequestBody.Binary(bytes("avatar!"), "image/png")) + assertNotEquals(body, RequestBody.Binary(bytes("avatar"), "image/jpeg")) + + // --- RequestBody.Multipart ------------------------------------------------ + + test("two multipart bodies carrying equal-but-distinct arrays are equal and hash alike"): + val one = RequestBody.Multipart("attachment", "notes.txt", bytes("hi"), "text/plain") + val two = RequestBody.Multipart("attachment", "notes.txt", bytes("hi"), "text/plain") + + assert(!one.bytes.eq(two.bytes), "the two arrays must be distinct objects, or the test proves nothing") + assertEquals(one, two) + assertEquals(one.hashCode, two.hashCode) + assertEquals(Set[RequestBody](one, two).size, 1) + + test("every field of a multipart body counts, not only the bytes"): + val part = RequestBody.Multipart("attachment", "notes.txt", bytes("hi"), "text/plain") + + assertNotEquals(part, RequestBody.Multipart("attachment", "notes.txt", bytes("ho"), "text/plain")) + assertNotEquals(part, RequestBody.Multipart("file", "notes.txt", bytes("hi"), "text/plain")) + assertNotEquals(part, RequestBody.Multipart("attachment", "other.txt", bytes("hi"), "text/plain")) + assertNotEquals(part, RequestBody.Multipart("attachment", "notes.txt", bytes("hi"), "text/markdown")) + + test("a binary body is not equal to a multipart body that happens to carry the same bytes"): + val binary = RequestBody.Binary(bytes("hi"), "text/plain") + val multipart = RequestBody.Multipart("attachment", "notes.txt", bytes("hi"), "text/plain") + + assertNotEquals[RequestBody, RequestBody](binary, multipart) + assertNotEquals[RequestBody, RequestBody](multipart, binary) + + // --- the cases that carry no bytes --------------------------------------- + + test("the byte-free cases still compare structurally, and the empty body is still a value"): + // RequestBody stopped being an enum so that two of its cases could write + // their own equals; the other three must be unaffected by that. + assertEquals(RequestBody.Json("""{"a":1}"""), RequestBody.Json("""{"a":1}""")) + assertNotEquals(RequestBody.Json("""{"a":1}"""), RequestBody.Json("""{"a":2}""")) + assertEquals( + RequestBody.Text("# heading", RequestBody.TextMediaType), + RequestBody.Text("# heading", "text/plain; charset=utf-8"), + ) + assertEquals[RequestBody, RequestBody](RequestBody.Empty, RequestBody.Empty) diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/CodebergResponsePagingSuite.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/CodebergResponsePagingSuite.scala index 3ad852b..4801a78 100644 --- a/modules/core/test/src/com/worxbend/codeberg4s/core/CodebergResponsePagingSuite.scala +++ b/modules/core/test/src/com/worxbend/codeberg4s/core/CodebergResponsePagingSuite.scala @@ -11,7 +11,7 @@ final class CodebergResponsePagingSuite extends FunSuite: "; rel=\"last\"" private def responseWith(headers: (String, List[String])*): CodebergResponse = - CodebergResponse(200, headers.toMap, "[]") + CodebergResponse(200, headers.toMap, ResponseBody.utf8("[]")) private def page(value: Int): PageNumber = PageNumber.from(value).getOrElse(PageNumber.First) diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/CodebergResponseSuite.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/CodebergResponseSuite.scala index cde64ed..ced6da7 100644 --- a/modules/core/test/src/com/worxbend/codeberg4s/core/CodebergResponseSuite.scala +++ b/modules/core/test/src/com/worxbend/codeberg4s/core/CodebergResponseSuite.scala @@ -19,7 +19,7 @@ final class CodebergResponseSuite extends FunSuite: assertEquals(headers("x-request-id" -> " abc123 ").requestId, Some("abc123")) test("only the first value of a repeated header is used"): - val response = CodebergResponse(200, Map("x-total-count" -> List("42", "7")), "") + val response = CodebergResponse(200, Map("x-total-count" -> List("42", "7")), ResponseBody.Empty) assertEquals(response.totalCount, Some(42)) @@ -53,4 +53,4 @@ final class CodebergResponseSuite extends FunSuite: assertEquals(headers("x-request-id" -> "abc123").requestId, Some("abc123")) private def headers(entries: (String, String)*): CodebergResponse = - CodebergResponse(200, entries.map((name, value) => (name, List(value))).toMap, "") + CodebergResponse(200, entries.map((name, value) => (name, List(value))).toMap, ResponseBody.Empty) diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/PagesSuite.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/PagesSuite.scala index 6d44e50..c2409c4 100644 --- a/modules/core/test/src/com/worxbend/codeberg4s/core/PagesSuite.scala +++ b/modules/core/test/src/com/worxbend/codeberg4s/core/PagesSuite.scala @@ -9,7 +9,7 @@ import munit.FunSuite final class PagesSuite extends FunSuite: private def responseWith(headers: (String, List[String])*): CodebergResponse = - CodebergResponse(200, headers.toMap, "[]") + CodebergResponse(200, headers.toMap, ResponseBody.utf8("[]")) private def page(value: Int): PageNumber = PageNumber.from(value).getOrElse(PageNumber.First) diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/PaginationProps.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/PaginationProps.scala deleted file mode 100644 index b4f7283..0000000 --- a/modules/core/test/src/com/worxbend/codeberg4s/core/PaginationProps.scala +++ /dev/null @@ -1,156 +0,0 @@ -package com.worxbend.codeberg4s.core - -import com.worxbend.codeberg4s.CodebergError -import com.worxbend.codeberg4s.ValidationError -import com.worxbend.codeberg4s.paging.Page -import com.worxbend.codeberg4s.paging.PageNumber -import com.worxbend.codeberg4s.paging.PageParams -import com.worxbend.codeberg4s.paging.PageSize -import com.worxbend.codeberg4s.syntax.discard - -import org.scalacheck.Gen -import org.scalacheck.Prop -import org.scalacheck.Prop.AnyOperators -import org.scalacheck.Prop.forAll - -import scala.collection.mutable.ListBuffer - -/** The pagination driver's invariants, over an arbitrary collection rather than the three-page example. - * - * Four things must hold for every shape of collection, and each one is a way a paginating client goes wrong in the - * field. '''Exactly once, in order''' — a driver that refetched or reordered a page would silently duplicate or - * scramble a caller's data. '''One fetch per page''' — an extra request per page doubles the load on the instance and - * burns the caller's rate limit. '''Laziness''' — page `n + 1` must not be requested before page `n` has been folded, - * or a listing of a large repository becomes an outage. '''Nothing partial''' — a fetch that fails must end the walk - * with that failure and not with a prefix that looks like a complete answer. - * - * The pages are built to advertise a next page for as long as one exists, which is the only end-of-collection signal - * Forgejo emits that can be trusted; see `Pages` for why a short page does not mean the last page. - */ -final class PaginationProps extends PropertyBase: - - private val pagination: Pagination[Exec.Result] = new Pagination[Exec.Result] - - private val size: PageSize = PageSize.from(10).getOrElse(PageSize.Default) - - private def numbered(value: Int): PageNumber = - PageNumber.from(value).getOrElse(PageNumber.First) - - /** Collections of one to six pages, each holding between zero and five items. */ - private val collections: Gen[Vector[Vector[Int]]] = - Gen - .choose(1, 6) - .flatMap(count => Gen.listOfN(count, Gen.listOf(Gen.choose(0, 999)).map(_.toVector))) - .map(_.toVector) - - /** The same, with every page guaranteed non-empty, so nothing but the missing `rel="next"` ends the walk. */ - private val inhabitedCollections: Gen[Vector[Vector[Int]]] = - Gen - .choose(1, 6) - .flatMap(count => Gen.listOfN(count, Gen.nonEmptyListOf(Gen.choose(0, 999)).map(_.toVector))) - .map(_.toVector) - - /** Page `index` of `collection`, advertising a following page for as long as one exists. */ - private def pageAt(collection: Vector[Vector[Int]], index: Int): Page[Int] = - Page( - items = collection(index - 1), - params = PageParams(numbered(index), size), - totalCount = Some(collection.map(_.size).sum), - nextPage = if index < collection.size then Some(numbered(index + 1)) else None, - prevPage = numbered(index).previous, - ) - - /** A fetch that serves `collection` and records the page numbers it was asked for. */ - private def fetcher(collection: Vector[Vector[Int]], log: ListBuffer[String]): PageParams => Exec.Result[Page[Int]] = - params => - val index = params.page.value - log.append(s"fetch-$index").discard - if index >= 1 && index <= collection.size then Right(pageAt(collection, index)) - else Left(CodebergError.Validation(ValidationError("page", s"no page $index in this fixture"))) - - private def numbersIn(log: ListBuffer[String], prefix: String): List[Int] = - log.toList.filter(_.startsWith(prefix)).flatMap(_.stripPrefix(prefix).toIntOption) - - property("every item of every page is folded exactly once, in the order the server sent it".tag(Property)): - forAll(inhabitedCollections) { collection => - val log = ListBuffer.empty[String] - - val collected = pagination.listAll(PageParams.First)(fetcher(collection, log)) - - (collected ?= Right(collection.flatten)).label("the concatenation of every page, in order") && - (numbersIn(log, "fetch-") ?= (1 to collection.size).toList).label("one fetch per page, in order") - } - - property("listAll is foldPages with concatenation".tag(Property)): - forAll(collections) { collection => - val listLog = ListBuffer.empty[String] - val foldLog = ListBuffer.empty[String] - - val collected = pagination.listAll(PageParams.First)(fetcher(collection, listLog)) - val folded = pagination.foldPages(PageParams.First, Vector.empty[Int])(fetcher(collection, foldLog)) { - (accumulated, page) => accumulated ++ page.items - } - - (collected ?= folded).label("same result") && - (listLog.toList ?= foldLog.toList).label("same fetches") - } - - property("no page is requested before the previous one has been folded".tag(Property)): - forAll(inhabitedCollections) { collection => - val log = ListBuffer.empty[String] - - val total = pagination.foldPages(PageParams.First, 0)(fetcher(collection, log)) { (running, page) => - log.append(s"fold-${page.params.page.value}").discard - running + page.size - } - - val expected = (1 to collection.size).toList.flatMap(index => List(s"fetch-$index", s"fold-$index")) - - (log.toList ?= expected).label("fetch and fold must strictly alternate") && - (total ?= Right(collection.map(_.size).sum)) - } - - property("an empty page ends the walk even while the response still advertises a next page".tag(Property)): - forAll(inhabitedCollections, Gen.choose(1, 7)) { (collection, position) => - val stopAt = math.min(position, collection.size + 1) - val withHole = collection.take(stopAt - 1) ++ Vector(Vector.empty[Int]) ++ collection.drop(stopAt - 1) - val log = ListBuffer.empty[String] - - val collected = pagination.listAll(PageParams.First)(fetcher(withHole, log)) - - (collected ?= Right(withHole.take(stopAt - 1).flatten)).label(s"items before the empty page at $stopAt") && - (numbersIn(log, "fetch-") ?= (1 to stopAt).toList).label("the empty page is the last one fetched") - } - - property("a failed fetch ends the walk with that failure and discards what was folded".tag(Property)): - forAll(inhabitedCollections, Gen.choose(1, 6)) { (collection, position) => - val failAt = math.min(position, collection.size) - val log = ListBuffer.empty[String] - val failure: CodebergError = - CodebergError.Validation(ValidationError("page", s"page $failAt is unavailable")) - - val serve = fetcher(collection, log) - - val collected = pagination.listAll(PageParams.First) { params => - if params.page.value >= failAt then Left(failure) else serve(params) - } - - (collected ?= Left(failure)).label("the failure, never a partial result") && - (numbersIn(log, "fetch-") ?= (1 until failAt).toList).label("nothing is fetched past the failure") - } - - property("the window's size is carried to every page of the walk".tag(Property)): - forAll(inhabitedCollections, Gen.choose(1, 50)) { (collection, requested) => - val window = PageParams(PageNumber.First, PageSize.from(requested).getOrElse(PageSize.Default)) - val sizes = ListBuffer.empty[Int] - - val collected = pagination.foldPages(window, 0) { params => - sizes.append(params.size.value).discard - val index = params.page.value - if index >= 1 && index <= collection.size then Right(pageAt(collection, index)) - else Left(CodebergError.Validation(ValidationError("page", s"no page $index"))) - }((running, page) => running + page.size) - - Prop.propBoolean(collected.isRight) && - (sizes.toList ?= List.fill(collection.size)(requested)).label("every request must keep the window's size") - } diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/PaginationSuite.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/PaginationSuite.scala deleted file mode 100644 index 9ae26f2..0000000 --- a/modules/core/test/src/com/worxbend/codeberg4s/core/PaginationSuite.scala +++ /dev/null @@ -1,98 +0,0 @@ -package com.worxbend.codeberg4s.core - -import com.worxbend.codeberg4s.CodebergError -import com.worxbend.codeberg4s.ValidationError -import com.worxbend.codeberg4s.paging.Page -import com.worxbend.codeberg4s.paging.PageNumber -import com.worxbend.codeberg4s.paging.PageParams -import com.worxbend.codeberg4s.syntax.discard - -import munit.FunSuite - -import scala.collection.mutable.ListBuffer - -final class PaginationSuite extends FunSuite: - - private val pagination: Pagination[Exec.Result] = new Pagination[Exec.Result] - - test("a single page is fetched once and no second page is asked for"): - val log = ListBuffer.empty[String] - val pages = Vector(pageOf(1, Vector(1, 2), lastIndex = 1)) - - val result = pagination.listAll(PageParams.First)(fetcher(pages, log)) - - assertEquals(result, Right(Vector(1, 2))) - assertEquals(log.toList, List("fetch-1")) - - test("three pages are walked in order and their items concatenated"): - val log = ListBuffer.empty[String] - val pages = threePages - - val result = pagination.listAll(PageParams.First)(fetcher(pages, log)) - - assertEquals(result, Right(Vector(1, 2, 3, 4, 5))) - assertEquals(log.toList, List("fetch-1", "fetch-2", "fetch-3")) - - test("an empty page ends the walk even when the response still offers a next page"): - val log = ListBuffer.empty[String] - val pages = Vector(pageOf(1, Vector.empty[Int], lastIndex = 3)) - - val result = pagination.listAll(PageParams.First)(fetcher(pages, log)) - - assertEquals(result, Right(Vector.empty[Int])) - assertEquals(log.toList, List("fetch-1")) - - test("a page is never fetched before the previous one has been folded"): - val log = ListBuffer.empty[String] - val pages = threePages - - val result = pagination.foldPages(PageParams.First, 0)(fetcher(pages, log)): (total, page) => - log.append(s"fold-${page.params.page.value}").discard - total + page.size - - assertEquals(result, Right(5)) - assertEquals(log.toList, List("fetch-1", "fold-1", "fetch-2", "fold-2", "fetch-3", "fold-3")) - - test("the fold sees the pagination metadata, not just the items"): - val log = ListBuffer.empty[String] - val pages = threePages - - val result = pagination.foldPages(PageParams.First, List.empty[Int])(fetcher(pages, log)): (seen, page) => - seen.appended(page.params.page.value) - - assertEquals(result, Right(List(1, 2, 3))) - - test("a failed fetch ends the walk with that failure"): - val log = ListBuffer.empty[String] - val pages = Vector(pageOf(1, Vector(1), lastIndex = 2)) - - val result = pagination.listAll(PageParams.First)(fetcher(pages, log)) - - assert(result.isLeft) - assertEquals(log.toList, List("fetch-1", "fetch-2")) - - private def threePages: Vector[Page[Int]] = Vector( - pageOf(1, Vector(1, 2), lastIndex = 3), - pageOf(2, Vector(3, 4), lastIndex = 3), - pageOf(3, Vector(5), lastIndex = 3), - ) - - /** Serves the prepared pages and records every request, so an eager walk shows up as an extra `fetch-` entry. */ - private def fetcher(pages: Vector[Page[Int]], log: ListBuffer[String]): PageParams => Exec.Result[Page[Int]] = - params => - val index = params.page.value - log.append(s"fetch-$index").discard - if index >= 1 && index <= pages.size then Right(pages(index - 1)) - else Left(CodebergError.Validation(ValidationError("page", s"page $index does not exist"))) - - private def pageOf(index: Int, items: Vector[Int], lastIndex: Int): Page[Int] = - Page( - items = items, - params = PageParams.First.at(pageNumber(index)), - totalCount = None, - nextPage = if index < lastIndex then Some(pageNumber(index + 1)) else None, - prevPage = if index > 1 then Some(pageNumber(index - 1)) else None, - ) - - private def pageNumber(index: Int): PageNumber = - if index <= 1 then PageNumber.First else pageNumber(index - 1).next diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/RecordingTelemetry.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/RecordingTelemetry.scala index 688fffd..3411883 100644 --- a/modules/core/test/src/com/worxbend/codeberg4s/core/RecordingTelemetry.scala +++ b/modules/core/test/src/com/worxbend/codeberg4s/core/RecordingTelemetry.scala @@ -19,6 +19,7 @@ final class RecordingTelemetry(failing: Boolean) extends Telemetry[Exec.Result]: private val recorded = ListBuffer.empty[String] private val contexts = ListBuffer.empty[CallContext] + private val failures = ListBuffer.empty[CodebergError] override def onRequest(ctx: CallContext): Exec.Result[Unit] = record(ctx, s"request ${ctx.operation}") @@ -27,6 +28,7 @@ final class RecordingTelemetry(failing: Boolean) extends Telemetry[Exec.Result]: record(ctx, s"response $status") override def onError(ctx: CallContext, error: CodebergError): Exec.Result[Unit] = + failures.append(error).discard record(ctx, s"error ${RecordingTelemetry.nameOf(error)}") /** Every callback, rendered as a short label, in the order it fired. */ @@ -35,6 +37,14 @@ final class RecordingTelemetry(failing: Boolean) extends Telemetry[Exec.Result]: /** The call context each callback received, in the same order as [[events]]. */ def seen: Vector[CallContext] = contexts.toVector + /** The failures the sink was handed, whole rather than as labels. + * + * A sink is the first thing to see a failure — the hook fires while the pipeline settles the attempt — so it is also + * the first place a body excerpt could be logged. Keeping the values, and not only their case names, is what lets a + * test assert on what a real sink would have printed. + */ + def observed: Vector[CodebergError] = failures.toVector + private def record(ctx: CallContext, event: String): Exec.Result[Unit] = recorded.append(event).discard contexts.append(ctx).discard @@ -54,3 +64,4 @@ object RecordingTelemetry: case CodebergError.DecodingFailed(_, _, _, _) => "DecodingFailed" case CodebergError.Validation(_) => "Validation" case CodebergError.RetriesExhausted(_, _, _) => "RetriesExhausted" + case CodebergError.WalkTruncated(_, _) => "WalkTruncated" diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/RedactionSuite.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/RedactionSuite.scala index f12f833..4fe1e0f 100644 --- a/modules/core/test/src/com/worxbend/codeberg4s/core/RedactionSuite.scala +++ b/modules/core/test/src/com/worxbend/codeberg4s/core/RedactionSuite.scala @@ -44,6 +44,18 @@ final class RedactionSuite extends FunSuite: test("no path and no query renders the base uri alone"): assertEquals(Redaction.uri(base, Nil, Nil), base) + test("user information in the base uri never reaches the rendered uri"): + val rendered = Redaction.uri("https://user:hunter2@forge.example/api/v1", List("repos"), Nil) + + assertEquals(rendered, "https://forge.example/api/v1/repos") + assert(!rendered.contains("hunter2"), s"the password survived in: $rendered") + + test("a query already on the base uri is dropped rather than rendered"): + assertEquals(Redaction.uri(s"$base?token=s3cret", List("repos"), Nil), s"$base/repos") + + test("a fragment already on the base uri is dropped rather than rendered"): + assertEquals(Redaction.uri(s"$base#frag", Nil, Nil), base) + test("an authorization header is masked"): val masked = Redaction.headers(List("authorization" -> "token s3cret", "accept" -> "application/json")) diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/ResponseBodySuite.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/ResponseBodySuite.scala new file mode 100644 index 0000000..6ea21d8 --- /dev/null +++ b/modules/core/test/src/com/worxbend/codeberg4s/core/ResponseBodySuite.scala @@ -0,0 +1,167 @@ +package com.worxbend.codeberg4s.core + +import munit.FunSuite + +import java.nio.charset.StandardCharsets + +/** [[ResponseBody]] is where the library stopped turning every response into text. + * + * Two things here are worth more than the rest. The first is that the charset is '''read''' rather than assumed: sttp + * used to pick it off `Content-Type` inside `asStringAlways`, and moving the byte-to-text step out of the transport + * would have silently thrown that away. The second is [[ResponseBody.excerpt]], which bounds a number of characters + * over an array of bytes — the operation with the trap in it, since slicing bytes at a fixed length cuts a multi-byte + * character in half. + */ +final class ResponseBodySuite extends FunSuite: + + /** Three bytes in UTF-8, one character. */ + private val euro: String = "€" + + /** Four bytes in UTF-8, two characters — a surrogate pair. */ + private val grin: String = "😀" + + /** What a decoder writes where it could not read a character, and therefore the marker of a byte slice that cut one + * in half. + */ + private val replacement: Char = '�' + + private def utf8(text: String): ResponseBody = ResponseBody.utf8(text) + + // --- bytes and text ------------------------------------------------------- + + test("the bytes come back exactly as they went in"): + val raw = Array[Byte](-1, -2, 0, 65) + + assertEquals(ResponseBody.of(raw, StandardCharsets.UTF_8).bytes.toList, raw.toList) + + test("bytes are not copied, because copying is the cost this type exists to avoid"): + val raw = Array[Byte](1, 2, 3) + + assert(ResponseBody.of(raw, StandardCharsets.UTF_8).bytes.eq(raw)) + + test("text decodes with the charset the response declared"): + val latin1 = ResponseBody.of("café".getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.ISO_8859_1) + + assertEquals(latin1.text, "café") + + test("bytes that are invalid in the declared charset become replacement characters, not a failure"): + val broken = ResponseBody.of(Array[Byte](-1, -2), StandardCharsets.UTF_8) + + assert(broken.text.forall(_.equals(replacement)), s"expected only replacement characters, got ${broken.text}") + + test("size and isEmpty report the bytes, not the decoded text"): + assertEquals(utf8(euro).size, 3) + assert(!utf8(euro).isEmpty) + assert(ResponseBody.Empty.isEmpty) + assertEquals(ResponseBody.Empty.size, 0) + + // --- blankness ------------------------------------------------------------ + + test("a body of ASCII whitespace is blank, and so is an empty one"): + assert(ResponseBody.Empty.isBlank) + assert(utf8("").isBlank) + assert(utf8(" \t\r\n \f").isBlank) + + test("a body with anything else in it is not blank"): + assert(!utf8("{}").isBlank) + assert(!utf8(" x ").isBlank) + assert(!utf8(euro).isBlank) + + // --- the UTF-8 requirement ------------------------------------------------ + + test("utf8Bytes hands back the same array when the response declared UTF-8, so the JSON path copies nothing"): + val body = utf8("""{"a":1}""") + + assert(body.utf8Bytes.eq(body.bytes)) + + test("utf8Bytes transcodes when the response declared something else"): + val latin1 = ResponseBody.of("café".getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.ISO_8859_1) + + assertEquals(String(latin1.utf8Bytes, StandardCharsets.UTF_8), "café") + assert(!latin1.utf8Bytes.eq(latin1.bytes)) + + // --- charset negotiation -------------------------------------------------- + + test("a response with no content type is read as UTF-8"): + assertEquals(ResponseBody.charsetOf(None), StandardCharsets.UTF_8) + + test("a content type with no charset parameter is read as UTF-8"): + assertEquals(ResponseBody.charsetOf(Some("application/json")), StandardCharsets.UTF_8) + + test("the declared charset is honoured, whatever case it was written in"): + assertEquals(ResponseBody.charsetOf(Some("text/plain; CharSet=ISO-8859-1")), StandardCharsets.ISO_8859_1) + + test("a quoted charset value is unquoted, as RFC 9110 allows"): + assertEquals(ResponseBody.charsetOf(Some("""text/plain; charset="utf-8"""")), StandardCharsets.UTF_8) + + test("a charset parameter after another parameter is still found"): + assertEquals(ResponseBody.charsetOf(Some("text/plain; boundary=x; charset=US-ASCII")), StandardCharsets.US_ASCII) + + test("an unknown or illegal charset name falls back to UTF-8 rather than failing the request"): + assertEquals(ResponseBody.charsetOf(Some("text/plain; charset=klingon-1")), StandardCharsets.UTF_8) + assertEquals(ResponseBody.charsetOf(Some("text/plain; charset=not a name")), StandardCharsets.UTF_8) + assertEquals(ResponseBody.charsetOf(Some("text/plain; charset=")), StandardCharsets.UTF_8) + + test("what Forgejo actually sends is read as UTF-8"): + assertEquals(ResponseBody.charsetOf(Some("application/json;charset=utf-8")), StandardCharsets.UTF_8) + + // --- excerpt -------------------------------------------------------------- + + test("a body shorter than the bound comes back whole"): + assertEquals(utf8("""{"a":1}""").excerpt(512), """{"a":1}""") + + test("a body longer than the bound is cut at that many characters"): + assertEquals(utf8("x".repeat(2000)).excerpt(512), "x".repeat(512)) + + test("the bound counts characters, so a three-byte character is one of them"): + // The whole point. 512 euro signs are 1,536 bytes; a bound applied to bytes + // would have answered 512 bytes, which is 170 characters and a broken one. + assertEquals(utf8(euro.repeat(2000)).excerpt(512), euro.repeat(512)) + + test("cutting never splits a multi-byte character"): + val excerpt = utf8(euro.repeat(2000)).excerpt(512) + + assert(!excerpt.contains(replacement), "a replacement character means the byte slice cut a character in half") + + test("a four-byte character, which is two characters, is also counted correctly"): + val excerpt = utf8(grin.repeat(2000)).excerpt(512) + + assertEquals(excerpt, grin.repeat(256)) + assert(!excerpt.contains(replacement)) + + test("a bound of zero or less yields nothing"): + assertEquals(utf8("payload").excerpt(0), "") + assertEquals(utf8("payload").excerpt(-1), "") + + test("an empty body excerpts to nothing"): + assertEquals(ResponseBody.Empty.excerpt(512), "") + + test("a charset that is not ASCII-compatible still excerpts the right number of characters"): + val utf16 = ResponseBody.of("x".repeat(2000).getBytes(StandardCharsets.UTF_16), StandardCharsets.UTF_16) + + assertEquals(utf16.excerpt(512), "x".repeat(512)) + + // --- value semantics ------------------------------------------------------ + + test("two bodies with the same bytes and charset are equal, and hash alike"): + val one = utf8("payload") + val two = utf8("payload") + + assertEquals(one, two) + assertEquals(one.hashCode, two.hashCode) + + test("bodies differing in bytes or in charset are not equal"): + val ascii = ResponseBody.of("payload".getBytes(StandardCharsets.US_ASCII), StandardCharsets.US_ASCII) + + assertNotEquals(utf8("payload"), utf8("other")) + assertNotEquals[Any, Any](utf8("payload"), ascii) + + test("a body is not equal to something that is not a body"): + assertNotEquals[Any, Any](utf8("payload"), "payload") + + test("toString reports the size and the charset and never the payload"): + val rendered = utf8("s3cret-token").toString + + assert(!rendered.contains("s3cret"), s"a body must not render its payload, got $rendered") + assert(rendered.contains("12 B"), rendered) + assert(rendered.contains("UTF-8"), rendered) diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/RetryEngineProps.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/RetryEngineProps.scala index 51f7cc2..bc854bc 100644 --- a/modules/core/test/src/com/worxbend/codeberg4s/core/RetryEngineProps.scala +++ b/modules/core/test/src/com/worxbend/codeberg4s/core/RetryEngineProps.scala @@ -81,7 +81,11 @@ final class RetryEngineProps extends PropertyBase: .suchThat(status => !StatusMapping.isRetryable(status)) .map(status => CodebergError.Api(context, status, ApiErrorBody.Empty)), Gen - .oneOf[TransportCause](TransportCause.Tls("certificate expired"), TransportCause.Interrupted("cancelled")) + .oneOf[TransportCause]( + TransportCause.Tls("certificate expired"), + TransportCause.Interrupted("cancelled"), + TransportCause.ResponseTooLarge("Stream length limit of 16777216 bytes exceeded"), + ) .map(cause => CodebergError.Transport(context, cause)), Gen.const(CodebergError.DecodingFailed(context, "{", JsonPath.Root, "unexpected end of input")), Gen.const(CodebergError.Validation(ValidationError("owner", "must not be blank"))), diff --git a/modules/core/test/src/com/worxbend/codeberg4s/core/RetryEngineSuite.scala b/modules/core/test/src/com/worxbend/codeberg4s/core/RetryEngineSuite.scala index eb8318c..78812a5 100644 --- a/modules/core/test/src/com/worxbend/codeberg4s/core/RetryEngineSuite.scala +++ b/modules/core/test/src/com/worxbend/codeberg4s/core/RetryEngineSuite.scala @@ -55,6 +55,19 @@ final class RetryEngineSuite extends FunSuite: assertEquals(result, Left(CodebergError.RetriesExhausted(context, 3, last))) assertEquals(attempts.toList, List(1, 2, 3)) + test("a terminal failure after a retryable one is reported unwrapped"): + val timer = FakeTimer(0L) + val attempts = ListBuffer.empty[Int] + val terminal = apiFailure(404) + + val result = engine(RetryPolicy.Default, timer, JitterSource.Deterministic).run("issues.list"): number => + attempts.append(number).discard + if number < 2 then Right(Left(apiFailure(503))) else Right(Left(terminal)) + + assertEquals(result, Left(terminal)) + assertEquals(attempts.toList, List(1, 2)) + assertEquals(timer.sleeps, Vector(250.millis)) + test("the backoff doubles after every failed attempt"): val timer = FakeTimer(0L) @@ -133,6 +146,23 @@ final class RetryEngineSuite extends FunSuite: assertEquals(result, Left(last)) assertEquals(attempts.toList, List(1)) + test("an oversized response body is not downloaded a second time"): + // The point of this one is arithmetic, not taxonomy. If ResponseTooLarge were + // retryable — which it would be if it had been folded into TransportCause.Unknown + // — the default policy would fetch the oversized body once per attempt, so a + // bound meant to cap a call at N bytes would let through maxAttempts * N. + val timer = FakeTimer(0L) + val attempts = ListBuffer.empty[Int] + val last = + CodebergError.Transport(context, TransportCause.ResponseTooLarge("Stream length limit of 16777216 bytes exceeded")) + + val result = engine(RetryPolicy.Default, timer, JitterSource.Deterministic) + .run("issues.list")(failing(attempts, last)) + + assertEquals(result, Left(last)) + assertEquals(attempts.toList, List(1)) + assert(timer.sleeps.isEmpty) + test("a policy of one attempt reports the failure unwrapped"): val timer = FakeTimer(0L) val attempts = ListBuffer.empty[Int] diff --git a/modules/domain/src/com/worxbend/codeberg4s/ApiErrorBody.scala b/modules/domain/src/com/worxbend/codeberg4s/ApiErrorBody.scala index f8eeafe..4e2d204 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/ApiErrorBody.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/ApiErrorBody.scala @@ -13,7 +13,7 @@ package com.worxbend.codeberg4s * @param errors * per-field problems reported by validation endpoints; empty when the server sent none */ -final case class ApiErrorBody(message: Option[String], url: Option[String], errors: List[String]) +final case class ApiErrorBody private[codeberg4s] (message: Option[String], url: Option[String], errors: List[String]) object ApiErrorBody: diff --git a/modules/domain/src/com/worxbend/codeberg4s/BaseUri.scala b/modules/domain/src/com/worxbend/codeberg4s/BaseUri.scala index 98c2ecb..2714e13 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/BaseUri.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/BaseUri.scala @@ -21,7 +21,11 @@ object BaseUri: * * Accepts an absolute `http://` or `https://` URI with something after the scheme, trims surrounding whitespace and * removes any trailing slashes. Rejects an empty or blank value, a value carrying a control character, a scheme this - * library cannot speak, and a scheme with no authority. + * library cannot speak, a scheme with no authority, and — because a base URI is prefixed to every rendered URI, and + * every rendered URI can reach a log file — user information before the host, a query string and a fragment. + * + * A rejection message never repeats the offending value: user information is a credential, and reporting it would + * put it exactly where this validation exists to keep it out of. * * @return * the normalised URI, or a [[ValidationError]] on the `"baseUri"` field @@ -36,6 +40,10 @@ object BaseUri: case Some(scheme) => val normalised = trimmed.replaceAll(TrailingSlashes, "") if normalised.length <= scheme.length then Left(invalid("must have a host after the scheme")) + else if hasUserInfo(normalised, scheme) then + Left(invalid("must not carry user information before the host")) + else if normalised.indexOf('?') >= 0 then Left(invalid("must not carry a query string")) + else if normalised.indexOf('#') >= 0 then Left(invalid("must not carry a fragment")) else Right(normalised) private def schemeOf(value: String): Option[String] = @@ -43,6 +51,24 @@ object BaseUri: else if value.startsWith(HttpPrefix) then Some(HttpPrefix) else None + /** Whether the authority carries `user:password@` before the host. + * + * Only the authority is inspected — the run between the scheme and the first `/`, `?` or `#`. An `@` further along + * is an ordinary path character, as in `https://forge.example/api/v1/@me`, and is left alone. + */ + private def hasUserInfo(value: String, scheme: String): Boolean = + value.substring(scheme.length, authorityEnd(value, scheme)).indexOf('@') >= 0 + + /** The index at which the authority ends: the first `/`, `?` or `#` after the scheme, or the end of the value. */ + private def authorityEnd(value: String, scheme: String): Int = + val boundary = value.indexWhere(endsAuthority, scheme.length) + if boundary < 0 then value.length else boundary + + private def endsAuthority(char: Char): Boolean = + char match + case '/' | '?' | '#' => true + case _ => false + private def invalid(message: String): ValidationError = ValidationError("baseUri", message) diff --git a/modules/domain/src/com/worxbend/codeberg4s/CodebergConfig.scala b/modules/domain/src/com/worxbend/codeberg4s/CodebergConfig.scala index 2d9a825..54ebd69 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/CodebergConfig.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/CodebergConfig.scala @@ -9,8 +9,11 @@ import scala.concurrent.duration.FiniteDuration /** Everything a client needs to talk to one Forgejo deployment. * - * Built once near the client's construction and passed by constructor from there on. Every field is a validated domain - * type rather than a raw primitive, so a misconfigured client fails at construction instead of on the first call. + * Built once near the client's construction and passed by constructor from there on. Every field naming a domain + * concept — the base URI, the credentials, the page size, the user agent — is a validated type rather than a raw + * primitive, so a misconfigured client fails at construction instead of on the first call. The two timeouts and the + * two response-body bounds are quantities rather than domain concepts and are carried as they are; a negative or zero + * value is not rejected here, and would make every call fail. * * `toString` is safe to log: the credential types inside [[auth.Auth]] redact themselves. * @@ -28,6 +31,12 @@ import scala.concurrent.duration.FiniteDuration * how long to wait for the connection to be established * @param readTimeout * how long to wait for the response once the request has been sent + * @param maxResponseBodyBytes + * the most bytes a textual response may carry before the call fails with [[TransportCause.ResponseTooLarge]]; see + * [[DefaultMaxResponseBodyBytes]] + * @param maxDownloadBodyBytes + * the same bound for the archive-downloading operations under `client.downloads`, which is larger for the reason + * [[DefaultMaxDownloadBodyBytes]] gives */ final case class CodebergConfig( baseUri: BaseUri, @@ -37,6 +46,8 @@ final case class CodebergConfig( defaultPageSize: PageSize, connectTimeout: FiniteDuration, readTimeout: FiniteDuration, + maxResponseBodyBytes: Long, + maxDownloadBodyBytes: Long, ) object CodebergConfig: @@ -47,18 +58,53 @@ object CodebergConfig: /** Time allowed for a response body once the request has been sent, when using the Codeberg defaults. */ val DefaultReadTimeout: FiniteDuration = 30.seconds + /** 16 MiB — the most a textual response may carry before the call is abandoned. + * + * This library reads a whole response into memory; it does not stream. Without a bound, the only thing standing + * between a misbehaving or hostile instance and the client's heap is how long the caller is prepared to wait, so + * every request carries this one. + * + * The number is derived rather than picked. The largest legitimate JSON body Forgejo produces is a file-contents + * response, whose payload is a repository blob base64-encoded — and `GET /api/v1/settings/api` reports + * `default_max_blob_size` as `10485760`, 10 MiB (`docs/HAZARDS.md` §5 records the capture). Base64 costs four bytes + * per three, so 10 MiB of blob reaches roughly 13.4 MiB on the wire, and 16 MiB clears that with room for the + * surrounding fields. + * + * That number is per-instance configuration, not a protocol constant. A client talking to a self-hosted Forgejo that + * raises it should read the instance's own value back from [[miscellaneous.ServerApiSettings.maxBlobSizeBytes]] and + * raise this setting to match. + */ + val DefaultMaxResponseBodyBytes: Long = 16L * 1024 * 1024 + + /** 50 MiB — the same bound for the two archive downloads, deliberately larger than [[DefaultMaxResponseBodyBytes]]. + * + * `client.downloads` fetches a CI artifact or a run's logs as a ZIP. A ZIP is not a JSON document bounded by + * `default_max_blob_size`; it is whatever a workflow uploaded, so the reasoning behind the textual bound says + * nothing about it and a shared number would have had to be wrong for one of the two — either small enough to reject + * ordinary artifacts, or large enough to make the bound on JSON meaningless. + * + * 50 MiB is where the project already drew this line: `SECURITY.md` and `docs/ROADMAP.md` both record "attachment + * streaming above 50 MB" as out of scope for v1, which is to say that archives above that size are the case this + * library does not undertake to serve. The default now enforces what those documents describe instead of leaving it + * to the heap. Raise it if you download bigger artifacts and have the memory for them. + */ + val DefaultMaxDownloadBodyBytes: Long = 50L * 1024 * 1024 + /** Codeberg defaults for everything except authentication. * * Uses [[BaseUri.Codeberg]], [[retry.RetryPolicy.Default]], [[UserAgent.Default]], [[paging.PageSize.Default]], - * [[DefaultConnectTimeout]] and [[DefaultReadTimeout]]. Copy the result to change one field. + * [[DefaultConnectTimeout]], [[DefaultReadTimeout]], [[DefaultMaxResponseBodyBytes]] and + * [[DefaultMaxDownloadBodyBytes]]. Copy the result to change one field. */ def apply(auth: Auth): CodebergConfig = new CodebergConfig( - baseUri = BaseUri.Codeberg, - auth = auth, - retry = RetryPolicy.Default, - userAgent = UserAgent.Default, - defaultPageSize = PageSize.Default, - connectTimeout = DefaultConnectTimeout, - readTimeout = DefaultReadTimeout, + baseUri = BaseUri.Codeberg, + auth = auth, + retry = RetryPolicy.Default, + userAgent = UserAgent.Default, + defaultPageSize = PageSize.Default, + connectTimeout = DefaultConnectTimeout, + readTimeout = DefaultReadTimeout, + maxResponseBodyBytes = DefaultMaxResponseBodyBytes, + maxDownloadBodyBytes = DefaultMaxDownloadBodyBytes, ) diff --git a/modules/domain/src/com/worxbend/codeberg4s/CodebergError.scala b/modules/domain/src/com/worxbend/codeberg4s/CodebergError.scala index b42052a..3fe36a5 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/CodebergError.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/CodebergError.scala @@ -1,5 +1,7 @@ package com.worxbend.codeberg4s +import com.worxbend.codeberg4s.paging.PageParams + /** Every failure this library reports, as one closed family. * * Recoverable failures are values: no operation throws for a `404`, a timeout, or a malformed payload. Each remote @@ -13,6 +15,8 @@ package com.worxbend.codeberg4s * - [[CodebergError.Validation]] — the request was rejected before it was built; fix the argument. * - [[CodebergError.RetriesExhausted]] — the retry engine gave up; `last` is the failure that ended it, never * discarded. + * - [[CodebergError.WalkTruncated]] — a walk over every page hit its page cap with pages still to come; the answer + * it would otherwise have returned was incomplete, so it is not returned at all. */ enum CodebergError: @@ -31,6 +35,23 @@ enum CodebergError: /** The retry engine ran out of attempts. `last` preserves the failure of the final attempt. */ case RetriesExhausted(ctx: CallContext, attempts: Int, last: CodebergError) + /** A walk over every page of a collection stopped at its page cap while the server was still offering another page. + * + * This is not a remote failure — nothing went wrong on the wire — which is why it carries no [[CallContext]]. It + * says that the result the walk was assembling covers only `pagesVisited` pages of a longer collection, so handing + * that result back would be handing back a short answer indistinguishable from a complete one. + * + * `resumeFrom` is the window the walk was about to request, page size included. Passing it back as the starting + * window continues exactly where this walk stopped, which is what makes the failure recoverable rather than merely + * informative. + * + * @param pagesVisited + * how many pages were fetched and folded before the cap was reached + * @param resumeFrom + * the page the walk would have requested next + */ + case WalkTruncated(pagesVisited: Int, resumeFrom: PageParams) + object CodebergError: /** Upper bound, in characters, on any single free-form fragment [[describe]] embeds — body snippets, server messages @@ -62,6 +83,11 @@ object CodebergError: s"invalid ${problem.field}: ${bound(problem.message)}" case RetriesExhausted(ctx, attempts, last) => s"${renderContext(ctx)} gave up after $attempts attempts; last failure: ${bound(last.describe)}" + case WalkTruncated(pagesVisited, resumeFrom) => + // Every fragment is a number this library produced, so there is nothing + // here for `bound` to protect against. + s"page walk stopped after $pagesVisited pages with more pages still offered; " + + s"resume at page ${resumeFrom.page.value} with limit ${resumeFrom.size.value}" /** The most detail fragments any one rendering embeds, so the whole string stays bounded rather than merely each * piece of it. diff --git a/modules/domain/src/com/worxbend/codeberg4s/ContentType.scala b/modules/domain/src/com/worxbend/codeberg4s/ContentType.scala new file mode 100644 index 0000000..7883861 --- /dev/null +++ b/modules/domain/src/com/worxbend/codeberg4s/ContentType.scala @@ -0,0 +1,47 @@ +package com.worxbend.codeberg4s + +/** Validation shared by every media type this library writes into a header. + * + * This is a security boundary, not a convenience. A media type reaches the wire as the value of a `Content-Type` + * header — for an upload, the header of the multipart part itself — and an HTTP header value ends at the first + * carriage return or newline. A value carrying one would close that header early and let everything after it be read + * as headers of the caller's choosing, or as the start of a part the caller was never offered. That is header + * injection, and the only reliable place to stop it is before the value becomes bytes. + * + * '''It lives in the root package for the reason [[PathSegment]] does''': a rule that only one package can reach gets + * copied into the packages that cannot, and copies drift. [[com.worxbend.codeberg4s.issues.UploadAttachment]] and + * [[com.worxbend.codeberg4s.repositories.publishing.UploadAsset]] both call [[from]], which is the check that gives a + * caller a named [[ValidationError]] before any request exists. The sttp adapter calls [[isSafe]] a second time on the + * exact string it is about to write. The two are not redundant: the domain check is the one a caller can act on, and + * the transport check is the last point at which the value is still a Scala `String` rather than wire bytes. + */ +private[codeberg4s] object ContentType: + + /** Whether `value` can be written verbatim as a header value. + * + * True when `value` holds at least one non-whitespace character and no control character. Carriage return and + * newline are control characters, so the injection case falls out of the general rule rather than needing a clause + * of its own. + */ + def isSafe(value: String): Boolean = !value.isBlank && !value.exists(_.isControl) + + /** Trims `value` and accepts it only when the result is safe to write as a header value. + * + * The trim happens first, so a media type that arrived with trailing whitespace — including the newline a caller + * gets from reading a line out of a file — is accepted as its trimmed form rather than refused. A control character + * anywhere the trim does not reach is a rejection, which is the case that matters: `text/plain\r\nX-Injected: 1` has + * the injection in the middle, where no amount of trimming removes it. + * + * @param field + * the field name to report in a [[ValidationError]], for example `"mediaType"` + * @return + * the trimmed media type, or a [[ValidationError]] naming `field` + */ + def from(field: String, value: String): Either[ValidationError, String] = + val trimmed = value.trim + // Written as "accept, then explain" rather than as a chain of rejections, + // because [[isSafe]] is the whole rule and the transport applies exactly + // it. The two branches below only choose which half of it was broken. + if isSafe(trimmed) then Right(trimmed) + else if trimmed.isEmpty then Left(ValidationError(field, "must not be blank")) + else Left(ValidationError(field, "must not contain a control character")) diff --git a/modules/domain/src/com/worxbend/codeberg4s/PathSegment.scala b/modules/domain/src/com/worxbend/codeberg4s/PathSegment.scala new file mode 100644 index 0000000..7496f5d --- /dev/null +++ b/modules/domain/src/com/worxbend/codeberg4s/PathSegment.scala @@ -0,0 +1,65 @@ +package com.worxbend.codeberg4s + +/** Validation shared by every identifier that becomes part of a URI path. + * + * This is a security boundary, not a convenience: an identifier such as [[com.worxbend.codeberg4s.repositories.Owner]] + * or [[com.worxbend.codeberg4s.users.Username]] is interpolated into a request path, so a value containing `/` would + * let a caller reach an endpoint the API surface never offered, and a control character would corrupt the request + * line. Both are rejected here, once, rather than at each call site. + * + * '''It lives in the root package so that "once" is true.''' The rule used to sit in + * `com.worxbend.codeberg4s.repositories` and be visible only there, which meant the three identifiers outside that + * package — [[com.worxbend.codeberg4s.users.Username]], [[com.worxbend.codeberg4s.organizations.OrgName]] and + * [[com.worxbend.codeberg4s.users.social.AccessTokenName]] — spelled the same four checks out inline. Four copies of a + * security rule is four places to forget the next clause, which is exactly what happened: the traversal check reached + * `segmented` and not the copies. A rule that every package can reach cannot drift that way. + * + * [[from]] is for an identifier that must occupy exactly one segment. [[segmented]] is for the two that legitimately + * span several — [[com.worxbend.codeberg4s.repositories.BranchName]] and + * [[com.worxbend.codeberg4s.repositories.ContentPath]], whose routes Forgejo matches with a wildcard. Both reject the + * traversal segments `.` and `..`: they carry no slash, so a slash rule alone never sees them, and they survive + * percent-encoding untouched, so one would reach the request path as a dot segment rather than as a name. + */ +private[codeberg4s] object PathSegment: + + /** Trims `value` and accepts it only if it can stand alone as one path segment. + * + * Rejects an empty or blank value, a value containing `/`, a value containing any control character, and the + * traversal segments `.` and `..`. + * + * @param field + * the field name to report in a [[ValidationError]] + */ + def from(field: String, value: String): Either[ValidationError, String] = + val trimmed = value.trim + if trimmed.isEmpty then Left(ValidationError(field, "must not be blank")) + else if trimmed.contains('/') then Left(ValidationError(field, "must not contain a slash")) + else if trimmed.exists(_.isControl) then Left(ValidationError(field, "must not contain a control character")) + else if isTraversal(trimmed) then Left(ValidationError(field, "must not be '.' or '..'")) + else Right(trimmed) + + /** Trims `value` and accepts it only if every `/`-separated part can stand alone as one path segment. + * + * Rejects an empty or blank value, any control character, a leading or trailing `/`, an empty segment such as the + * middle of `a//b`, and a `.` or `..` segment. The last of those is the one that matters: `..` survives + * percent-encoding untouched, so a value carrying it would walk out of the route it was meant for. + * + * @param field + * the field name to report in a [[ValidationError]] + */ + def segmented(field: String, value: String): Either[ValidationError, String] = + val trimmed = value.trim + if trimmed.isEmpty then Left(ValidationError(field, "must not be blank")) + else if trimmed.exists(_.isControl) then Left(ValidationError(field, "must not contain a control character")) + else if trimmed.startsWith("/") || trimmed.endsWith("/") then + Left(ValidationError(field, "must not start or end with a slash")) + else + val parts = trimmed.split('/') + if parts.exists(_.isEmpty) then Left(ValidationError(field, "must not contain an empty segment")) + else if parts.exists(isTraversal) then Left(ValidationError(field, "must not contain a '.' or '..' segment")) + else Right(trimmed) + + private def isTraversal(segment: String): Boolean = + segment match + case "." | ".." => true + case _ => false diff --git a/modules/domain/src/com/worxbend/codeberg4s/ServerVersion.scala b/modules/domain/src/com/worxbend/codeberg4s/ServerVersion.scala index ae767e8..207c53a 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/ServerVersion.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/ServerVersion.scala @@ -14,4 +14,4 @@ package com.worxbend.codeberg4s * @param raw * the string exactly as the instance reported it, never blank */ -final case class ServerVersion(raw: String) +final case class ServerVersion private[codeberg4s] (raw: String) diff --git a/modules/domain/src/com/worxbend/codeberg4s/TransportCause.scala b/modules/domain/src/com/worxbend/codeberg4s/TransportCause.scala index 984f5b7..c323c6c 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/TransportCause.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/TransportCause.scala @@ -1,11 +1,16 @@ package com.worxbend.codeberg4s -/** Why a request never produced an HTTP response. +/** Why a request produced no usable HTTP response. * * The transport adapter classifies the exception it caught into one of these cases; anything it cannot classify * becomes [[TransportCause.Unknown]] rather than being dropped. A status code — including `5xx` — is never a transport * cause: that is [[CodebergError.Api]]. * + * All but one of these mean nothing arrived at all. [[TransportCause.ResponseTooLarge]] is the exception: a response + * did begin to arrive, and reading it was abandoned once it passed the bound in [[CodebergConfig]]. It is here rather + * than under [[CodebergError.Api]] because the status is not what went wrong and the body was never completed, so + * there is nothing to hand a status-mapping decision. + * * Each case carries a short `detail` taken from the underlying exception message. Details are for humans; callers * branch on the case, not on the text. */ @@ -26,6 +31,18 @@ enum TransportCause: /** The calling thread was interrupted while the request was in flight. */ case Interrupted(detail: String) + /** The response body passed the byte bound configured in [[CodebergConfig]] and was abandoned part-read. + * + * A case of its own rather than an [[Unknown]] because retrying differs: an unclassified failure may be transient + * and is attempted again, whereas an instance that answered with too many bytes will answer with too many bytes + * again. Repeating it would download the oversized body once per attempt — the opposite of what a bound is for — so + * this case is excluded from retrying. + * + * Which bound was passed depends on the operation: [[CodebergConfig.maxDownloadBodyBytes]] for the archive downloads + * under `client.downloads`, [[CodebergConfig.maxResponseBodyBytes]] for everything else. + */ + case ResponseTooLarge(detail: String) + /** The transport failed in a way this library does not classify. */ case Unknown(detail: String) @@ -37,4 +54,5 @@ enum TransportCause: case Tls(detail) => s"TLS failure ($detail)" case Dns(detail) => s"name resolution failed ($detail)" case Interrupted(detail) => s"interrupted ($detail)" + case ResponseTooLarge(detail) => s"response body too large ($detail)" case Unknown(detail) => s"unclassified transport failure ($detail)" diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/Comment.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/Comment.scala index 0c3f828..26e37a1 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/Comment.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/Comment.scala @@ -28,7 +28,7 @@ import java.time.Instant * @param pullRequestUrl * the API URL of the pull request this comment belongs to, absent when the comment is on a plain issue */ -final case class Comment( +final case class Comment private[codeberg4s] ( id: CommentId, body: Option[String], author: Option[User], diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/Issue.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/Issue.scala index d83f127..d6acb6f 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/Issue.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/Issue.scala @@ -52,7 +52,7 @@ import java.time.Instant * @param dueDate * the deadline set on the issue, absent when it has none */ -final case class Issue( +final case class Issue private[codeberg4s] ( id: Long, number: IssueNumber, title: String, diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/IssueAttachment.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/IssueAttachment.scala index e71d427..6e7db93 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/IssueAttachment.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/IssueAttachment.scala @@ -19,8 +19,8 @@ import java.time.Instant * * This library returns metadata only. [[browserDownloadUrl]] is the supported route to the content: hand it to an HTTP * client that can stream, exactly as `com.worxbend.codeberg4s.repositories.actions.ActionArtifact.archiveDownloadUrl` - * does. Nothing here downloads, because [[com.worxbend.codeberg4s.core.CodebergResponse]] carries a body as `String` - * and an arbitrary file is not text. + * does. Nothing here downloads. That was once a limitation of the response type, which carried text and so could not + * carry a file; it is now a decision, since a response body is bytes. * * @param id * the instance-wide identifier, and the only way to address the attachment again; see [[AttachmentId]] @@ -39,7 +39,7 @@ import java.time.Instant * @param createdAt * when the attachment was uploaded */ -final case class IssueAttachment( +final case class IssueAttachment private[codeberg4s] ( id: AttachmentId, name: String, size: Long, diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/IssueDeadline.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/IssueDeadline.scala index 280b329..2eb219a 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/IssueDeadline.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/IssueDeadline.scala @@ -17,4 +17,4 @@ import java.time.Instant * @param dueDate * the deadline the instance now holds, absent when it reported none */ -final case class IssueDeadline(dueDate: Option[Instant]) +final case class IssueDeadline private[codeberg4s] (dueDate: Option[Instant]) diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/IssueSubscription.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/IssueSubscription.scala index 0e7aaa9..f70f032 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/IssueSubscription.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/IssueSubscription.scala @@ -28,7 +28,7 @@ import java.time.Instant * @param createdAt * when the subscription was recorded */ -final case class IssueSubscription( +final case class IssueSubscription private[codeberg4s] ( isSubscribed: Boolean, isIgnored: Boolean, url: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/Label.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/Label.scala index 741a29f..4989e3c 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/Label.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/Label.scala @@ -22,7 +22,7 @@ package com.worxbend.codeberg4s.issues * @param url * the API URL of the label itself, not a browser URL — Forgejo sends no `html_url` for labels */ -final case class Label( +final case class Label private[codeberg4s] ( id: LabelId, name: String, color: Option[LabelColor], diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/LabelColor.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/LabelColor.scala index 7489e5e..734220d 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/LabelColor.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/LabelColor.scala @@ -2,8 +2,6 @@ package com.worxbend.codeberg4s.issues import com.worxbend.codeberg4s.ValidationError -import scala.util.matching.Regex - import java.util.Locale /** The background colour of a [[Label]], as a hexadecimal RGB triplet. @@ -21,21 +19,41 @@ opaque type LabelColor = String object LabelColor: - private val Hexadecimal: Regex = "^#?([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$".r - /** Parses a label colour. * * Trims surrounding whitespace, drops a leading `#`, and lowercases the digits. Rejects anything that is not three * or six hexadecimal digits. * + * Checked by counting the digits and then scanning them, rather than by a regular expression. Every label Forgejo + * returns carries a colour, so this runs once per label on every issue and label listing; a `Pattern` match + * allocates a `Matcher` and a match result per call, and the grammar it encodes — a length and an alphabet — is + * cheaper to state directly. + * * @return * the normalised colour, or a [[ValidationError]] on the `"labelColor"` field */ def from(value: String): Either[ValidationError, LabelColor] = - Hexadecimal.findFirstMatchIn(value.trim) match - case Some(digits) => Right(digits.group(1).toLowerCase(Locale.ROOT)) - case None => - Left(ValidationError("labelColor", "must be three or six hexadecimal digits, optionally prefixed with '#'")) + val trimmed = value.trim + val digits = if trimmed.startsWith("#") then trimmed.substring(1) else trimmed + if isTripletLength(digits.length) && digits.forall(isHexadecimalDigit) then + Right(digits.toLowerCase(Locale.ROOT)) + else Left(ValidationError("labelColor", "must be three or six hexadecimal digits, optionally prefixed with '#'")) + + /** Whether `length` is one of the two digit counts Forgejo takes: three-digit shorthand or a six-digit triplet. */ + private def isTripletLength(length: Int): Boolean = + length match + case 3 | 6 => true + case _ => false + + /** Whether `digit` is one of `0`-`9`, `a`-`f` or `A`-`F`. + * + * Spelled out as range comparisons rather than delegating to `Character.digit`, which also accepts the non-ASCII + * decimal digits of every Unicode script — `٣` and `३` are digits to the JDK, and neither belongs in a colour. + */ + private def isHexadecimalDigit(digit: Char): Boolean = + (digit >= '0' && digit <= '9') || + (digit >= 'a' && digit <= 'f') || + (digit >= 'A' && digit <= 'F') extension (color: LabelColor) diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/Milestone.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/Milestone.scala index b6c803d..63741f0 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/Milestone.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/Milestone.scala @@ -23,7 +23,7 @@ import java.time.Instant * @param dueOn * the deadline the milestone was given, absent when it has none. Forgejo's `due_on`, sent as `null` when unset */ -final case class Milestone( +final case class Milestone private[codeberg4s] ( id: MilestoneId, title: String, description: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/Reaction.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/Reaction.scala index f9621eb..c6c384d 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/Reaction.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/Reaction.scala @@ -20,4 +20,4 @@ import java.time.Instant * @param createdAt * when the reaction was recorded */ -final case class Reaction(content: ReactionContent, user: Option[User], createdAt: Option[Instant]) +final case class Reaction private[codeberg4s] (content: ReactionContent, user: Option[User], createdAt: Option[Instant]) diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/TimelineEvent.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/TimelineEvent.scala index 61b58d3..46dbe5c 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/TimelineEvent.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/TimelineEvent.scala @@ -56,7 +56,7 @@ import java.time.Instant * @param trackedTime * the time entry a timetracking event recorded */ -final case class TimelineEvent( +final case class TimelineEvent private[codeberg4s] ( id: CommentId, eventType: Option[String], body: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/TrackedTime.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/TrackedTime.scala index c44408b..c740585 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/TrackedTime.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/TrackedTime.scala @@ -34,7 +34,7 @@ import java.time.Instant * @param createdAt * when the entry was recorded, which an import may backdate */ -final case class TrackedTime( +final case class TrackedTime private[codeberg4s] ( id: TrackedTimeId, issue: Option[Issue], spent: FiniteDuration, diff --git a/modules/domain/src/com/worxbend/codeberg4s/issues/UploadAttachment.scala b/modules/domain/src/com/worxbend/codeberg4s/issues/UploadAttachment.scala index a375cc9..0ae373b 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/issues/UploadAttachment.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/issues/UploadAttachment.scala @@ -1,8 +1,10 @@ package com.worxbend.codeberg4s.issues +import com.worxbend.codeberg4s.ContentType import com.worxbend.codeberg4s.ValidationError import java.time.Instant +import java.util.Arrays /** A file to attach to an issue or to a comment — the `multipart/form-data` half of * `POST /repos/{owner}/{repo}/issues/{index}/assets` and of the matching comment route. @@ -23,13 +25,21 @@ import java.time.Instant * [[content]] is '''not''' copied, here or at the transport boundary, for the reason * `com.worxbend.codeberg4s.repositories.publishing.UploadAsset` gives: an attachment can be large and copying it to * gain an immutability guarantee the caller can already provide is the wrong trade. A caller must therefore not mutate - * the array after handing it over. For the same reason the generated `equals` compares [[content]] by reference, so - * two structurally identical uploads are not equal; nothing in this library depends on that. + * the array after handing it over. + * + * Equality is a separate question from copying, and is answered '''on the bytes''': see [[UploadAttachment.equals]]. + * + * ==Construction== + * + * The constructor is private, so [[UploadAttachment.of]] is the only way to obtain one and the checks it performs + * cannot be stepped around by calling the generated `apply` or `copy`. Reading the fields and pattern matching are + * unaffected. * * ==Error contract== * - * Construction produces [[ValidationError]] on the `"fileName"` field and nothing else; it performs no I/O and never - * reads a file. Turning a path into bytes is the caller's job, deliberately: this library owns no filesystem effect. + * [[UploadAttachment.of]] produces a [[ValidationError]] on the `"fileName"` field, [[as]] one on the `"mediaType"` + * field, and nothing else here can fail. No member performs I/O or reads a file. Turning a path into bytes is the + * caller's job, deliberately: this library owns no filesystem effect. * * @param fileName * the file name announced in the multipart part @@ -42,7 +52,7 @@ import java.time.Instant * @param updatedAt * the `updated_at` query parameter, which an import uses to backdate the attachment */ -final case class UploadAttachment( +final case class UploadAttachment private ( fileName: String, content: Array[Byte], mediaType: String, @@ -53,8 +63,17 @@ final case class UploadAttachment( /** Records the attachment under `attachment` instead of under [[fileName]]. */ def named(attachment: String): UploadAttachment = copy(storedName = Some(attachment)) - /** Declares the part's content type, for an instance or a proxy that acts on it. */ - def as(media: String): UploadAttachment = copy(mediaType = media) + /** Declares the part's own `Content-Type`, for an instance or a proxy that acts on it. + * + * The value is written into the multipart body as a header, so it is checked the way [[fileName]] is: trimmed, then + * refused when it is blank or carries a control character. A carriage return or a newline in it would end the part's + * header line and let whatever follows be read as headers of the caller's choosing. + * + * @return + * the upload sent under `media`, or a [[ValidationError]] on the `"mediaType"` field + */ + def as(media: String): Either[ValidationError, UploadAttachment] = + ContentType.from("mediaType", media).map(checked => copy(mediaType = checked)) /** Backdates the attachment, which is what an importer wants and nothing else does. */ def recordedAt(moment: Instant): UploadAttachment = copy(updatedAt = Some(moment)) @@ -62,6 +81,28 @@ final case class UploadAttachment( /** How many bytes would be sent. */ def size: Int = content.length + /** Structural, on the names, the media type and the timestamp first and then on the bytes. + * + * Written out for the reason `com.worxbend.codeberg4s.repositories.publishing.UploadAsset.equals` gives: an array's + * own `equals` in Scala is identity, so the generated equality would report two uploads built from byte-identical + * files as different and hash them differently. Comparing the bytes copies nothing, so the aliasing decision above + * stands; the cheap fields are tested first so the scan is reached only when everything else already matched. + * + * The class is `final`, so no subclass can exist and the type test below is the whole of the compiler-generated + * `canEqual`; calling `canEqual` as well would add nothing. Removing `final` would change that. + */ + override def equals(other: Any): Boolean = + other match + case that: UploadAttachment => + fileName.equals(that.fileName) && mediaType.equals(that.mediaType) && + storedName.equals(that.storedName) && updatedAt.equals(that.updatedAt) && + Arrays.equals(content, that.content) + case _ => false + + override def hashCode(): Int = + 31 * (31 * (31 * (31 * fileName.hashCode + mediaType.hashCode) + storedName.hashCode) + updatedAt.hashCode) + + Arrays.hashCode(content) + object UploadAttachment: /** The fallback content type for a file whose type the caller does not know. diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/GitignoreTemplate.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/GitignoreTemplate.scala index e81d943..ec60f29 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/GitignoreTemplate.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/GitignoreTemplate.scala @@ -17,7 +17,7 @@ package com.worxbend.codeberg4s.miscellaneous * the file's contents, verbatim, newlines and comments included. This is the answer to the question the call asked, * so a payload without it does not decode */ -final case class GitignoreTemplate( +final case class GitignoreTemplate private[codeberg4s] ( name: Option[String], source: String, ) diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/LicenseTemplate.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/LicenseTemplate.scala index 838dcd7..4076415 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/LicenseTemplate.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/LicenseTemplate.scala @@ -22,7 +22,7 @@ package com.worxbend.codeberg4s.miscellaneous * @param url * the API URL of the template itself */ -final case class LicenseTemplate( +final case class LicenseTemplate private[codeberg4s] ( body: String, name: Option[String], key: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/LicenseTemplateSummary.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/LicenseTemplateSummary.scala index 7bde36a..3325755 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/LicenseTemplateSummary.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/LicenseTemplateSummary.scala @@ -19,7 +19,7 @@ package com.worxbend.codeberg4s.miscellaneous * @param url * the API URL of the template itself, as the instance rendered it */ -final case class LicenseTemplateSummary( +final case class LicenseTemplateSummary private[codeberg4s] ( name: TemplateName, key: Option[String], url: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/NodeInfo.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/NodeInfo.scala index 8e66254..fcf4f80 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/NodeInfo.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/NodeInfo.scala @@ -36,7 +36,7 @@ package com.worxbend.codeberg4s.miscellaneous * @param hasOpenRegistrations * whether anyone may create an account without an invitation */ -final case class NodeInfo( +final case class NodeInfo private[codeberg4s] ( version: String, software: NodeInfoSoftware, protocols: Vector[String], @@ -57,7 +57,7 @@ final case class NodeInfo( * @param homepage * the project's own site */ -final case class NodeInfoSoftware( +final case class NodeInfoSoftware private[codeberg4s] ( name: String, version: Option[String], repository: Option[String], @@ -75,7 +75,7 @@ final case class NodeInfoSoftware( * @param outbound * services the instance can publish content to */ -final case class NodeInfoServices( +final case class NodeInfoServices private[codeberg4s] ( inbound: Vector[String], outbound: Vector[String], ) @@ -93,7 +93,7 @@ final case class NodeInfoServices( * @param localComments * comments created on this instance */ -final case class NodeInfoUsage( +final case class NodeInfoUsage private[codeberg4s] ( users: Option[NodeInfoUsers], localPosts: Option[Long], localComments: Option[Long], @@ -111,7 +111,7 @@ final case class NodeInfoUsage( * @param activeMonth * accounts that signed in within the last month */ -final case class NodeInfoUsers( +final case class NodeInfoUsers private[codeberg4s] ( total: Option[Long], activeHalfyear: Option[Long], activeMonth: Option[Long], diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/RenderedMarkdown.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/RenderedMarkdown.scala index d987dd5..cc5d7de 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/RenderedMarkdown.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/RenderedMarkdown.scala @@ -17,4 +17,4 @@ package com.worxbend.codeberg4s.miscellaneous * @param html * the response body verbatim, exactly as the instance rendered it */ -final case class RenderedMarkdown(html: String) +final case class RenderedMarkdown private[codeberg4s] (html: String) diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerApiSettings.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerApiSettings.scala index 8ac30ea..98865f8 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerApiSettings.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerApiSettings.scala @@ -22,7 +22,7 @@ package com.worxbend.codeberg4s.miscellaneous * @param maxBlobSizeBytes * the largest blob the contents endpoints will inline, in bytes. Absent on an instance that does not report it */ -final case class ServerApiSettings( +final case class ServerApiSettings private[codeberg4s] ( maxResponseItems: Long, defaultPagingNum: Long, gitTreesPerPage: Option[Long], diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerAttachmentSettings.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerAttachmentSettings.scala index ec31e4d..e795d11 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerAttachmentSettings.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerAttachmentSettings.scala @@ -17,7 +17,7 @@ package com.worxbend.codeberg4s.miscellaneous * @param maxFiles * how many attachments one upload may carry */ -final case class ServerAttachmentSettings( +final case class ServerAttachmentSettings private[codeberg4s] ( enabled: Boolean, allowedTypes: Vector[String], maxSizeMib: Option[Long], diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerRepositorySettings.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerRepositorySettings.scala index 1b169e2..1a1e7a0 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerRepositorySettings.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerRepositorySettings.scala @@ -26,7 +26,7 @@ package com.worxbend.codeberg4s.miscellaneous * @param lfsDisabled * Git LFS is not served */ -final case class ServerRepositorySettings( +final case class ServerRepositorySettings private[codeberg4s] ( mirrorsDisabled: Boolean, httpGitDisabled: Boolean, migrationsDisabled: Boolean, diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerUiSettings.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerUiSettings.scala index 38e741c..b6a39b1 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerUiSettings.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/ServerUiSettings.scala @@ -20,7 +20,7 @@ package com.worxbend.codeberg4s.miscellaneous * @param defaultTheme * the theme a signed-out visitor sees, absent when the instance did not report one */ -final case class ServerUiSettings( +final case class ServerUiSettings private[codeberg4s] ( allowedReactions: Vector[String], customEmojis: Vector[String], defaultTheme: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/SigningKey.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/SigningKey.scala index 0452b17..0ffaedc 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/SigningKey.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/SigningKey.scala @@ -10,7 +10,7 @@ package com.worxbend.codeberg4s.miscellaneous * @param armored * the ASCII-armored key block exactly as the instance sent it, never blank */ -final case class SigningKey(armored: String) +final case class SigningKey private[codeberg4s] (armored: String) object SigningKey: diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/SshSigningKey.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/SshSigningKey.scala index 0e1a64c..cec356d 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/SshSigningKey.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/SshSigningKey.scala @@ -16,7 +16,7 @@ package com.worxbend.codeberg4s.miscellaneous * @param openSsh * the public key in OpenSSH authorized-key format, exactly as the instance sent it, never blank */ -final case class SshSigningKey(openSsh: String) +final case class SshSigningKey private[codeberg4s] (openSsh: String) object SshSigningKey: diff --git a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/TemplateLabel.scala b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/TemplateLabel.scala index 3848ebb..f4ede9d 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/TemplateLabel.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/miscellaneous/TemplateLabel.scala @@ -33,7 +33,7 @@ import com.worxbend.codeberg4s.issues.LabelColor * @param isExclusive * whether the seeded label would be exclusive within its `scope/` prefix */ -final case class TemplateLabel( +final case class TemplateLabel private[codeberg4s] ( name: String, color: Option[LabelColor], description: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/notifications/NotificationSubject.scala b/modules/domain/src/com/worxbend/codeberg4s/notifications/NotificationSubject.scala index 39f8f0d..e00f4d2 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/notifications/NotificationSubject.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/notifications/NotificationSubject.scala @@ -41,7 +41,7 @@ package com.worxbend.codeberg4s.notifications * @param latestCommentHtmlUrl * the browser URL of the most recent comment, verbatim, with the same empty-string convention */ -final case class NotificationSubject( +final case class NotificationSubject private[codeberg4s] ( subjectType: NotificationSubjectType, title: Option[String], state: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/notifications/NotificationThread.scala b/modules/domain/src/com/worxbend/codeberg4s/notifications/NotificationThread.scala index 6dfb5b7..55c1156 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/notifications/NotificationThread.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/notifications/NotificationThread.scala @@ -50,7 +50,7 @@ import java.time.Instant * when the thread last changed. Absent when the instance omitted it or sent one of Forgejo's zero-time sentinels; * see [[com.worxbend.codeberg4s.codec.Timestamps]] */ -final case class NotificationThread( +final case class NotificationThread private[codeberg4s] ( id: NotificationThreadId, subject: Option[NotificationSubject], repository: Option[Repository], diff --git a/modules/domain/src/com/worxbend/codeberg4s/organizations/BlockedUser.scala b/modules/domain/src/com/worxbend/codeberg4s/organizations/BlockedUser.scala index 65f745a..7f3376b 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/organizations/BlockedUser.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/organizations/BlockedUser.scala @@ -53,4 +53,4 @@ object BlockId: * @param createdAt * when the block was recorded, absent when the instance sent no timestamp or the Go zero-time sentinel */ -final case class BlockedUser(blockId: BlockId, createdAt: Option[Instant]) +final case class BlockedUser private[codeberg4s] (blockId: BlockId, createdAt: Option[Instant]) diff --git a/modules/domain/src/com/worxbend/codeberg4s/organizations/OrgName.scala b/modules/domain/src/com/worxbend/codeberg4s/organizations/OrgName.scala index dee8a5d..6ab003a 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/organizations/OrgName.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/organizations/OrgName.scala @@ -1,5 +1,6 @@ package com.worxbend.codeberg4s.organizations +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError /** The handle that names an organisation — the `{org}` of `/orgs/{org}`. @@ -23,14 +24,17 @@ object OrgName: /** Parses an organisation name. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..`. * * '''This is a security boundary, not a convenience.''' An `OrgName` is interpolated into a request path, so a value * containing `/` would let a caller reach an endpoint the API surface never offered — `client.organizations.get` on * `"forgejo/../../admin"` — and a control character would corrupt the request line. Both are rejected here, once, - * rather than at each call site. Forgejo's own rules for what an organisation may be called are narrower still, but - * they are the instance's business: this type promises only that the value cannot forge a path. + * rather than at each call site. A bare `.` or `..` is rejected for the same reason and needs saying separately: it + * carries no slash, so the slash rule never sees it, and it survives percent-encoding untouched, so it would reach + * the request path as a dot segment rather than as an organisation name. Forgejo's own rules for what an + * organisation may be called are narrower still, but they are the instance's business: this type promises only that + * the value cannot forge a path. * * The rules are deliberately no narrower than that, because `golden/organization/org-list.json` contains * organisations named `_CYBER_STONES_`, `-_` and `-_-`. An identifier validator that assumed alphanumerics would @@ -40,11 +44,7 @@ object OrgName: * the trimmed name, or a [[ValidationError]] on the `"orgName"` field */ def from(value: String): Either[ValidationError, OrgName] = - val trimmed = value.trim - if trimmed.isEmpty then Left(ValidationError(Field, "must not be blank")) - else if trimmed.contains('/') then Left(ValidationError(Field, "must not contain a slash")) - else if trimmed.exists(_.isControl) then Left(ValidationError(Field, "must not contain a control character")) - else Right(trimmed) + PathSegment.from(Field, value) extension (name: OrgName) diff --git a/modules/domain/src/com/worxbend/codeberg4s/organizations/Organization.scala b/modules/domain/src/com/worxbend/codeberg4s/organizations/Organization.scala index df96ae5..3215660 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/organizations/Organization.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/organizations/Organization.scala @@ -49,7 +49,7 @@ import java.time.Instant * @param createdAt * when the organisation was created */ -final case class Organization( +final case class Organization private[codeberg4s] ( id: Long, name: OrgName, fullName: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/organizations/OrganizationPermissions.scala b/modules/domain/src/com/worxbend/codeberg4s/organizations/OrganizationPermissions.scala index 4cbd803..900f76e 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/organizations/OrganizationPermissions.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/organizations/OrganizationPermissions.scala @@ -31,7 +31,7 @@ package com.worxbend.codeberg4s.organizations * @param canCreateRepository * whether the account may create a repository under the organisation */ -final case class OrganizationPermissions( +final case class OrganizationPermissions private[codeberg4s] ( isOwner: Boolean, isAdmin: Boolean, canWrite: Boolean, diff --git a/modules/domain/src/com/worxbend/codeberg4s/organizations/QuotaInfo.scala b/modules/domain/src/com/worxbend/codeberg4s/organizations/QuotaInfo.scala index dd0d499..e8fa2a3 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/organizations/QuotaInfo.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/organizations/QuotaInfo.scala @@ -27,7 +27,7 @@ package com.worxbend.codeberg4s.organizations * @param used * what the organisation has consumed; [[QuotaUsage.Empty]] when the payload said nothing */ -final case class QuotaInfo(groups: Vector[QuotaGroup], used: QuotaUsage) +final case class QuotaInfo private[codeberg4s] (groups: Vector[QuotaGroup], used: QuotaUsage) /** One quota group — a named bundle of rules Forgejo applies together. * @@ -37,7 +37,7 @@ final case class QuotaInfo(groups: Vector[QuotaGroup], used: QuotaUsage) * @param rules * the rules in the group, empty when none came back */ -final case class QuotaGroup(name: Option[String], rules: Vector[QuotaRule]) +final case class QuotaGroup private[codeberg4s] (name: Option[String], rules: Vector[QuotaRule]) /** One quota rule — a limit, and the subjects it counts towards. * @@ -53,10 +53,10 @@ final case class QuotaGroup(name: Option[String], rules: Vector[QuotaRule]) * newer Forgejo emits must not cost the caller the whole rule. Converting one for use as a query argument is the * caller's explicit step through [[QuotaSubject.from]] */ -final case class QuotaRule(name: Option[String], limit: Option[Long], subjects: Vector[String]) +final case class QuotaRule private[codeberg4s] (name: Option[String], limit: Option[Long], subjects: Vector[String]) /** What an organisation has consumed. One field today, because `QuotaUsed` declares one property. */ -final case class QuotaUsage(size: QuotaSizes) +final case class QuotaUsage private[codeberg4s] (size: QuotaSizes) object QuotaUsage: @@ -72,7 +72,11 @@ object QuotaUsage: * @param git * Git object storage that is billed separately, which today means LFS */ -final case class QuotaSizes(repositories: QuotaRepositorySizes, assets: QuotaAssetSizes, git: QuotaGitSizes) +final case class QuotaSizes private[codeberg4s] ( + repositories: QuotaRepositorySizes, + assets: QuotaAssetSizes, + git: QuotaGitSizes, +) object QuotaSizes: @@ -86,7 +90,7 @@ object QuotaSizes: * @param privateBytes * storage used by private repositories */ -final case class QuotaRepositorySizes(publicBytes: Option[Long], privateBytes: Option[Long]) +final case class QuotaRepositorySizes private[codeberg4s] (publicBytes: Option[Long], privateBytes: Option[Long]) object QuotaRepositorySizes: @@ -102,7 +106,7 @@ object QuotaRepositorySizes: * @param packageBytes * storage used by packages — the spec's `packages.all`, which is the only property that object has */ -final case class QuotaAssetSizes( +final case class QuotaAssetSizes private[codeberg4s] ( artifactBytes: Option[Long], attachments: QuotaAttachmentSizes, packageBytes: Option[Long], @@ -120,7 +124,7 @@ object QuotaAssetSizes: * @param releaseBytes * storage used by attachments on releases */ -final case class QuotaAttachmentSizes(issueBytes: Option[Long], releaseBytes: Option[Long]) +final case class QuotaAttachmentSizes private[codeberg4s] (issueBytes: Option[Long], releaseBytes: Option[Long]) object QuotaAttachmentSizes: @@ -133,7 +137,7 @@ object QuotaAttachmentSizes: * storage used by Git LFS objects. The wire key is `LFS`, in capitals — the one upper-case key in this whole model, * because Go's field is `LFS` and the spec's generator left it alone */ -final case class QuotaGitSizes(lfsBytes: Option[Long]) +final case class QuotaGitSizes private[codeberg4s] (lfsBytes: Option[Long]) object QuotaGitSizes: diff --git a/modules/domain/src/com/worxbend/codeberg4s/organizations/QuotaUsedItems.scala b/modules/domain/src/com/worxbend/codeberg4s/organizations/QuotaUsedItems.scala index 1b27876..2530f63 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/organizations/QuotaUsedItems.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/organizations/QuotaUsedItems.scala @@ -20,7 +20,11 @@ package com.worxbend.codeberg4s.organizations * @param htmlUrl * a browser link to the action run containing the artifact */ -final case class QuotaArtifact(name: Option[String], sizeBytes: Option[Long], htmlUrl: Option[String]) +final case class QuotaArtifact private[codeberg4s] ( + name: Option[String], + sizeBytes: Option[Long], + htmlUrl: Option[String], +) /** Where an attachment hangs — the `contained_in` object of Forgejo's `QuotaUsedAttachment`. * @@ -32,7 +36,7 @@ final case class QuotaArtifact(name: Option[String], sizeBytes: Option[Long], ht * @param htmlUrl * the browser URL of the containing object */ -final case class QuotaAttachmentContext(apiUrl: Option[String], htmlUrl: Option[String]) +final case class QuotaAttachmentContext private[codeberg4s] (apiUrl: Option[String], htmlUrl: Option[String]) /** One attachment counting towards an organisation's quota — Forgejo's `QuotaUsedAttachment`. * @@ -48,7 +52,7 @@ final case class QuotaAttachmentContext(apiUrl: Option[String], htmlUrl: Option[ * @param containedIn * what it is attached to, absent when the instance sent no context object */ -final case class QuotaAttachment( +final case class QuotaAttachment private[codeberg4s] ( name: Option[String], sizeBytes: Option[Long], apiUrl: Option[String], @@ -72,7 +76,7 @@ final case class QuotaAttachment( * @param htmlUrl * a browser link to the package version */ -final case class QuotaPackage( +final case class QuotaPackage private[codeberg4s] ( name: Option[String], version: Option[String], packageType: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/organizations/Team.scala b/modules/domain/src/com/worxbend/codeberg4s/organizations/Team.scala index c07bd3e..0fc2835 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/organizations/Team.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/organizations/Team.scala @@ -44,7 +44,7 @@ package com.worxbend.codeberg4s.organizations * @param includesAllRepositories * whether the team reaches every repository of the organisation, present and future, rather than an explicit list */ -final case class Team( +final case class Team private[codeberg4s] ( id: TeamId, name: String, description: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/pulls/ChangedFile.scala b/modules/domain/src/com/worxbend/codeberg4s/pulls/ChangedFile.scala index 03ed60f..6f79343 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/pulls/ChangedFile.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/pulls/ChangedFile.scala @@ -35,7 +35,7 @@ import com.worxbend.codeberg4s.repositories.CommitFileStatus * @param rawUrl * the browser URL of the raw file at that same commit */ -final case class ChangedFile( +final case class ChangedFile private[codeberg4s] ( filename: String, status: Option[CommitFileStatus], additions: Long, diff --git a/modules/domain/src/com/worxbend/codeberg4s/pulls/PullRequest.scala b/modules/domain/src/com/worxbend/codeberg4s/pulls/PullRequest.scala index 3c0baff..2b9b10a 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/pulls/PullRequest.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/pulls/PullRequest.scala @@ -73,7 +73,7 @@ import java.time.Instant * @param dueDate * the deadline set on the pull request, absent when it has none */ -final case class PullRequest( +final case class PullRequest private[codeberg4s] ( id: Long, number: PullRequestNumber, title: String, diff --git a/modules/domain/src/com/worxbend/codeberg4s/pulls/PullRequestBranch.scala b/modules/domain/src/com/worxbend/codeberg4s/pulls/PullRequestBranch.scala index 6da434a..e23c919 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/pulls/PullRequestBranch.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/pulls/PullRequestBranch.scala @@ -29,7 +29,7 @@ import com.worxbend.codeberg4s.repositories.Repository * @param repository * the repository this end lives in, when the endpoint supplied it */ -final case class PullRequestBranch( +final case class PullRequestBranch private[codeberg4s] ( label: Option[String], ref: Option[BranchName], sha: Option[CommitSha], diff --git a/modules/domain/src/com/worxbend/codeberg4s/pulls/Review.scala b/modules/domain/src/com/worxbend/codeberg4s/pulls/Review.scala index 277e78f..334a8be 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/pulls/Review.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/pulls/Review.scala @@ -42,7 +42,7 @@ import java.time.Instant * the browser URL of the review, absent for a review request — `""` on the fixture's two request rows and a real * anchor on the approval */ -final case class Review( +final case class Review private[codeberg4s] ( id: ReviewId, state: Option[ReviewState], body: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/pulls/ReviewComment.scala b/modules/domain/src/com/worxbend/codeberg4s/pulls/ReviewComment.scala index 08cc013..0f4a744 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/pulls/ReviewComment.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/pulls/ReviewComment.scala @@ -56,7 +56,7 @@ import java.time.Instant * @param resolver * the account that marked the conversation resolved, absent while it is still open */ -final case class ReviewComment( +final case class ReviewComment private[codeberg4s] ( id: ReviewCommentId, reviewId: Option[ReviewId], body: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/ArchiveDownloadCount.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/ArchiveDownloadCount.scala index 42da554..829bd59 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/ArchiveDownloadCount.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/ArchiveDownloadCount.scala @@ -10,4 +10,4 @@ package com.worxbend.codeberg4s.repositories * @param tarGz * downloads of the `.tar.gz` archive */ -final case class ArchiveDownloadCount(zip: Long, tarGz: Long) +final case class ArchiveDownloadCount private[codeberg4s] (zip: Long, tarGz: Long) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/Branch.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/Branch.scala index 9aa7ce8..24025df 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/Branch.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/Branch.scala @@ -26,7 +26,7 @@ package com.worxbend.codeberg4s.repositories * the name of the protection rule that matched, absent when none did. Forgejo sends `""` rather than `null` for * "none", and that spelling is folded away before it reaches this model */ -final case class Branch( +final case class Branch private[codeberg4s] ( name: BranchName, commit: CommitSummary, isProtected: Boolean, diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/BranchName.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/BranchName.scala index b7297b2..8693968 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/BranchName.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/BranchName.scala @@ -1,5 +1,6 @@ package com.worxbend.codeberg4s.repositories +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError /** The name of a branch, as `GET /repos/{owner}/{repo}/branches/{branch}` spells it. diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/Commit.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/Commit.scala index 37667cc..4f9ca88 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/Commit.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/Commit.scala @@ -36,7 +36,7 @@ import java.time.Instant * @param stats * the line counts, when the endpoint reports them */ -final case class Commit( +final case class Commit private[codeberg4s] ( sha: CommitSha, url: Option[String], htmlUrl: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitDetails.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitDetails.scala index a815ba8..bb45632 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitDetails.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitDetails.scala @@ -20,7 +20,7 @@ package com.worxbend.codeberg4s.repositories * @param verification * the instance's signature verdict, when it reports one */ -final case class CommitDetails( +final case class CommitDetails private[codeberg4s] ( message: Option[String], url: Option[String], author: Option[GitIdentity], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitFile.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitFile.scala index f743e99..9f2f5be 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitFile.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitFile.scala @@ -11,4 +11,4 @@ package com.worxbend.codeberg4s.repositories * what the commit did to the file, absent when the instance sent a value this library does not recognise; see * [[CommitFileStatus.parse]] */ -final case class CommitFile(filename: String, status: Option[CommitFileStatus]) +final case class CommitFile private[codeberg4s] (filename: String, status: Option[CommitFileStatus]) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitRef.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitRef.scala index ef5e496..0c8f92e 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitRef.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitRef.scala @@ -16,4 +16,4 @@ import java.time.Instant * the commit time, when the endpoint reports one. Forgejo sends its zero-time sentinel here on trees, and * [[com.worxbend.codeberg4s.codec.Timestamps]] folds that into `None` */ -final case class CommitRef(sha: CommitSha, url: Option[String], created: Option[Instant]) +final case class CommitRef private[codeberg4s] (sha: CommitSha, url: Option[String], created: Option[Instant]) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitStats.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitStats.scala index 6a9179b..9c242a7 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitStats.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitStats.scala @@ -9,4 +9,4 @@ package com.worxbend.codeberg4s.repositories * @param deletions * lines removed */ -final case class CommitStats(total: Long, additions: Long, deletions: Long) +final case class CommitStats private[codeberg4s] (total: Long, additions: Long, deletions: Long) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitSummary.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitSummary.scala index 0d5205b..792a2d9 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitSummary.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitSummary.scala @@ -26,7 +26,7 @@ import java.time.Instant * @param timestamp * when the commit was made */ -final case class CommitSummary( +final case class CommitSummary private[codeberg4s] ( sha: CommitSha, message: Option[String], url: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitVerification.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitVerification.scala index d3d0da8..6ef0164 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitVerification.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/CommitVerification.scala @@ -19,7 +19,7 @@ package com.worxbend.codeberg4s.repositories * the exact bytes that were signed — the commit object as Git serialises it. Present so a caller can verify * independently rather than take [[isVerified]] on trust */ -final case class CommitVerification( +final case class CommitVerification private[codeberg4s] ( isVerified: Boolean, reason: Option[String], signature: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/ContentMeta.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/ContentMeta.scala index e4a936a..e58a2fb 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/ContentMeta.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/ContentMeta.scala @@ -26,7 +26,7 @@ import java.time.Instant * @param gitUrl * the API URL of the underlying Git object */ -final case class ContentMeta( +final case class ContentMeta private[codeberg4s] ( name: String, path: ContentPath, sha: CommitSha, diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/ContentPath.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/ContentPath.scala index 3ceb0c0..c54e53e 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/ContentPath.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/ContentPath.scala @@ -1,5 +1,6 @@ package com.worxbend.codeberg4s.repositories +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError /** A path to a file or directory inside a repository, as `GET /repos/{owner}/{repo}/contents/{filepath}` spells it. @@ -33,5 +34,10 @@ object ContentPath: /** The path split on `/`, for appending to a request path one segment at a time. */ def segments: List[String] = path.split('/').toList - /** The last segment — the file or directory's own name. */ - def name: String = path.split('/').lastOption.getOrElse(path) + /** The last segment — the file or directory's own name. + * + * `models/user.go` gives `user.go`; a path with no `/` at all is already its own name and is returned unchanged. + * Read off the last `/` rather than by splitting, so the answer costs one substring instead of an array holding + * every segment of the path. [[from]] has already rejected a trailing `/`, so the last segment is never empty. + */ + def name: String = path.substring(path.lastIndexOf('/') + 1) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/GitIdentity.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/GitIdentity.scala index 9795a2f..94274bb 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/GitIdentity.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/GitIdentity.scala @@ -25,7 +25,7 @@ import java.time.Instant * @param date * when the authorship or the commit was recorded, when the endpoint reports it */ -final case class GitIdentity( +final case class GitIdentity private[codeberg4s] ( name: Option[String], email: Option[String], username: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/Owner.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/Owner.scala index 9f3d6b3..f525c6a 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/Owner.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/Owner.scala @@ -1,5 +1,6 @@ package com.worxbend.codeberg4s.repositories +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError /** The user or organisation that owns a repository — the first segment of `owner/name`. @@ -13,8 +14,8 @@ object Owner: /** Parses an owner. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..`. * * @return * the trimmed owner, or a [[ValidationError]] on the `"owner"` field diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/PathSegment.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/PathSegment.scala deleted file mode 100644 index 3d9c11a..0000000 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/PathSegment.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.worxbend.codeberg4s.repositories - -import com.worxbend.codeberg4s.ValidationError - -/** Validation shared by every identifier that becomes part of a URI path. - * - * This is a security boundary, not a convenience: [[Owner]] and [[RepoName]] are interpolated into request paths, so a - * value containing `/` would let a caller reach an endpoint the API surface never offered, and a control character - * would corrupt the request line. Both are rejected here, once, rather than at each call site. - * - * [[from]] is for an identifier that must occupy exactly one segment. [[segmented]] is for the two that legitimately - * span several — [[BranchName]] and [[ContentPath]], whose routes Forgejo matches with a wildcard — and it rejects the - * traversal segments that would otherwise turn a slash into an escape hatch. - */ -private[repositories] object PathSegment: - - /** Trims `value` and accepts it only if it can stand alone as one path segment. - * - * Rejects an empty or blank value, a value containing `/`, and a value containing any control character. - * - * @param field - * the field name to report in a [[ValidationError]] - */ - def from(field: String, value: String): Either[ValidationError, String] = - val trimmed = value.trim - if trimmed.isEmpty then Left(ValidationError(field, "must not be blank")) - else if trimmed.contains('/') then Left(ValidationError(field, "must not contain a slash")) - else if trimmed.exists(_.isControl) then Left(ValidationError(field, "must not contain a control character")) - else Right(trimmed) - - /** Trims `value` and accepts it only if every `/`-separated part can stand alone as one path segment. - * - * Rejects an empty or blank value, any control character, a leading or trailing `/`, an empty segment such as the - * middle of `a//b`, and a `.` or `..` segment. The last of those is the one that matters: `..` survives - * percent-encoding untouched, so a value carrying it would walk out of the route it was meant for. - * - * @param field - * the field name to report in a [[ValidationError]] - */ - def segmented(field: String, value: String): Either[ValidationError, String] = - val trimmed = value.trim - if trimmed.isEmpty then Left(ValidationError(field, "must not be blank")) - else if trimmed.exists(_.isControl) then Left(ValidationError(field, "must not contain a control character")) - else if trimmed.startsWith("/") || trimmed.endsWith("/") then - Left(ValidationError(field, "must not start or end with a slash")) - else if trimmed.split('/').exists(_.isEmpty) then Left(ValidationError(field, "must not contain an empty segment")) - else if trimmed.split('/').exists(isTraversal) then - Left(ValidationError(field, "must not contain a '.' or '..' segment")) - else Right(trimmed) - - private def isTraversal(segment: String): Boolean = - segment match - case "." | ".." => true - case _ => false diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/Release.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/Release.scala index 2ac56b8..30ab84c 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/Release.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/Release.scala @@ -48,7 +48,7 @@ import java.time.Instant * @param archiveDownloads * how often the generated source archives have been downloaded, when the instance reports it */ -final case class Release( +final case class Release private[codeberg4s] ( id: ReleaseId, tagName: TagName, targetCommitish: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/ReleaseAsset.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/ReleaseAsset.scala index 71e8db8..1b7fabc 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/ReleaseAsset.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/ReleaseAsset.scala @@ -26,7 +26,7 @@ import java.time.Instant * where the asset can be fetched. Downloading it is outside this library: the body is arbitrarily large and belongs * in a stream, not in a `String` */ -final case class ReleaseAsset( +final case class ReleaseAsset private[codeberg4s] ( id: Long, name: String, size: Long, diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/RepoName.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/RepoName.scala index 87dfd13..876a807 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/RepoName.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/RepoName.scala @@ -1,5 +1,6 @@ package com.worxbend.codeberg4s.repositories +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError /** The repository half of `owner/name`. @@ -12,8 +13,8 @@ object RepoName: /** Parses a repository name. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..`. * * @return * the trimmed name, or a [[ValidationError]] on the `"repoName"` field diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/Repository.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/Repository.scala index 700c722..8719721 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/Repository.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/Repository.scala @@ -30,7 +30,7 @@ import java.time.Instant * @param archivedAt * absent unless the repository is archived; Forgejo sends the Unix epoch as its "never" sentinel */ -final case class Repository( +final case class Repository private[codeberg4s] ( id: Long, slug: RepoSlug, fullName: String, diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/RepositoryPermissions.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/RepositoryPermissions.scala index b3a3b4a..ace2c8a 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/RepositoryPermissions.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/RepositoryPermissions.scala @@ -13,4 +13,4 @@ package com.worxbend.codeberg4s.repositories * @param pull * may read; `true` for any repository a caller can see at all */ -final case class RepositoryPermissions(admin: Boolean, push: Boolean, pull: Boolean) +final case class RepositoryPermissions private[codeberg4s] (admin: Boolean, push: Boolean, pull: Boolean) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/Tag.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/Tag.scala index 6a97d9b..334f220 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/Tag.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/Tag.scala @@ -21,7 +21,7 @@ package com.worxbend.codeberg4s.repositories * @param archiveDownloads * how often those archives have been downloaded, when the instance reports it */ -final case class Tag( +final case class Tag private[codeberg4s] ( name: TagName, message: Option[String], commitSha: CommitSha, diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/TagName.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/TagName.scala index 39ad698..cdbeea9 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/TagName.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/TagName.scala @@ -1,5 +1,6 @@ package com.worxbend.codeberg4s.repositories +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError /** The name of a Git tag — `v16.0.2` on `golden/repository/tags-list.json`. diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/access/AccessNames.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/access/AccessNames.scala index 457119a..25b8f82 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/access/AccessNames.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/access/AccessNames.scala @@ -1,7 +1,7 @@ package com.worxbend.codeberg4s.repositories.access +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError -import com.worxbend.codeberg4s.repositories.PathSegment /** The name that addresses one branch protection rule — the `{name}` of * `/repos/{owner}/{repo}/branch_protections/{name}`, and the `rule_name` a rule reports. @@ -40,9 +40,10 @@ object BranchRuleName: /** Parses a branch protection rule name. * - * Trims surrounding whitespace. Rejects an empty or blank name, a name containing `/`, and a name containing a - * control character — see [[com.worxbend.codeberg4s.repositories.PathSegment]] for why that is a security boundary - * and not a convenience, and the type's own note for why the slash is rejected rather than encoded. + * Trims surrounding whitespace. Rejects an empty or blank name, a name containing `/`, a name containing a control + * character, and the traversal segments `.` and `..` — see [[com.worxbend.codeberg4s.repositories.PathSegment]] for + * why that is a security boundary and not a convenience, and the type's own note for why the slash is rejected + * rather than encoded. * * Glob characters are '''not''' rejected: `*` and `?` are what a rule name is made of, and they are legal in a URI * path segment. @@ -113,8 +114,8 @@ object TeamName: /** Parses a team name. * - * Trims surrounding whitespace. Rejects an empty or blank name, a name containing `/`, and a name containing a - * control character. + * Trims surrounding whitespace. Rejects an empty or blank name, a name containing `/`, a name containing a control + * character, and the traversal segments `.` and `..`. * * @return * the trimmed name, or a [[ValidationError]] on the `"teamName"` field diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/access/BranchProtection.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/access/BranchProtection.scala index 0042d2b..474cb67 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/access/BranchProtection.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/access/BranchProtection.scala @@ -46,7 +46,7 @@ import java.time.Instant * @param applyToAdmins * whether repository administrators are bound by this rule too. `false` — the default reading — means they are not */ -final case class BranchProtection( +final case class BranchProtection private[codeberg4s] ( ruleName: String, legacyBranchName: Option[String], enablePush: Boolean, diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/access/CollaboratorAccess.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/access/CollaboratorAccess.scala index 74ed39b..af4cbb8 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/access/CollaboratorAccess.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/access/CollaboratorAccess.scala @@ -36,7 +36,7 @@ import com.worxbend.codeberg4s.users.User * Forgejo's display name for the role, such as `Owner` or `Collaborator`. It is instance-configurable and localised, * so it stays text rather than becoming an enum this library would have to keep in step with a deployment */ -final case class CollaboratorAccess( +final case class CollaboratorAccess private[codeberg4s] ( user: User, permission: Option[TeamPermission], rawPermission: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/access/DeployKey.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/access/DeployKey.scala index 6709b67..8fc95bb 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/access/DeployKey.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/access/DeployKey.scala @@ -46,7 +46,7 @@ import java.time.Instant * model may over-state a key's power, never under-state it. Forgejo sends the field on every real payload, so the * case is theoretical */ -final case class DeployKey( +final case class DeployKey private[codeberg4s] ( id: DeployKeyId, key: String, keyId: Option[Long], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/access/TagProtection.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/access/TagProtection.scala index b7fb498..1539dd2 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/access/TagProtection.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/access/TagProtection.scala @@ -28,7 +28,7 @@ import java.time.Instant * @param whitelistTeams * the teams exempt from the rule, on the same reading */ -final case class TagProtection( +final case class TagProtection private[codeberg4s] ( id: TagProtectionId, namePattern: String, whitelistUsernames: Vector[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionArtifact.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionArtifact.scala index 2a7b5d3..089f582 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionArtifact.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionArtifact.scala @@ -11,11 +11,11 @@ import java.time.Instant * * ==Downloading== * - * This library does '''not''' implement `GET /repos/{owner}/{repo}/actions/artifacts/{artifact_id}/zip`, because it - * cannot do so honestly: the transport reads every response as text, and a ZIP that has been through a UTF-8 decoder - * is no longer a ZIP. [[archiveDownloadUrl]] is the supported route — hand it to an HTTP client that can stream bytes. - * Note that the URL is authenticated exactly like the API is, so the caller's own client must send the same - * credentials. + * `GET /repos/{owner}/{repo}/actions/artifacts/{artifact_id}/zip` is implemented, on + * `com.worxbend.codeberg4s.repositories.actions.ActionDownloadApi` — reached as `client.downloads`. It holds the whole + * archive in memory, because nothing in this library streams, so [[archiveDownloadUrl]] remains the route for an + * archive too large to want in the heap: hand it to an HTTP client that can stream bytes. Note that the URL is + * authenticated exactly like the API is, so the caller's own client must send the same credentials. * * @param id * the identifier the artifact endpoints address this artifact by @@ -33,7 +33,7 @@ import java.time.Instant * @param expiresAt * when the bytes are removed, absent when the instance did not report a retention window */ -final case class ActionArtifact( +final case class ActionArtifact private[codeberg4s] ( id: ArtifactId, name: Option[String], sizeInBytes: Option[Long], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionNames.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionNames.scala index 88002af..903bdbb 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionNames.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionNames.scala @@ -1,7 +1,7 @@ package com.worxbend.codeberg4s.repositories.actions +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError -import com.worxbend.codeberg4s.repositories.PathSegment /** The identifier of a registered runner, as the runner endpoints take it in a path. * @@ -17,9 +17,9 @@ object RunnerId: /** Parses a runner identifier from its string spelling. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character — see [[com.worxbend.codeberg4s.repositories.PathSegment]] for why that is a security boundary - * and not a convenience. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..` — see + * [[com.worxbend.codeberg4s.repositories.PathSegment]] for why that is a security boundary and not a convenience. * * @return * the identifier, or a [[ValidationError]] on the `"runnerId"` field @@ -55,9 +55,10 @@ object SecretName: /** Parses a secret name. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character. Forgejo applies further rules of its own — it rejects a name that starts with a digit, and one - * that uses a reserved `GITHUB_` or `GITEA_` prefix — which arrive as a `400`, not as a [[ValidationError]]. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..`. Forgejo applies further rules of its own — it rejects + * a name that starts with a digit, and one that uses a reserved `GITHUB_` or `GITEA_` prefix — which arrive as a + * `400`, not as a [[ValidationError]]. * * @return * the name, or a [[ValidationError]] on the `"secretName"` field @@ -81,8 +82,9 @@ object VariableName: /** Parses a variable name. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character. As with [[SecretName]], Forgejo's own naming rules are enforced remotely and arrive as a `400`. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..`. As with [[SecretName]], Forgejo's own naming rules are + * enforced remotely and arrive as a `400`. * * @return * the name, or a [[ValidationError]] on the `"variableName"` field @@ -107,9 +109,9 @@ object WorkflowFileName: /** Parses a workflow file name. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character. The extension is not checked: Forgejo accepts both `.yml` and `.yaml`, and a name this library - * refused would be a workflow the caller could not dispatch. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..`. The extension is not checked: Forgejo accepts both + * `.yml` and `.yaml`, and a name this library refused would be a workflow the caller could not dispatch. * * @return * the name, or a [[ValidationError]] on the `"workflowFileName"` field diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRun.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRun.scala index c600066..6ae14b7 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRun.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRun.scala @@ -57,7 +57,7 @@ import java.time.Instant * @param stoppedAt * when the run finished, absent while it is still going */ -final case class ActionRun( +final case class ActionRun private[codeberg4s] ( id: RunId, indexInRepo: Option[Long], title: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRunJob.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRunJob.scala index b48bb86..d005aa1 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRunJob.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRunJob.scala @@ -34,7 +34,7 @@ package com.worxbend.codeberg4s.repositories.actions * @param repoId * the numeric id of the repository the job ran for */ -final case class ActionRunJob( +final case class ActionRunJob private[codeberg4s] ( id: JobId, name: Option[String], runId: Option[RunId], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRunner.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRunner.scala index 93d1675..304704c 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRunner.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionRunner.scala @@ -32,7 +32,7 @@ package com.worxbend.codeberg4s.repositories.actions * the repository this runner belongs to; `0` on the wire, and absent here, when it belongs to a user or an * organisation */ -final case class ActionRunner( +final case class ActionRunner private[codeberg4s] ( id: RunnerId, uuid: Option[String], name: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionSecret.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionSecret.scala index 5940737..c224425 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionSecret.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionSecret.scala @@ -25,7 +25,7 @@ import java.time.Instant * when the secret was first set. Forgejo reports no modification time, so a secret that has been rewritten looks * exactly like one that has not */ -final case class ActionSecret( +final case class ActionSecret private[codeberg4s] ( name: SecretName, createdAt: Option[Instant], ) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionTask.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionTask.scala index 2cd492a..8d3117b 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionTask.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionTask.scala @@ -33,7 +33,7 @@ import java.time.Instant * @param runStartedAt * when execution began, absent while the task is still queued */ -final case class ActionTask( +final case class ActionTask private[codeberg4s] ( id: TaskId, name: Option[String], status: Option[ActionStatus], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionVariable.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionVariable.scala index e79841f..ff20df4 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionVariable.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/ActionVariable.scala @@ -20,7 +20,7 @@ package com.worxbend.codeberg4s.repositories.actions * @param repoId * the repository the variable belongs to; `0` on the wire, and absent here, for an owner-level variable */ -final case class ActionVariable( +final case class ActionVariable private[codeberg4s] ( name: VariableName, value: String, ownerId: Option[Long], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/DispatchedWorkflowRun.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/DispatchedWorkflowRun.scala index 17aa384..57355d5 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/DispatchedWorkflowRun.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/DispatchedWorkflowRun.scala @@ -20,7 +20,7 @@ package com.worxbend.codeberg4s.repositories.actions * @param jobs * the names of the jobs the run will execute, as the workflow file spells them */ -final case class DispatchedWorkflowRun( +final case class DispatchedWorkflowRun private[codeberg4s] ( id: Option[RunId], runNumber: Option[Long], jobs: Vector[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/RegisteredRunner.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/RegisteredRunner.scala index c575ec8..1921f34 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/RegisteredRunner.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/RegisteredRunner.scala @@ -21,7 +21,7 @@ package com.worxbend.codeberg4s.repositories.actions * the one-shot registration credential. Required: a registration response without it registers nothing, so decoding * fails rather than handing back a runner nobody can start */ -final case class RegisteredRunner( +final case class RegisteredRunner private[codeberg4s] ( id: Option[RunnerId], uuid: Option[String], token: RunnerRegistrationToken, diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/RunnerRegistrationToken.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/RunnerRegistrationToken.scala index eb74bba..4e397f3 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/RunnerRegistrationToken.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/actions/RunnerRegistrationToken.scala @@ -10,9 +10,11 @@ import com.worxbend.codeberg4s.ValidationError * `toString`, because an opaque alias over `String` cannot stop interpolation from printing it, and * [[RunnerRegistrationToken.reveal]] as the single way to observe the material. * - * Unlike [[SecretValue]] this one travels '''from''' the instance: it is decoded out of a response body, which means a - * decoding failure could otherwise have carried it into an error message. It cannot — see the class note on - * [[SecretValue]] for why — but the mask is what makes that true rather than merely likely. + * Unlike [[SecretValue]] this one travels '''from''' the instance: it is decoded out of a response body, so a decoding + * failure on that response would carry the raw payload — and therefore the material — into + * [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]], where no mask can reach it. The decoders that read the + * registration and registration-token responses are marked sensitive for that reason, and the pipeline reports a + * placeholder in place of the body excerpt on those two endpoints alone. * * Instances compare structurally on the underlying material. The comparison is not constant-time. */ diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/AdminIds.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/AdminIds.scala index 1a07f11..f66e736 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/AdminIds.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/AdminIds.scala @@ -1,7 +1,7 @@ package com.worxbend.codeberg4s.repositories.admin +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError -import com.worxbend.codeberg4s.repositories.PathSegment /** Validation shared by every identifier in this group that Forgejo expresses as a positive integer. * @@ -105,8 +105,8 @@ object MirrorName: /** Parses a push-mirror remote name. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..`. * * @return * the trimmed name, or a [[ValidationError]] on the `"mirrorName"` field diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/FileCommands.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/FileCommands.scala index 7a04696..e6026cb 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/FileCommands.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/FileCommands.scala @@ -330,7 +330,7 @@ object ChangeFiles: * @param verification * what Forgejo made of the commit's signature, when it signed one */ -final case class FileChangeSet( +final case class FileChangeSet private[codeberg4s] ( commit: Option[FileCommit], files: Vector[ContentEntry], verification: Option[CommitVerification], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/LanguageBreakdown.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/LanguageBreakdown.scala index 6fe1082..8701a12 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/LanguageBreakdown.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/LanguageBreakdown.scala @@ -17,7 +17,7 @@ import java.time.Instant * @param bytes * bytes per language name, exactly as the instance reported them */ -final case class LanguageBreakdown(bytes: Map[String, Long]): +final case class LanguageBreakdown private[codeberg4s] (bytes: Map[String, Long]): /** Every counted byte, across all languages. `0` for a repository with no analysis. */ def total: Long = bytes.values.sum @@ -56,7 +56,7 @@ object LanguageBreakdown: * @param updatedAt * when its repository count last changed */ -final case class TopicSummary( +final case class TopicSummary private[codeberg4s] ( id: TopicId, name: String, repositoryCount: Long, diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/PushMirror.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/PushMirror.scala index 0f4dbfe..44bd116 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/PushMirror.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/PushMirror.scala @@ -44,7 +44,7 @@ import java.time.Instant * @param lastUpdateAt * when it last ran, absent when it has never run */ -final case class PushMirror( +final case class PushMirror private[codeberg4s] ( remoteName: MirrorName, remoteAddress: Option[String], repoName: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/RepositoryActivity.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/RepositoryActivity.scala index 3dbb963..8bf277e 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/RepositoryActivity.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/RepositoryActivity.scala @@ -209,7 +209,7 @@ object ActivityOperation: * @param createdAt * when it happened */ -final case class RepositoryActivity( +final case class RepositoryActivity private[codeberg4s] ( id: ActivityId, actor: Option[User], operation: Option[ActivityOperation], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/WatchStatus.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/WatchStatus.scala index 5567571..6e09b95 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/WatchStatus.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/admin/WatchStatus.scala @@ -32,7 +32,7 @@ import java.time.Instant * @param createdAt * when the subscription started */ -final case class WatchStatus( +final case class WatchStatus private[codeberg4s] ( subscribed: Boolean, ignored: Boolean, reason: Option[String], @@ -64,7 +64,7 @@ final case class WatchStatus( * @param forkCommit * the commit this fork's branch is at, on the same terms as [[baseCommit]] */ -final case class ForkSyncInfo( +final case class ForkSyncInfo private[codeberg4s] ( allowed: Boolean, commitsBehind: Long, baseCommit: Option[String], @@ -87,4 +87,4 @@ final case class ForkSyncInfo( * @param pullRequests * whether one more pull request may be pinned */ -final case class IssuePinsAllowed(issues: Boolean, pullRequests: Boolean) +final case class IssuePinsAllowed private[codeberg4s] (issues: Boolean, pullRequests: Boolean) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/AnnotatedTag.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/AnnotatedTag.scala index 2670273..28d1f7a 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/AnnotatedTag.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/AnnotatedTag.scala @@ -35,7 +35,7 @@ import com.worxbend.codeberg4s.repositories.TagName * @param url * the API URL of the tag object, when the endpoint reports one */ -final case class AnnotatedTag( +final case class AnnotatedTag private[codeberg4s] ( name: TagName, sha: CommitSha, target: Option[GitObjectRef], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CombinedCommitStatus.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CombinedCommitStatus.scala index 1cabcec..f5ebf94 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CombinedCommitStatus.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CombinedCommitStatus.scala @@ -35,7 +35,7 @@ import com.worxbend.codeberg4s.repositories.Repository * @param url * the API URL of the combined status itself */ -final case class CombinedCommitStatus( +final case class CombinedCommitStatus private[codeberg4s] ( sha: CommitSha, state: Option[CommitStatusState], totalCount: Long, diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CommitComparison.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CommitComparison.scala index c12eb9b..eff469a 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CommitComparison.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CommitComparison.scala @@ -23,7 +23,11 @@ import com.worxbend.codeberg4s.repositories.CommitFile * the files the difference touches. Names and statuses only — the diff itself is a different endpoint, and * [[com.worxbend.codeberg4s.repositories.CommitFile]] says why */ -final case class CommitComparison(totalCommits: Long, commits: Vector[Commit], files: Vector[CommitFile]): +final case class CommitComparison private[codeberg4s] ( + totalCommits: Long, + commits: Vector[Commit], + files: Vector[CommitFile], +): /** Whether the instance returned fewer commits than it says the comparison holds — see the note on [[totalCommits]]. */ def isTruncated: Boolean = commits.size.toLong < totalCommits diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CommitStatus.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CommitStatus.scala index 8d20b8d..5341b2b 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CommitStatus.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CommitStatus.scala @@ -29,7 +29,7 @@ import java.time.Instant * @param url * the API URL of the status, when the endpoint reports one */ -final case class CommitStatus( +final case class CommitStatus private[codeberg4s] ( id: Long, state: Option[CommitStatusState], context: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CompareRange.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CompareRange.scala index 39f38f5..f1c7fa9 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CompareRange.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/CompareRange.scala @@ -1,7 +1,7 @@ package com.worxbend.codeberg4s.repositories.gitdata +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError -import com.worxbend.codeberg4s.repositories.PathSegment /** The `basehead` path parameter of `GET /repos/{owner}/{repo}/compare/{basehead}` — two refs joined by `...`. * diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/EditorConfigDefinitions.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/EditorConfigDefinitions.scala index 715f7cd..7d686ad 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/EditorConfigDefinitions.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/EditorConfigDefinitions.scala @@ -17,7 +17,7 @@ package com.worxbend.codeberg4s.repositories.gitdata * @param values * the properties exactly as the instance named them, in no particular order */ -final case class EditorConfigDefinitions(values: Map[String, String]): +final case class EditorConfigDefinitions private[codeberg4s] (values: Map[String, String]): /** The value of `property`, matched case-insensitively. * diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/FileChange.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/FileChange.scala index a04954b..7a34d5b 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/FileChange.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/FileChange.scala @@ -20,7 +20,7 @@ import com.worxbend.codeberg4s.repositories.ContentEntry * @param verification * the instance's signature verdict for the new commit, when it reports one */ -final case class FileChange( +final case class FileChange private[codeberg4s] ( commit: Option[FileCommit], content: Option[ContentEntry], verification: Option[CommitVerification], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/FileCommit.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/FileCommit.scala index 2dd526d..cfece28 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/FileCommit.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/FileCommit.scala @@ -33,7 +33,7 @@ import java.time.Instant * @param htmlUrl * the browser URL of the commit */ -final case class FileCommit( +final case class FileCommit private[codeberg4s] ( sha: CommitSha, message: Option[String], author: Option[GitIdentity], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitBlob.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitBlob.scala index ec2f5ec..359d796 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitBlob.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitBlob.scala @@ -22,4 +22,9 @@ import com.worxbend.codeberg4s.repositories.FileContent * @param url * the API URL of the blob, when the endpoint reports one */ -final case class GitBlob(sha: CommitSha, size: Long, content: Option[FileContent], url: Option[String]) +final case class GitBlob private[codeberg4s] ( + sha: CommitSha, + size: Long, + content: Option[FileContent], + url: Option[String], +) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitNote.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitNote.scala index 6890c6d..c292e6c 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitNote.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitNote.scala @@ -14,4 +14,4 @@ import com.worxbend.codeberg4s.repositories.Commit * @param commit * the commit the note is attached to, as the endpoint echoes it back. Absent when the instance sent no `commit` */ -final case class GitNote(message: Option[String], commit: Option[Commit]) +final case class GitNote private[codeberg4s] (message: Option[String], commit: Option[Commit]) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitObjectRef.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitObjectRef.scala index bc63cd2..fea4836 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitObjectRef.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitObjectRef.scala @@ -19,4 +19,4 @@ import com.worxbend.codeberg4s.repositories.CommitSha * @param url * the API URL of the object, when the endpoint reports one */ -final case class GitObjectRef(sha: CommitSha, kind: Option[GitObjectKind], url: Option[String]) +final case class GitObjectRef private[codeberg4s] (sha: CommitSha, kind: Option[GitObjectKind], url: Option[String]) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitReference.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitReference.scala index 8601525..5d51d82 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitReference.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitReference.scala @@ -19,4 +19,4 @@ package com.worxbend.codeberg4s.repositories.gitdata * @param target * the object the ref points at, absent when the instance sent no usable `object` */ -final case class GitReference(name: RefName, url: Option[String], target: Option[GitObjectRef]) +final case class GitReference private[codeberg4s] (name: RefName, url: Option[String], target: Option[GitObjectRef]) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitTreeEntry.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitTreeEntry.scala index d6a0d2d..e02074e 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitTreeEntry.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/GitTreeEntry.scala @@ -24,7 +24,7 @@ import com.worxbend.codeberg4s.repositories.ContentPath * @param url * the API URL of the entry's object, when the endpoint reports one */ -final case class GitTreeEntry( +final case class GitTreeEntry private[codeberg4s] ( path: ContentPath, sha: CommitSha, kind: Option[GitObjectKind], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/RefName.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/RefName.scala index 707fc71..28973d0 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/RefName.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/gitdata/RefName.scala @@ -1,7 +1,7 @@ package com.worxbend.codeberg4s.repositories.gitdata +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError -import com.worxbend.codeberg4s.repositories.PathSegment /** The name of a Git reference, whole or partial — `refs/heads/main`, `heads/main`, `tags/v1.2`, `main`. * diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/GitHook.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/GitHook.scala index aa68c99..ea85d1a 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/GitHook.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/GitHook.scala @@ -18,7 +18,7 @@ package com.worxbend.codeberg4s.repositories.hooks * the script itself, verbatim. Absent for a hook that has none, which is what an inactive hook usually is. '''Not * base64''' — the spec declares it a plain string, unlike a wiki page's content */ -final case class GitHook( +final case class GitHook private[codeberg4s] ( name: GitHookName, isActive: Option[Boolean], content: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/HookIds.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/HookIds.scala index a3e9379..d3aebd4 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/HookIds.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/HookIds.scala @@ -1,7 +1,7 @@ package com.worxbend.codeberg4s.repositories.hooks +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError -import com.worxbend.codeberg4s.repositories.PathSegment /** The instance-wide identifier of one webhook — the `{id}` of `/repos/{owner}/{repo}/hooks/{id}`. * @@ -52,9 +52,9 @@ object GitHookName: /** Parses a Git hook name. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character — see [[com.worxbend.codeberg4s.repositories.PathSegment]] for why that is a security boundary - * and not a convenience. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..` — see + * [[com.worxbend.codeberg4s.repositories.PathSegment]] for why that is a security boundary and not a convenience. * * @return * the name, or a [[ValidationError]] on the `"gitHookName"` field diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/IssueConfig.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/IssueConfig.scala index 96ca0e8..64686c6 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/IssueConfig.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/IssueConfig.scala @@ -17,7 +17,7 @@ package com.worxbend.codeberg4s.repositories.hooks * the alternatives offered alongside the templates, in the order the file lists them. Empty when the payload carried * no `contact_links` key, `null`, or an empty array */ -final case class IssueConfig( +final case class IssueConfig private[codeberg4s] ( blankIssuesEnabled: Option[Boolean], contactLinks: Vector[IssueContactLink], ) @@ -33,7 +33,7 @@ final case class IssueConfig( * @param about * the sentence explaining when to use this link instead of an issue */ -final case class IssueContactLink( +final case class IssueContactLink private[codeberg4s] ( name: String, url: String, about: Option[String], @@ -53,7 +53,7 @@ final case class IssueContactLink( * @param message * what was wrong, when something was. Absent for a valid config */ -final case class IssueConfigValidation( +final case class IssueConfigValidation private[codeberg4s] ( isValid: Boolean, message: Option[String], ) diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/IssueTemplate.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/IssueTemplate.scala index d57ea14..27544fa 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/IssueTemplate.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/IssueTemplate.scala @@ -31,7 +31,7 @@ import java.util.Locale * @param fields * the form fields, in file order, for a template that is a form. Empty for a Markdown template */ -final case class IssueTemplate( +final case class IssueTemplate private[codeberg4s] ( fileName: String, name: Option[String], about: Option[String], @@ -76,7 +76,7 @@ final case class IssueTemplate( * `IssueFormFieldVisible` is a bare `type: string` that enumerates nothing, and inventing a vocabulary here would be * exactly the guesswork `docs/HAZARDS.md` §1 warns against */ -final case class IssueFormField( +final case class IssueFormField private[codeberg4s] ( id: Option[String], fieldType: Option[IssueFormFieldType], attributes: Map[String, String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/RepositoryFlag.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/RepositoryFlag.scala index 8c7d67f..a9abf38 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/RepositoryFlag.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/RepositoryFlag.scala @@ -1,7 +1,7 @@ package com.worxbend.codeberg4s.repositories.hooks +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError -import com.worxbend.codeberg4s.repositories.PathSegment /** One administrative flag attached to a repository — the `{flag}` of `/repos/{owner}/{repo}/flags/{flag}`. * @@ -25,10 +25,11 @@ object RepositoryFlag: /** Parses a flag name. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character — see [[com.worxbend.codeberg4s.repositories.PathSegment]] for why that is a security boundary - * and not a convenience. Nothing else is checked: the vocabulary belongs to the instance, and a flag this library - * refused would be one the caller could not set. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..` — see + * [[com.worxbend.codeberg4s.repositories.PathSegment]] for why that is a security boundary and not a convenience. + * Nothing else is checked: the vocabulary belongs to the instance, and a flag this library refused would be one the + * caller could not set. * * @return * the flag, or a [[ValidationError]] on the `"repositoryFlag"` field diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/Webhook.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/Webhook.scala index b9c7c22..5f56cf2 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/Webhook.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/Webhook.scala @@ -43,7 +43,7 @@ import java.time.Instant * @param updatedAt * when the hook was last changed, on the same terms as [[createdAt]] */ -final case class Webhook( +final case class Webhook private[codeberg4s] ( id: HookId, hookType: Option[HookType], configuration: HookConfig, diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiCommit.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiCommit.scala index 413edaf..007ea37 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiCommit.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiCommit.scala @@ -27,7 +27,7 @@ import com.worxbend.codeberg4s.repositories.GitIdentity * the commit message, which is the `message` a caller passed to [[CreateWikiPage]] or [[EditWikiPage]] when the edit * came through this API */ -final case class WikiCommit( +final case class WikiCommit private[codeberg4s] ( sha: CommitSha, author: Option[GitIdentity], committer: Option[GitIdentity], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiPage.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiPage.scala index 5f653f0..2f44073 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiPage.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiPage.scala @@ -39,7 +39,7 @@ import com.worxbend.codeberg4s.repositories.FileContent * how many revisions the page has. Absent when the instance did not say; `0` is a value the instance can genuinely * send and is preserved */ -final case class WikiPage( +final case class WikiPage private[codeberg4s] ( title: String, content: Option[FileContent], sidebar: Option[String], @@ -67,7 +67,7 @@ final case class WikiPage( * @param lastCommit * the page's most recent revision */ -final case class WikiPageMeta( +final case class WikiPageMeta private[codeberg4s] ( title: String, htmlUrl: Option[String], subUrl: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiPageName.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiPageName.scala index 53554e8..2424c8f 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiPageName.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/hooks/WikiPageName.scala @@ -1,7 +1,7 @@ package com.worxbend.codeberg4s.repositories.hooks +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError -import com.worxbend.codeberg4s.repositories.PathSegment /** The name of a wiki page, as `GET /repos/{owner}/{repo}/wiki/page/{pageName}` spells it. * diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/publishing/Topic.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/publishing/Topic.scala index 5eea908..e86a7a3 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/publishing/Topic.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/publishing/Topic.scala @@ -1,7 +1,7 @@ package com.worxbend.codeberg4s.repositories.publishing +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError -import com.worxbend.codeberg4s.repositories.PathSegment /** One repository topic — `forge`, `forgejo`, `git`, `self-hosted` on `golden/repository/topics.json`. * @@ -25,9 +25,9 @@ object Topic: /** Parses a topic name. * - * Trims surrounding whitespace. Rejects a blank name, a name containing `/`, and a name containing a control - * character — the three things that would let a value escape its path segment. See the type's own note for what is - * deliberately '''not''' checked. + * Trims surrounding whitespace. Rejects a blank name, a name containing `/`, a name containing a control character, + * and the traversal segments `.` and `..` — the four things that would let a value escape its path segment. See the + * type's own note for what is deliberately '''not''' checked. * * @return * the trimmed name, or a [[ValidationError]] on the `"topic"` field diff --git a/modules/domain/src/com/worxbend/codeberg4s/repositories/publishing/UploadAsset.scala b/modules/domain/src/com/worxbend/codeberg4s/repositories/publishing/UploadAsset.scala index 0aa1803..0a6281f 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/repositories/publishing/UploadAsset.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/repositories/publishing/UploadAsset.scala @@ -1,7 +1,10 @@ package com.worxbend.codeberg4s.repositories.publishing +import com.worxbend.codeberg4s.ContentType import com.worxbend.codeberg4s.ValidationError +import java.util.Arrays + /** A file to attach to a release — the `multipart/form-data` half of `POST /repos/{owner}/{repo}/releases/{id}/assets`. * * Derived from that operation's parameters in `spec/swagger.v1.json`: a `name` query parameter, and a form part called @@ -21,14 +24,20 @@ import com.worxbend.codeberg4s.ValidationError * [[content]] is '''not''' copied, here or at the transport boundary, because a release asset is routinely hundreds of * megabytes — the first element of `golden/repository/release-latest.json` is 119 MB — and copying it twice to gain an * immutability guarantee the caller can already provide is the wrong trade. A caller must therefore not mutate the - * array after handing it over. For the same reason the generated `equals` compares [[content]] by reference, so two - * structurally identical uploads are not equal; nothing in this library depends on that. + * array after handing it over. + * + * Equality is a separate question from copying, and is answered '''on the bytes''': see [[UploadAsset.equals]]. + * + * ==Construction== + * + * The constructor is private, so [[UploadAsset.of]] is the only way to obtain one and the checks it performs cannot be + * stepped around by calling the generated `apply` or `copy`. Reading the fields and pattern matching are unaffected. * * ==Error contract== * - * Construction produces [[ValidationError]] on the `"fileName"` field and nothing else; it performs no I/O and never - * reads a file. Turning a path into bytes is the caller's job, and deliberately so: this library owns no filesystem - * effect. + * [[UploadAsset.of]] produces a [[ValidationError]] on the `"fileName"` field, [[as]] one on the `"mediaType"` field, + * and nothing else here can fail. No member performs I/O or reads a file. Turning a path into bytes is the caller's + * job, and deliberately so: this library owns no filesystem effect. * * @param fileName * the file name announced in the multipart part @@ -39,17 +48,50 @@ import com.worxbend.codeberg4s.ValidationError * @param name * the `name` query parameter, when the stored name should differ from [[fileName]] */ -final case class UploadAsset(fileName: String, content: Array[Byte], mediaType: String, name: Option[String]): +final case class UploadAsset private (fileName: String, content: Array[Byte], mediaType: String, name: Option[String]): /** Stores the attachment under `attachment` instead of under [[fileName]]. */ def named(attachment: String): UploadAsset = copy(name = Some(attachment)) - /** Declares the part's content type, for an instance or a proxy that acts on it. */ - def as(media: String): UploadAsset = copy(mediaType = media) + /** Declares the part's own `Content-Type`, for an instance or a proxy that acts on it. + * + * The value is written into the multipart body as a header, so it is checked the way [[fileName]] is: trimmed, then + * refused when it is blank or carries a control character. A carriage return or a newline in it would end the part's + * header line and let whatever follows be read as headers of the caller's choosing. + * + * @return + * the upload sent under `media`, or a [[ValidationError]] on the `"mediaType"` field + */ + def as(media: String): Either[ValidationError, UploadAsset] = + ContentType.from("mediaType", media).map(checked => copy(mediaType = checked)) /** How many bytes would be sent. */ def size: Int = content.length + /** Structural, on the names and the media type first and then on the bytes. + * + * Written out because an array's own `equals` in Scala is '''identity''': the equality a case class generates would + * compare [[content]] by reference, so two uploads built from byte-identical files would be unequal and would hash + * differently — a wrong answer with no warning attached, in an assertion or in a `Set`. Comparing the bytes copies + * nothing, so it does not undo the aliasing decision above; it only costs a scan, and only when everything cheaper + * already matched, which is why the three cheap fields are tested first. + * + * A consequence worth knowing for a very large asset: [[hashCode]] has to read the whole array, so using an upload + * as a key in a hashed collection reads the file's bytes once per lookup. + * + * The class is `final`, so no subclass can exist and the type test below is the whole of the compiler-generated + * `canEqual`; calling `canEqual` as well would add nothing. Removing `final` would change that. + */ + override def equals(other: Any): Boolean = + other match + case that: UploadAsset => + fileName.equals(that.fileName) && mediaType.equals(that.mediaType) && name.equals(that.name) && + Arrays.equals(content, that.content) + case _ => false + + override def hashCode(): Int = + 31 * (31 * (31 * fileName.hashCode + mediaType.hashCode) + name.hashCode) + Arrays.hashCode(content) + object UploadAsset: /** The fallback content type for a file whose type the caller does not know. diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/PublicKey.scala b/modules/domain/src/com/worxbend/codeberg4s/users/PublicKey.scala index 99d7c6e..d30b3c6 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/PublicKey.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/PublicKey.scala @@ -32,7 +32,7 @@ import java.time.Instant * whether the account holder proved possession of the private half, which Forgejo tracks separately from * registration */ -final case class PublicKey( +final case class PublicKey private[codeberg4s] ( id: Long, key: String, title: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/User.scala b/modules/domain/src/com/worxbend/codeberg4s/users/User.scala index 0a5ff94..81980f7 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/User.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/User.scala @@ -30,7 +30,7 @@ import java.time.Instant * @param lastLoginAt * absent unless the caller is an administrator; Forgejo reports the zero-time sentinel otherwise */ -final case class User( +final case class User private[codeberg4s] ( id: Long, login: String, fullName: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/Username.scala b/modules/domain/src/com/worxbend/codeberg4s/users/Username.scala index ed399b3..d881814 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/Username.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/Username.scala @@ -1,5 +1,6 @@ package com.worxbend.codeberg4s.users +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError /** The handle that names a person — the `{username}` of `/users/{username}`. @@ -23,24 +24,23 @@ object Username: /** Parses a username. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..`. * * '''This is a security boundary, not a convenience.''' A `Username` is interpolated into a request path, so a value * containing `/` would let a caller reach an endpoint the API surface never offered — `client.users.keys` on * `"someone/../../admin"` — and a control character would corrupt the request line. Both are rejected here, once, - * rather than at each call site. Forgejo's own rules for what an account may be called are narrower still, but they - * are the instance's business: this type promises only that the value cannot forge a path. + * rather than at each call site. A bare `.` or `..` is rejected for the same reason and needs saying separately: it + * carries no slash, so the slash rule never sees it, and it survives percent-encoding untouched, so it would reach + * the request path as a dot segment rather than as an account name. Forgejo's own rules for what an account may be + * called are narrower still, but they are the instance's business: this type promises only that the value cannot + * forge a path. * * @return * the trimmed username, or a [[ValidationError]] on the `"username"` field */ def from(value: String): Either[ValidationError, Username] = - val trimmed = value.trim - if trimmed.isEmpty then Left(ValidationError(Field, "must not be blank")) - else if trimmed.contains('/') then Left(ValidationError(Field, "must not contain a slash")) - else if trimmed.exists(_.isControl) then Left(ValidationError(Field, "must not contain a control character")) - else Right(trimmed) + PathSegment.from(Field, value) extension (username: Username) diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/account/ClientSecret.scala b/modules/domain/src/com/worxbend/codeberg4s/users/account/ClientSecret.scala index 2cee9da..31096cb 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/account/ClientSecret.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/account/ClientSecret.scala @@ -28,14 +28,14 @@ import com.worxbend.codeberg4s.ValidationError * a redacted URI and never a body, so no [[com.worxbend.codeberg4s.CodebergError]] built '''from a value''' can carry * the material. `AccountSecrecySuite` asserts every one of those paths. * - * '''There is one residual path, and it is the pipeline's rather than this type's.''' - * [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]] carries a bounded snippet of the '''raw response body''', - * taken before any conversion — so a `201` whose payload carries a `client_secret` and also fails to convert for some - * other reason produces a failure whose snippet contains the credential in the clear. That is inherent to reporting - * what could not be decoded, applies equally to every credential-bearing response in this library, and is bounded at - * [[com.worxbend.codeberg4s.CodebergError.MaxSnippetLength]]. `UserApplicationApiSuite` pins the behaviour so it - * cannot change unnoticed. The consequence for an application: a `DecodingFailed` from a creation is not safe to log - * verbatim, while every '''successful''' result is. + * '''The raw body is closed off too, and that is the pipeline's doing rather than this type's.''' + * [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]] ordinarily carries a bounded snippet of the raw response + * body, taken before any conversion — so a `201` whose payload carries a `client_secret` and fails to convert for some + * other reason would report the credential in the clear, mask or no mask. The decoder the creation and the update pass + * to the pipeline is therefore marked `Decode.sensitive`, and the pipeline substitutes a fixed placeholder naming the + * size of the withheld body. The consequence for an application: a `DecodingFailed` from any call in this group is + * safe to log, and so is every successful result. `UserApplicationApiSuite` asserts both, and asserts that a + * '''read''' keeps its snippet — a read carries no secret to withhold. * * Instances compare structurally on the underlying material, so a value stays comparable in a test. The comparison is * not constant-time; this type guards against accidental disclosure, not against a timing oracle. diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/account/Email.scala b/modules/domain/src/com/worxbend/codeberg4s/users/account/Email.scala index 846f9f5..1295df6 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/account/Email.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/account/Email.scala @@ -27,7 +27,7 @@ package com.worxbend.codeberg4s.users.account * @param username * the account's login, on the same terms as [[userId]] */ -final case class Email( +final case class Email private[codeberg4s] ( address: EmailAddress, isPrimary: Boolean, isVerified: Boolean, diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/account/OAuth2Application.scala b/modules/domain/src/com/worxbend/codeberg4s/users/account/OAuth2Application.scala index 65e03c1..5e27465 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/account/OAuth2Application.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/account/OAuth2Application.scala @@ -37,7 +37,7 @@ import java.time.Instant * @param createdAt * when the application was registered, absent when the instance sent no timestamp or the zero-time sentinel */ -final case class OAuth2Application( +final case class OAuth2Application private[codeberg4s] ( id: OAuth2ApplicationId, name: Option[String], clientId: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/account/QuotaInfo.scala b/modules/domain/src/com/worxbend/codeberg4s/users/account/QuotaInfo.scala index c7c53ae..850d607 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/account/QuotaInfo.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/account/QuotaInfo.scala @@ -21,7 +21,7 @@ package com.worxbend.codeberg4s.users.account * @param used * what the account is currently storing, broken down by what is storing it */ -final case class QuotaInfo( +final case class QuotaInfo private[codeberg4s] ( groups: Vector[QuotaGroup], used: QuotaUsedSize, ): @@ -41,7 +41,7 @@ final case class QuotaInfo( * @param rules * the limits the group imposes. Empty when the payload carried none */ -final case class QuotaGroup( +final case class QuotaGroup private[codeberg4s] ( name: Option[String], rules: Vector[QuotaRule], ) @@ -59,7 +59,7 @@ final case class QuotaGroup( * what the rule counts; see [[QuotaSubject]] for why the vocabulary is not enumerated. A subject the instance sent * that cannot be one — blank, or carrying a control character — is dropped rather than failing the whole rule */ -final case class QuotaRule( +final case class QuotaRule private[codeberg4s] ( name: Option[String], limit: Option[Long], subjects: Vector[QuotaSubject], @@ -95,7 +95,7 @@ final case class QuotaRule( * @param packages * the size of the account's published packages */ -final case class QuotaUsedSize( +final case class QuotaUsedSize private[codeberg4s] ( publicRepositories: Option[Long], privateRepositories: Option[Long], gitLfs: Option[Long], diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/account/QuotaUsage.scala b/modules/domain/src/com/worxbend/codeberg4s/users/account/QuotaUsage.scala index 81ec3f0..e9a9bcc 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/account/QuotaUsage.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/account/QuotaUsage.scala @@ -20,7 +20,7 @@ package com.worxbend.codeberg4s.users.account * a browser link to the run that produced the artifact, which is the closest thing this payload has to a way back to * the object */ -final case class QuotaUsedArtifact( +final case class QuotaUsedArtifact private[codeberg4s] ( name: Option[String], size: Option[Long], htmlUrl: Option[String], @@ -41,7 +41,7 @@ final case class QuotaUsedArtifact( * where the attachment hangs — the issue, comment or release it belongs to. Absent when the payload carried no * `contained_in` object at all */ -final case class QuotaUsedAttachment( +final case class QuotaUsedAttachment private[codeberg4s] ( name: Option[String], size: Option[Long], apiUrl: Option[String], @@ -60,7 +60,7 @@ final case class QuotaUsedAttachment( * @param htmlUrl * the browser link to the containing object */ -final case class AttachmentContainer( +final case class AttachmentContainer private[codeberg4s] ( apiUrl: Option[String], htmlUrl: Option[String], ) @@ -82,7 +82,7 @@ final case class AttachmentContainer( * @param htmlUrl * a browser link to the package version */ -final case class QuotaUsedPackage( +final case class QuotaUsedPackage private[codeberg4s] ( name: Option[String], version: Option[String], packageType: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/account/UserSettings.scala b/modules/domain/src/com/worxbend/codeberg4s/users/account/UserSettings.scala index c988c29..357e784 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/account/UserSettings.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/account/UserSettings.scala @@ -45,7 +45,7 @@ package com.worxbend.codeberg4s.users.account * @param showsRepoUnitHints * whether the repository view offers hints for units that are enabled but empty */ -final case class UserSettings( +final case class UserSettings private[codeberg4s] ( fullName: Option[String], website: Option[String], location: Option[String], diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/social/AccessToken.scala b/modules/domain/src/com/worxbend/codeberg4s/users/social/AccessToken.scala index a070123..2152c3d 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/social/AccessToken.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/social/AccessToken.scala @@ -1,5 +1,6 @@ package com.worxbend.codeberg4s.users.social +import com.worxbend.codeberg4s.PathSegment import com.worxbend.codeberg4s.ValidationError import com.worxbend.codeberg4s.auth.ApiToken import com.worxbend.codeberg4s.repositories.RepoSlug @@ -25,19 +26,16 @@ object AccessTokenName: /** Parses a token name. * - * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, and a value containing a - * control character — everything that would forge or corrupt a request path. + * Trims surrounding whitespace. Rejects an empty or blank value, a value containing `/`, a value containing a + * control character, and the traversal segments `.` and `..` — everything that would forge or corrupt a request + * path. The dot segments need stating separately because they carry no slash, so the slash rule never sees them, and + * they survive percent-encoding untouched. * * @return * the trimmed name, or a [[ValidationError]] on the `"accessTokenName"` field */ def from(value: String): Either[ValidationError, AccessTokenName] = - val trimmed = value.trim - - if trimmed.isEmpty then Left(ValidationError(Field, "must not be blank")) - else if trimmed.contains('/') then Left(ValidationError(Field, "must not contain a slash")) - else if trimmed.exists(_.isControl) then Left(ValidationError(Field, "must not contain a control character")) - else Right(trimmed) + PathSegment.from(Field, value) extension (name: AccessTokenName) @@ -103,7 +101,7 @@ enum AccessTokenRef: * @param createdAt * when the token was issued */ -final case class AccessToken( +final case class AccessToken private[codeberg4s] ( id: AccessTokenId, name: Option[AccessTokenName], scopes: Vector[TokenScope], @@ -130,16 +128,18 @@ final case class AccessToken( * ==Where the material can and cannot reach== * * It reaches the caller and nothing else. [[com.worxbend.codeberg4s.CallContext]] carries a redacted URI and never a - * response body; [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]] snippets a body only on the failure path, - * and a body that failed to decode produced no token; and the mask makes every accidental rendering safe even so. - * `UserTokenApiSuite` asserts all of that rather than asserting the intention. + * response body; the mask makes every accidental rendering safe; and the one channel that would otherwise have carried + * the material — the body snippet on [[com.worxbend.codeberg4s.CodebergError.DecodingFailed]], which is the '''raw''' + * payload and therefore survives every mask — is replaced by a placeholder, because the decoder this endpoint uses + * declares itself sensitive to the pipeline. `UserTokenApiSuite` asserts all of that rather than asserting the + * intention. * * @param token * the credential, available on this response and never again * @param details * everything a listing would also have shown: the identifier, the name, the scopes and the repository restriction */ -final case class CreatedAccessToken(token: ApiToken, details: AccessToken) +final case class CreatedAccessToken private[codeberg4s] (token: ApiToken, details: AccessToken) /** What `POST /users/{username}/tokens` needs to mint a token. * diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/social/BlockedUser.scala b/modules/domain/src/com/worxbend/codeberg4s/users/social/BlockedUser.scala index e23da99..5db71ca 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/social/BlockedUser.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/social/BlockedUser.scala @@ -24,4 +24,4 @@ import java.time.Instant * @param createdAt * when the block was put in place */ -final case class BlockedUser(blockId: BlockId, createdAt: Option[Instant]) +final case class BlockedUser private[codeberg4s] (blockId: BlockId, createdAt: Option[Instant]) diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/social/GpgKey.scala b/modules/domain/src/com/worxbend/codeberg4s/users/social/GpgKey.scala index 959e406..f54da2f 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/social/GpgKey.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/social/GpgKey.scala @@ -59,7 +59,7 @@ object OpenPgpKeyId: * whether the instance has confirmed the address belongs to the account. Absent on the wire reads as `false`, the * answer that claims the least */ -final case class GpgKeyEmail(email: String, isVerified: Boolean) +final case class GpgKeyEmail private[codeberg4s] (email: String, isVerified: Boolean) /** A GPG key an account has registered, used to verify commit and tag signatures. * @@ -109,7 +109,7 @@ final case class GpgKeyEmail(email: String, isVerified: Boolean) * when the key expires, absent for a key that does not — Forgejo spells "never" as the Go zero time, which * `com.worxbend.codeberg4s.codec.Timestamps` folds into absence */ -final case class GpgKey( +final case class GpgKey private[codeberg4s] ( id: GpgKeyId, keyId: Option[OpenPgpKeyId], primaryKeyId: Option[OpenPgpKeyId], diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/social/HeatmapEntry.scala b/modules/domain/src/com/worxbend/codeberg4s/users/social/HeatmapEntry.scala index ccc2492..b248918 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/social/HeatmapEntry.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/social/HeatmapEntry.scala @@ -26,4 +26,4 @@ import java.time.Instant * how many contributions fell in it. Forgejo counts commits, issues, pull requests and reviews; which of those it * counts on a given release is not specified */ -final case class HeatmapEntry(at: Instant, contributions: Long) +final case class HeatmapEntry private[codeberg4s] (at: Instant, contributions: Long) diff --git a/modules/domain/src/com/worxbend/codeberg4s/users/social/StopWatch.scala b/modules/domain/src/com/worxbend/codeberg4s/users/social/StopWatch.scala index f192002..974c299 100644 --- a/modules/domain/src/com/worxbend/codeberg4s/users/social/StopWatch.scala +++ b/modules/domain/src/com/worxbend/codeberg4s/users/social/StopWatch.scala @@ -40,7 +40,7 @@ import java.time.Instant * @param createdAt * when the stopwatch was started */ -final case class StopWatch( +final case class StopWatch private[codeberg4s] ( issueIndex: Long, issueTitle: Option[String], repository: Option[RepoSlug], diff --git a/modules/domain/test/src/com/worxbend/codeberg4s/BaseUriSuite.scala b/modules/domain/test/src/com/worxbend/codeberg4s/BaseUriSuite.scala index 9f4590a..28242df 100644 --- a/modules/domain/test/src/com/worxbend/codeberg4s/BaseUriSuite.scala +++ b/modules/domain/test/src/com/worxbend/codeberg4s/BaseUriSuite.scala @@ -31,6 +31,24 @@ final class BaseUriSuite extends FunSuite: test("rejects an embedded control character"): assertEquals(field(BaseUri.from("https://codeberg.org/api\nv1")), Some("baseUri")) + test("rejects user information, which would otherwise be reproduced in every error"): + assertEquals(field(BaseUri.from("https://u:p@h/api/v1")), Some("baseUri")) + + test("the rejection of user information does not echo the credential back"): + val rejected = message(BaseUri.from("https://user:hunter2@h/api/v1")) + + assert(rejected.isDefined, "a base URI carrying a password must be rejected") + assert(!rejected.exists(_.contains("hunter2")), s"the message repeated the password: $rejected") + + test("rejects a query string"): + assertEquals(field(BaseUri.from("https://h/api/v1?token=x")), Some("baseUri")) + + test("rejects a fragment"): + assertEquals(field(BaseUri.from("https://h/api/v1#frag")), Some("baseUri")) + + test("an at sign inside the path is not user information"): + assertEquals(value(BaseUri.from("https://h/api/v1/@me")), Some("https://h/api/v1/@me")) + test("the Codeberg constant points at the public api root"): assertEquals(BaseUri.Codeberg.value, "https://codeberg.org/api/v1") @@ -39,3 +57,6 @@ final class BaseUriSuite extends FunSuite: private def field(result: Either[ValidationError, ?]): Option[String] = result.swap.toOption.map(_.field) + + private def message(result: Either[ValidationError, ?]): Option[String] = + result.swap.toOption.map(_.message) diff --git a/modules/domain/test/src/com/worxbend/codeberg4s/CodebergConfigSuite.scala b/modules/domain/test/src/com/worxbend/codeberg4s/CodebergConfigSuite.scala index 6e8aff9..76d4dc0 100644 --- a/modules/domain/test/src/com/worxbend/codeberg4s/CodebergConfigSuite.scala +++ b/modules/domain/test/src/com/worxbend/codeberg4s/CodebergConfigSuite.scala @@ -29,6 +29,24 @@ final class CodebergConfigSuite extends FunSuite: assertEquals(config.defaultPageSize.value, PageSize.Default.value) assertEquals(config.connectTimeout, CodebergConfig.DefaultConnectTimeout) assertEquals(config.readTimeout, CodebergConfig.DefaultReadTimeout) + assertEquals(config.maxResponseBodyBytes, CodebergConfig.DefaultMaxResponseBodyBytes) + assertEquals(config.maxDownloadBodyBytes, CodebergConfig.DefaultMaxDownloadBodyBytes) + + test("the default response-body bound clears the largest JSON body Forgejo can produce"): + // `default_max_blob_size` is 10 MiB (docs/HAZARDS.md §4), and a file-contents + // response carries that blob base64-encoded, which costs four bytes per three. + val largestBlobBase64 = 10L * 1024 * 1024 * 4 / 3 + + assert( + CodebergConfig.DefaultMaxResponseBodyBytes > largestBlobBase64, + s"${CodebergConfig.DefaultMaxResponseBodyBytes} would reject a legitimate $largestBlobBase64-byte body", + ) + + test("the download bound is larger than the textual one, because an archive is not a JSON document"): + assert( + CodebergConfig.DefaultMaxDownloadBodyBytes > CodebergConfig.DefaultMaxResponseBodyBytes, + "a download bound at or below the textual bound would make the two settings pointless", + ) test("a config carrying a token does not leak it through toString"): val rendered = CodebergConfig(Auth.Token(token(Secret))).toString diff --git a/modules/domain/test/src/com/worxbend/codeberg4s/IdentifierProps.scala b/modules/domain/test/src/com/worxbend/codeberg4s/IdentifierProps.scala index ba9b86b..e18e01a 100644 --- a/modules/domain/test/src/com/worxbend/codeberg4s/IdentifierProps.scala +++ b/modules/domain/test/src/com/worxbend/codeberg4s/IdentifierProps.scala @@ -98,10 +98,16 @@ final class IdentifierProps extends PropertyBase: private def isHexadecimal(value: String): Boolean = value.forall(digit => "0123456789abcdef".contains(digit)) - /** Types that must occupy exactly one URI path segment: no slash gets through, at any cost. */ + /** Types that must occupy exactly one URI path segment: no slash and no dot segment gets through, at any cost. */ private val singleSegment: Vector[StringIdentifier] = val promise: String => Boolean = - value => value.nonEmpty && !value.contains('/') && !value.exists(_.isControl) && isTrimmed(value) + value => + value.nonEmpty && + !value.contains('/') && + !value.exists(_.isControl) && + isTrimmed(value) && + !value.equals(".") && + !value.equals("..") Vector( StringIdentifier("Owner", "owner", PropertyBase.plainSegment, promise, text => Owner.from(text).map(_.value)), StringIdentifier( diff --git a/modules/domain/test/src/com/worxbend/codeberg4s/TransportCauseSuite.scala b/modules/domain/test/src/com/worxbend/codeberg4s/TransportCauseSuite.scala index a0793b7..ae8103f 100644 --- a/modules/domain/test/src/com/worxbend/codeberg4s/TransportCauseSuite.scala +++ b/modules/domain/test/src/com/worxbend/codeberg4s/TransportCauseSuite.scala @@ -2,9 +2,9 @@ package com.worxbend.codeberg4s import munit.FunSuite -/** [[TransportCause.describe]] is what a human reads when a request never produced a response, and its six arms are six - * near-identical one-liners — exactly the shape a copy-paste gets wrong. Each is asserted whole, so an arm that named - * the wrong failure or dropped the detail fails here. +/** [[TransportCause.describe]] is what a human reads when a request produced no usable response, and its seven arms are + * seven near-identical one-liners — exactly the shape a copy-paste gets wrong. Each is asserted whole, so an arm that + * named the wrong failure or dropped the detail fails here. */ final class TransportCauseSuite extends FunSuite: @@ -25,6 +25,9 @@ final class TransportCauseSuite extends FunSuite: test("an interruption says so, and is not reported as a timeout"): assertEquals(TransportCause.Interrupted(Detail).describe, s"interrupted ($Detail)") + test("an oversized body says the body was too large, not that the connection failed"): + assertEquals(TransportCause.ResponseTooLarge(Detail).describe, s"response body too large ($Detail)") + test("an unclassified failure says it is unclassified rather than guessing"): assertEquals(TransportCause.Unknown(Detail).describe, s"unclassified transport failure ($Detail)") @@ -43,5 +46,6 @@ final class TransportCauseSuite extends FunSuite: TransportCause.Tls(Detail), TransportCause.Dns(Detail), TransportCause.Interrupted(Detail), + TransportCause.ResponseTooLarge(Detail), TransportCause.Unknown(Detail), ) diff --git a/modules/domain/test/src/com/worxbend/codeberg4s/auth/SecretProps.scala b/modules/domain/test/src/com/worxbend/codeberg4s/auth/SecretProps.scala index d6cb031..930d98d 100644 --- a/modules/domain/test/src/com/worxbend/codeberg4s/auth/SecretProps.scala +++ b/modules/domain/test/src/com/worxbend/codeberg4s/auth/SecretProps.scala @@ -9,6 +9,7 @@ import com.worxbend.codeberg4s.JsonPath import com.worxbend.codeberg4s.PropertyBase import com.worxbend.codeberg4s.TransportCause import com.worxbend.codeberg4s.ValidationError +import com.worxbend.codeberg4s.paging.PageParams import org.scalacheck.Gen import org.scalacheck.Prop @@ -84,6 +85,12 @@ final class SecretProps extends PropertyBase: 3, CodebergError.Transport(context, TransportCause.Timeout(s"$token")), ), + // The one case with no free-form position and no CallContext: its + // rendering is built from two numbers. Listed anyway, so that the claim + // "every case of the error ADT" stays a claim about the whole ADT and a + // later case that does carry text is added beside a neighbour rather + // than into a gap nobody notices. + "WalkTruncated" -> CodebergError.WalkTruncated(3, PageParams.First), ) /** Every way this library can turn a credential into text, named so a failure says which path leaked. */ diff --git a/modules/domain/test/src/com/worxbend/codeberg4s/issues/IssueTailCommandSuite.scala b/modules/domain/test/src/com/worxbend/codeberg4s/issues/IssueTailCommandSuite.scala index a946197..63da5ed 100644 --- a/modules/domain/test/src/com/worxbend/codeberg4s/issues/IssueTailCommandSuite.scala +++ b/modules/domain/test/src/com/worxbend/codeberg4s/issues/IssueTailCommandSuite.scala @@ -143,11 +143,31 @@ final class IssueTailCommandSuite extends FunSuite: // --- UploadAttachment ----------------------------------------------------- test("every upload builder sets its own field and leaves every sibling alone"): + // Compared field by field rather than against a `copy`: the constructor is + // private now, which is the point of the type, so `copy` is not reachable + // from a test either. val upload = populatedUpload - assertEquals(upload.named("failing-run.txt"), upload.copy(storedName = Some("failing-run.txt"))) - assertEquals(upload.as("application/json"), upload.copy(mediaType = "application/json")) - assertEquals(upload.recordedAt(Monday), upload.copy(updatedAt = Some(Monday))) + assertEquals( + fieldsOf(upload.named("failing-run.txt")), + (upload.fileName, Some("failing-run.txt"), upload.mediaType, upload.updatedAt), + ) + assertEquals( + fieldsOf(orFail(upload.as("application/json"))), + (upload.fileName, upload.storedName, "application/json", upload.updatedAt), + ) + assertEquals( + fieldsOf(upload.recordedAt(Monday)), + (upload.fileName, upload.storedName, upload.mediaType, Some(Monday)), + ) + + test("a media type that would inject a header into the part is refused, on the mediaType field"): + val upload = orFail(UploadAttachment.of("run.txt", bytes)) + + assertEquals(fieldOf(upload.as("text/plain\r\nX-Injected: 1")), "mediaType") + assertEquals(messageOf(upload.as("text/plain\r\nX-Injected: 1")), "must not contain a control character") + assertEquals(messageOf(upload.as(s"text/${Bell}plain")), "must not contain a control character") + assertEquals(messageOf(upload.as(" ")), "must not be blank") test("the stored name is a second name, and never overwrites the one the multipart part announces"): val upload = orFail(UploadAttachment.of("build/logs/run.txt", bytes)).named("failing-run.txt") @@ -178,6 +198,32 @@ final class IssueTailCommandSuite extends FunSuite: assertEquals(fieldOf(UploadAttachment.of("a\rb.log", bytes)), "fileName") assertEquals(fieldOf(UploadAttachment.of(s"a${Bell}b.log", bytes)), "fileName") + test("two uploads built from equal-but-distinct arrays are equal and hash alike"): + // `bytes` hands over a fresh array on every call, so these two uploads + // hold no array in common. An array's own equality is identity, so until + // the type compared its content they were unequal to each other. + val one = populatedUpload + val two = populatedUpload + + assert(!one.content.eq(two.content), "the two arrays must be distinct objects, or the test proves nothing") + assertEquals(one, two) + assertEquals(one.hashCode, two.hashCode) + assertEquals(Set(one, two).size, 1) + + test("an upload differing in one field, the bytes included, is not equal to the original"): + val upload = populatedUpload + + assertNotEquals(upload, uploadCarrying("other".getBytes(StandardCharsets.UTF_8))) + assertNotEquals(upload, upload.named("failing-run.txt")) + assertNotEquals(upload, orFail(upload.as("application/json"))) + assertNotEquals(upload, upload.recordedAt(Monday)) + + test("an upload is not equal to a value of some other type"): + // A hand-written equals owns this case. Getting it wrong turns a harmless + // collection lookup into a ClassCastException at the call site. + assertNotEquals[Any, Any](populatedUpload, "build.log") + assertNotEquals[Any, Any](populatedUpload, 0) + // --- AddTrackedTime ------------------------------------------------------- test("a duration one nanosecond off a whole second is refused, which a truncating check would accept"): @@ -272,8 +318,17 @@ final class IssueTailCommandSuite extends FunSuite: state = Some(IssueStateChange.Reopen), ) - private def populatedUpload: UploadAttachment = - orFail(UploadAttachment.of("run.txt", bytes)).named("run.txt").as("text/plain").recordedAt(Friday) + private def populatedUpload: UploadAttachment = uploadCarrying(bytes) + + /** The same fully populated upload, over content the caller chooses — so that a test can vary the bytes and nothing + * else. + */ + private def uploadCarrying(content: Array[Byte]): UploadAttachment = + val named = orFail(UploadAttachment.of("run.txt", content)).named("run.txt") + orFail(named.as("text/plain")).recordedAt(Friday) + + private def fieldsOf(upload: UploadAttachment): (String, Option[String], String, Option[Instant]) = + (upload.fileName, upload.storedName, upload.mediaType, upload.updatedAt) private def populatedTrackedTime: AddTrackedTime = orFail(AddTrackedTime.of(90.seconds)).attributedTo("crystal").recordedAt(Friday) diff --git a/modules/domain/test/src/com/worxbend/codeberg4s/issues/LabelColorSuite.scala b/modules/domain/test/src/com/worxbend/codeberg4s/issues/LabelColorSuite.scala index 19103b6..8e2209e 100644 --- a/modules/domain/test/src/com/worxbend/codeberg4s/issues/LabelColorSuite.scala +++ b/modules/domain/test/src/com/worxbend/codeberg4s/issues/LabelColorSuite.scala @@ -32,3 +32,20 @@ final class LabelColorSuite extends FunSuite: test("an empty colour is rejected"): assert(LabelColor.from("").isLeft) + + /** The one case where dropping the regular expression changed an answer, pinned so it cannot drift back. + * + * `from` used to match `^#?([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$`. In a Java regular expression `$` matches not only at + * the end of the input but also immediately before a line terminator that ends it, and Java counts NEL (U+0085), + * LINE SEPARATOR (U+2028) and PARAGRAPH SEPARATOR (U+2029) as line terminators. `String.trim` strips only characters + * up to and including U+0020, so those three reached the matcher intact and were then silently swallowed by `$`: + * `"eb6420\u0085"` came back as an accepted `"eb6420"`. U+0085 is a control character — the kind this module rejects + * everywhere a value reaches a request — so accepting it here was a hole, not a convenience. + */ + test("a trailing Unicode line separator is rejected rather than quietly swallowed"): + assert(LabelColor.from("eb6420\u0085").isLeft) + assert(LabelColor.from("eb6420\u2028").isLeft) + assert(LabelColor.from("eb6420\u2029").isLeft) + + test("whitespace up to U+0020 is still trimmed, line separator or not"): + assertEquals(LabelColor.from("\neb6420\t").map(_.value), Right("eb6420")) diff --git a/modules/domain/test/src/com/worxbend/codeberg4s/repositories/publishing/UploadAssetSuite.scala b/modules/domain/test/src/com/worxbend/codeberg4s/repositories/publishing/UploadAssetSuite.scala index 502977b..2863614 100644 --- a/modules/domain/test/src/com/worxbend/codeberg4s/repositories/publishing/UploadAssetSuite.scala +++ b/modules/domain/test/src/com/worxbend/codeberg4s/repositories/publishing/UploadAssetSuite.scala @@ -11,6 +11,8 @@ final class UploadAssetSuite extends FunSuite: private val Bytes: Array[Byte] = "checksums".getBytes(StandardCharsets.UTF_8) + private val Bell: String = 7.toChar.toString + test("an ordinary file name is accepted and the media type defaults to octet-stream"): val upload = accepted("forgejo-16.0.2-linux-amd64") @@ -49,7 +51,23 @@ final class UploadAssetSuite extends FunSuite: assertEquals(upload.name, Some("forgejo-16.0.2-linux-amd64")) test("the media type can be stated"): - assertEquals(accepted("notes.txt").as("text/plain").mediaType, "text/plain") + assertEquals(accepted("notes.txt").as("text/plain").map(_.mediaType), Right("text/plain")) + + test("surrounding whitespace in the media type is trimmed"): + assertEquals(accepted("notes.txt").as(" text/plain ").map(_.mediaType), Right("text/plain")) + + test("a blank media type is rejected, and the rejection names the mediaType field"): + assertEquals((refusedMedia(" ").field, refusedMedia(" ").message), ("mediaType", "must not be blank")) + + test("a media type carrying a line break is rejected, because it would inject a header into the part"): + val injected = refusedMedia("text/plain\r\nX-Injected: 1") + + assertEquals((injected.field, injected.message), ("mediaType", "must not contain a control character")) + + test("any other control character in the media type is rejected too"): + // Mid-string on purpose: `trim` removes every character below a space, + // so a control character at either end never reaches the check. + assertEquals(refusedMedia(s"text/${Bell}plain").message, "must not contain a control character") test("the bytes are held, not copied — the caller keeps ownership of the array"): val content = "one".getBytes(StandardCharsets.UTF_8) @@ -59,12 +77,53 @@ final class UploadAssetSuite extends FunSuite: assert(upload.content eq content, "the constructor copied the array, which the Scaladoc promises it does not") - private def accepted(fileName: String): UploadAsset = - UploadAsset.of(fileName, Bytes) match + test("two uploads built from equal-but-distinct arrays are equal and hash alike"): + // Holding the caller's array rather than copying it says nothing about + // equality: an array's own equality is identity, so these two uploads were + // unequal until the type started comparing its content. + val one = accepted("checksums.txt", "sha256".getBytes(StandardCharsets.UTF_8)) + val two = accepted("checksums.txt", "sha256".getBytes(StandardCharsets.UTF_8)) + + assert(!one.content.eq(two.content), "the two arrays must be distinct objects, or the test proves nothing") + assertEquals(one, two) + assertEquals(one.hashCode, two.hashCode) + assertEquals(Set(one, two).size, 1) + + test("an upload differing in one field, the bytes included, is not equal to the original"): + val upload = accepted("checksums.txt", "sha256".getBytes(StandardCharsets.UTF_8)) + + assertNotEquals(upload, accepted("checksums.txt", "sha512".getBytes(StandardCharsets.UTF_8))) + assertNotEquals(upload, accepted("other.txt", "sha256".getBytes(StandardCharsets.UTF_8))) + assertNotEquals(upload, upload.named("release-checksums")) + assertNotEquals(upload, retyped(upload, "text/plain")) + + test("an upload is not equal to a value of some other type"): + // A hand-written equals has to answer this case itself, and answering it + // wrongly is how a type ends up throwing a ClassCastException out of a + // collection lookup rather than returning false. + val upload = accepted("checksums.txt", "sha256".getBytes(StandardCharsets.UTF_8)) + + assertNotEquals[Any, Any](upload, "checksums.txt") + assertNotEquals[Any, Any](upload, 0) + + private def accepted(fileName: String): UploadAsset = accepted(fileName, Bytes) + + private def accepted(fileName: String, content: Array[Byte]): UploadAsset = + UploadAsset.of(fileName, content) match case Right(upload) => upload case Left(error) => fail(s"expected $fileName to be accepted, got ${error.message}") + private def retyped(upload: UploadAsset, media: String): UploadAsset = + upload.as(media) match + case Right(value) => value + case Left(error) => fail(s"expected $media to be accepted, got ${error.message}") + private def rejected(fileName: String): ValidationError = UploadAsset.of(fileName, Bytes) match case Left(error) => error case Right(upload) => fail(s"expected $fileName to be rejected, got ${upload.fileName}") + + private def refusedMedia(media: String): ValidationError = + accepted("notes.txt").as(media) match + case Left(error) => error + case Right(upload) => fail(s"expected the media type to be rejected, got ${upload.mediaType}") diff --git a/modules/domain/test/src/com/worxbend/codeberg4s/users/UsernameSuite.scala b/modules/domain/test/src/com/worxbend/codeberg4s/users/UsernameSuite.scala index 2647448..e294aa8 100644 --- a/modules/domain/test/src/com/worxbend/codeberg4s/users/UsernameSuite.scala +++ b/modules/domain/test/src/com/worxbend/codeberg4s/users/UsernameSuite.scala @@ -28,6 +28,18 @@ final class UsernameSuite extends FunSuite: test("rejects an embedded control character, which would corrupt the request line"): assertEquals(rejection(Username.from("earl\nwarren")), Some(("username", "must not contain a control character"))) + test("rejects a bare dot, which would reach the path as a segment rather than a name"): + assertEquals(rejection(Username.from(".")), Some(("username", "must not be '.' or '..'"))) + + test("rejects a bare double dot, which carries no slash for the slash rule to catch"): + assertEquals(rejection(Username.from("..")), Some(("username", "must not be '.' or '..'"))) + + test("rejects a dot segment that was padded with whitespace, because the value is trimmed first"): + assertEquals(rejection(Username.from(" .. ")), Some(("username", "must not be '.' or '..'"))) + + test("accepts a name that merely starts with a dot, which is an ordinary handle"): + assertEquals(Username.from(".hidden").toOption.map(_.value), Some(".hidden")) + /** The field and reason of a rejection, or `None` when the value was accepted. */ private def rejection(result: Either[ValidationError, ?]): Option[(String, String)] = result.swap.toOption.map(error => (error.field, error.message)) diff --git a/modules/domain/test/src/com/worxbend/codeberg4s/users/social/SocialDomainSuite.scala b/modules/domain/test/src/com/worxbend/codeberg4s/users/social/SocialDomainSuite.scala index f36eb9a..7586464 100644 --- a/modules/domain/test/src/com/worxbend/codeberg4s/users/social/SocialDomainSuite.scala +++ b/modules/domain/test/src/com/worxbend/codeberg4s/users/social/SocialDomainSuite.scala @@ -112,6 +112,12 @@ final class SocialDomainSuite extends FunSuite: assert(AccessTokenName.from("ci\tdeploy").isLeft, "a control character was accepted") assert(AccessTokenName.from(" ").isLeft, "a blank name was accepted") + test("a bare dot segment is refused, since no slash rule would ever see it"): + assert(AccessTokenName.from(".").isLeft, "'.' was accepted into a path segment") + assert(AccessTokenName.from("..").isLeft, "'..' was accepted into a path segment") + assertEquals(fieldOf(AccessTokenName.from("..")), "accessTokenName") + assertEquals(orFail(AccessTokenName.from(".ci")).value, ".ci") + test("a token reference renders whichever spelling it carries"): val byId = AccessTokenRef.ById(orFail(AccessTokenId.from(42L))) val byName = AccessTokenRef.ByName(orFail(AccessTokenName.from("ci"))) diff --git a/modules/examples/src/com/worxbend/codeberg4s/examples/HandlingErrors.scala b/modules/examples/src/com/worxbend/codeberg4s/examples/HandlingErrors.scala index 17076d0..7832aff 100644 --- a/modules/examples/src/com/worxbend/codeberg4s/examples/HandlingErrors.scala +++ b/modules/examples/src/com/worxbend/codeberg4s/examples/HandlingErrors.scala @@ -40,13 +40,14 @@ import scala.concurrent.duration.FiniteDuration * ==There is no `NotFound` case== * * This is the single most common wrong assumption about the library, so the program prints the proof. The ADT has - * exactly five cases: + * exactly six cases: * * - `Transport` — nothing reached the server; * - `Api` — the server answered, and `status` is the HTTP status; * - `DecodingFailed` — a 2xx body did not match the model; * - `Validation` — an argument was rejected before a request was built; - * - `RetriesExhausted` — the retry engine gave up, wrapping the failure that ended it. + * - `RetriesExhausted` — the retry engine gave up, wrapping the failure that ended it; + * - `WalkTruncated` — a walk over every page hit its page cap with pages still to come. * * A `404` is `Api(ctx, 404, body)`. A `429` is `Api(ctx, 429, body)`, or a `RetriesExhausted` wrapping one once the * policy has run out of attempts. Nothing else exists to match on. @@ -127,7 +128,11 @@ object HandlingErrors: s"$status from ${ctx.operation} after ${ctx.durationMs}ms: $message" case CodebergError.Transport(ctx, cause) => - s"nothing reached the instance for ${ctx.operation}: ${cause.describe}" + // Almost always "nothing arrived". The one case where something did is + // TransportCause.ResponseTooLarge — the body passed the configured bound + // and reading it was abandoned — so the wording stays neutral and lets + // `describe` say which it was. + s"no usable response for ${ctx.operation}: ${cause.describe}" case CodebergError.DecodingFailed(ctx, snippet, path, cause) => s"${ctx.operation} answered 2xx but ${path.render} did not decode ($cause); body began $snippet" @@ -139,3 +144,12 @@ object HandlingErrors: // `last` is never discarded, so a 429 that outlived the policy is still // an Api(429) once this case is unwrapped. s"${ctx.operation} gave up after $attempts attempts; the last failure was: ${classify(last)}" + + case CodebergError.WalkTruncated(pagesVisited, resumeFrom) => + // The only case with no CallContext, because nothing went wrong on the + // wire: every one of those pages arrived. What failed is the walk's + // promise to cover the whole collection, so the partial answer is + // refused rather than returned, and `resumeFrom` is where a caller who + // wants the rest starts the next walk. + s"the page walk covered $pagesVisited pages and the collection was still going; " + + s"resume at page ${resumeFrom.page.value}" diff --git a/modules/examples/src/com/worxbend/codeberg4s/examples/SharingABackend.scala b/modules/examples/src/com/worxbend/codeberg4s/examples/SharingABackend.scala index eae4d13..9e4532b 100644 --- a/modules/examples/src/com/worxbend/codeberg4s/examples/SharingABackend.scala +++ b/modules/examples/src/com/worxbend/codeberg4s/examples/SharingABackend.scala @@ -5,10 +5,9 @@ import com.worxbend.codeberg4s.CodebergClient import com.worxbend.codeberg4s.CodebergConfig import com.worxbend.codeberg4s.auth.Auth import com.worxbend.codeberg4s.syntax.discard +import com.worxbend.codeberg4s.transport.SttpHttpPort import sttp.client4.Backend -import sttp.client4.BackendOptions -import sttp.client4.httpclient.HttpClientFutureBackend import scala.concurrent.Await import scala.concurrent.ExecutionContext @@ -44,8 +43,8 @@ import scala.concurrent.duration.FiniteDuration * * sttp models the connect timeout as a property of the backend, not of a request, so * [[com.worxbend.codeberg4s.CodebergConfig.connectTimeout]] is '''ignored''' by `usingBackend` — configure it on the - * backend, as the `BackendOptions` below do. [[com.worxbend.codeberg4s.CodebergConfig.readTimeout]] is applied per - * request and is honoured either way. + * backend, as the `defaultBackend` call below does. [[com.worxbend.codeberg4s.CodebergConfig.readTimeout]] is applied + * per request and is honoured either way. * * ==Why share one== * @@ -64,13 +63,17 @@ object SharingABackend: private val ConnectTimeout: FiniteDuration = 10.seconds def main(args: Array[String]): Unit = - given ExecutionContext = ExecutionContext.global + given executionContext: ExecutionContext = ExecutionContext.global // This program creates the backend, so this program closes it. The connect - // timeout is set here because usingBackend cannot apply the one in the + // timeout is passed here because usingBackend cannot apply the one in the // config — see the class comment. + // + // SttpHttpPort.defaultBackend rather than sttp's own HttpClientFutureBackend + // because closing the latter does not release the JDK HTTP client under it, + // so the pool this program is careful to close would outlive it anyway. val backend: Backend[Future] = - HttpClientFutureBackend(BackendOptions.Default.connectionTimeout(ConnectTimeout)) + SttpHttpPort.defaultBackend(ConnectTimeout, executionContext) // Two clients, one pool. They differ in configuration, not in transport. val codeberg: CodebergClient = CodebergClient.usingBackend(CodebergConfig(Auth.Anonymous), backend) @@ -94,9 +97,10 @@ object SharingABackend: codeberg.close() mirrored.foreach(client => client.close()) - // Then the thing this program owns. sttp's shutdown is asynchronous, so - // this returns a Future; nothing here observes it, and `discard` says so - // at the call site rather than letting -Wvalue-discard be switched off. + // Then the thing this program owns. Closing a backend returns a Future + // because sttp models it as an effect; the shutdown it starts does not + // block, nothing here observes the Future, and `discard` says so at the + // call site rather than letting -Wvalue-discard be switched off. backend.close().discard /** An optional second instance to talk to, so the example has a reason to share a pool. diff --git a/modules/it/src/com/worxbend/codeberg4s/it/IntegrationConfig.scala b/modules/it/src/com/worxbend/codeberg4s/it/IntegrationConfig.scala index 0d5e8b5..5656658 100644 --- a/modules/it/src/com/worxbend/codeberg4s/it/IntegrationConfig.scala +++ b/modules/it/src/com/worxbend/codeberg4s/it/IntegrationConfig.scala @@ -29,6 +29,9 @@ object IntegrationConfig: val ReadTimeout: FiniteDuration = 60.seconds /** Builds the configuration for one instance. + * + * The response-body bounds are left at the library defaults on purpose: a suite that raised them would stop + * exercising the limit real callers get. * * @param baseUri * the API root, `…/api/v1`, of the instance under test @@ -40,11 +43,13 @@ object IntegrationConfig: */ def forInstance(baseUri: BaseUri, auth: Auth, retry: RetryPolicy): CodebergConfig = CodebergConfig( - baseUri = baseUri, - auth = auth, - retry = retry, - userAgent = UserAgent.Default, - defaultPageSize = PageSize.Default, - connectTimeout = ConnectTimeout, - readTimeout = ReadTimeout, + baseUri = baseUri, + auth = auth, + retry = retry, + userAgent = UserAgent.Default, + defaultPageSize = PageSize.Default, + connectTimeout = ConnectTimeout, + readTimeout = ReadTimeout, + maxResponseBodyBytes = CodebergConfig.DefaultMaxResponseBodyBytes, + maxDownloadBodyBytes = CodebergConfig.DefaultMaxDownloadBodyBytes, ) diff --git a/modules/transport/src/com/worxbend/codeberg4s/transport/SttpHttpPort.scala b/modules/transport/src/com/worxbend/codeberg4s/transport/SttpHttpPort.scala index addae10..3521931 100644 --- a/modules/transport/src/com/worxbend/codeberg4s/transport/SttpHttpPort.scala +++ b/modules/transport/src/com/worxbend/codeberg4s/transport/SttpHttpPort.scala @@ -1,6 +1,7 @@ package com.worxbend.codeberg4s.transport import com.worxbend.codeberg4s.CodebergConfig +import com.worxbend.codeberg4s.ContentType import com.worxbend.codeberg4s.TransportCause import com.worxbend.codeberg4s.auth.Auth import com.worxbend.codeberg4s.core.BinaryHttpPort @@ -9,18 +10,22 @@ import com.worxbend.codeberg4s.core.CodebergRequest import com.worxbend.codeberg4s.core.CodebergResponse import com.worxbend.codeberg4s.core.HttpPort import com.worxbend.codeberg4s.core.RequestBody +import com.worxbend.codeberg4s.core.ResponseBody import com.worxbend.codeberg4s.core.TransportFailure +import sttp.capabilities.Effect +import sttp.capabilities.StreamMaxLengthExceededException import sttp.client4.Backend import sttp.client4.BackendOptions +import sttp.client4.GenericRequest import sttp.client4.PartialRequest import sttp.client4.Request import sttp.client4.Response import sttp.client4.asByteArrayAlways -import sttp.client4.asStringAlways import sttp.client4.basicRequest import sttp.client4.httpclient.HttpClientFutureBackend import sttp.client4.multipart +import sttp.client4.wrappers.DelegateBackend import sttp.model.Header import sttp.model.HeaderNames import sttp.model.MediaType @@ -31,14 +36,19 @@ import scala.annotation.tailrec import scala.concurrent.ExecutionContext import scala.concurrent.Future import scala.concurrent.duration.FiniteDuration +import scala.jdk.DurationConverters.ScalaDurationOps import scala.util.control.NonFatal +import java.net.Authenticator import java.net.ConnectException +import java.net.PasswordAuthentication import java.net.SocketException import java.net.SocketTimeoutException import java.net.UnknownHostException +import java.net.http.HttpClient import java.net.http.HttpTimeoutException import java.util.Locale +import java.util.concurrent.Executor /** The sttp implementation of [[com.worxbend.codeberg4s.core.HttpPort]] for `Future`. * @@ -47,28 +57,39 @@ import java.util.Locale * are called here and nowhere else. * * '''Failure contract.''' Every HTTP status — including `5xx` — is a `Right`, because deciding what a status means - * belongs to [[com.worxbend.codeberg4s.core.StatusMapping]]. A `Left` means no response arrived at all, classified + * belongs to [[com.worxbend.codeberg4s.core.StatusMapping]]. A `Left` means no complete response arrived, classified * into a [[com.worxbend.codeberg4s.TransportCause]] by walking the exception's cause chain; sttp wraps the original * `java.net` exception in an `SttpClientException`, so the outermost type is never the interesting one. An * unclassified non-fatal exception becomes [[com.worxbend.codeberg4s.TransportCause.Unknown]] rather than being - * dropped, and a fatal error stays fatal. + * dropped, and a fatal error stays fatal. Every cause but [[com.worxbend.codeberg4s.TransportCause.ResponseTooLarge]] + * means nothing arrived at all; see "Response size" below for the one that does not. * * '''Security contract.''' Nothing here logs, and nothing here renders a credential: the `Authorization` header is * built and handed straight to sttp, `toString` is deliberately opaque, and a [[TransportFailure]] carries only the * message of the exception that caused it. Credentials never reach the request URI, so the URI in an sttp exception - * message is safe. + * message is safe. The configured credential is also the only one that can be sent: a per-request header map naming + * `Authorization` or `Proxy-Authorization` has that entry dropped, and the configured credential is applied after + * every remaining header, so no request can carry two credentials or a caller-chosen one. * - * '''Resource ownership.''' `backend` belongs to whoever created it. This class never closes it, not even on failure — - * see [[SttpHttpPort.defaultBackend]]. + * '''Resource ownership.''' `backend` belongs to whoever created it. This class never closes it, not even on failure. + * A backend built by [[SttpHttpPort.defaultBackend]] owns a JDK `java.net.http.HttpClient` and releases it when it is + * closed; see that method for what "released" means and for why sttp's own backend does not manage to do it. * * '''Timeouts.''' `config.readTimeout` is applied per request. `config.connectTimeout` is a property of the backend in * sttp, so it is honoured only by a backend built through [[SttpHttpPort.defaultBackend]]; a caller who supplies their * own backend configures the connect timeout on that backend. * + * '''Response size.''' Every request carries a byte bound, because this library reads whole bodies into memory and + * never streams. [[send]] applies [[com.worxbend.codeberg4s.CodebergConfig.maxResponseBodyBytes]] and [[sendBinary]] + * applies the larger [[com.worxbend.codeberg4s.CodebergConfig.maxDownloadBodyBytes]]; passing either abandons the + * response as [[com.worxbend.codeberg4s.TransportCause.ResponseTooLarge]]. Unlike the connect timeout this holds for a + * caller-supplied backend too, since sttp models it per request. + * * @param backend * the sttp backend requests are sent on, owned and closed by the caller * @param config - * the instance to talk to, the credentials to use, the user agent to send and the read timeout to apply + * the instance to talk to, the credentials to use, the user agent to send, and the read timeout and response-body + * bounds to apply */ final class SttpHttpPort( backend: Backend[Future], @@ -89,72 +110,117 @@ final class SttpHttpPort( .left .map(_ => TransportFailure(TransportCause.Unknown(SttpHttpPort.UnparseableBaseUri))) + /** The part of an sttp request that is the same for every request this port will ever send. + * + * An sttp request is an immutable value, so each builder call allocates a new one. The user agent and the read + * timeout are read straight off [[com.worxbend.codeberg4s.CodebergConfig]], which cannot change once the port + * exists, so applying them per request rebuilt the identical pair of values on every call. They are applied once + * here instead and [[build]] starts from the result. + * + * The user agent sits here even though configuration has to win over a caller who sets `User-Agent` — normally that + * would mean applying it last, since sttp's `header` replaces by default. It does not need to: `User-Agent` is one + * of the names [[SttpHttpPort.callerHeaders]] drops, so by the time the caller's headers are applied there is + * nothing left that could overwrite this one. + */ + private val template: PartialRequest[Either[String, String]] = + basicRequest + .header(HeaderNames.UserAgent, config.userAgent.value) + .readTimeout(config.readTimeout) + /** Sends `request`, never throwing and never logging. * * `redactedUri` is deliberately unused: it exists so an adapter that reports what it dialled reports the safe * rendering, and this adapter reports nothing at all. */ override def send(request: CodebergRequest, redactedUri: String): Future[Either[TransportFailure, CodebergResponse]] = - root match - case Left(failure) => Future.successful(Left(failure)) - case Right(uri) => dispatch(build(request, uri)) + dispatch(request, config.maxResponseBodyBytes, SttpHttpPort.succeed) - /** Sends `request` and keeps the response body as bytes. + /** Sends `request` and reports what came back as a [[com.worxbend.codeberg4s.core.BinaryResponse]] instead. + * + * Identical to [[send]] apart from the response type it assembles and the body bound it applies, because both read + * the body the same way now. Two methods remain because core still has two response types; see + * [[com.worxbend.codeberg4s.core.BinaryResponse]] for why that is expected to change. * - * Reads with `asByteArrayAlways` rather than `asStringAlways`, so an archive survives. Only the few endpoints that - * answer a ZIP go through here; everything else keeps the textual path, which is cheaper and is what the JSON and - * `text/plain` endpoints want. + * The bound is [[com.worxbend.codeberg4s.CodebergConfig.maxDownloadBodyBytes]] rather than + * [[com.worxbend.codeberg4s.CodebergConfig.maxResponseBodyBytes]]: this is the path the ZIP-fetching operations + * under `client.downloads` take, and a CI artifact is legitimately far bigger than the largest JSON document Forgejo + * will produce. */ override def sendBinary( request: CodebergRequest, redactedUri: String, ): Future[Either[TransportFailure, BinaryResponse]] = - root match - case Left(failure) => Future.successful(Left(failure)) - case Right(uri) => dispatchBinary(buildBinary(request, uri)) + dispatch(request, config.maxDownloadBodyBytes, SttpHttpPort.succeedBinary) /** Deliberately opaque: this object holds the configured credentials, so it renders nothing about its state. */ override def toString: String = "SttpHttpPort" - private def dispatchBinary( - request: Request[Array[Byte]] - ): Future[Either[TransportFailure, BinaryResponse]] = - request - .send(backend) - .map(SttpHttpPort.succeedBinary) - .recover: - case error: InterruptedException => SttpHttpPort.fail(error) - case NonFatal(error) => SttpHttpPort.fail(error) - - private def buildBinary(request: CodebergRequest, uri: Uri): Request[Array[Byte]] = - withAuth(SttpHttpPort.withBody(request.body, basicRequest)) - .headers(request.headers.map((name, value) => Header(name, value))*) - .header(HeaderNames.UserAgent, config.userAgent.value) - .readTimeout(config.readTimeout) - .method(Method(request.method.wireName), SttpHttpPort.target(uri, request)) - .response(asByteArrayAlways) - - private def dispatch(request: Request[String]): Future[Either[TransportFailure, CodebergResponse]] = - request - .send(backend) - .map(SttpHttpPort.succeed) - .recover: - case error: InterruptedException => SttpHttpPort.fail(error) - case NonFatal(error) => SttpHttpPort.fail(error) - - private def build(request: CodebergRequest, uri: Uri): Request[String] = - withAuth(SttpHttpPort.withBody(request.body, basicRequest)) - .headers(request.headers.map((name, value) => Header(name, value))*) - .header(HeaderNames.UserAgent, config.userAgent.value) - .readTimeout(config.readTimeout) - .method(Method(request.method.wireName), SttpHttpPort.target(uri, request)) - .response(asStringAlways) + /** Builds one request, sends it, and turns whatever arrived into `A`, or into a classified transport failure. + * + * `onResponse` is the only thing that differed between the textual and the byte-carrying path, so the build, the + * send, the recovery and the exception classification are written once instead of twice. + * + * Two things can go wrong before a socket is touched: the configured base URI may not parse, and [[build]] may + * refuse the body. Both are already `Left` values, so they short-circuit here into an already-completed `Future` and + * the backend never sees the request. + */ + private def dispatch[A]( + request: CodebergRequest, + maxBodyBytes: Long, + onResponse: Response[Array[Byte]] => Either[TransportFailure, A], + ): Future[Either[TransportFailure, A]] = + root.flatMap(uri => build(request, uri, maxBodyBytes)) match + case Left(failure) => Future.successful(Left(failure)) + case Right(built) => + built + .send(backend) + .map(onResponse) + .recover: + case error: InterruptedException => SttpHttpPort.fail(error) + case NonFatal(error) => SttpHttpPort.fail(error) + + /** Builds the sttp request, reading '''every''' response body as bytes, or refuses to build it at all. + * + * `asByteArrayAlways` rather than `asStringAlways`, and that single word is the point of this path. With + * `asStringAlways`, sttp decodes the socket bytes into a `String`, and the JSON parser then encodes that `String` + * straight back into a `byte[]` in order to read it — two full copies of every payload before a single field is + * looked at. The charset sttp would have applied is not lost: it is read off `Content-Type` into + * [[com.worxbend.codeberg4s.core.ResponseBody]], which applies it if and when something actually asks for text. + * + * `maxResponseBodyLength` is what keeps "read the whole body into memory" from meaning "read as much as the peer + * cares to send". sttp stops reading at `maxBodyBytes` and fails the request with a + * `sttp.capabilities.StreamMaxLengthExceededException`, which [[SttpHttpPort.classify]] turns into + * [[com.worxbend.codeberg4s.TransportCause.ResponseTooLarge]]. Without it the only bound on a response is + * `config.readTimeout` multiplied by the peer's bandwidth, which is not a bound. + * + * The `Left` comes from [[SttpHttpPort.withBody]], which refuses a body whose media type cannot be written as a + * header. + * + * '''The order the headers go on is the security-relevant part.''' sttp's `header` defaults to + * `DuplicateHeaderBehavior.Replace`, so a name written twice keeps the value written last. The order below is + * therefore, from first to last: the user agent, from [[template]]; the body's own `Content-Type`, from + * [[SttpHttpPort.withBody]]; the caller's headers, which is what lets `POST /markdown/raw` send a `text/plain` + * content type over a body core models as JSON; and finally the credential, which nothing after it can overwrite + * because there is nothing after it. + */ + private def build( + request: CodebergRequest, + uri: Uri, + maxBodyBytes: Long, + ): Either[TransportFailure, Request[Array[Byte]]] = + SttpHttpPort + .withBody(request.body, template) + .map: carrying => + withAuth(carrying.headers(SttpHttpPort.callerHeaders(request.headers)*)) + .maxResponseBodyLength(maxBodyBytes) + .method(Method(request.method.wireName), SttpHttpPort.target(uri, request)) + .response(asByteArrayAlways) /** The one sanctioned call site of `reveal`. * * `Auth.Token` becomes `Authorization: token `, which is the spec's `AuthorizationHeaderToken` scheme — a * `Bearer` prefix is rejected by Forgejo. The header is applied after the caller's own headers so configuration - * always wins. + * always wins; see [[build]] for why "after" is what decides that with sttp's replace-by-default semantics. */ private def withAuth(request: PartialRequest[Either[String, String]]): PartialRequest[Either[String, String]] = config.auth match @@ -168,6 +234,14 @@ object SttpHttpPort: /** The detail reported when sttp cannot parse the configured base URI. Never echoes the value. */ val UnparseableBaseUri: String = "the configured base URI is not a valid request target" + /** The detail reported when a multipart part's media type cannot be written as a header. Never echoes the value. + * + * Reaching this means a [[com.worxbend.codeberg4s.core.RequestBody.Multipart]] was built from a raw string rather + * than from one of the upload commands, which already refuse a blank or control-carrying media type. The request is + * not sent either way. + */ + val UnsafeMultipartMediaType: String = "the multipart part's media type is blank or contains a control character" + /** The longest exception message copied into a [[com.worxbend.codeberg4s.TransportCause]]. * * A misbehaving proxy can fail with a multi-kilobyte message, and a transport detail is meant to fit in a log line. @@ -186,22 +260,167 @@ object SttpHttpPort: /** A `Future` backend built on the JDK HTTP client, with an explicit connect timeout and execution context. * - * '''Ownership.''' As above — the caller closes it. + * '''Ownership.''' As above — the caller closes it, and closing it really does release the connection pool. + * + * '''Why the client is built here rather than by sttp.''' `HttpClientFutureBackend(options)` decides whether it may + * release the `java.net.http.HttpClient` it creates by testing whether the `ExecutionContext` it was handed is also + * a `java.util.concurrent.Executor` — if it is, sttp assumes the executor is the caller's to shut down and sets its + * internal `closeClient` flag to `false`, after which `close()` releases nothing. Every ordinary `ExecutionContext` + * — `ExecutionContext.global`, one from `ExecutionContext.fromExecutor`, the one a test framework supplies — is an + * `ExecutionContextExecutor`, so that flag is always `false` and the pool always survived `close()`. Building the + * client here and handing it to `HttpClientFutureBackend.usingClient` moves the decision to this library, which + * knows it created the client and may therefore end it. + * + * The client is configured exactly as sttp configures its own: the connect timeout below, no redirect following + * (sttp's `FollowRedirectsBackend` wrapper does that itself, and a client that also followed them would apply the + * policy twice), the system proxy that `BackendOptions.Default` reads out of the standard `http.proxyHost` family of + * properties, and `executionContext` as the client's executor when it is one. * * @param connectTimeout - * how long to wait for a connection to be established; sttp configures this per backend, not per request + * how long to wait for a connection to be established; the JDK models this per client, not per request * @param executionContext - * where response callbacks run + * where response callbacks run, and the client's executor when it happens to be an `Executor` as well */ def defaultBackend(connectTimeout: FiniteDuration, executionContext: ExecutionContext): Backend[Future] = - HttpClientFutureBackend(BackendOptions.Default.connectionTimeout(connectTimeout))(using executionContext) + owning(defaultHttpClient(connectTimeout, executionContext), executionContext) + + /** The JDK HTTP client [[defaultBackend]] builds for itself, before it is wrapped in a backend. + * + * Internal: it exists apart from [[defaultBackend]] so a test can hold the client and ask it whether closing the + * backend terminated it. Callers outside the library have no reason to want the two halves separately. + */ + private[codeberg4s] def defaultHttpClient( + connectTimeout: FiniteDuration, + executionContext: ExecutionContext, + ): HttpClient = + val configured = HttpClient + .newBuilder() + .followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(connectTimeout.toJava) + + val executing = executionContext match + case executor: Executor => configured.executor(executor) + case _ => configured + + BackendOptions.Default.proxy.fold(executing)(proxy => proxied(executing, proxy)).build() + + /** An sttp backend on `client` that shuts `client` down when it is closed. + * + * Internal, and the other half of [[defaultHttpClient]]: together they are [[defaultBackend]]. + */ + private[codeberg4s] def owning(client: HttpClient, executionContext: ExecutionContext): Backend[Future] = + OwnedClientBackend(HttpClientFutureBackend.usingClient(client)(using executionContext), client) + + /** Points the builder at a proxy, and answers that proxy's authentication challenge when it has credentials. */ + private def proxied(builder: HttpClient.Builder, proxy: BackendOptions.Proxy): HttpClient.Builder = + val routed = builder.proxy(proxy.asJavaProxySelector) + proxy.auth.fold(routed)(credentials => routed.authenticator(ProxyAuthenticator(credentials))) + + /** What to answer a `requestor` that is asking for credentials: the proxy's own, and only if it is the proxy asking. + * + * Internal because it is the decision [[ProxyAuthenticator]] exists to make, and a test can then assert it without + * standing up a proxy. The `None` branch is the one that matters: a `java.net.Authenticator` is consulted for + * origin-server challenges too, and answering one of those with the proxy's password would disclose it to whatever + * host the request was aimed at. + */ + private[codeberg4s] def proxyCredentialsFor( + requestor: Authenticator.RequestorType, + credentials: BackendOptions.ProxyAuth, + ): Option[PasswordAuthentication] = + requestor match + case Authenticator.RequestorType.PROXY => + Some(PasswordAuthentication(credentials.username, credentials.password.toCharArray)) + case _ => None + + /** A backend that releases the JDK HTTP client underneath it, which is the one thing sttp's own `close` will not do. + * + * `close()` calls `shutdown()` and not `close()`. The JDK's `HttpClient.close()` waits until every in-flight request + * has finished, and [[com.worxbend.codeberg4s.CodebergClient.close]] is documented as returning promptly so that an + * ordinary `finally` block stays cheap. `shutdown()` starts the same orderly shutdown — requests already submitted + * run to completion, no new one is accepted — and returns without waiting for it. + * + * The client's executor is not touched, because it is `executionContext`, and that belongs to the caller. Only a + * client the JDK gave its own default executor loses one, and that executor was never anybody else's. + */ + private final class OwnedClientBackend(delegate: Backend[Future], client: HttpClient) + extends DelegateBackend[Future, Any](delegate), + Backend[Future]: + + override def send[T](request: GenericRequest[T, Any & Effect[Future]]): Future[Response[T]] = + delegate.send(request) + + override def close(): Future[Unit] = + client.shutdown() + delegate.close() + + /** Hands [[proxyCredentialsFor]]'s answer to the JDK. + * + * `java.net.Authenticator`'s contract for "I hold no credentials for this challenge" is a `null` return, so `orNull` + * here is the single point at which the `Option` that carries that answer meets the Java side. + */ + private final class ProxyAuthenticator(proxyAuth: BackendOptions.ProxyAuth) extends Authenticator: + + override protected def getPasswordAuthentication: PasswordAuthentication = + proxyCredentialsFor(getRequestorType, proxyAuth).orNull private def target(uri: Uri, request: CodebergRequest): Uri = uri.addPath(request.path).addParams(request.query*) + /** The header names this port owns, lowercased so a lookup can be case-insensitive the way HTTP is. + * + * A header name is case-insensitive on the wire, so `authorization` and `Authorization` are one header and a set + * membership test has to see them as one. `Locale.ROOT` rather than the default locale because the default one may + * be Turkish, where lowercasing `I` produces a dotless `ı` and the comparison silently stops matching. + */ + private val PortOwnedHeaders: Set[String] = + Set(HeaderNames.Authorization, HeaderNames.ProxyAuthorization, HeaderNames.UserAgent) + .map(_.toLowerCase(Locale.ROOT)) + + /** The caller's headers with the ones this port owns removed, ready to hand to sttp. + * + * '''Why a credential header is dropped and not merely overwritten.''' + * [[com.worxbend.codeberg4s.core.CodebergRequest]] documents that its `headers` never carry a credential, and every + * one of those maps is built inside this library, so today none of them does. Nothing structural stops one: a new + * endpoint is an ordinary `List[(String, String)]` away from putting `Authorization` there, and the result would be + * a request authenticated as something other than what [[com.worxbend.codeberg4s.auth.Auth]] configured — or, with a + * different spelling of the name, two credentials on one request. Removing the entry here makes that outcome + * unreachable rather than unlikely, and costs nothing, because no endpoint has a reason to set these names. + * + * `User-Agent` is in the same set for a plainer reason: it is configuration too, and dropping it here is what lets + * [[SttpHttpPort.template]] apply the configured one first and still win. + * + * Every other name is passed through untouched, which is the point — a per-request `Content-Type` is a legitimate + * override and `POST /markdown/raw` depends on it. + */ + private def callerHeaders(headers: List[(String, String)]): List[Header] = + headers.collect: + case (name, value) if !PortOwnedHeaders.contains(name.toLowerCase(Locale.ROOT)) => Header(name, value) + + /** Attaches the body, unless the body carries a media type that must not become a header. + * + * '''Defence in depth, and deliberately a second copy of a rule the domain already enforces.''' + * [[com.worxbend.codeberg4s.issues.UploadAttachment.as]] and + * [[com.worxbend.codeberg4s.repositories.publishing.UploadAsset.as]] refuse such a value at construction, and that + * is the check a caller should ever see, because it names the field and happens before a request exists. This one + * exists because [[com.worxbend.codeberg4s.core.RequestBody.Multipart]] is a plain case class that any code inside + * the library can build with a bare `String`, and this method is the last point at which that string is still a + * Scala value rather than wire bytes. + * + * A refusal is a [[TransportFailure]] and not an exception: a request that was never sent is exactly what + * [[com.worxbend.codeberg4s.TransportCause]] describes, and the pipeline above already knows how to report one. + */ private def withBody( body: Option[RequestBody], request: PartialRequest[Either[String, String]], + ): Either[TransportFailure, PartialRequest[Either[String, String]]] = + body match + case Some(RequestBody.Multipart(_, _, _, mediaType)) if !ContentType.isSafe(mediaType) => + Left(TransportFailure(TransportCause.Unknown(UnsafeMultipartMediaType))) + case other => Right(attach(other, request)) + + private def attach( + body: Option[RequestBody], + request: PartialRequest[Either[String, String]], ): PartialRequest[Either[String, String]] = body match case None => request @@ -218,8 +437,15 @@ object SttpHttpPort: private def succeedBinary(response: Response[Array[Byte]]): Either[TransportFailure, BinaryResponse] = Right(BinaryResponse(response.code.code, lowercased(response.headers), response.body)) - private def succeed(response: Response[String]): Either[TransportFailure, CodebergResponse] = - Right(CodebergResponse(response.code.code, lowercased(response.headers), response.body)) + /** The charset the response declared is captured here, next to the bytes, and applied nowhere yet. + * + * sttp used to make this decision inside `asStringAlways`; it now belongs to + * [[com.worxbend.codeberg4s.core.ResponseBody]], which is where a reader that wants text asks for it. Reading the + * header at this point rather than later matters because a `ResponseBody` outlives the sttp `Response` it came from. + */ + private def succeed(response: Response[Array[Byte]]): Either[TransportFailure, CodebergResponse] = + val body = ResponseBody.of(response.body, ResponseBody.charsetOf(response.contentType)) + Right(CodebergResponse(response.code.code, lowercased(response.headers), body)) /** Generic in the success type so the textual and binary paths share one classification. */ private def fail[A](error: Throwable): Either[TransportFailure, A] = @@ -242,14 +468,18 @@ object SttpHttpPort: @tailrec private def classify(error: Throwable): TransportCause = error match - case _: UnknownHostException => TransportCause.Dns(detail(error)) - case _: SocketTimeoutException => TransportCause.Timeout(detail(error)) - case _: HttpTimeoutException => TransportCause.Timeout(detail(error)) - case _: javax.net.ssl.SSLException => TransportCause.Tls(detail(error)) - case _: ConnectException => TransportCause.ConnectionFailed(detail(error)) - case _: SocketException => TransportCause.ConnectionFailed(detail(error)) - case _: InterruptedException => TransportCause.Interrupted(detail(error)) - case _ => + case _: UnknownHostException => TransportCause.Dns(detail(error)) + case _: SocketTimeoutException => TransportCause.Timeout(detail(error)) + case _: HttpTimeoutException => TransportCause.Timeout(detail(error)) + case _: javax.net.ssl.SSLException => TransportCause.Tls(detail(error)) + case _: ConnectException => TransportCause.ConnectionFailed(detail(error)) + case _: SocketException => TransportCause.ConnectionFailed(detail(error)) + case _: InterruptedException => TransportCause.Interrupted(detail(error)) + // Thrown by sttp when the body passes the request's maxResponseBodyLength. + // It arrives wrapped in an SttpClientException.ReadException, which is why + // this is a cause-chain walk and not a match on the outermost type. + case _: StreamMaxLengthExceededException => TransportCause.ResponseTooLarge(detail(error)) + case _ => Option(error.getCause).filterNot(_.eq(error)) match case Some(cause) => classify(cause) case None => TransportCause.Unknown(detail(error)) diff --git a/modules/transport/test/src/com/worxbend/codeberg4s/transport/DefaultBackendSuite.scala b/modules/transport/test/src/com/worxbend/codeberg4s/transport/DefaultBackendSuite.scala new file mode 100644 index 0000000..1bd35d0 --- /dev/null +++ b/modules/transport/test/src/com/worxbend/codeberg4s/transport/DefaultBackendSuite.scala @@ -0,0 +1,101 @@ +package com.worxbend.codeberg4s.transport + +import com.worxbend.codeberg4s.syntax.discard + +import sttp.client4.BackendOptions + +import munit.FunSuite + +import scala.concurrent.ExecutionContext +import scala.concurrent.ExecutionContextExecutorService +import scala.concurrent.duration.DurationInt +import scala.concurrent.duration.FiniteDuration +import scala.jdk.DurationConverters.ScalaDurationOps +import scala.jdk.OptionConverters.RichOptional + +import java.net.Authenticator +import java.net.http.HttpClient +import java.util.concurrent.Executor +import java.util.concurrent.Executors + +/** What [[SttpHttpPort.defaultBackend]] builds, and what closing it actually releases. + * + * These are the only tests in this module that create a real `java.net.http.HttpClient` rather than a `BackendStub`, + * because the behaviour under test is a lifecycle and a stub has none. No test here sends a request, so nothing + * touches the network: the JDK starts the client's selector thread when the client is built, and `isTerminated` + * reports on it without a socket ever being opened. + */ +final class DefaultBackendSuite extends FunSuite: + + test("closing a backend that owns its JDK client terminates that client"): + withExecutor: executor => + val client = SttpHttpPort.defaultHttpClient(DefaultBackendSuite.ConnectTimeout, executor) + val backend = SttpHttpPort.owning(client, executor) + + assert(!client.isTerminated, "a client that was never closed already reports itself terminated") + + backend.close().discard + + assert( + client.awaitTermination(DefaultBackendSuite.TerminationLimit.toJava), + "close() left the JDK HTTP client running", + ) + + test("closing the backend leaves the caller's execution context running"): + withExecutor: executor => + val backend = + SttpHttpPort.owning(SttpHttpPort.defaultHttpClient(DefaultBackendSuite.ConnectTimeout, executor), executor) + + backend.close().discard + + assert(!executor.isShutdown, "closing the backend shut down an execution context the caller owns") + + test("the client is built the way sttp's redirect wrapper needs it"): + withExecutor: executor => + val client = SttpHttpPort.defaultHttpClient(DefaultBackendSuite.ConnectTimeout, executor) + + // sttp wraps every HttpClientFutureBackend in FollowRedirectsBackend, + // which applies the redirect policy itself. A JDK client that followed + // redirects as well would apply it twice, which is why sttp's own + // defaultClient sets NEVER and why this one has to agree. + assertEquals(client.followRedirects, HttpClient.Redirect.NEVER) + assertEquals(client.connectTimeout.toScala, Some(DefaultBackendSuite.ConnectTimeout.toJava)) + assertEquals(client.executor.toScala, Option[Executor](executor)) + + test("a proxy's credentials answer the proxy's own challenge"): + val answer = SttpHttpPort.proxyCredentialsFor(Authenticator.RequestorType.PROXY, DefaultBackendSuite.ProxyAuth) + + assertEquals(answer.map(_.getUserName), Some("alice")) + assertEquals(answer.map(_.getPassword.mkString), Some("s3cret")) + + test("a proxy's credentials are withheld from an origin server's challenge"): + // A java.net.Authenticator is consulted for origin-server challenges too. + // Answering one with the proxy's password would hand that password to + // whichever host the request was aimed at. + assertEquals( + SttpHttpPort.proxyCredentialsFor(Authenticator.RequestorType.SERVER, DefaultBackendSuite.ProxyAuth), + None, + ) + + /** Runs `use` against an execution context that is also an `ExecutorService`, and shuts that service down afterwards. + * + * Being an `Executor` is the point rather than an incidental detail: it is the property that made sttp decline to + * release the HTTP client, so a fixture without it would exercise the one case the defect never reached. + */ + private def withExecutor(use: ExecutionContextExecutorService => Unit): Unit = + val executor = ExecutionContext.fromExecutorService(Executors.newSingleThreadExecutor()) + + try use(executor) + finally executor.shutdown() + +object DefaultBackendSuite: + + private val ConnectTimeout: FiniteDuration = 3.seconds + + /** Credentials that must reach a proxy and no one else. */ + private val ProxyAuth: BackendOptions.ProxyAuth = BackendOptions.ProxyAuth("alice", "s3cret") + + /** Generous on purpose. An idle client terminates in single-digit milliseconds; this is a hang detector, not a race + * the test is trying to win. + */ + private val TerminationLimit: FiniteDuration = 10.seconds diff --git a/modules/transport/test/src/com/worxbend/codeberg4s/transport/RequestBodySuite.scala b/modules/transport/test/src/com/worxbend/codeberg4s/transport/RequestBodySuite.scala index c7a32d3..ab0c3ab 100644 --- a/modules/transport/test/src/com/worxbend/codeberg4s/transport/RequestBodySuite.scala +++ b/modules/transport/test/src/com/worxbend/codeberg4s/transport/RequestBodySuite.scala @@ -3,9 +3,12 @@ package com.worxbend.codeberg4s.transport import com.worxbend.codeberg4s.BaseUri import com.worxbend.codeberg4s.CodebergConfig import com.worxbend.codeberg4s.HttpMethod +import com.worxbend.codeberg4s.TransportCause import com.worxbend.codeberg4s.auth.Auth import com.worxbend.codeberg4s.core.CodebergRequest +import com.worxbend.codeberg4s.core.CodebergResponse import com.worxbend.codeberg4s.core.RequestBody +import com.worxbend.codeberg4s.core.TransportFailure import sttp.client4.GenericRequest import sttp.client4.MultipartBody @@ -16,6 +19,7 @@ import sttp.model.HeaderNames import munit.FunSuite import scala.concurrent.ExecutionContext +import scala.util.Success import java.nio.charset.StandardCharsets @@ -32,7 +36,14 @@ final class RequestBodySuite extends FunSuite: private val config: CodebergConfig = CodebergConfig(Auth.Anonymous).copy(baseUri = BaseUri.Codeberg) - private def send(body: RequestBody): GenericRequest[?, ?] = + private val Payload: Array[Byte] = "hi".getBytes(StandardCharsets.UTF_8) + + /** Sends `body` and reports both what the port answered and every request the backend actually saw. + * + * The backend list matters for the refusal case: a body the transport rejects must never reach a socket, and an + * assertion on the answer alone would not notice a request that was sent and then reported as failed. + */ + private def attempt(body: RequestBody): (Either[TransportFailure, CodebergResponse], List[GenericRequest[?, ?]]) = val recording = RecordingBackend(BackendStub.asynchronousFuture.whenAnyRequest.thenRespondOk()) val port = SttpHttpPort(recording, config) val request = CodebergRequest( @@ -43,14 +54,18 @@ final class RequestBodySuite extends FunSuite: headers = Nil, body = Some(body), ) - port.send(request, "probe").value.discard - recording.allInteractions.map(_._1).head + port.send(request, "probe").value match + case Some(Success(answer)) => (answer, recording.allInteractions.map(_._1)) + case other => fail(s"expected the send to have completed, got $other") + + private def send(body: RequestBody): GenericRequest[?, ?] = + attempt(body) match + case (_, request :: _) => request + case (answer, Nil) => fail(s"the backend saw no request; the port answered $answer") private def contentTypeOf(request: GenericRequest[?, ?]): Option[String] = request.header(HeaderNames.ContentType) - extension [A](value: A) private def discard: Unit = () - test("a JSON body is sent as application/json"): val request = send(RequestBody.Json("""{"a":1}""")) @@ -90,6 +105,30 @@ final class RequestBodySuite extends FunSuite: assertEquals(multipart.parts.map(_.fileName).toList, List(Some("notes.txt"))) case other => fail(s"expected a multipart body, got ${other.show}") + test("a multipart media type carrying a line break is refused, and no request reaches the backend"): + // Defence in depth: UploadAttachment.as and UploadAsset.as already refuse + // this, so getting here means a Multipart was built from a raw string. The + // value would have become the part's own Content-Type header, and the CRLF + // in it would have ended that header and opened one of the caller's + // choosing. + val (answer, seen) = + attempt(RequestBody.Multipart("attachment", "notes.txt", Payload, "text/plain\r\nX-Injected: 1")) + + val expected: Either[TransportFailure, CodebergResponse] = + Left(TransportFailure(TransportCause.Unknown(SttpHttpPort.UnsafeMultipartMediaType))) + + assertEquals(answer, expected) + assert(seen.isEmpty, s"the request was sent anyway: $seen") + + test("a blank multipart media type is refused for the same reason"): + val (answer, seen) = attempt(RequestBody.Multipart("attachment", "notes.txt", Payload, " ")) + + val expected: Either[TransportFailure, CodebergResponse] = + Left(TransportFailure(TransportCause.Unknown(SttpHttpPort.UnsafeMultipartMediaType))) + + assertEquals(answer, expected) + assert(seen.isEmpty, s"the request was sent anyway: $seen") + test("an empty body sends no content"): val request = send(RequestBody.Empty) diff --git a/modules/transport/test/src/com/worxbend/codeberg4s/transport/SttpHttpPortSuite.scala b/modules/transport/test/src/com/worxbend/codeberg4s/transport/SttpHttpPortSuite.scala index c860980..968a10f 100644 --- a/modules/transport/test/src/com/worxbend/codeberg4s/transport/SttpHttpPortSuite.scala +++ b/modules/transport/test/src/com/worxbend/codeberg4s/transport/SttpHttpPortSuite.scala @@ -12,16 +12,21 @@ import com.worxbend.codeberg4s.auth.Password import com.worxbend.codeberg4s.core.CodebergRequest import com.worxbend.codeberg4s.core.CodebergResponse import com.worxbend.codeberg4s.core.RequestBody +import com.worxbend.codeberg4s.core.ResponseBody import com.worxbend.codeberg4s.core.TransportFailure +import sttp.capabilities.StreamMaxLengthExceededException import sttp.client4.Backend import sttp.client4.GenericRequest +import sttp.client4.SttpClientException +import sttp.client4.basicRequest import sttp.client4.testing.BackendStub import sttp.client4.testing.RecordingBackend import sttp.client4.testing.ResponseStub import sttp.model.Header import sttp.model.Method import sttp.model.StatusCode +import sttp.model.Uri import munit.FunSuite @@ -84,6 +89,42 @@ final class SttpHttpPortSuite extends FunSuite: send(port, awkwardRequest).map: _ => assertEquals(headerOf(backend, "authorization"), None) + test("a caller's Authorization header cannot override the configured credential"): + val backend = recording(respondingOk) + val port = SttpHttpPort(backend, configFor(Auth.Token(token(Secret)))) + val request = awkwardRequest.copy(headers = List("Authorization" -> "token someone-elses-token")) + + send(port, request).map: _ => + assertEquals(valuesOf(backend, "authorization"), List(s"token $Secret")) + + test("a caller's Authorization header is dropped whatever case it spells the name in"): + val backend = recording(respondingOk) + val port = SttpHttpPort(backend, configFor(Auth.Anonymous)) + val request = awkwardRequest.copy(headers = List("authorization" -> "token someone-elses-token")) + + send(port, request).map: _ => + assertEquals(valuesOf(backend, "authorization"), Nil) + + test("a caller's Proxy-Authorization header never reaches the wire"): + val backend = recording(respondingOk) + val port = SttpHttpPort(backend, configFor(Auth.Anonymous)) + val request = awkwardRequest.copy(headers = List("Proxy-Authorization" -> "Basic c29tZTpvbmU=")) + + send(port, request).map: _ => + assertEquals(valuesOf(backend, "proxy-authorization"), Nil) + + test("a caller's Content-Type still overrides the one the body implies"): + val backend = recording(respondingOk) + val port = SttpHttpPort(backend, configFor(Auth.Anonymous)) + val request = awkwardRequest.copy( + method = HttpMethod.Post, + headers = List("Content-Type" -> "text/plain; charset=utf-8"), + body = Some(RequestBody.Json("# Title")), + ) + + send(port, request).map: _ => + assertEquals(valuesOf(backend, "content-type"), List("text/plain; charset=utf-8")) + test("the configured user agent is sent"): val agent = orFail(UserAgent.from("codeberg4s-test/1.0")) val config = configFor(Auth.Anonymous).copy(userAgent = agent) @@ -123,7 +164,7 @@ final class SttpHttpPortSuite extends FunSuite: val port = SttpHttpPort(backend, configFor(Auth.Anonymous)) send(port, awkwardRequest).map: result => - assertEquals(result, Right(CodebergResponse(500, Map.empty, "upstream exploded"))) + assertEquals(result, Right(CodebergResponse(500, Map.empty, ResponseBody.utf8("upstream exploded")))) test("response header names are lowercased and repeated values are kept in order"): val headers = List(Header("X-Total-Count", "1590"), Header("Link", "; rel=\"next\""), Header("Link", "")) @@ -162,6 +203,47 @@ final class SttpHttpPortSuite extends FunSuite: causeOf(new InterruptedException("interrupted")).map: cause => assertEquals(cause, TransportCause.Interrupted("interrupted")) + test("a body that passed the configured bound is classified as too large, not as unknown"): + // Unknown is retryable and ResponseTooLarge is not, so misclassifying this one + // would re-download the oversized body on every remaining attempt. + causeOf(StreamMaxLengthExceededException(1024L)).map: cause => + assertEquals(cause, TransportCause.ResponseTooLarge("Stream length limit of 1024 bytes exceeded")) + + test("the same failure is recognised through the sttp exception that wraps it"): + // This is the shape a real backend produces: sttp maps the internal exception + // to SttpClientException.ReadException before it reaches the recover block, so + // matching only the outermost type would classify every oversized body as unknown. + val wrapped = SttpClientException.ReadException(sttpRequest, StreamMaxLengthExceededException(1024L)) + + causeOf(wrapped).map: cause => + assertEquals(cause, TransportCause.ResponseTooLarge("Stream length limit of 1024 bytes exceeded")) + + test("a textual request carries the configured response-body bound"): + val backend = recording(respondingOk) + val port = SttpHttpPort(backend, configFor(Auth.Anonymous)) + + send(port, awkwardRequest).map: _ => + assertEquals(sent(backend).options.maxResponseBodyLength, Some(CodebergConfig.DefaultMaxResponseBodyBytes)) + + test("a download carries the larger download bound instead"): + val backend = recording(respondingOk) + val port = SttpHttpPort(backend, configFor(Auth.Anonymous)) + + port.sendBinary(awkwardRequest, "https://forge.example/api/v1/repos/ow%20ner").map: _ => + assertEquals(sent(backend).options.maxResponseBodyLength, Some(CodebergConfig.DefaultMaxDownloadBodyBytes)) + + test("both bounds are taken from the config rather than hardcoded"): + val config = configFor(Auth.Anonymous).copy(maxResponseBodyBytes = 111L, maxDownloadBodyBytes = 222L) + val textual = recording(respondingOk) + val binary = recording(respondingOk) + + for + _ <- send(SttpHttpPort(textual, config), awkwardRequest) + _ <- SttpHttpPort(binary, config).sendBinary(awkwardRequest, "https://forge.example/api/v1") + yield + assertEquals(sent(textual).options.maxResponseBodyLength, Some(111L)) + assertEquals(sent(binary).options.maxResponseBodyLength, Some(222L)) + test("an exception this library does not recognise is unknown, never dropped"): causeOf(new IllegalStateException("something else entirely")).map: cause => assertEquals(cause, TransportCause.Unknown("something else entirely")) @@ -229,6 +311,10 @@ final class SttpHttpPortSuite extends FunSuite: private def responding(status: Int, headers: List[Header], body: String): BackendStub[Future] = BackendStub.asynchronousFuture.whenAnyRequest.thenRespond(ResponseStub.adjust(body, StatusCode(status), headers)) + /** A minimal sttp request, only so an `SttpClientException` can be built the way a real backend builds one. */ + private def sttpRequest: GenericRequest[?, ?] = + basicRequest.get(Uri.unsafeParse("https://forge.example/api/v1")) + private def failingWith(error: Throwable): BackendStub[Future] = BackendStub.asynchronousFuture.whenAnyRequest.thenThrow(error) @@ -243,6 +329,15 @@ final class SttpHttpPortSuite extends FunSuite: private def headerOf(backend: RecordingBackend, name: String): Option[String] = sent(backend).headers.find(_.is(name)).map(_.value) + /** Every value sent under `name`, in order. + * + * [[headerOf]] reports the first match and so cannot tell "sent once" from "sent twice with different values", which + * is exactly the difference the credential tests are about. `Header.is` compares the name case-insensitively, the + * way HTTP does. + */ + private def valuesOf(backend: RecordingBackend, name: String): List[String] = + sent(backend).headers.filter(_.is(name)).map(_.value).toList + // --- validated fixtures --------------------------------------------------- private def token(value: String): ApiToken = diff --git a/scripts/alloc-bench.sc b/scripts/alloc-bench.sc new file mode 100644 index 0000000..b76f4fd --- /dev/null +++ b/scripts/alloc-bench.sc @@ -0,0 +1,576 @@ +//> using scala 3.8.4 + +// scripts/alloc-bench.sc — bytes allocated and wall-clock time per JSON decode operation. +// +// Usage (scripts/alloc-bench.sh is the entry point; it resolves the compiled +// codec classpath, which this script needs at COMPILE time and therefore cannot +// resolve for itself): +// +// scripts/alloc-bench.sh # every operation +// scripts/alloc-bench.sh timestamps repo # only operations whose name contains one of these +// scripts/alloc-bench.sh --rounds=11 --scale=4 # more rounds, four times the iterations +// +// or, when the classpath is already in $CP: +// +// scala-cli run scripts/alloc-bench.sc --extra-jars "$CP" -- [filters] +// +// --------------------------------------------------------------------------- +// DELIBERATELY NOT WIRED INTO verify.sh +// --------------------------------------------------------------------------- +// This is a measurement tool, not a gate. A timing assertion in CI fails on a +// noisy runner and passes on a quiet one, which trains everybody to re-run the +// build until it goes green — the exact habit that makes a real regression +// invisible. Allocation counts are far steadier than times, so a gate on them +// would be more defensible one day, but it is not free either: escape analysis +// moves the number with the JIT's mood (see the limitations below), so such a +// gate would still be a threshold on JVM behaviour rather than on this code. +// +// Run this by hand before and after a change, quote both numbers in the commit +// message, and let a human read them. +// +// --------------------------------------------------------------------------- +// WHAT IS MEASURED +// --------------------------------------------------------------------------- +// parse.page-50 Json.parse of a 50-repository page — the parse step alone +// decode.page-50 Json.decode[Vector[RepositoryDto]] of the same page from a +// String — the parse plus the assembly of 50 wide DTOs +// decode.page-50-bytes the same decode from the bytes the socket produced, +// which is the path a response takes today +// decode.page-50-viastring the same decode the way it worked before a response +// body became bytes: decode the socket's bytes into a String +// (what sttp's asStringAlways did), then hand that String to +// the parser, which encodes it back into a byte[] to read it. +// The gap between this row and decode.page-50-bytes is the +// cost that carrying bytes removed +// decode.repo-wide Json.decode[RepositoryDto] of one repository object +// assemble.repo-wide the same DTO built from an ALREADY-PARSED document, so the +// parse is excluded and only field lookup and assembly remain +// decode.org-narrow Json.decode[OrganizationDto] of one organisation object +// assemble.org-narrow the same, already parsed +// timestamps.parse Timestamps.parse of a real timestamp lifted out of the +// repository fixture, with the answer stored where it +// cannot be optimised away — see "the sink" below +// timestamps.parse-ea the same call with the answer thrown away, so that +// HotSpot's escape analysis is free to delete the Instant; +// whether it does depends on what else ran in the same JVM +// — see the profile-pollution limitation below +// parse.ints-1000 Json.parse of a 1000-element array of integers +// parse.bools-1000 Json.parse of a 1000-element array of booleans +// baseline.noop an operation that does nothing, driven through the same +// loop — that row is the harness measuring itself, and it +// must read 0.0 B/op or nothing above it is trustworthy +// +// The narrow and the wide DTO are measured apart on purpose. RepositoryDto +// models 63 fields of a 64-key object; OrganizationDto models 12 of a 12-key +// object. Anything that changes how a field is looked up — the Map that +// JsonFields wraps today — behaves differently at those two widths, and one +// average over a mixed payload would hide which way each of them moved. +// +// --------------------------------------------------------------------------- +// METHODOLOGY +// --------------------------------------------------------------------------- +// Allocation is read from com.sun.management.ThreadMXBean, an OpenJDK extension +// to the standard java.lang.management.ThreadMXBean: +// +// getCurrentThreadAllocatedBytes — the running total of heap bytes this thread +// has allocated since it started +// +// One measurement is: read the counter, run the operation N times in a +// tail-recursive loop, read the counter again, divide the difference by N. Wall +// clock is System.nanoTime around the same loop. +// +// Warm-up is three full rounds per operation, discarded, before seven measured +// rounds; both counts are adjustable (--warmup, --rounds, --scale). Three rounds +// is enough for HotSpot to reach its top compilation tier on every operation +// here, because even the smallest of them runs hundreds of thousands of times +// per round. +// +// The two reported figures are summarised differently on purpose: +// +// B/op is the MEDIAN of the measured rounds. Allocation is close to +// deterministic once warm — the spread column normally reads "exact" — +// and the median ignores the odd round that ran extra code. +// ns/op is the FASTEST measured round, not the median. Wall clock on a shared +// machine measures the machine as much as the code: a round that caught +// a garbage collection, a background compilation or another process is +// slower for a reason that has nothing to do with the operation, and +// those reasons only ever add time. The fastest round is the least +// contaminated estimate available without a quiet machine, and the +// spread column beside it says how much the other rounds disagreed — +// when that number is large, the machine was busy, not the code slow. +// +// Every operation answers an int checksum derived from its result, the loop adds +// those up, and the total is printed. Without that, nothing stops the JIT from +// deleting work whose result is never read. +// +// Before measuring anything the harness runs each operation once and refuses to +// continue if it answered a failure: a decode that fails bails out early and +// would report a fraction of the real cost as though it were the whole cost. +// +// THE SINK. An object that never leaves the method that made it can be taken +// apart by HotSpot's escape analysis and never allocated at all. That is a +// genuine saving when it happens in the library, and a measurement artefact when +// it happens only because a benchmark threw the answer away. Timestamps.parse is +// the case where it matters: measured with its answer discarded it allocates +// nothing, and in the library the Instant it returns is stored in a Repository +// and therefore does allocate. So the primary timestamp operation writes its +// answer into a one-element array allocated once, up front — the standard +// blackhole trick, and the smallest thing that makes the object escape. The +// paired -ea operation does not, so the gap between the two rows is the size of +// the effect rather than a claim to have avoided it. +// +// --------------------------------------------------------------------------- +// KNOWN LIMITATIONS — READ THESE BEFORE QUOTING A NUMBER +// --------------------------------------------------------------------------- +// THIS IS NOT JMH. It is a single-threaded allocation counter with a loop around +// it. It does not fork a JVM per benchmark, does not blackhole its results +// beyond the checksum, does not measure a distribution, does not report error +// bars, and cannot detect that a result was constant-folded. JMH exists and does +// all of that; if a number from here ever decides a design argument, reach for +// JMH rather than arguing about this script. +// +// More specifically: +// +// * THE COUNTER IS PER THREAD. Work moved onto another thread — a parallel +// collection, a future, an executor — allocates just as much and shows up +// here as free. Everything measured below is synchronous today. +// +// * ESCAPE ANALYSIS COUNTS AS A SAVING. An allocation HotSpot scalar-replaces +// is never counted, which is right (it costs nothing at run time) and also +// means the number depends on JIT state, on inlining decisions and on the +// JDK build. Numbers from two different JDKs are not comparable. That is why +// the JDK is printed in the report header, and why the two timestamp rows +// are there: they are the same call measured either side of the effect. +// +// * THE DRIVING CALL SITE IS MEGAMORPHIC. Every operation goes through one +// virtual call in the loop, so an operation is never inlined into its +// caller. That is representative for Json.decode, which is far too large for +// any real caller to inline, and pessimistic for something as small as +// Timestamps.parse, whose Instant might well be scalar-replaced at a real +// call site and is counted here. +// +// * EVERY OPERATION SHARES ONE JVM, so they pollute each other's profiles. +// This is not hypothetical here, and the size of it was measured rather than +// assumed: run on its own, timestamps.parse-ea reports 0.0 B/op, because +// Timestamps.parse has one caller, gets inlined, and its Instant is scalar- +// replaced. Run in the full table, where timestamps.parse calls it too, both +// rows report 40.0 B/op — one Instant (24 B) plus one Some (16 B) — because +// the shared method no longer inlines the same way. Forking a JVM per +// benchmark is precisely what JMH does about this and what this script does +// not. Compare a full table against a full table, never a filtered run +// against an unfiltered one. +// +// * TIMES ARE WALL CLOCK ON WHATEVER MACHINE THIS RAN ON, with no attempt to +// pin a CPU, quiet the machine, or account for turbo and thermal drift. Read +// ns/op as an order of magnitude and a direction of travel: the same row has +// been seen to move by half between two consecutive full runs on an idle +// laptop. Bytes are the number worth arguing about, and even they are not +// perfectly reproducible across runs — the small operations have been +// observed to move by 16 bytes, one object, from one run to the next. A +// one-object difference at these sizes is noise, not a result. +// +// * THE SPREAD COLUMNS say how far the measured rounds of ONE run disagreed. +// "exact" means every round agreed to the byte. They say nothing about how +// far two different runs would disagree; only running it again says that. +// +// * ONLY HEAP BYTES ARE COUNTED. Direct byte buffers, memory-mapped files and +// native allocation are invisible here. +// +// --------------------------------------------------------------------------- +// INPUTS — REAL PAYLOADS, WITH ONE DOCUMENTED EXCEPTION +// --------------------------------------------------------------------------- +// The page is built by repeating modules/codec/test/resources/golden/repository/ +// repo-single.json fifty times inside one array. That file is a verbatim, +// byte-for-byte capture from the live Codeberg instance (3,404 bytes; see the +// golden MANIFEST), so the page has the field spellings, the null parent, the Go +// zero-time sentinels and the pretty-printed whitespace a real listing has. The +// organisation object is golden/organization/org-single.json, likewise verbatim. +// The timestamp is read out of the parsed repository rather than typed in here, +// so it cannot drift from the fixture. +// +// The two 1000-element scalar arrays are the exception: no captured fixture +// holds a scalar array anywhere near that long, so they are generated. They +// measure the array and number readers of JsonValue at a length where the +// per-element cost is visible, and their rows are labelled "synth". +// +// Exit codes: 0 measured, 2 could not measure — a missing fixture, a JVM without +// the allocation counter, an operation that failed its pre-flight, or a bad +// argument. There is no exit code 1: nothing here can breach a threshold, +// because there is no threshold. + +import com.worxbend.codeberg4s.codec.Json +import com.worxbend.codeberg4s.codec.JsonDecoder +import com.worxbend.codeberg4s.codec.JsonValue +import com.worxbend.codeberg4s.codec.Timestamps +import com.worxbend.codeberg4s.organizations.wire.OrganizationDto +import com.worxbend.codeberg4s.repositories.wire.RepositoryDto + +import com.sun.management.ThreadMXBean + +import scala.annotation.tailrec +import scala.sys.process.Process +import scala.sys.process.ProcessLogger +import scala.util.Try + +import java.lang.management.ManagementFactory +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +/** Reports the reason and gives up. Every failure here is "could not measure", never "measured something bad". */ +def bail(reason: String): Nothing = + Console.err.println() + Console.err.println(s" alloc-bench: $reason") + sys.exit(2) + +// -------------------------------------------------------------------------- +// Arguments +// -------------------------------------------------------------------------- + +val Usage: String = "usage: scripts/alloc-bench.sh [--rounds=N] [--warmup=N] [--scale=F] [name-filter ...]" + +val KnownFlags: Seq[String] = Seq("--rounds=", "--warmup=", "--scale=") + +def flagValue(prefix: String): Option[String] = + args.find(_.startsWith(prefix)).map(_.drop(prefix.length)) + +def intFlag(prefix: String, fallback: Int): Int = + flagValue(prefix) match + case None => fallback + case Some(raw) => + Try(raw.toInt).toOption + .filter(_ > 0) + .getOrElse(bail(s"$prefix expects a positive integer, got '$raw'\n $Usage")) + +def doubleFlag(prefix: String, fallback: Double): Double = + flagValue(prefix) match + case None => fallback + case Some(raw) => + Try(raw.toDouble).toOption + .filter(_ > 0.0) + .getOrElse(bail(s"$prefix expects a positive number, got '$raw'\n $Usage")) + +args.find(argument => argument.startsWith("--") && !KnownFlags.exists(argument.startsWith)) match + case Some(unknown) => bail(s"unknown option '$unknown'\n $Usage") + case None => () + +val rounds: Int = intFlag("--rounds=", 7) +val warmup: Int = intFlag("--warmup=", 3) +val scale: Double = doubleFlag("--scale=", 1.0) +val filters: Seq[String] = args.filterNot(_.startsWith("--")).toSeq + +// -------------------------------------------------------------------------- +// Inputs +// -------------------------------------------------------------------------- + +/** Where the captured bodies live. Overridable so the harness can be pointed at a checkout elsewhere. */ +val GoldenRoot: Path = Paths.get(sys.env.getOrElse("GOLDEN_ROOT", "modules/codec/test/resources/golden")) + +def fixture(relative: String): String = + val path = GoldenRoot.resolve(relative) + if !Files.isRegularFile(path) then + bail(s"missing golden fixture $path\n run this from the repository root, or set GOLDEN_ROOT") + else String(Files.readAllBytes(path), StandardCharsets.UTF_8) + +/** Repository objects in the measured page. Fifty is one Forgejo listing page at the default limit. */ +val PageRepetitions: Int = 50 + +/** Length of the two generated scalar arrays — the only inputs here that are not a capture. */ +val ScalarArrayLength: Int = 1000 + +val repoSingle: String = fixture("repository/repo-single.json") +val orgSingle: String = fixture("organization/org-single.json") + +val page: String = Vector.fill(PageRepetitions)(repoSingle).mkString("[", ",", "]") + +/** The same page as it actually arrives: bytes off a socket, before anything has decoded them. + * + * This is what the transport now hands the parser. The two decode.page rows below read this array two different ways, + * which is the whole before/after of carrying a body as bytes. + */ +val pageBytes: Array[Byte] = page.getBytes(StandardCharsets.UTF_8) + +val integerArray: String = (0 until ScalarArrayLength).mkString("[", ",", "]") + +val booleanArray: String = + val alternating = (0 until ScalarArrayLength).map(index => if index % 2 == 0 then "true" else "false") + alternating.mkString("[", ",", "]") + +def parsedOrBail(label: String, body: String): JsonValue = + Json.parse(body) match + case Right(value) => value + case Left(failure) => bail(s"$label did not parse: ${failure.message}") + +val repoValue: JsonValue = parsedOrBail("repo-single.json", repoSingle) +val orgValue: JsonValue = parsedOrBail("org-single.json", orgSingle) + +val repoDecoder: JsonDecoder[RepositoryDto] = JsonDecoder[RepositoryDto] +val orgDecoder: JsonDecoder[OrganizationDto] = JsonDecoder[OrganizationDto] + +/** A real timestamp, read out of the parsed fixture rather than written here, so it cannot drift from the capture. */ +val timestamp: String = + repoValue + .field("updated_at") + .flatMap(_.strOpt) + .getOrElse(bail("repo-single.json carries no string updated_at to measure Timestamps.parse against")) + +// -------------------------------------------------------------------------- +// Operations +// -------------------------------------------------------------------------- + +/** One measurable operation. + * + * `run` answers an int checksum derived from the result, and a negative checksum means the operation failed. A + * primitive int rather than the result itself is deliberate: a generic return type would box on every iteration and + * add an allocation the harness would then charge to the code under test. + * + * @param name + * how the row is labelled, and what a filter argument matches against + * @param input + * the size of what this operation consumes, for the report + * @param baseIterations + * calls per round before `--scale` is applied, chosen so that a round takes roughly a tenth of a second + */ +abstract class Op(val name: String, val input: String, val baseIterations: Int): + def run(): Int + + final def iterations: Int = math.max(1, (baseIterations * scale).toInt) + +def sizeOf(text: String): String = f"${text.length}%,d B" + +/** Length of an optional wire string, as a checksum contribution. Written out rather than `fold`ed because a generic + * combinator would box the int on every iteration. + */ +def widthOf(value: Option[String]): Int = value match + case Some(text) => text.length + case None => 0 + +val parsePage: Op = new Op("parse.page-50", sizeOf(page), 100): + + def run(): Int = Json.parse(page) match + case Right(JsonValue.Arr(values)) => values.size + case Right(_) => 0 + case Left(_) => -1 + +val decodePage: Op = new Op("decode.page-50", sizeOf(page), 60): + + def run(): Int = Json.decode[Vector[RepositoryDto]](page) match + case Right(repositories) => repositories.size + case Left(_) => -1 + +/** The path a page takes today: the socket's bytes, straight into the parser. */ +val decodePageFromBytes: Op = new Op("decode.page-50-bytes", sizeOf(page), 60): + + def run(): Int = Json.decode[Vector[RepositoryDto]](pageBytes) match + case Right(repositories) => repositories.size + case Left(_) => -1 + +/** The path a page took before the body became bytes, measured end to end and in one place. + * + * The socket's bytes are decoded into a `String` — what sttp's `asStringAlways` did — and that `String` is handed to + * the parser, which encodes it back into a `byte[]` to read it. Two copies of the payload. The gap between this row + * and `decode.page-50-bytes` is the size of what carrying bytes removed; `decode.page-50` is the same work minus + * sttp's half, kept so that the number can still be compared with runs from before the change. + */ +val decodePageViaString: Op = new Op("decode.page-50-viastring", sizeOf(page), 60): + + def run(): Int = Json.decode[Vector[RepositoryDto]](String(pageBytes, StandardCharsets.UTF_8)) match + case Right(repositories) => repositories.size + case Left(_) => -1 + +val decodeWide: Op = new Op("decode.repo-wide", sizeOf(repoSingle), 3000): + + def run(): Int = Json.decode[RepositoryDto](repoSingle) match + case Right(repository) => repository.topics.size + case Left(_) => -1 + +val assembleWide: Op = new Op("assemble.repo-wide", "parsed", 6000): + + def run(): Int = repoDecoder.decode(repoValue) match + case Right(repository) => repository.topics.size + case Left(_) => -1 + +val decodeNarrow: Op = new Op("decode.org-narrow", sizeOf(orgSingle), 20000): + + def run(): Int = Json.decode[OrganizationDto](orgSingle) match + case Right(organisation) => widthOf(organisation.name) + case Left(_) => -1 + +val assembleNarrow: Op = new Op("assemble.org-narrow", "parsed", 40000): + + def run(): Int = orgDecoder.decode(orgValue) match + case Right(organisation) => widthOf(organisation.name) + case Left(_) => -1 + +/** Where the escaping operations put their answer, so that HotSpot cannot prove the object dies in `run` and delete it. + * + * Allocated once, before any measurement, and never read: a store into it is a card mark and nothing else. This is the + * one piece of mutable state in the harness and it exists because JMH's blackholes are not available here. + */ +val sink: Array[AnyRef] = new Array[AnyRef](1) + +val parseTimestamp: Op = new Op("timestamps.parse", sizeOf(timestamp), 300000): + + def run(): Int = + val parsed = Timestamps.parse(timestamp) + sink(0) = parsed + parsed match + case Some(instant) => instant.getNano + case None => -1 + +val parseTimestampDiscarded: Op = new Op("timestamps.parse-ea", sizeOf(timestamp), 300000): + + def run(): Int = Timestamps.parse(timestamp) match + case Some(instant) => instant.getNano + case None => -1 + +val parseIntegers: Op = new Op("parse.ints-1000", s"${sizeOf(integerArray)} synth", 2000): + + def run(): Int = Json.parse(integerArray) match + case Right(JsonValue.Arr(values)) => values.size + case Right(_) => 0 + case Left(_) => -1 + +val parseBooleans: Op = new Op("parse.bools-1000", s"${sizeOf(booleanArray)} synth", 2000): + + def run(): Int = Json.parse(booleanArray) match + case Right(JsonValue.Arr(values)) => values.size + case Right(_) => 0 + case Left(_) => -1 + +val noop: Op = new Op("baseline.noop", "—", 2000000): + def run(): Int = 1 + +val operations: Vector[Op] = + Vector( + parsePage, + decodePage, + decodePageFromBytes, + decodePageViaString, + decodeWide, + assembleWide, + decodeNarrow, + assembleNarrow, + parseTimestamp, + parseTimestampDiscarded, + parseIntegers, + parseBooleans, + noop, + ) + +val selected: Vector[Op] = + if filters.isEmpty then operations + else operations.filter(operation => filters.exists(filter => operation.name.contains(filter))) + +if selected.isEmpty then + bail(s"no operation matches ${filters.mkString(", ")}\n known: ${operations.map(_.name).mkString(", ")}") + +// -------------------------------------------------------------------------- +// Measurement +// -------------------------------------------------------------------------- + +val threads: ThreadMXBean = ManagementFactory.getThreadMXBean match + case bean: ThreadMXBean => bean + case other => + bail( + s"this JVM's ThreadMXBean is ${other.getClass.getName}, not the com.sun.management extension\n" + + " the harness needs getCurrentThreadAllocatedBytes, which is an OpenJDK/HotSpot extension" + ) + +if !threads.isThreadAllocatedMemorySupported then bail("this JVM does not support per-thread allocation counting") + +if !threads.isThreadAllocatedMemoryEnabled then threads.setThreadAllocatedMemoryEnabled(true) + +/** What one round of one operation cost. `checksum` exists so that the work cannot be proved dead. */ +final case class Round(bytes: Long, nanos: Long, checksum: Int) + +@tailrec +def drive(operation: Op, remaining: Int, checksum: Int): Int = + if remaining <= 0 then checksum else drive(operation, remaining - 1, checksum + operation.run()) + +def measure(operation: Op, iterations: Int): Round = + val startBytes = threads.getCurrentThreadAllocatedBytes + val startNanos = System.nanoTime() + val checksum = drive(operation, iterations, 0) + val endNanos = System.nanoTime() + val endBytes = threads.getCurrentThreadAllocatedBytes + Round(endBytes - startBytes, endNanos - startNanos, checksum) + +/** Two back-to-back reads of the counter. Anything but zero means the reads themselves allocate, and every row below + * carries that much noise per round. + */ +val counterOverhead: Long = + val before = threads.getCurrentThreadAllocatedBytes + val after = threads.getCurrentThreadAllocatedBytes + after - before + +def median(values: Vector[Long]): Double = + val sorted = values.sorted + val size = sorted.length + if size % 2 == 1 then sorted(size / 2).toDouble + else (sorted(size / 2 - 1) + sorted(size / 2)).toDouble / 2.0 + +/** How far apart the measured rounds were, as a percentage of their median. "exact" means they agreed to the byte. */ +def spreadOf(values: Vector[Long]): String = + if values.min.equals(values.max) then "exact" + else f"${(values.max - values.min).toDouble / math.max(1.0, median(values)) * 100.0}%.1f%%" + +final case class Result(operation: Op, iterations: Int, warmChecksum: Int, rounds: Vector[Round]): + def bytesPerOp: Double = median(rounds.map(_.bytes)) / iterations.toDouble + def nanosPerOp: Double = rounds.map(_.nanos).min.toDouble / iterations.toDouble + def checksum: Int = warmChecksum + rounds.map(_.checksum).sum + def byteSpread: String = spreadOf(rounds.map(_.bytes)) + def nanoSpread: String = spreadOf(rounds.map(_.nanos)) + +def profile(operation: Op): Result = + val iterations = operation.iterations + // The warm-up checksum is kept and printed for the same reason the measured one is: so that none of it is dead code. + val warmed = (1 to warmup).map(_ => drive(operation, iterations, 0)).sum + sys.runtime.gc() + Result(operation, iterations, warmed, (1 to rounds).map(_ => measure(operation, iterations)).toVector) + +// A failing operation does a fraction of the work and would be reported as though it had done all of it. +selected.foreach: operation => + if operation.run() < 0 then + bail(s"${operation.name} answered a failure on its pre-flight run — the harness would time the failure path") + +// -------------------------------------------------------------------------- +// Report +// -------------------------------------------------------------------------- + +def amount(value: Double): String = if value >= 1000.0 then f"$value%,.0f" else f"$value%.1f" + +def commit: String = + Try(Process(Seq("git", "rev-parse", "--short", "HEAD")).!!(ProcessLogger(_ => ())).trim) + .filter(_.nonEmpty) + .getOrElse("unknown commit") + +val jvm: String = s"${sys.props.getOrElse("java.vm.name", "?")} ${sys.props.getOrElse("java.runtime.version", "?")}" + +println() +println(" allocation harness — heap bytes this thread allocated per operation, and wall-clock time") +println(s" at $commit · $jvm") +println(s" $rounds measured round(s) after $warmup warm-up round(s) · iteration scale $scale") +println(" B/op is the median of those rounds, ns/op the fastest of them — see the header for why they differ") +println(" NOT JMH, NOT A GATE — read the header of scripts/alloc-bench.sc before quoting a number") +println() +println( + f" ${"operation"}%-20s ${"input"}%-15s ${"iters"}%9s " + + f"${"B/op"}%12s ${"B spread"}%9s ${"ns/op best"}%12s ${"ns spread"}%10s" +) +println(" " + "-" * 95) + +val results: Vector[Result] = selected.map(profile) + +results.foreach: result => + println( + f" ${result.operation.name}%-20s ${result.operation.input}%-15s ${result.iterations}%,9d " + + f"${amount(result.bytesPerOp)}%12s ${result.byteSpread}%9s " + + f"${amount(result.nanosPerOp)}%12s ${result.nanoSpread}%10s" + ) + +println() +println(f" counter overhead between two back-to-back reads: $counterOverhead%,d B (zero is the expected answer)") +println(s" checksum ${results.map(_.checksum).sum} — printed so the JIT cannot prove the measured work is dead") +println() diff --git a/scripts/alloc-bench.sh b/scripts/alloc-bench.sh new file mode 100755 index 0000000..62cc9b7 --- /dev/null +++ b/scripts/alloc-bench.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# +# Allocation and wall-clock harness for the JSON decode path — the entry point +# for scripts/alloc-bench.sc, which holds the measurement methodology and its +# limitations. Read that header before quoting any number this prints. +# +# scripts/alloc-bench.sh # every operation +# scripts/alloc-bench.sh timestamps repo # only operations whose name contains one of these +# scripts/alloc-bench.sh --rounds=11 --scale=4 # more rounds, four times the iterations +# scripts/alloc-bench.sh --help # the harness's own header, in full +# +# This is NOT part of verify.sh and must not become part of it: it is a +# measurement tool, not a gate, and a timing assertion in CI is a flaky test. +# Run it by hand before and after a change and quote both numbers. +# +# Why a wrapper exists at all: the harness names types from `modules.codec`, so +# it needs that module's classes and its jsoniter dependency on the classpath at +# COMPILE time, and a Scala script cannot put them there for itself. This script +# asks Mill where they are (`./mill show modules.codec.runClasspath`, which +# compiles the module first if it is stale) and hands the answer to scala-cli. +# +# Exit codes are the harness's own: 0 measured, 2 could not measure. + +set -euo pipefail + +cd "$(dirname "$0")/.." + +readonly MILL="./mill" +readonly HARNESS="scripts/alloc-bench.sc" + +for arg in "$@"; do + case "$arg" in + # Print the harness's header block, whatever length it has grown to. A fixed + # line range goes stale the first time somebody documents a new operation. + -h|--help) + awk 'NR > 1 && /^\/\//{ sub(/^\/\/ ?/, ""); print; next } NR > 1 && NF { exit }' "$HARNESS" + exit 0 + ;; + esac +done + +command -v scala-cli >/dev/null 2>&1 || { + echo "alloc-bench: scala-cli is not on PATH — the same tool scripts/crap.sc needs" >&2 + exit 2 +} + +# `show` prints a JSON array whose entries are Mill path references, e.g. +# "qref:v1:3cca3705:/home/…/jsoniter-scala-core_3-2.39.1.jar" +# so the path is everything from the first slash. Mill's progress output goes to +# stderr and is kept: if the build fails, its message is what explains why. +classpath=$( + "$MILL" show modules.codec.runClasspath | + grep -oE '/[^"]+' | + grep -vE '/(scala-library|scala3-library_3)-[0-9.]+\.jar$' | + paste -sd: +) || { + echo "alloc-bench: could not resolve modules.codec.runClasspath (see above)" >&2 + exit 2 +} + +[[ -n "$classpath" ]] || { + echo "alloc-bench: modules.codec.runClasspath came back empty" >&2 + exit 2 +} + +# The two standard-library jars are dropped above because scala-cli puts its own +# on the classpath, and two copies make the compiler warn that several versions +# of the standard library are present — noise on every single run. + +exec scala-cli run "$HARNESS" --server=false --extra-jars "$classpath" -- "$@" diff --git a/site/src/guides/03-errors.md b/site/src/guides/03-errors.md index 5aef602..0c4f66a 100644 --- a/site/src/guides/03-errors.md +++ b/site/src/guides/03-errors.md @@ -33,14 +33,22 @@ guessing wrong in a library is worse than handing you the number. ### `TransportCause` -`Transport` carries why nothing arrived: -`ConnectionFailed`, `Timeout`, `Tls`, `Dns`, `Interrupted`, `Unknown`. Each +`Transport` carries why no usable response came back: `ConnectionFailed`, +`Timeout`, `Tls`, `Dns`, `Interrupted`, `ResponseTooLarge`, `Unknown`. Each holds a short `detail` string taken from the underlying exception. Branch on the case, never on the text. A status code — including `500` — is never a transport cause. If the server answered anything at all, you get `Api`. +`ResponseTooLarge` is the one case where something did begin to arrive. This +library reads whole bodies into memory, so every request carries a byte bound +(`CodebergConfig.maxResponseBodyBytes`, and `maxDownloadBodyBytes` for the ZIP +downloads); a body that passes it is abandoned part-read, which leaves no status +to map and no body to decode. It is also the one transport cause besides `Tls` +and `Interrupted` that is never retried — repeating the call would download the +oversized body again on every attempt. + ## The two rails Every operation exists twice. diff --git a/site/src/guides/07-testing-your-code.md b/site/src/guides/07-testing-your-code.md index 9b02a7e..898f017 100644 --- a/site/src/guides/07-testing-your-code.md +++ b/site/src/guides/07-testing-your-code.md @@ -321,9 +321,16 @@ def refusing()(using ExecutionContext): BackendStub[Future] = The transport adapter classifies the exception by walking its cause chain, so a `ConnectException` becomes `TransportCause.ConnectionFailed`, a -`SocketTimeoutException` becomes `TransportCause.Timeout`, and an -`UnknownHostException` becomes `TransportCause.Dns`. Assert on the case, not on -the `detail` string. +`SocketTimeoutException` becomes `TransportCause.Timeout`, an +`UnknownHostException` becomes `TransportCause.Dns`, and sttp's +`StreamMaxLengthExceededException` — thrown when a response body passes the +bound in `CodebergConfig` — becomes `TransportCause.ResponseTooLarge`. Assert on +the case, not on the `detail` string. + +One caveat if you are testing the oversized-body path: `BackendStub` does not +apply `maxResponseBodyLength`, so failing the stub with a +`StreamMaxLengthExceededException` is how you produce that cause. A real backend +is what enforces the bound. **A rate limit.** `StatusCode(429)`, optionally with a `Retry-After` header: diff --git a/site/src/guides/09-self-hosted.md b/site/src/guides/09-self-hosted.md index 18afc7e..cee6d84 100644 --- a/site/src/guides/09-self-hosted.md +++ b/site/src/guides/09-self-hosted.md @@ -27,13 +27,15 @@ val selfHosted: Either[ValidationError, CodebergConfig] = agent <- UserAgent.from("my-app/1.0") size <- PageSize.from(50) yield CodebergConfig( - baseUri = base, - auth = Auth.Anonymous, - retry = RetryPolicy.Default, - userAgent = agent, - defaultPageSize = size, - connectTimeout = 10.seconds, - readTimeout = 30.seconds, + baseUri = base, + auth = Auth.Anonymous, + retry = RetryPolicy.Default, + userAgent = agent, + defaultPageSize = size, + connectTimeout = 10.seconds, + readTimeout = 30.seconds, + maxResponseBodyBytes = CodebergConfig.DefaultMaxResponseBodyBytes, + maxDownloadBodyBytes = CodebergConfig.DefaultMaxDownloadBodyBytes, ) ``` @@ -235,6 +237,39 @@ honoured either way. Raise the read timeout for endpoints that do real work on the instance: generating an archive, comparing two distant commits, migrating a repository. +## Response size on an instance you configured + +This library reads a whole response into memory rather than streaming it, so +every request carries a byte bound. A body that passes the bound is abandoned +part-read and reported as +`CodebergError.Transport(ctx, TransportCause.ResponseTooLarge(detail))` — and +that cause is deliberately **not** retried, because repeating the call would +download the oversized body once per attempt. + +```scala mdoc:compile-only +import com.worxbend.codeberg4s.CodebergConfig +import com.worxbend.codeberg4s.auth.Auth + +val roomierBodies: CodebergConfig = + CodebergConfig(Auth.Anonymous).copy( + maxResponseBodyBytes = 64L * 1024 * 1024, + maxDownloadBodyBytes = 512L * 1024 * 1024, + ) +``` + +The defaults are 16 MiB for a textual response and 50 MiB for the two ZIP +downloads under `client.downloads`. Two settings rather than one, +because the reasoning behind them is different: the textual bound is derived +from `default_max_blob_size` — 10 MiB on codeberg.org, base64-encoded into a +file-contents response at four bytes per three — while a CI artifact is whatever +a workflow uploaded and no such number bounds it. + +That first number is per-instance configuration, not a protocol constant. If +your Forgejo raises `default_max_blob_size`, read the instance's own value back +from `client.misc.apiSettings()` — it is `maxBlobSizeBytes` on the returned +`ServerApiSettings` — and raise `maxResponseBodyBytes` to match, or fetching a +large file will fail as `ResponseTooLarge`. + ## Self-signed certificates This library does not configure TLS. It builds an sttp JDK-HTTP-client backend diff --git a/verify.sh b/verify.sh index 9601c4b..67e9e6f 100755 --- a/verify.sh +++ b/verify.sh @@ -53,18 +53,38 @@ readonly COVERED_MODULES=(modules.domain modules.core modules.codec) # change that adds one group fails, and a change that removes ten is told to # bank the win by lowering this number. # -# MEASURED, NOT RECALLED: `scripts/cpd.sh --report` on 2026-08-02 against +# MEASURED, NOT RECALLED: `scripts/cpd.sh --report` on 2026-08-09 against # modules/{domain,core,codec,transport,client}/src with PMD 7.26.0 at 40 -# tokens — 323 groups over 1195 locations (599 in codec, 541 in client, 45 in -# domain, 8 in core, 2 in transport). docs/LEDGER.md still says 62; that figure -# predates the long tail, which took the surface from 61 operations to 439. +# tokens — 363 groups over 1342 locations (762 in codec, 535 in client, 39 in +# domain, 6 in core, none in transport). +# +# The ten groups between 373 and 363 came off with the credential-redaction +# change: the three copies of the Actions runner registration decoders no +# longer read as one repeated shape, and UserTokenApi lost the bespoke +# error-rewriting helper that the pipeline now makes unnecessary. Banked here +# rather than left as headroom, per the paragraph above. +# +# The baseline recorded before that was 323 groups over 1195 locations, +# measured on 2026-08-02. Everything between the two numbers is codec: the +# upickle-to-jsoniter rewrite (commit 48f64fe) replaced hand-written readers +# and writers with per-DTO codec definitions that repeat the same shape once +# per field, so codec's share of the reported locations went from 599 to 764 +# while every other module stayed where it was. Recording the higher number +# registered that debt; it did not forgive it, and it was not a licence to +# add more. +# +# The five groups between 378 and 373 were then paid off rather than +# tolerated: four copies of the same element-decoding fold became one shared +# helper, and the four inline copies of the path-segment security rule became +# one call to PathSegment. Both are why this number is a recorded measurement +# and not a threshold — a threshold would have absorbed the win silently. # # The number is specific to PMD 7.26.0 at 40 tokens. Change either and remeasure # rather than guessing which way the count moved. # # Deliberately not overridable from the environment: moving the baseline has to # appear in a diff, with a commit message saying why. -readonly CPD_BASELINE_GROUPS=323 +readonly CPD_BASELINE_GROUPS=363 with_slow=false nightly=false @@ -202,15 +222,41 @@ boundary_violation() { # scala.concurrent.duration is fine everywhere — FiniteDuration is how timeouts # and backoff are typed. It is Future and ExecutionContext that must not appear # below the client module. -readonly FORBIDDEN_BELOW_CLIENT='^import (sttp|upickle|ujson|scala\.concurrent\.(Future|ExecutionContext|Await|Promise|blocking))' +# +# The JSON alternatives are the other half of PLAN.md §3.1: domain and core know +# neither the transport nor a JSON library. Until this commit the list read +# `upickle|ujson`, which stopped being a boundary the moment commit 48f64fe +# removed upickle — the library actually on the classpath, +# `com.github.plokhotnyuk.jsoniter_scala`, could have been imported straight +# into domain or core and this step would still have printed "boundaries clean". +# The vendor prefix is matched rather than the full package so a future +# `jsoniter_scala.macros` import is caught by the same alternative. +# +# upickle, ujson, circe, play-json, zio-json, Jackson, json4s and Gson are named +# even though none of them is a dependency. A pattern for a library that is not +# there costs one alternation and catches the day somebody adds it, which is the +# only day this check has anything to say. +# +# Matching imports rather than the build graph is deliberate: the graph already +# gives domain and core no `mvnDeps` at all, so a violation needs a `mvnDeps` +# edit as well as an import. This grep is what makes the import half fail +# loudly, and it is also what covers the case where the offending symbol arrives +# through a module that is on the graph. +readonly FORBIDDEN_JSON='com\.github\.plokhotnyuk|upickle|ujson|io\.circe|play\.api\.libs\.json|zio\.json|com\.fasterxml\.jackson|org\.json4s|com\.google\.gson' +readonly FORBIDDEN_BELOW_CLIENT="^import (sttp|$FORBIDDEN_JSON|scala\\.concurrent\\.(Future|ExecutionContext|Await|Promise|blocking))" boundaries_ok=true boundary_violation modules/domain/src "$FORBIDDEN_BELOW_CLIENT" \ 'domain must depend on nothing but the standard library' || boundaries_ok=false boundary_violation modules/core/src "$FORBIDDEN_BELOW_CLIENT" \ - 'core must not know about sttp, upickle or Future' || boundaries_ok=false + 'core must not know about sttp, a JSON library or Future' || boundaries_ok=false boundary_violation modules/codec/src '^import sttp' \ 'codec must not know about the transport' || boundaries_ok=false +# The mirror image of the line above, and stated in docs/adr/0003 as the reason +# the unused sttp-upickle integration was dropped: the transport reads every +# body as a String and hands it to codec, so the JSON library stays out of it. +boundary_violation modules/transport/src "^import ($FORBIDDEN_JSON)" \ + 'transport must not know about a JSON library — it reads bodies as String' || boundaries_ok=false boundary_violation 'modules/domain/src modules/core/src modules/codec/src modules/transport/src modules/client/src' \ 'Await\.(result|ready)' 'Await is banned in production code' || boundaries_ok=false boundary_violation 'modules/domain/src modules/core/src modules/codec/src modules/transport/src modules/client/src' \ @@ -225,7 +271,8 @@ for module in "${COVERED_MODULES[@]}"; do "$MILL" "${module}.scoverage.xmlReport" || fail "coverage report for $module" done if [[ -f scripts/coverage-gate.sc ]]; then - scala-cli run scripts/coverage-gate.sc -- "${COVERED_MODULES[@]}" || fail "coverage thresholds" + scala-cli run scripts/coverage-gate.sc --server=false -- "${COVERED_MODULES[@]}" || + fail "coverage thresholds" else echo " (scripts/coverage-gate.sc absent — thresholds not enforced yet)" fi @@ -289,7 +336,7 @@ if $with_slow; then announce "CRAP (coverage-weighted complexity)" if [[ -f scripts/crap.sc ]]; then set +e - scala-cli run scripts/crap.sc -- "${COVERED_MODULES[@]}" + scala-cli run scripts/crap.sc --server=false -- "${COVERED_MODULES[@]}" crap_status=$? set -e case "$crap_status" in