diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..d0a5635 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,288 @@ +# Tag-triggered publish flow for knowledge. +# +# push of a v*.*.* tag drives: +# publish-api-image \ +# -> resolve-image-lock -> publish-deploy-artifact -> register-service +# publish-worker-image / +# +# Two images (knowledge-api, knowledge-ingest-worker) are built in parallel and +# their digests are merged into a single platform/images.lock.json before the +# deploy artifact is rendered. +# +# Boundary rule (R1-2): the image lock crosses the reusable-workflow boundary +# as an UPLOADED ARTIFACT (image-lock-artifact), never as a runner-local path. +name: Publish + +on: + push: + tags: + - 'v*.*.*' + workflow_dispatch: + inputs: + tag: + description: 'Tag to publish (e.g. v0.3.0)' + required: true + type: string + +permissions: {} + +jobs: + # -- 1a. Build and push knowledge-api image -------------------------------- + publish-api-image: + uses: JorisJonkers-dev/github-workflows/.github/workflows/container-publish.yml@4b444f9a50edf76e4aea4148d73407c34a1e43a4 # v0.12.1 + with: + image-name: knowledge/knowledge-api + dockerfile: api/Dockerfile + version: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }} + permissions: + contents: read + packages: write + + # -- 1b. Build and push knowledge-ingest-worker image ---------------------- + publish-worker-image: + uses: JorisJonkers-dev/github-workflows/.github/workflows/container-publish.yml@4b444f9a50edf76e4aea4148d73407c34a1e43a4 # v0.12.1 + with: + image-name: knowledge/knowledge-ingest-worker + dockerfile: ingest-worker/Dockerfile + version: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }} + permissions: + contents: read + packages: write + + # -- 2. Resolve the image lock (upload artifact, never path) -------------- + resolve-image-lock: + needs: [publish-api-image, publish-worker-image] + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + outputs: + images-lock-artifact: images-lock-${{ github.run_id }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && format('refs/tags/{0}', inputs.tag) || github.ref }} + + - name: resolve-api-digest + id: api-digest + env: + GH_TOKEN: ${{ github.token }} + IMAGE: ghcr.io/jorisjonkers-dev/knowledge/knowledge-api + TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }} + run: | + set -euo pipefail + printf '%s' "$GH_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + digest="$(docker buildx imagetools inspect "${IMAGE}:${TAG}" --format '{{json .Manifest}}' | jq -r '.digest')" + if [ -z "$digest" ] || [ "$digest" = "null" ]; then + echo "E_FLOATING_IMAGE: could not resolve a digest for ${IMAGE}:${TAG}" >&2 + exit 1 + fi + echo "image-digest=$digest" >> "$GITHUB_OUTPUT" + + - name: resolve-worker-digest + id: worker-digest + env: + GH_TOKEN: ${{ github.token }} + IMAGE: ghcr.io/jorisjonkers-dev/knowledge/knowledge-ingest-worker + TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }} + run: | + set -euo pipefail + digest="$(docker buildx imagetools inspect "${IMAGE}:${TAG}" --format '{{json .Manifest}}' | jq -r '.digest')" + if [ -z "$digest" ] || [ "$digest" = "null" ]; then + echo "E_FLOATING_IMAGE: could not resolve a digest for ${IMAGE}:${TAG}" >&2 + exit 1 + fi + echo "image-digest=$digest" >> "$GITHUB_OUTPUT" + + - name: lock-images + env: + API_DIGEST: ${{ steps.api-digest.outputs.image-digest }} + WORKER_DIGEST: ${{ steps.worker-digest.outputs.image-digest }} + run: | + set -euo pipefail + for digest in "$API_DIGEST" "$WORKER_DIGEST"; do + case "$digest" in + sha256:*) ;; + *) echo "E_FLOATING_IMAGE: digest is not a digest ref (got: $digest)" >&2; exit 1 ;; + esac + done + printf '{"knowledge-api":"ghcr.io/jorisjonkers-dev/knowledge/knowledge-api@%s","knowledge-ingest-worker":"ghcr.io/jorisjonkers-dev/knowledge/knowledge-ingest-worker@%s"}\n' \ + "$API_DIGEST" "$WORKER_DIGEST" > platform/images.lock.json + echo "Locked api: ghcr.io/jorisjonkers-dev/knowledge/knowledge-api@${API_DIGEST}" + echo "Locked worker: ghcr.io/jorisjonkers-dev/knowledge/knowledge-ingest-worker@${WORKER_DIGEST}" + + - name: verify-lock + run: | + set -euo pipefail + for key in knowledge-api knowledge-ingest-worker; do + ref=$(jq -r --arg k "$key" '.[$k]' platform/images.lock.json) + case "$ref" in + *@sha256:*) echo "Digest pin verified: $ref" ;; + *) echo "E_FLOATING_IMAGE: lock ref for $key is not digest-pinned: $ref" >&2; exit 1 ;; + esac + done + + - name: upload-image-lock + uses: actions/upload-artifact@v5 + with: + name: images-lock-${{ github.run_id }} + path: platform/images.lock.json + retention-days: 1 + + # -- 3. Render and publish the deploy artifact ---------------------------- + publish-deploy-artifact: + needs: [resolve-image-lock] + uses: JorisJonkers-dev/github-workflows/.github/workflows/deploy-artifact.yml@4b444f9a50edf76e4aea4148d73407c34a1e43a4 # v0.12.1 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && format('refs/tags/{0}', inputs.tag) || github.ref }} + artifact-name: knowledge + artifact-version: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }} + schema-version: 0.16.0 + context-ref: ghcr.io/jorisjonkers-dev/cluster-deploy-context-public@sha256:0423d9bcfc091f3b6e9bd6bdd91128bb0278b97e9c9de1907e6b0817c5500232 + deploy-dir: platform + image-lock-artifact: ${{ needs.resolve-image-lock.outputs.images-lock-artifact }} + environments: production + permissions: + contents: read + packages: write + id-token: write + attestations: write + + # -- 4. Open/refresh the homelab-deploy registry PR ------------------------ + register-service: + needs: [publish-deploy-artifact] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: mint-app-token + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: homelab-deploy + + - name: assert-token-minted + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + if [ -z "$APP_TOKEN" ]; then + echo "E_APP_TOKEN_MISSING: App token was not minted for register-service." >&2 + exit 1 + fi + + - uses: actions/checkout@v7 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && format('refs/tags/{0}', inputs.tag) || github.ref }} + + - name: build-registry-payload + # Updates only bump spec.artifact; every other field (including + # pvcDecisions and migrationJobDecisions) is carried from the existing + # registry file and never overwritten. + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + ARTIFACT_REF: ${{ needs.publish-deploy-artifact.outputs.artifact-ref }} + ARTIFACT_DIGEST: ${{ needs.publish-deploy-artifact.outputs.artifact-digest }} + SERVICE_NAME: knowledge + REPO_OWNER: ${{ github.repository_owner }} + SOURCE_REPOSITORY: ${{ github.repository }} + TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }} + run: | + set -euo pipefail + branch="registry-update/${SERVICE_NAME}/${TAG}" + registry_path="registry/${SERVICE_NAME}.yaml" + repo="${REPO_OWNER}/homelab-deploy" + + if ! gh api "repos/${repo}/git/refs/heads/${branch}" >/dev/null 2>&1; then + main_sha="$(gh api "repos/${repo}/git/refs/heads/main" --jq '.object.sha')" + gh api --method POST "repos/${repo}/git/refs" \ + -f ref="refs/heads/${branch}" -f sha="$main_sha" >/dev/null + fi + + current_sha="" + if gh api "repos/${repo}/contents/${registry_path}?ref=${branch}" > current.json 2>/dev/null; then + current_sha="$(jq -r '.sha' current.json)" + jq -r '.content' current.json | base64 -d > current.yaml + fi + + python3 - "$SERVICE_NAME" <<'PY' + import os, sys, yaml + + service = sys.argv[1] + artifact = { + "ref": os.environ["ARTIFACT_REF"], + "digest": os.environ["ARTIFACT_DIGEST"], + "contractPath": "artifact-contract.yaml", + } + + if os.path.exists("current.yaml"): + # Update: bump only spec.artifact; preserve every other field + # (including pvcDecisions and migrationJobDecisions). + doc = yaml.safe_load(open("current.yaml")) + doc["spec"]["artifact"] = artifact + else: + # First registration: full spec derived from platform/deployment.yml. + dep = yaml.safe_load(open("platform/deployment.yml")) + spec = dep.get("spec", {}) + platform = spec.get("platform") or {} + layer = platform.get("layer") + if not layer: + sys.exit("E_UNKNOWN_LAYER: platform/deployment.yml spec.platform.layer is " + "required for first registration") + workloads = spec.get("workloads") or [] + health = (workloads[0].get("health") if workloads else None) or {} + # Determine healthClass: stateful if any workload is stateful. + health_class = "stateless" + for w in workloads: + if w.get("stateful"): + health_class = "stateful" + break + health_class = (health.get("class") + or health.get("timeoutClass") + or health_class) + doc = { + "apiVersion": "deployment.jorisjonkers.dev/deploy-unit-registration/v1", + "kind": "DeployUnitRegistration", + "metadata": { + "name": service, + "owner": os.environ["REPO_OWNER"], + }, + "spec": { + "artifact": artifact, + "namespace": spec["namespace"], + "layer": layer, + "sourceRepository": os.environ["SOURCE_REPOSITORY"], + "environments": spec.get("environments") or ["production"], + "healthClass": health_class, + "prune": {"default": True}, + "allowedClusterScope": [], + }, + } + + yaml.safe_dump(doc, open("registration.yaml", "w"), + sort_keys=False, default_flow_style=False) + PY + + if [ -n "$current_sha" ]; then + gh api --method PUT "repos/${repo}/contents/${registry_path}" \ + -f message="chore(registry): update ${SERVICE_NAME} to ${TAG}" \ + -f content="$(base64 -w0 registration.yaml)" \ + -f sha="$current_sha" -f branch="$branch" >/dev/null + else + gh api --method PUT "repos/${repo}/contents/${registry_path}" \ + -f message="chore(registry): register ${SERVICE_NAME}" \ + -f content="$(base64 -w0 registration.yaml)" \ + -f branch="$branch" >/dev/null + fi + + open_prs="$(gh api "repos/${repo}/pulls?head=${REPO_OWNER}:${branch}&state=open" --jq 'length')" + if [ "$open_prs" -eq 0 ]; then + gh api --method POST "repos/${repo}/pulls" \ + -f title="chore(registry): update ${SERVICE_NAME} to ${TAG}" \ + -f body="Automated registry update from publish.yml. Artifact: ${ARTIFACT_REF}" \ + -f head="$branch" -f base="main" >/dev/null + else + echo "Updated existing registry PR for branch ${branch}" + fi diff --git a/PLATFORM.md b/PLATFORM.md new file mode 100644 index 0000000..cf53e72 --- /dev/null +++ b/PLATFORM.md @@ -0,0 +1,64 @@ +# knowledge — platform integration guide + +## Quick start: local validate + +```bash +./platform/render-local.sh +``` + +Review `out/scorecard.md` for any failures before committing. + +## Full deployment flow + +1. **Local validate** — `./platform/render-local.sh`; all scorecard fields must pass (or be `not_applicable`). +2. **Tag release** — merge to `main`; release-please opens a release PR; merging it pushes a `v*.*.*` tag. +3. **Automated publish** — `publish.yml` runs on the tag: builds both images → locks image digests → + publishes the deploy artifact to GHCR → opens a registry PR in homelab-deploy. +4. **Renovate pin PR** — Renovate opens a PR in homelab-deploy updating the registry entry digest. +5. **Deploy preview** — on any PR touching `platform/`, `deploy-preview.yml` posts a scorecard comment. +6. **Merge → gates** — homelab-deploy runs Compose Gate, Leak Scan, Stack Integration Gate, Pipeline Complete. +7. **Auto-deploy** — on merge to homelab-deploy main, `publish-deploy-branch.yml` pushes to `platform/production`. + +## Stateful workload notes + +`knowledge-ingest-worker` is declared `stateful: true` with `migrationPolicy.strategy: none`. +The vault migration job (`knowledge-vault-migrate-2026-05-18`) has already run and is decommissioned +(owner decision 2026-07-12). `strategy: none` satisfies the schema requirement without triggering a new +pre-deploy job. The registry entry in homelab-deploy carries `pvcDecisions` for the +`knowledge-vault-clone` PVC — this must not be removed without owner approval. + +See `knowledge-onboarding-plan.md` in the migration workspace for the full state-move-plan. + +## Files + +| File | Purpose | +|------|---------| +| `platform/deployment.yml` | Deployment contract (v2): namespace, `platform.layer`, workloads, health, routes, rollback policy | +| `platform/images.lock.json` | Digest-pinned image references (filled by `publish.yml`; stub in repo) | +| `.github/workflows/publish.yml` | Tag-triggered dual-image publish and registry PR | +| `.github/workflows/release.yml` | release-please with mandatory App token | + +## Readiness scorecard (SC-11) + +Every render evaluates these fields; `fail` blocks deployment, `not_applicable` +means the check does not apply to this service: + +| Field | not_applicable when | +|---|---| +| `schema_pinned` | never | +| `context_pinned` | never | +| `no_latest_images` | never | +| `health_declared` | never | +| `route_owner_authmode_declared` | no workload declares routes | +| `rollback_retention_acknowledged` | never | +| `no_raw_secrets` | never | +| `stateful_policy_declared` | all workloads are stateless | +| `raw_manifests_guarded` | no workload enables rawManifests | +| `npm_signatures_verified` | never | + +## Reference + +- Full platform design: [DEPLOY-PLATFORM-PLAN.md](https://github.com/JorisJonkers-dev/platform-blueprints/blob/main/DEPLOY-PLATFORM-PLAN.md) +- Schema docs: `deploy-config-schema --help` +- Composition authority: [homelab-deploy](https://github.com/JorisJonkers-dev/homelab-deploy) +- State-move plan: `homelab-deploy/knowledge-state-adopt.yml` \ No newline at end of file diff --git a/platform/deployment.yml b/platform/deployment.yml new file mode 100644 index 0000000..ecc2ad9 --- /dev/null +++ b/platform/deployment.yml @@ -0,0 +1,72 @@ +# Deployment contract for knowledge (deployment v2). +# Validated by deploy-preview.yml on every PR touching platform/ and rendered +# into the deploy artifact by publish.yml. Run ./platform/render-local.sh for +# a local CI-parity check. +apiVersion: deployment.jorisjonkers.dev/v2 +kind: Deployment +metadata: + name: knowledge +spec: + schemaVersion: 0.16.0 + namespace: knowledge-system + # platform.layer is REQUIRED for the first homelab-deploy registration + # (register-service derives the registry layer from it). + platform: + layer: apps-core + workloads: + - name: knowledge-api + health: + path: /api/actuator/health/readiness + port: 8080 + timeoutClass: stateless + mandatory: true + rollbackTargetRetention: + minimumDays: 90 + acknowledged: true + routes: + - host: kb.jorisjonkers.dev + owner: knowledge-api + authMode: sso + expose: + tier: public-frankfurt + rules: + - path: /mcp + port: http + operation: exact + auth: + scope: anonymous + - path: /mcp/ + port: http + operation: prefix + auth: + scope: anonymous + - path: /install.sh + port: http + operation: exact + auth: + scope: anonymous + - path: /install-agents.sh + port: http + operation: exact + auth: + scope: anonymous + - path: / + port: http + operation: prefix + middleware: + - forward-auth + auth: + scope: authenticated + + - name: knowledge-ingest-worker + stateful: true + # Migration is complete (knowledge-vault-migrate-2026-05-18 has run and + # is decommissioned per owner decision 2026-07-12). strategy: none + # satisfies the schema requirement that stateful workloads declare a + # migrationPolicy, without triggering a new pre-deploy job. + migrationPolicy: + required: false + strategy: none + rollbackTargetRetention: + minimumDays: 90 + acknowledged: true diff --git a/platform/images.lock.json b/platform/images.lock.json new file mode 100644 index 0000000..16c8524 --- /dev/null +++ b/platform/images.lock.json @@ -0,0 +1,4 @@ +{ + "knowledge-api": "ghcr.io/jorisjonkers-dev/knowledge/knowledge-api@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "knowledge-ingest-worker": "ghcr.io/jorisjonkers-dev/knowledge/knowledge-ingest-worker@sha256:0000000000000000000000000000000000000000000000000000000000000000" +}