diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 49013d13..9287de11 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @wandb/delivery-tooling-team \ No newline at end of file +* @wandb/on-prem-team \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a16e42c1..022c83d6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,9 +14,3 @@ updates: reviewers: - wandb/delivery-tooling-team - - package-ecosystem: github-actions - directory: /.github/workflows - schedule: - interval: daily - reviewers: - - wandb/delivery-tooling-team diff --git a/.github/renovate.json5 b/.github/renovate.json5 new file mode 100644 index 00000000..94cee6c7 --- /dev/null +++ b/.github/renovate.json5 @@ -0,0 +1,4 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["github>wandb/renovate-config"] +} diff --git a/.github/workflows/chart-validation.yaml b/.github/workflows/chart-validation.yaml new file mode 100644 index 00000000..6d406a76 --- /dev/null +++ b/.github/workflows/chart-validation.yaml @@ -0,0 +1,71 @@ +name: Chart Validation + +on: + push: + branches: [v2, main] + pull_request: + branches: [v2, main] + +jobs: + chart-validation: + name: Chart Validation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Install Helm + uses: azure/setup-helm@bf6a7d304bc2fdb57e0331155b7ebf2c504acf0a # v4 + with: + version: v3.19.0 + + - name: Install chart-testing + uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 + with: + version: v3.14.0 + + - name: Resolve chart dependencies + run: | + helm repo add ci-wandb https://charts.wandb.ai/ + helm repo add ci-moco https://cybozu-go.github.io/moco/ + helm repo add ci-ot-container-kit https://ot-container-kit.github.io/helm-charts + helm repo add ci-seaweedfs https://seaweedfs.github.io/seaweedfs-operator/ + helm repo add ci-prometheus-community https://prometheus-community.github.io/helm-charts + helm repo add ci-altinity https://helm.altinity.com + helm repo add ci-victoria-metrics https://victoriametrics.github.io/helm-charts/ + helm repo add ci-grafana https://grafana.github.io/helm-charts + helm dependency build deploy/operator + git diff --exit-code deploy/operator/Chart.lock + + - name: Run chart-testing + run: ct lint --all --config deploy/ct.yaml + + - name: Validate values schema + run: | + set -euo pipefail + helm lint --strict deploy/operator + for profile in deploy/operator/profiles/*.yaml; do + extra_args=() + if [[ "${profile}" == *telemetry-forward.yaml ]]; then + extra_args+=(--set-string telemetry.forwarding.otlp.endpoint=https://example.invalid:4317) + fi + helm lint --strict deploy/operator --values "${profile}" "${extra_args[@]}" + done + + - name: Render representative configurations + run: | + set -euo pipefail + helm template wandb-operator deploy/operator \ + --namespace wandb-operators \ + --include-crds >/dev/null + for profile in deploy/operator/profiles/*.yaml; do + extra_args=() + if [[ "${profile}" == *telemetry-forward.yaml ]]; then + extra_args+=(--set-string telemetry.forwarding.otlp.endpoint=https://example.invalid:4317) + fi + helm template wandb-operator deploy/operator \ + --namespace wandb-operators \ + --include-crds \ + --values "${profile}" \ + "${extra_args[@]}" >/dev/null + done diff --git a/.github/workflows/docker-build-scan.yml b/.github/workflows/docker-build-scan.yml index f11386cb..7f74719c 100644 --- a/.github/workflows/docker-build-scan.yml +++ b/.github/workflows/docker-build-scan.yml @@ -2,22 +2,22 @@ name: Docker Build and Security Scan on: push: - branches: [main] + branches: [v2, main] pull_request: - branches: [main] + branches: [v2, main] jobs: build-and-scan: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Build Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . push: false @@ -27,7 +27,7 @@ jobs: cache-to: type=gha,mode=max - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master + uses: aquasecurity/trivy-action@c07df6fec6fa692e6fd1200d50aaa1fdd66f03c8 # master with: image-ref: wandb/operator:${{ github.sha }} format: "table" diff --git a/.github/workflows/internal-chart-publish.yaml b/.github/workflows/internal-chart-publish.yaml index 9b6dbc35..1d05842e 100644 --- a/.github/workflows/internal-chart-publish.yaml +++ b/.github/workflows/internal-chart-publish.yaml @@ -5,30 +5,51 @@ on: jobs: release: - name: Release + name: Publish prerelease chart runs-on: ubuntu-latest permissions: - contents: 'read' - id-token: 'write' + contents: read + id-token: write + env: + CHART_REPOSITORY: us-docker.pkg.dev/wandb-production/public/wandb/charts steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - persist-credentials: false fetch-depth: 0 + persist-credentials: false + + - name: Validate prerelease chart version + id: chart + shell: bash + run: | + set -euo pipefail + version="$(awk '$1 == "version:" { print $2; exit }' deploy/operator/Chart.yaml | tr -d '\"')" + if [[ ! "${version}" =~ ^2\.[0-9]+\.[0-9]+-[0-9A-Za-z][0-9A-Za-z.-]*$ ]]; then + echo "Internal chart publishing requires a v2 prerelease version; got ${version}" >&2 + exit 1 + fi + echo "version=${version}" >> "${GITHUB_OUTPUT}" + + - name: Install Helm + uses: azure/setup-helm@bf6a7d304bc2fdb57e0331155b7ebf2c504acf0a # v4 + with: + version: v3.19.0 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + - name: Install chart-testing + uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 + with: + version: v3.14.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + - name: Lint charts + run: ct lint --all --config deploy/ct.yaml - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3.0.1 + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 - id: auth name: Authenticate to Google Cloud - uses: google-github-actions/auth@v3 + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 with: create_credentials_file: 'true' token_format: access_token @@ -36,44 +57,36 @@ jobs: workload_identity_provider: ${{ secrets.CI_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.CI_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} - - name: Authorize Docker to use Google Container Registry - run: | - gcloud auth configure-docker us-docker.pkg.dev - - - uses: actions/setup-go@v4 - with: - go-version: 1.25 + - name: Authorize Docker for Artifact Registry + run: gcloud auth configure-docker us-docker.pkg.dev --quiet - - name: Install Helm - uses: azure/setup-helm@v4 - with: - version: v3.19.0 - - - name: Set up chart-testing - uses: helm/chart-testing-action@v2.6.1 - with: - version: v3.14.0 - - - name: Run chart-testing (list-changed) - id: list-changed + - name: Reject existing chart version + env: + VERSION: ${{ steps.chart.outputs.version }} + shell: bash run: | - changed=$(ct list-changed --config deploy/ct.yaml || true) - if [[ -n "$changed" ]]; then - echo "changed=true" >> $GITHUB_OUTPUT + set -euo pipefail + artifact="${CHART_REPOSITORY}/operator:${VERSION}" + set +e + output="$(gcloud artifacts docker images describe "${artifact}" --format='value(image_summary.digest)' 2>&1)" + status=$? + set -e + if [[ ${status} -eq 0 ]]; then + echo "Refusing to overwrite existing chart ${artifact}" >&2 + exit 1 + fi + if ! grep -Eqi 'NOT_FOUND|not found' <<< "${output}"; then + echo "Could not safely determine whether ${artifact} exists:" >&2 + echo "${output}" >&2 + exit 1 fi - - name: Run chart-testing (lint) - run: ct lint --config deploy/ct.yaml - - - name: Install Ginkgo - run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - - - name: Build and Push to GAR - if: steps.list-changed.outputs.changed == 'true' + - name: Package and publish prerelease chart + env: + VERSION: ${{ steps.chart.outputs.version }} run: | - VERSION=$(grep "^version:" deploy/operator/Chart.yaml | awk '{print $2}') + set -euo pipefail helm dependency build deploy/operator - helm package deploy/operator - helm push operator-${VERSION}.tgz oci://$REPOSITORY - env: - REPOSITORY: us-docker.pkg.dev/wandb-production/public/wandb/charts \ No newline at end of file + mkdir -p dist + helm package deploy/operator --destination dist + helm push "dist/operator-${VERSION}.tgz" "oci://${CHART_REPOSITORY}" diff --git a/.github/workflows/internal-image-publish.yaml b/.github/workflows/internal-image-publish.yaml index 596c1e38..5cce00de 100644 --- a/.github/workflows/internal-image-publish.yaml +++ b/.github/workflows/internal-image-publish.yaml @@ -5,35 +5,38 @@ on: inputs: image_tag: type: string - description: 'Tags for Images in GAR' + description: 'Development tag in the form dev--<7-to-40-character-sha>' required: true jobs: release: - name: Release + name: Publish development image runs-on: ubuntu-latest permissions: - contents: 'read' - id-token: 'write' + contents: read + id-token: write steps: + - name: Validate development tag + env: + VERSION: ${{ inputs.image_tag }} + run: | + if [[ ! "${VERSION}" =~ ^dev-[a-z0-9][a-z0-9._-]*-[0-9a-f]{7,40}$ ]]; then + echo "Development tags must use dev--<7-to-40-character-sha>" >&2 + exit 1 + fi + - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - persist-credentials: false fetch-depth: 0 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + persist-credentials: false - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3.0.1 + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 - id: auth name: Authenticate to Google Cloud - uses: google-github-actions/auth@v3 + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 with: create_credentials_file: 'true' token_format: access_token @@ -41,21 +44,11 @@ jobs: workload_identity_provider: ${{ secrets.CI_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.CI_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} - - name: Authorize Docker to use Google Container Registry - run: | - gcloud auth configure-docker us-docker.pkg.dev - - - uses: actions/setup-go@v4 - with: - go-version: 1.25 - - - name: Install Ginkgo - run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + - name: Authorize Docker for Artifact Registry + run: gcloud auth configure-docker us-docker.pkg.dev --quiet - - name: Build and Push to GAR - run: | - export IMG=$IMAGE_TAG_BASE:$VERSION - make docker-build docker-push + - name: Build and publish development image env: - IMAGE_TAG_BASE: us-docker.pkg.dev/wandb-production/public/wandb/operator - VERSION: ${{ github.event.inputs.image_tag }} \ No newline at end of file + IMAGE_REPOSITORY: us-docker.pkg.dev/wandb-production/public/wandb/operator + VERSION: ${{ inputs.image_tag }} + run: make docker-build docker-push IMG="${IMAGE_REPOSITORY}:${VERSION}" diff --git a/.github/workflows/pr-title.yaml b/.github/workflows/pr-title.yaml index b9437ecb..15705a24 100644 --- a/.github/workflows/pr-title.yaml +++ b/.github/workflows/pr-title.yaml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: # https://github.com/amannn/action-semantic-pull-request/releases - - uses: amannn/action-semantic-pull-request@v4.2.0 + - uses: amannn/action-semantic-pull-request@0eb081bc9c35210408951834a444794406eff6f8 # v4.2.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 42b355b8..c706ad44 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,91 +1,168 @@ -name: Release +name: Release v2 on: - workflow_dispatch: push: - branches: - - main + tags: + - "v2.*.*" + +permissions: + contents: write + id-token: write + +concurrency: + group: release-v2 + cancel-in-progress: false + +env: + CHART_REPOSITORY: us-docker.pkg.dev/wandb-production/public/wandb/charts + IMAGE_REPOSITORY: us-docker.pkg.dev/wandb-production/public/wandb/operator jobs: release: - name: Release - runs-on: ubuntu-latest - # Skip running release workflow on forks + name: Release v2 if: github.repository_owner == 'wandb' + runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 + - name: Checkout release tag + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - persist-credentials: false fetch-depth: 0 + fetch-tags: true + persist-credentials: false - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + - name: Validate tag and release versions + id: release + shell: bash + run: | + set -euo pipefail - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} + tag="${GITHUB_REF_NAME}" - - name: Login to quay.io - uses: docker/login-action@v2 - with: - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_TOKEN }} - registry: quay.io + if [[ ! "${tag}" =~ ^v2\.([0-9]+)\.([0-9]+)(\-.*)?$ ]]; then + echo "Expected a stable v2 tag in the form v2.x.y; got ${tag}" >&2 + exit 1 + fi - - name: Versioning - id: release - uses: cycjimmy/semantic-release-action@v3 + git fetch --no-tags --force origin "refs/tags/${tag}:refs/tags/${tag}" + + if [[ "$(git cat-file -t "refs/tags/${tag}")" != "tag" ]]; then + echo "Release tag ${tag} must be annotated" >&2 + exit 1 + fi + + git fetch --no-tags --force origin refs/heads/main:refs/remotes/origin/main + + tagged_commit="$(git rev-list -n 1 "${tag}")" + if ! git merge-base --is-ancestor "${tagged_commit}" refs/remotes/origin/main; then + echo "Tagged commit ${tagged_commit} is not reachable from origin/main" >&2 + exit 1 + fi + + version="${tag#v}" + chart_version="$(awk '$1 == "version:" { print $2; exit }' deploy/operator/Chart.yaml | tr -d '\"')" + app_version="$(awk '$1 == "appVersion:" { print $2; exit }' deploy/operator/Chart.yaml | tr -d '\"')" + image_tag="$(awk ' + $0 == "wandb-operator:" { in_operator = 1; next } + in_operator && $0 == " image:" { in_image = 1; next } + in_image && $1 == "tag:" { print $2; exit } + ' deploy/operator/values.yaml | tr -d '\"')" + + for value in "${chart_version}" "${app_version}" "${image_tag}"; do + if [[ "${value}" != "${version}" ]]; then + echo "Chart version, appVersion, and operator image tag must all equal ${version}" >&2 + echo "Found chart=${chart_version}, appVersion=${app_version}, image=${image_tag}" >&2 + exit 1 + fi + done + + echo "tag=${tag}" >> "${GITHUB_OUTPUT}" + echo "tagged_commit=${tagged_commit}" >> "${GITHUB_OUTPUT}" + echo "version=${version}" >> "${GITHUB_OUTPUT}" + + - name: Install Helm + uses: azure/setup-helm@bf6a7d304bc2fdb57e0331155b7ebf2c504acf0a # v4 with: - semantic_version: 19.0.2 - extra_plugins: | - @semantic-release/changelog@6.0.1 - @semantic-release/git@10.0.1 - conventional-changelog-conventionalcommits@4.6.3 - env: - GITHUB_TOKEN: ${{ secrets.GH_SECRET }} + version: v3.19.0 + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 - - uses: actions/setup-go@v4 + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 with: - go-version: 1.25 + create_credentials_file: 'true' + token_format: access_token + project_id: wandb-production + workload_identity_provider: ${{ secrets.CI_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.CI_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} - - name: Install Ginkgo - run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + - name: Authorize Docker for Artifact Registry + run: gcloud auth configure-docker us-docker.pkg.dev --quiet - - name: Build and Push Latest - if: steps.release.outputs.new_release_version - run: | - export IMG=$IMAGE_TAG_BASE:$VERSION - make docker-build docker-push - env: - IMAGE_TAG_BASE: wandb/controller - VERSION: latest +# - name: Reject existing production artifacts +# env: +# VERSION: ${{ steps.release.outputs.version }} +# shell: bash +# run: | +# set -euo pipefail +# +# assert_absent() { +# local artifact=$1 +# local output +# local status +# +# set +e +# output="$(gcloud artifacts docker images describe "${artifact}" --format='value(image_summary.digest)' 2>&1)" +# status=$? +# set -e +# +# if [[ ${status} -eq 0 ]]; then +# echo "Refusing to overwrite existing artifact ${artifact}" >&2 +# exit 1 +# fi +# if ! grep -Eqi 'NOT_FOUND|not found' <<< "${output}"; then +# echo "Could not safely determine whether ${artifact} exists:" >&2 +# echo "${output}" >&2 +# exit 1 +# fi +# } +# +# assert_absent "${IMAGE_REPOSITORY}:${VERSION}" +# assert_absent "${CHART_REPOSITORY}/operator:${VERSION}" - - name: Tag and Push to Docker Hub - if: steps.release.outputs.new_release_version - run: | - docker tag wandb/controller:latest wandb/controller:${{ steps.release.outputs.new_release_version }} - docker push wandb/controller:${{ steps.release.outputs.new_release_version }} + - name: Build and publish image + env: + VERSION: ${{ steps.release.outputs.version }} + run: make docker-build docker-push IMG="${IMAGE_REPOSITORY}:${VERSION}" - docker tag wandb/controller:latest wandb/controller:${{ steps.release.outputs.new_release_major_version }}.${{ steps.release.outputs.new_release_minor_version }} - docker push wandb/controller:${{ steps.release.outputs.new_release_major_version }}.${{ steps.release.outputs.new_release_minor_version }} + - name: Install chart-testing + uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 + with: + version: v3.14.0 - docker tag wandb/controller:latest wandb/controller:${{ steps.release.outputs.new_release_major_version }} - docker push wandb/controller:${{ steps.release.outputs.new_release_major_version }} + - name: Lint charts + run: ct lint --all --config deploy/ct.yaml + - name: Package and publish chart + env: + VERSION: ${{ steps.release.outputs.version }} + run: | + set -euo pipefail + helm dependency build deploy/operator + mkdir -p dist + helm package deploy/operator --destination dist + helm push "dist/operator-${VERSION}.tgz" "oci://${CHART_REPOSITORY}" - - name: Tag and Push to Quay.io - if: steps.release.outputs.new_release_version + - name: Create GitHub release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.release.outputs.tag }} + TAGGED_COMMIT: ${{ steps.release.outputs.tagged_commit }} run: | - docker tag wandb/controller:latest quay.io/wandb_tools/wandb-k8s-operator:${{ steps.release.outputs.new_release_version }} - docker push quay.io/wandb_tools/wandb-k8s-operator:${{ steps.release.outputs.new_release_version }} - - docker tag wandb/controller:latest quay.io/wandb_tools/wandb-k8s-operator:${{ steps.release.outputs.new_release_major_version }}.${{ steps.release.outputs.new_release_minor_version }} - docker push quay.io/wandb_tools/wandb-k8s-operator:${{ steps.release.outputs.new_release_major_version }}.${{ steps.release.outputs.new_release_minor_version }} - - docker tag wandb/controller:latest quay.io/wandb_tools/wandb-k8s-operator:${{ steps.release.outputs.new_release_major_version }} - docker push quay.io/wandb_tools/wandb-k8s-operator:${{ steps.release.outputs.new_release_major_version }} + gh release create "${TAG}" \ + --repo "${GITHUB_REPOSITORY}" \ + --target "${TAGGED_COMMIT}" \ + --verify-tag \ + --generate-notes \ + --title "${TAG}" diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 7c4f7938..0f6bd009 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -2,36 +2,54 @@ name: "Run Tests" on: push: branches: + - v2 - main pull_request: branches: + - v2 - main jobs: test: name: Test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 with: - go-version: 1.25 + go-version: 1.26.3 - name: Install dependencies run: go mod download - name: Install Ginkgo - run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + run: go install github.com/onsi/ginkgo/v2/ginkgo@v2.28.1 + - name: Install Helm + uses: azure/setup-helm@bf6a7d304bc2fdb57e0331155b7ebf2c504acf0a # v4 + with: + version: v3.19.0 + - name: Resolve chart dependencies + run: | + helm repo add ci-wandb https://charts.wandb.ai/ + helm repo add ci-moco https://cybozu-go.github.io/moco/ + helm repo add ci-ot-container-kit https://ot-container-kit.github.io/helm-charts + helm repo add ci-seaweedfs https://seaweedfs.github.io/seaweedfs-operator/ + helm repo add ci-prometheus-community https://prometheus-community.github.io/helm-charts + helm repo add ci-altinity https://helm.altinity.com + helm repo add ci-victoria-metrics https://victoriametrics.github.io/helm-charts/ + helm repo add ci-grafana https://grafana.github.io/helm-charts + helm dependency build deploy/operator + git diff --exit-code deploy/operator/Chart.lock - name: Tests - run: make test-coverage + run: make test build: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 with: - go-version: 1.25 + go-version: 1.26.3 - name: Build run: make build @@ -39,11 +57,11 @@ jobs: name: Dependency Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 with: - go-version: 1.25 + go-version: 1.26.3 - name: Install dependencies run: go mod download - name: Check for changes in go.mod or go.sum diff --git a/.releaserc.json b/.releaserc.json deleted file mode 100644 index abb5694d..00000000 --- a/.releaserc.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "branches": ["main"], - "ci": false, - "plugins": [ - ["@semantic-release/commit-analyzer", { "preset": "conventionalcommits" }], - [ - "@semantic-release/release-notes-generator", - { "preset": "conventionalcommits" } - ], - [ - "@semantic-release/github", - { - "successComment": "This ${issue.pull_request ? 'PR is included' : 'issue has been resolved'} in version ${nextRelease.version} :tada:", - "labels": false, - "releasedLabels": false - } - ], - [ - "@semantic-release/changelog", - { - "changelogFile": "CHANGELOG.md", - "changelogTitle": "# Changelog\n\nAll notable changes to this project will be documented in this file." - } - ], - [ - "@semantic-release/git", - { - "assets": ["CHANGELOG.md"], - "message": "chore(release): version ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}" - } - ] - ] -} diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d076425..e730dc9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [1.22.0](https://github.com/wandb/operator/compare/v1.21.3...v1.22.0) (2026-04-30) + + +### Features + +* Add OCI Helm chart registry support and upgrade to Helm v4 ([#147](https://github.com/wandb/operator/issues/147)) ([fcb72b8](https://github.com/wandb/operator/commit/fcb72b8652c6370fb05aa6a805707540c1f57d81)) + ### [1.21.3](https://github.com/wandb/operator/compare/v1.21.2...v1.21.3) (2025-12-04) diff --git a/Dockerfile b/Dockerfile index b553568a..554e717b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,10 +27,19 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o ma RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o crd-installer ./cmd/crd-installer FROM registry.access.redhat.com/ubi9/ubi-minimal + WORKDIR / COPY --from=manager-builder /workspace/manager . COPY --from=manager-builder /workspace/crd-installer . +RUN mkdir -p /helm/.cache/helm /helm/.config/helm /helm/.local/share/helm && \ + chown -R 65532:65532 /helm + +USER 65532:65532 + +ENV HELM_CACHE_HOME=/helm/.cache/helm +ENV HELM_CONFIG_HOME=/helm/.config/helm +ENV HELM_DATA_HOME=/helm/.local/share/helm ENV OPERATOR_MODE=production ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 23314e42..b66628d1 100644 --- a/Makefile +++ b/Makefile @@ -138,6 +138,56 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes lint-config: golangci-lint ## Verify golangci-lint linter configuration $(GOLANGCI_LINT) config verify +REGISTRY_PORT ?= 5050 +REGISTRY_NAME ?= registry +REGISTRY_USER ?= admin +REGISTRY_PASS ?= admin123 +REGISTRY_AUTH_DIR ?= /tmp/registry-auth +CHART_REPO ?= https://charts.wandb.ai +CHART_NAME ?= operator-wandb +OCI_MANIFEST ?= hack/testing-manifests/wandb/oci.yaml +CHART_VERSION ?= $(shell grep 'version:' $(OCI_MANIFEST) | head -1 | sed 's/.*"\(.*\)"/\1/') + +.PHONY: local-registry +local-registry: ## Start a local OCI registry with basic auth for testing. + @if $(CONTAINER_TOOL) ps --filter name=$(REGISTRY_NAME) --format '{{.Names}}' | grep -q $(REGISTRY_NAME); then \ + echo "Registry already running on port $(REGISTRY_PORT)"; \ + else \ + $(CONTAINER_TOOL) rm -f $(REGISTRY_NAME) >/dev/null 2>&1 || true; \ + mkdir -p $(REGISTRY_AUTH_DIR) && \ + $(CONTAINER_TOOL) run --rm --entrypoint sh registry:2 -c \ + "apk add --no-cache apache2-utils >/dev/null 2>&1 && htpasswd -Bbn $(REGISTRY_USER) $(REGISTRY_PASS)" \ + > $(REGISTRY_AUTH_DIR)/htpasswd && \ + $(CONTAINER_TOOL) run -d --name $(REGISTRY_NAME) -p $(REGISTRY_PORT):5000 \ + -v $(REGISTRY_AUTH_DIR):/auth \ + -e REGISTRY_AUTH=htpasswd \ + -e REGISTRY_AUTH_HTPASSWD_REALM=Registry \ + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + registry:2; \ + echo "Registry started on port $(REGISTRY_PORT) (user: $(REGISTRY_USER))"; \ + fi + +.PHONY: local-registry-push +local-registry-push: local-registry ## Push a chart from the wandb helm repo to the local OCI registry. + @tmpdir=$$(mktemp -d) && \ + helm pull $(CHART_NAME) --repo $(CHART_REPO) --version $(CHART_VERSION) -d $$tmpdir && \ + helm push $$tmpdir/$(CHART_NAME)-$(CHART_VERSION).tgz oci://localhost:$(REGISTRY_PORT)/wandb \ + --plain-http --username $(REGISTRY_USER) --password $(REGISTRY_PASS) && \ + rm -rf $$tmpdir + +WANDB_NAMESPACE ?= default + +.PHONY: local-registry-secret +local-registry-secret: ## Create a Kubernetes secret with registry credentials for the operator. + @kubectl -n $(WANDB_NAMESPACE) create secret generic oci-registry-creds \ + --from-literal=HELM_USERNAME=$(REGISTRY_USER) \ + --from-literal=HELM_PASSWORD=$(REGISTRY_PASS) \ + --dry-run=client -o yaml | kubectl apply -f - + +.PHONY: local-registry-stop +local-registry-stop: ## Stop and remove the local OCI registry. + @$(CONTAINER_TOOL) rm -f $(REGISTRY_NAME) 2>/dev/null || true + ##@ Build .PHONY: build diff --git a/README.md b/README.md index e81e1d35..ab78ed55 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ A Kubernetes operator for deploying and managing self-hosted [Weights & Biases](https://wandb.ai) on your own cluster. +This branch contains Operator v2. Maintenance and release history for the +legacy Operator v1 line are available on the [`v1` branch](https://github.com/wandb/operator/tree/v1). + ## Description The operator turns a single `WeightsAndBiases` custom resource into a fully running @@ -15,7 +18,7 @@ It supports two modes for backing infrastructure: - **Managed**: the operator provisions and operates dependencies in-cluster through bundled component operators: - 1. MySQL via [Moco](https://github.com/cybozu-go/moco) + 1. MySQL via [Moco](https://github.com/cybozu-go/moco) 2. Redis 3. Kafka via [Strimzi](https://strimzi.io/) 4. Object Storage via @@ -76,11 +79,50 @@ The operator reconciles the resource, brings up the requested backing services, and rolls out the W&B application. See [`deploy/operator/values.yaml`](deploy/operator/values.yaml) for the available chart options and which component operators are enabled. +### Custom CA certificates (air-gapped / private registry) + +When the operator must reach a private OCI registry served with a self-signed or +corporate CA — for example to pull the server manifest in an air-gapped install — +provide the CA at install time via `wandb-operator.caCerts`. The certs are mounted +into the operator pod and added to `SSL_CERT_DIR` additively, so the system trust +bundle (and any public roots) is preserved. + +Provide the CA one of three ways: + +```yaml +# values.yaml — pick ONE source +wandb-operator: + caCerts: + # 1. inline PEM blocks (synthesized into a Secret by the chart) + certs: + - | + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- + # 2. or an existing Secret whose keys are PEM certs + existingSecret: my-registry-ca + # 3. or an existing ConfigMap whose keys are PEM certs + existingConfigMap: my-registry-ca +``` + +```bash +# e.g. mount a CA file straight from disk +helm install wandb-operator oci://.../charts/operator \ + --namespace wandb-operators --create-namespace \ + --set-file 'wandb-operator.caCerts.certs[0]=./registry-ca.crt' +``` + +This trusts the CA for the operator's own egress (the server-manifest pull). To +trust a CA on the W&B **application** workloads instead, use +`spec.global.customCACerts` / `spec.global.caCertsConfigMap` on the +`WeightsAndBiases` resource. + ## Documentation - [Configuration API](docs/config-api.md) - [Infrastructure Connection Settings](docs/infra-connection-settings.md) - [Monitoring and Telemetry Guide](docs/monitoring.md) +- [Deploying on OpenShift](docs/openshift.md) ## Development diff --git a/Tiltfile b/Tiltfile index 0a0f45a5..a22ebac6 100644 --- a/Tiltfile +++ b/Tiltfile @@ -6,6 +6,7 @@ GENERATED_DIR = "hack/testing-manifests/wandb/.generated" GENERATED_WANDB_CR = GENERATED_DIR + "/tilt-wandb-cr.yaml" GENERATED_OPERATOR_VALUES = GENERATED_DIR + "/tilt-operator-values.yaml" +GENERATED_CUSTOM_CA_CONFIGMAP = GENERATED_DIR + "/tilt-custom-ca-configmap.yaml" GATEWAY_API_CRDS_URL = "https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.0/standard-install.yaml" IMG = "controller:latest" @@ -35,7 +36,7 @@ settings = { "wandbName": "wandb", "wandbNamespace": "wandb", "wandbHostname": "http://localhost:8080", - "wandbVersion": "0.80.0", + "wandbVersion": "0.83.0-clickhouse-keeper.2", "size": "dev", "retentionPolicy": "detach", "licenseFile": "", @@ -51,6 +52,11 @@ settings = { "observabilityMode": "off", "logFormat": "pretty", # pretty, text, json + + "useExternalMysql": False, + "useExternalRedis": False, + "useExternalObjectStore": False, + "useCustomCA": False, } if os.path.exists("tilt-settings.json"): @@ -176,6 +182,18 @@ settings["openshiftSCC"] = as_bool(settings.get("openshiftSCC")) if settings["manifestSource"] == "local": settings["localManifestPath"] = validate_local_manifest_path(settings.get("localManifestPath")) +USE_EXTERNAL_MYSQL = as_bool(settings.get("useExternalMysql")) +USE_EXTERNAL_REDIS = as_bool(settings.get("useExternalRedis")) +USE_EXTERNAL_OBJECT_STORE = as_bool(settings.get("useExternalObjectStore")) +USE_CUSTOM_CA = as_bool(settings.get("useCustomCA")) +USE_EXTERNAL_INFRA = USE_EXTERNAL_MYSQL or USE_EXTERNAL_REDIS or USE_EXTERNAL_OBJECT_STORE +USE_TEST_INFRA_TLS = USE_CUSTOM_CA and (USE_EXTERNAL_MYSQL or USE_EXTERNAL_REDIS) + +if (USE_EXTERNAL_INFRA or USE_CUSTOM_CA) and not as_bool(settings.get("includeCR")): + fail("useExternalMysql/useExternalRedis/useExternalObjectStore/useCustomCA require includeCR=True") +if (USE_EXTERNAL_INFRA or USE_CUSTOM_CA) and settings.get("wandbCR", "") != "": + fail("useExternalMysql/useExternalRedis/useExternalObjectStore/useCustomCA patch generated CRs; use crFile instead of wandbCR") + watch_settings(ignore=["**/.git", "**/*.out", GENERATED_DIR + "/**"]) update_settings(k8s_upsert_timeout_secs=300) @@ -197,7 +215,6 @@ if IS_CRC: os.putenv("PATH", "./bin:" + os.getenv("PATH")) -load("ext://restart_process", "docker_build_with_restart") load("ext://helm_resource", "helm_repo", "helm_resource") def operator_dockerfile(): @@ -211,32 +228,15 @@ def operator_dockerfile(): if settings.get("manifestSource") == "local": lines.append("ADD %s /server-manifest" % settings.get("localManifestPath")) - if settings.get("openshiftSCC"): - lines += [ - "", - "RUN mkdir -p /helm/.cache/helm /helm/.config/helm /helm/.local/share/helm && \\", - " chgrp -R 0 /helm && chmod -R g=u /helm", - ] - else: - lines += [ - "", - "RUN mkdir -p /helm/.cache/helm /helm/.config/helm /helm/.local/share/helm", - ] - - lines += [ + lines.extend([ "", + "RUN mkdir -p /helm/.cache/helm /helm/.config/helm /helm/.local/share/helm && chown -R 65532:65532 /helm", + "USER 65532:65532", "ENV HELM_CACHE_HOME=/helm/.cache/helm", "ENV HELM_CONFIG_HOME=/helm/.config/helm", "ENV HELM_DATA_HOME=/helm/.local/share/helm", - ] - - if settings.get("openshiftSCC"): - lines += [ - "", - "USER 1001", - ] - - lines.append("") + "", + ]) return "\n".join(lines) @@ -327,29 +327,22 @@ def build_operator_values(telemetry_namespace): }, } + manager_entrypoint = ["/manager", "--log-format=" + settings.get("logFormat")] + + values["wandb-operator"]["containers"]["operator"]["command"] = manager_entrypoint + if settings.get("openshiftSCC"): values["wandb-operator"]["podSecurityContext"] = { - "runAsNonRoot": True, "runAsUser": None, "runAsGroup": None, "fsGroup": None, "fsGroupChangePolicy": None, - "seccompProfile": { - "type": "RuntimeDefault", - }, } values["wandb-operator"]["containers"]["operator"]["env"] = { "OPENSHIFT": { "value": "true", }, } - values["wandb-operator"]["containers"]["operator"]["securityContext"] = { - "allowPrivilegeEscalation": False, - "readOnlyRootFilesystem": True, - "capabilities": { - "drop": ["ALL"], - }, - } values["altinity-clickhouse-operator"] = { "crdHook": { "enabled": False, @@ -390,6 +383,12 @@ def build_wandb_cr(): cmd += helper_flag("ingress-class", settings.get("ingressClass")) cmd += helper_bool_flag("create-ca", settings.get("createCA")) cmd += helper_flag("issuer-name", settings.get("issuerName", "")) + cmd += helper_bool_flag("external-mysql", USE_EXTERNAL_MYSQL) + cmd += helper_bool_flag("external-redis", USE_EXTERNAL_REDIS) + cmd += helper_bool_flag("external-objectstore", USE_EXTERNAL_OBJECT_STORE) + cmd += helper_bool_flag("custom-ca", USE_CUSTOM_CA) + if USE_CUSTOM_CA: + cmd += helper_flag("custom-ca-configmap-out", GENERATED_CUSTOM_CA_CONFIGMAP) local(cmd) return GENERATED_WANDB_CR @@ -495,6 +494,15 @@ if CREATE_WANDB_NAMESPACE: labels=[GROUP_DEPENDENCIES], ) +if USE_CUSTOM_CA: + k8s_yaml(GENERATED_CUSTOM_CA_CONFIGMAP) + k8s_resource( + new_name="Custom-CA-ConfigMap", + objects=["wandb-user-ca-certs:configmap:%s" % WANDB_NAMESPACE], + resource_deps=["WandB-Namespace"], + labels=[GROUP_WANDB_APP], + ) + local_resource( "Operator-Codegen", "make manifests generate", @@ -586,6 +594,32 @@ helm_resource( labels=[GROUP_DEPENDENCIES], ) +if USE_EXTERNAL_INFRA: + test_infra_deps = ["WandB-Namespace"] + if USE_TEST_INFRA_TLS: + test_infra_deps.append("cert-manager") + + helm_resource( + "Test-Infra", + chart="./hack/testing-manifests/test-infra", + release_name="test-infra", + namespace=WANDB_NAMESPACE, + flags=[ + "--create-namespace", + "--wait", + "--timeout=10m", + "--set=mysql.enabled=%s" % bool_string(USE_EXTERNAL_MYSQL), + "--set=redis.enabled=%s" % bool_string(USE_EXTERNAL_REDIS), + "--set=seaweedfs.enabled=%s" % bool_string(USE_EXTERNAL_OBJECT_STORE), + "--set=tls.enabled=%s" % bool_string(USE_TEST_INFRA_TLS), + "--set=mysql.tls.enabled=%s" % bool_string(USE_CUSTOM_CA and USE_EXTERNAL_MYSQL), + "--set=redis.tls.enabled=%s" % bool_string(USE_CUSTOM_CA and USE_EXTERNAL_REDIS), + ], + deps=["hack/testing-manifests/test-infra/"], + resource_deps=test_infra_deps, + labels=[GROUP_WANDB_APP], + ) + if LOCAL_NETWORKING_MODE == "gateway": nginx_gateway_flags = [ "--create-namespace", @@ -632,14 +666,23 @@ if LOCAL_NETWORKING_MODE == "ingress": labels=[GROUP_DEPENDENCIES], ) +kube_state_metrics_flags = [ + "--create-namespace", + "--version=5.27.0", +] +if settings.get("openshiftSCC"): + # Null the chart's hardcoded 65534 IDs so restricted-v2 assigns valid ones. + kube_state_metrics_flags += [ + "--set=securityContext.runAsUser=null", + "--set=securityContext.runAsGroup=null", + "--set=securityContext.fsGroup=null", + ] + helm_resource( "kube-state-metrics", chart="oci://ghcr.io/prometheus-community/charts/kube-state-metrics", namespace="kube-state-metrics", - flags=[ - "--create-namespace", - "--version=5.27.0", - ], + flags=kube_state_metrics_flags, labels=[GROUP_DEPENDENCIES], ) @@ -708,6 +751,10 @@ if as_bool(settings.get("includeCR")): wandb_deps.append("nginx-gateway-fabric") if LOCAL_NETWORKING_MODE == "ingress": wandb_deps.append("ingress-nginx-controller") + if USE_EXTERNAL_INFRA: + wandb_deps.append("Test-Infra") + if USE_CUSTOM_CA: + wandb_deps.append("Custom-CA-ConfigMap") if str(WANDB_HOSTNAME).startswith("https://") and as_bool(settings.get("createCA")): build_wandb_ca(WANDB_NAME, WANDB_NAMESPACE) @@ -722,6 +769,69 @@ if as_bool(settings.get("includeCR")): labels=[GROUP_WANDB_APP], ) + if settings.get("openshiftSCC"): + # Dev-only frontend-nginx SCC; not shipped (prod uses its own ingress). + # It needs its fixed image UID: restricted-v2 assigns an arbitrary UID + # and nonroot-v2 rejects named user, so clone restricted-v2 + RunAsAny. + frontend_scc_name = "wandb-frontend-anyuid-v2" + k8s_yaml_object({ + "apiVersion": "security.openshift.io/v1", + "kind": "SecurityContextConstraints", + "metadata": { + "name": frontend_scc_name, + "labels": { + "app.kubernetes.io/managed-by": "tilt", + }, + }, + "allowHostDirVolumePlugin": False, + "allowHostIPC": False, + "allowHostNetwork": False, + "allowHostPID": False, + "allowHostPorts": False, + "allowPrivilegeEscalation": False, + "allowPrivilegedContainer": False, + "allowedCapabilities": ["NET_BIND_SERVICE"], + "readOnlyRootFilesystem": False, + "requiredDropCapabilities": ["ALL"], + "runAsUser": {"type": "RunAsAny"}, + "seLinuxContext": {"type": "MustRunAs"}, + "seccompProfiles": ["runtime/default"], + "fsGroup": {"type": "MustRunAs"}, + "supplementalGroups": {"type": "RunAsAny"}, + "volumes": [ + "configMap", + "csi", + "downwardAPI", + "emptyDir", + "ephemeral", + "image", + "persistentVolumeClaim", + "projected", + "secret", + ], + "users": [ + "system:serviceaccount:%s:wandb-app" % WANDB_NAMESPACE, + ], + }) + k8s_resource( + new_name="OpenShift-Frontend-SCC", + objects=["%s:securitycontextconstraints" % frontend_scc_name], + resource_deps=["WandB-Namespace"], + labels=[GROUP_WANDB_APP], + ) + # Stamp required-scc on frontend Deployment; reconcile merge keeps it. + local_resource( + "OpenShift-Frontend-SCC-Bind", + cmd=( + "until kubectl -n %s get deploy/frontend >/dev/null 2>&1; do " % WANDB_NAMESPACE + + "echo 'waiting for frontend deployment...'; sleep 3; done && " + + "kubectl -n %s patch deploy/frontend --type=merge -p " % WANDB_NAMESPACE + + shell_quote('{"spec":{"template":{"metadata":{"annotations":{"openshift.io/required-scc":"%s"}}}}}' % frontend_scc_name) + ), + resource_deps=["Wandb", "OpenShift-Frontend-SCC"], + labels=[GROUP_WANDB_APP], + ) + endpoint_port = url_port(WANDB_HOSTNAME) endpoint_host = url_host(WANDB_HOSTNAME) @@ -757,6 +867,7 @@ if as_bool(settings.get("includeCR")): }, labels=[GROUP_WANDB_APP], ) + if settings.get("observabilityMode") == "full": managed_endpoint_resource( name="Telemetry-Endpoint-Grafana", @@ -808,23 +919,20 @@ if settings.get("observabilityMode") == "full": labels=[GROUP_TELEMETRY], ) -manager_entrypoint = ["/manager", "--log-format=" + settings.get("logFormat")] - docker_only = ["./tilt_bin/manager", "./tilt_bin/crd-installer"] -live_update_steps = [ - sync("./tilt_bin/manager", "/manager"), - sync("./tilt_bin/crd-installer", "/crd-installer"), -] if settings.get("manifestSource") == "local": - docker_only.append(repo_path(settings.get("localManifestPath"))) - live_update_steps.append(sync(settings.get("localManifestPath"), "/server-manifest")) - -docker_build_with_restart( + path = repo_path(settings.get("localManifestPath")) + if not path.endswith(".yaml"): + paths = listdir(path, True) + for path in paths: + docker_only.append(path) + else: + docker_only.append(path) + +docker_build( IMG, ".", dockerfile_contents=operator_dockerfile(), - entrypoint=manager_entrypoint, only=docker_only, - live_update=live_update_steps, ) diff --git a/api/v1/weightsandbiases_conversion_mapping.go b/api/v1/weightsandbiases_conversion_mapping.go index e7ed7614..d1c94b76 100644 --- a/api/v1/weightsandbiases_conversion_mapping.go +++ b/api/v1/weightsandbiases_conversion_mapping.go @@ -33,20 +33,22 @@ import ( // Default Secret keys used by v1's legacy ref blocks when only the Secret // name was specified. const ( - defaultMySQLPasswordSecretKey = "MYSQL_PASSWORD" - defaultRedisPasswordSecretKey = "REDIS_PASSWORD" - defaultOIDCClientSecretKey = "OIDC_SECRET" - defaultBucketAccessKeyName = "ACCESS_KEY" - defaultBucketSecretKeyName = "SECRET_KEY" + defaultMySQLPasswordSecretKey = "MYSQL_PASSWORD" + defaultRedisPasswordSecretKey = "REDIS_PASSWORD" + defaultOIDCClientSecretKey = "OIDC_SECRET" + defaultBucketAccessKeyName = "ACCESS_KEY" + defaultBucketSecretKeyName = "SECRET_KEY" + defaultClickHousePasswordSecretKey = "CLICKHOUSE_PASSWORD" ) // Annotations carrying v1 literals the reconciler materializes into Secrets // post-conversion (the webhook is stateless and can't create them itself). const ( - OIDCPendingAnnotation = "legacy.operator.wandb.com/oidc-pending" - MySQLPendingAnnotation = "legacy.operator.wandb.com/mysql-pending" - RedisPendingAnnotation = "legacy.operator.wandb.com/redis-pending" - BucketPendingAnnotation = "legacy.operator.wandb.com/bucket-pending" + OIDCPendingAnnotation = "legacy.operator.wandb.com/oidc-pending" + MySQLPendingAnnotation = "legacy.operator.wandb.com/mysql-pending" + RedisPendingAnnotation = "legacy.operator.wandb.com/redis-pending" + BucketPendingAnnotation = "legacy.operator.wandb.com/bucket-pending" + ClickHousePendingAnnotation = "legacy.operator.wandb.com/clickhouse-pending" ) var validSizes = map[string]appsv2.Size{ @@ -84,6 +86,9 @@ func applyValueMappings(src *WeightsAndBiases, dst *appsv2.WeightsAndBiases) err if err := mapIngress(values, dst); err != nil { return err } + if err := mapLegacyOverrides(values, dst); err != nil { + return err + } globalMap, found, err := unstructured.NestedMap(values, "global") if err != nil { @@ -106,6 +111,9 @@ func applyGlobalMappings(globalMap map[string]interface{}, dst *appsv2.WeightsAn if err := mapSize(globalMap, dst); err != nil { return err } + if err := mapCustomCACerts(globalMap, dst); err != nil { + return err + } if err := mapOIDC(globalMap, dst); err != nil { return err } @@ -118,6 +126,25 @@ func applyGlobalMappings(globalMap map[string]interface{}, dst *appsv2.WeightsAn if err := mapBucket(globalMap, dst); err != nil { return err } + if err := mapClickHouse(globalMap, dst); err != nil { + return err + } + + return nil +} + +func mapCustomCACerts(globalMap map[string]interface{}, dst *appsv2.WeightsAndBiases) error { + if certs, found, err := unstructured.NestedStringSlice(globalMap, "customCACerts"); err != nil { + return fmt.Errorf("spec.values.global.customCACerts: %w", err) + } else if found { + dst.Spec.Global.CustomCACerts = certs + } + + if configMap, found, err := unstructured.NestedString(globalMap, "caCertsConfigMap"); err != nil { + return fmt.Errorf("spec.values.global.caCertsConfigMap: %w", err) + } else if found { + dst.Spec.Global.CACertsConfigMap = configMap + } return nil } @@ -360,7 +387,10 @@ func mapBucket(globalMap map[string]interface{}, dst *appsv2.WeightsAndBiases) e return nil } - dst.Spec.ObjectStore.ExternalObjectStore = &appsv2.ObjectStoreConnection{} + conn := &appsv2.ObjectStoreConnection{} + dst.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{ + appsv2.DefaultInstanceName: {ExternalObjectStore: conn}, + } if sec, ok, err := unstructured.NestedMap(bucket, "secret"); err != nil { return fmt.Errorf("spec.values.global.bucket.secret: %w", err) @@ -375,11 +405,11 @@ func mapBucket(globalMap map[string]interface{}, dst *appsv2.WeightsAndBiases) e if secretKeyName == "" { secretKeyName = defaultBucketSecretKeyName } - dst.Spec.ObjectStore.ExternalObjectStore.AccessKey = corev1.SecretKeySelector{ + conn.AccessKey = corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: name}, Key: accessKeyName, } - dst.Spec.ObjectStore.ExternalObjectStore.SecretKey = corev1.SecretKeySelector{ + conn.SecretKey = corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: name}, Key: secretKeyName, } @@ -482,7 +512,9 @@ func mapMySQL(globalMap map[string]interface{}, dst *appsv2.WeightsAndBiases) er } } - dst.Spec.MySQL.ExternalMysql = conn + dst.Spec.MySQL = map[string]appsv2.MySQLSpec{ + appsv2.DefaultInstanceName: {ExternalMysql: conn}, + } if len(remaining) > 0 { return writeAnnotation(dst, MySQLPendingAnnotation, remaining) @@ -490,6 +522,92 @@ func mapMySQL(globalMap map[string]interface{}, dst *appsv2.WeightsAndBiases) er return nil } +// clickHouseFields maps each v1 global.clickhouse. to a +// *ClickHouseConnection setter (v1 `port` is the HTTP interface). +var clickHouseFields = []struct { + v1Key string + setRef func(*appsv2.ClickHouseConnection, corev1.SecretKeySelector) +}{ + {"host", func(c *appsv2.ClickHouseConnection, s corev1.SecretKeySelector) { c.Host = s }}, + {"port", func(c *appsv2.ClickHouseConnection, s corev1.SecretKeySelector) { c.HTTPPort = s }}, + {"database", func(c *appsv2.ClickHouseConnection, s corev1.SecretKeySelector) { c.Database = s }}, + {"user", func(c *appsv2.ClickHouseConnection, s corev1.SecretKeySelector) { c.Username = s }}, + {"password", func(c *appsv2.ClickHouseConnection, s corev1.SecretKeySelector) { c.Password = s }}, +} + +// mapClickHouse routes v1 global.clickhouse to externalClickhouse (like +// mapMySQL); external is asserted only when a connection field is present. +func mapClickHouse(globalMap map[string]interface{}, dst *appsv2.WeightsAndBiases) error { + chMap, found, err := unstructured.NestedMap(globalMap, "clickhouse") + if err != nil { + return fmt.Errorf("spec.values.global.clickhouse: %w", err) + } + if !found || len(chMap) == 0 { + return nil + } + + conn := &appsv2.ClickHouseConnection{} + remaining := map[string]string{} + sawField := false + + for _, f := range clickHouseFields { + raw, ok := chMap[f.v1Key] + if !ok { + continue + } + ref, literal, classifyErr := classifyValueFromOrLiteral(raw) + if classifyErr != nil { + return fmt.Errorf("spec.values.global.clickhouse.%s: %w", f.v1Key, classifyErr) + } + switch { + case ref != nil: + f.setRef(conn, *ref) + sawField = true + case literal != "": + remaining[f.v1Key] = literal + sawField = true + } + } + + if ps, ok, err := unstructured.NestedMap(chMap, "passwordSecret"); err != nil { + return fmt.Errorf("spec.values.global.clickhouse.passwordSecret: %w", err) + } else if ok { + name, _, err := unstructured.NestedString(ps, "name") + if err != nil { + return fmt.Errorf("spec.values.global.clickhouse.passwordSecret.name: %w", err) + } + alreadyHasPassword := conn.Password.Name != "" + if name != "" && !alreadyHasPassword { + key, _, err := unstructured.NestedString(ps, "passwordKey") + if err != nil { + return fmt.Errorf("spec.values.global.clickhouse.passwordSecret.passwordKey: %w", err) + } + if key == "" { + key = defaultClickHousePasswordSecretKey + } + conn.Password = corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: name}, + Key: key, + } + delete(remaining, "password") + sawField = true + } + } + + if !sawField { + return nil + } + + dst.Spec.ClickHouse = map[string]appsv2.ClickHouseSpec{ + appsv2.DefaultInstanceName: {ExternalClickHouse: conn}, + } + + if len(remaining) > 0 { + return writeAnnotation(dst, ClickHousePendingAnnotation, remaining) + } + return nil +} + // redisFields maps each v1 global.redis. to a *RedisConnection setter. var redisFields = []struct { v1Key string @@ -577,7 +695,9 @@ func mapRedis(globalMap map[string]interface{}, dst *appsv2.WeightsAndBiases) er } } - dst.Spec.Redis.ExternalRedis = conn + dst.Spec.Redis = map[string]appsv2.RedisSpec{ + appsv2.DefaultInstanceName: {ExternalRedis: conn}, + } if len(remaining) > 0 { return writeAnnotation(dst, RedisPendingAnnotation, remaining) diff --git a/api/v1/weightsandbiases_conversion_overrides.go b/api/v1/weightsandbiases_conversion_overrides.go new file mode 100644 index 00000000..204d3714 --- /dev/null +++ b/api/v1/weightsandbiases_conversion_overrides.go @@ -0,0 +1,367 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + appsv2 "github.com/wandb/operator/api/v2" + serverManifest "github.com/wandb/operator/pkg/wandb/manifest" +) + +const ( + legacyDefaultSize = "small" + + manifestFetchTimeout = 15 * time.Second + manifestFailureCooldown = time.Minute +) + +// conversionManifestGetter is a test seam over the real manifest resolver. +var conversionManifestGetter = serverManifest.GetServerManifest + +// Failure cooldown per (repository, version): GetServerManifest caches +// successes in its on-disk ORAS store but retries the remote on every failure, +// which would stall each v1 write for the fetch timeout. +var ( + manifestFailuresMu sync.Mutex + manifestFailures = map[string]manifestFailure{} +) + +type manifestFailure struct { + err error + until time.Time +} + +// SetConversionManifestGetter swaps the resolver and clears the failure +// cooldowns. For tests; nil restores the default. +func SetConversionManifestGetter(getter func(ctx context.Context, repository, version string) (serverManifest.Manifest, error)) { + manifestFailuresMu.Lock() + defer manifestFailuresMu.Unlock() + if getter == nil { + getter = serverManifest.GetServerManifest + } + conversionManifestGetter = getter + manifestFailures = map[string]manifestFailure{} +} + +// legacyManifestApps maps each manifest application name to the v1 values key +// holding its section (legacyKey when set, else the name). v1 values carry no +// repository field, so the defaulting webhook's default repository is used. +func legacyManifestApps(version string) (map[string]string, error) { + repository := appsv2.DefaultManifestRepository + key := repository + "|" + version + + manifestFailuresMu.Lock() + getter := conversionManifestGetter + if failure, ok := manifestFailures[key]; ok && time.Now().Before(failure.until) { + manifestFailuresMu.Unlock() + return nil, failure.err + } + manifestFailuresMu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), manifestFetchTimeout) + defer cancel() + + m, err := getter(ctx, repository, version) + if err != nil { + manifestFailuresMu.Lock() + manifestFailures[key] = manifestFailure{err: err, until: time.Now().Add(manifestFailureCooldown)} + manifestFailuresMu.Unlock() + return nil, err + } + + apps := make(map[string]string, len(m.Applications)) + for name, app := range m.Applications { + valuesKey := app.LegacyKey + if valuesKey == "" { + valuesKey = name + } + apps[name] = valuesKey + } + return apps, nil +} + +// mapLegacyOverrides extracts global and per-application env/extraEnv and +// resource overrides from v1 values into spec.wandb.legacyOverrides, with the +// server manifest for the converted version (set by mapVersion earlier) as +// the authority on which sections are applications. +func mapLegacyOverrides(values map[string]interface{}, dst *appsv2.WeightsAndBiases) error { + overrides := map[string]appsv2.LegacyOverrides{} + + globalMap, _, err := unstructured.NestedMap(values, "global") + if err != nil { + return fmt.Errorf("spec.values.global: %w", err) + } + if len(globalMap) > 0 { + env, err := legacyEnvFromSection(globalMap, "global") + if err != nil { + return err + } + if len(env) > 0 { + overrides[appsv2.LegacyOverridesGlobalKey] = appsv2.LegacyOverrides{Env: env} + } + } + + globalSize, _, err := unstructured.NestedString(globalMap, "size") + if err != nil { + return fmt.Errorf("spec.values.global.size: %w", err) + } + + if err := mapPerAppLegacyOverrides(values, dst.Spec.Wandb.Version, globalSize, overrides); err != nil { + return err + } + + if len(overrides) > 0 { + dst.Spec.Wandb.LegacyOverrides = overrides + } + return nil +} + +// mapPerAppLegacyOverrides is best-effort: a manifest fetch failure must never +// make v1 objects unservable, so it logs and skips instead of erroring. +func mapPerAppLegacyOverrides(values map[string]interface{}, version, globalSize string, overrides map[string]appsv2.LegacyOverrides) error { + if version == "" { + logger.Info("no version derived from v1 values; skipping per-application legacy overrides") + return nil + } + apps, err := legacyManifestApps(version) + if err != nil { + logger.Error(err, "failed to resolve server manifest; skipping per-application legacy overrides", + "version", version) + return nil + } + + appNames := make([]string, 0, len(apps)) + for name := range apps { + appNames = append(appNames, name) + } + sort.Strings(appNames) + + for _, name := range appNames { + key := apps[name] + section, found, err := unstructured.NestedMap(values, key) + if err != nil { + return fmt.Errorf("spec.values.%s: %w", key, err) + } + if !found { + continue + } + + env, err := legacyEnvFromSection(section, key) + if err != nil { + return err + } + resources, err := legacyResourcesFromSection(section, key, globalSize) + if err != nil { + return err + } + if len(env) == 0 && resources == nil { + continue + } + overrides[name] = appsv2.LegacyOverrides{Env: env, Resources: resources} + } + + logUnmappedLegacySections(values, apps, version) + return nil +} + +// logUnmappedLegacySections logs override-shaped sections no manifest +// application reads (e.g. the v1 monolith `app`, `console`) — not converted, +// preserved only in the v1-values annotation. +func logUnmappedLegacySections(values map[string]interface{}, apps map[string]string, version string) { + mappedKeys := make(map[string]struct{}, len(apps)) + for _, valuesKey := range apps { + mappedKeys[valuesKey] = struct{}{} + } + + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + + for _, key := range keys { + if key == "global" { + continue + } + if _, ok := mappedKeys[key]; ok { + continue + } + section, ok := values[key].(map[string]interface{}) + if !ok || !hasLegacyOverrideShape(section) { + continue + } + logger.Info("legacy values section does not map to any application in the server manifest; its env/resources are not converted", + "section", key, "version", version) + } +} + +// hasLegacyOverrideShape reports whether a section carries keys the extraction +// reads. Flat `resources` is excluded: infra sections (mysql, redis, …) +// legitimately carry it. +func hasLegacyOverrideShape(section map[string]interface{}) bool { + for _, key := range []string{"env", "extraEnv", "sizing"} { + if _, ok := section[key]; ok { + return true + } + } + return false +} + +// legacyEnvFromSection merges env over extraEnv (the chart's precedence) into +// a name-sorted EnvVar list, keeping round-trips deterministic. +func legacyEnvFromSection(section map[string]interface{}, sectionName string) ([]corev1.EnvVar, error) { + merged := map[string]interface{}{} + for _, sub := range []string{"extraEnv", "env"} { + m, found, err := unstructured.NestedMap(section, sub) + if err != nil { + return nil, fmt.Errorf("spec.values.%s.%s: %w", sectionName, sub, err) + } + if !found { + continue + } + for k, v := range m { + merged[k] = v + } + } + if len(merged) == 0 { + return nil, nil + } + + vars := make([]corev1.EnvVar, 0, len(merged)) + for name, raw := range merged { + envVar, ok, err := legacyEnvVar(name, raw, sectionName) + if err != nil { + return nil, err + } + if ok { + vars = append(vars, envVar) + } + } + if len(vars) == 0 { + return nil, nil + } + sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) + return vars, nil +} + +// legacyEnvVar renders one helm env entry: map values decode strictly as +// EnvVar bodies (malformed fails conversion, like other mappers), scalars +// coerce as helm's toString did, and `{{ }}` templates drop with a log. +func legacyEnvVar(name string, raw interface{}, sectionName string) (corev1.EnvVar, bool, error) { + if body, isMap := raw.(map[string]interface{}); isMap { + payload, err := json.Marshal(body) + if err != nil { + return corev1.EnvVar{}, false, fmt.Errorf("spec.values.%s env %s: %w", sectionName, name, err) + } + dec := json.NewDecoder(bytes.NewReader(payload)) + dec.DisallowUnknownFields() + var envVar corev1.EnvVar + if err := dec.Decode(&envVar); err != nil { + return corev1.EnvVar{}, false, fmt.Errorf("spec.values.%s env %s: %w", sectionName, name, err) + } + envVar.Name = name + if strings.Contains(envVar.Value, "{{") { + logger.Info("dropping legacy env var with helm template value", + "section", sectionName, "name", name) + return corev1.EnvVar{}, false, nil + } + return envVar, true, nil + } + + s, ok := scalarToString(raw) + if !ok { + logger.Info("dropping legacy env var with non-scalar value", + "section", sectionName, "name", name) + return corev1.EnvVar{}, false, nil + } + if strings.Contains(s, "{{") { + logger.Info("dropping legacy env var with helm template value", + "section", sectionName, "name", name) + return corev1.EnvVar{}, false, nil + } + return corev1.EnvVar{Name: name, Value: s}, true, nil +} + +// legacyResourcesFromSection deep-merges sizing.default → sizing. → flat resources, mirroring the chart. Sections that set nothing +// yield nil so v2 manifest sizing applies untouched. +func legacyResourcesFromSection(section map[string]interface{}, sectionName, globalSize string) (*corev1.ResourceRequirements, error) { + size, _, err := unstructured.NestedString(section, "size") + if err != nil { + return nil, fmt.Errorf("spec.values.%s.size: %w", sectionName, err) + } + if size == "" { + size = globalSize + } + if size == "" { + size = legacyDefaultSize + } + + merged := map[string]interface{}{} + for _, path := range [][]string{ + {"sizing", "default", "resources"}, + {"sizing", size, "resources"}, + {"resources"}, + } { + m, found, err := unstructured.NestedMap(section, path...) + if err != nil { + return nil, fmt.Errorf("spec.values.%s.%s: %w", sectionName, strings.Join(path, "."), err) + } + if found { + mergeLegacyValueMaps(merged, m) + } + } + if len(merged) == 0 { + return nil, nil + } + + payload, err := json.Marshal(merged) + if err != nil { + return nil, fmt.Errorf("spec.values.%s resources: %w", sectionName, err) + } + dec := json.NewDecoder(bytes.NewReader(payload)) + dec.DisallowUnknownFields() + var resources corev1.ResourceRequirements + if err := dec.Decode(&resources); err != nil { + return nil, fmt.Errorf("spec.values.%s resources: %w", sectionName, err) + } + return &resources, nil +} + +// mergeLegacyValueMaps deep-merges src into dst like helm merges values maps. +func mergeLegacyValueMaps(dst, src map[string]interface{}) { + for k, v := range src { + if srcMap, ok := v.(map[string]interface{}); ok { + if dstMap, ok := dst[k].(map[string]interface{}); ok { + mergeLegacyValueMaps(dstMap, srcMap) + continue + } + } + dst[k] = v + } +} diff --git a/api/v1/weightsandbiases_conversion_overrides_test.go b/api/v1/weightsandbiases_conversion_overrides_test.go new file mode 100644 index 00000000..c85734af --- /dev/null +++ b/api/v1/weightsandbiases_conversion_overrides_test.go @@ -0,0 +1,495 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "errors" + "os" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + appsv2 "github.com/wandb/operator/api/v2" + serverManifest "github.com/wandb/operator/pkg/wandb/manifest" +) + +const testLegacyVersion = "0.83.0-test" + +// disableConversionManifestFetch keeps unit tests off the network; tests opt +// in via withConversionManifest*. +func disableConversionManifestFetch() { + SetConversionManifestGetter(func(_ context.Context, _, _ string) (serverManifest.Manifest, error) { + return serverManifest.Manifest{}, errors.New("manifest fetch disabled in unit tests") + }) +} + +func TestMain(m *testing.M) { + disableConversionManifestFetch() + os.Exit(m.Run()) +} + +// withConversionManifest installs a fake resolver returning the given +// applications and returns its call counter. +func withConversionManifest(t *testing.T, apps map[string]serverManifest.Application) *atomic.Int32 { + t.Helper() + var calls atomic.Int32 + SetConversionManifestGetter(func(_ context.Context, _, _ string) (serverManifest.Manifest, error) { + calls.Add(1) + return serverManifest.Manifest{Applications: apps}, nil + }) + t.Cleanup(disableConversionManifestFetch) + return &calls +} + +// withConversionManifestApps is withConversionManifest for plain names with no +// legacyKey. +func withConversionManifestApps(t *testing.T, names ...string) *atomic.Int32 { + t.Helper() + apps := make(map[string]serverManifest.Application, len(names)) + for _, name := range names { + apps[name] = serverManifest.Application{Name: name} + } + return withConversionManifest(t, apps) +} + +// withVersion adds the app.image.tag mapVersion reads, so per-app extraction +// has a version to resolve the manifest with. +func withVersion(values map[string]interface{}) map[string]interface{} { + values["app"] = map[string]interface{}{ + "image": map[string]interface{}{"tag": testLegacyVersion}, + } + return values +} + +func TestConvertTo_LegacyOverridesAbsent(t *testing.T) { + withConversionManifestApps(t, "api") + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "global": map[string]interface{}{"host": "https://wandb.example.com"}, + })) + require.NoError(t, src.ConvertTo(dst)) + require.Nil(t, dst.Spec.Wandb.LegacyOverrides) +} + +func TestConvertTo_LegacyOverridesGlobalEnvPrecedence(t *testing.T) { + // No version in values: global env must convert without any manifest fetch. + SetConversionManifestGetter(func(_ context.Context, _, _ string) (serverManifest.Manifest, error) { + t.Fatal("manifest must not be resolved when no version is derived") + return serverManifest.Manifest{}, nil + }) + t.Cleanup(disableConversionManifestFetch) + + dst := &appsv2.WeightsAndBiases{} + src := newV1(map[string]interface{}{ + "global": map[string]interface{}{ + "env": map[string]interface{}{ + "BOTH": "from-env", + "ENV_ONLY": "env-value", + }, + "extraEnv": map[string]interface{}{ + "BOTH": "from-extra-env", + "EXTRA_ONLY": "extra-value", + }, + }, + }) + require.NoError(t, src.ConvertTo(dst)) + + require.Contains(t, dst.Spec.Wandb.LegacyOverrides, appsv2.LegacyOverridesGlobalKey) + global := dst.Spec.Wandb.LegacyOverrides[appsv2.LegacyOverridesGlobalKey] + require.Equal(t, []corev1.EnvVar{ + {Name: "BOTH", Value: "from-env"}, + {Name: "ENV_ONLY", Value: "env-value"}, + {Name: "EXTRA_ONLY", Value: "extra-value"}, + }, global.Env) + require.Nil(t, global.Resources) +} + +func TestConvertTo_LegacyOverridesScalarCoercion(t *testing.T) { + withConversionManifestApps(t, "parquet") + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "parquet": map[string]interface{}{ + "env": map[string]interface{}{ + "BOOL": true, + "INT": int64(8083), + "FLOAT": 1.5, + "STR": "plain", + }, + }, + })) + require.NoError(t, src.ConvertTo(dst)) + + require.Equal(t, []corev1.EnvVar{ + {Name: "BOOL", Value: "true"}, + {Name: "FLOAT", Value: "1.5"}, + {Name: "INT", Value: "8083"}, + {Name: "STR", Value: "plain"}, + }, dst.Spec.Wandb.LegacyOverrides["parquet"].Env) +} + +func TestConvertTo_LegacyOverridesValueFromBody(t *testing.T) { + withConversionManifestApps(t, "api") + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "api": map[string]interface{}{ + "env": map[string]interface{}{ + "API_KEY": map[string]interface{}{ + "valueFrom": map[string]interface{}{ + "secretKeyRef": map[string]interface{}{ + "name": "observability", + "key": "api-key", + }, + }, + }, + }, + }, + })) + require.NoError(t, src.ConvertTo(dst)) + + env := dst.Spec.Wandb.LegacyOverrides["api"].Env + require.Len(t, env, 1) + require.Equal(t, "API_KEY", env[0].Name) + require.NotNil(t, env[0].ValueFrom) + require.NotNil(t, env[0].ValueFrom.SecretKeyRef) + require.Equal(t, "observability", env[0].ValueFrom.SecretKeyRef.Name) + require.Equal(t, "api-key", env[0].ValueFrom.SecretKeyRef.Key) +} + +func TestConvertTo_LegacyOverridesMalformedBodyFails(t *testing.T) { + withConversionManifestApps(t, "api") + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "api": map[string]interface{}{ + "env": map[string]interface{}{ + "BROKEN": map[string]interface{}{ + "valueFrom": map[string]interface{}{ + "secretKeyReff": map[string]interface{}{"name": "x", "key": "y"}, + }, + }, + }, + }, + })) + err := src.ConvertTo(dst) + require.Error(t, err) + require.Contains(t, err.Error(), "spec.values.api env BROKEN") +} + +func TestConvertTo_LegacyOverridesTemplateValuesDropped(t *testing.T) { + withConversionManifestApps(t, "executor") + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "global": map[string]interface{}{ + "extraEnv": map[string]interface{}{ + "TEMPLATED": "{{ .Release.Name }}-suffix", + "KEPT": "plain", + "INTERP": "$(OTHER_VAR)/path", + }, + }, + "executor": map[string]interface{}{ + "env": map[string]interface{}{ + "ONLY_TEMPLATED": `{{ include "wandb.executor.taskQueue" . }}`, + }, + }, + })) + require.NoError(t, src.ConvertTo(dst)) + + global := dst.Spec.Wandb.LegacyOverrides[appsv2.LegacyOverridesGlobalKey] + require.Equal(t, []corev1.EnvVar{ + {Name: "INTERP", Value: "$(OTHER_VAR)/path"}, + {Name: "KEPT", Value: "plain"}, + }, global.Env) + + // executor's only entry was templated, so the whole section is absent. + require.NotContains(t, dst.Spec.Wandb.LegacyOverrides, "executor") +} + +func TestConvertTo_LegacyOverridesManifestLegacyKey(t *testing.T) { + // Renamed apps declare their v1 values key via legacyKey; no rename table. + withConversionManifest(t, map[string]serverManifest.Application{ + "nginx-proxy": {Name: "nginx-proxy", LegacyKey: "nginx"}, + "weave-trace-evaluate-model-worker": { + Name: "weave-trace-evaluate-model-worker", + LegacyKey: "weave-evaluate-model-worker", + }, + "parquet": {Name: "parquet"}, + }) + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "nginx": map[string]interface{}{ + "env": map[string]interface{}{"NGINX_VAR": "1"}, + }, + "weave-evaluate-model-worker": map[string]interface{}{ + "env": map[string]interface{}{"WORKER_VAR": "2"}, + }, + "parquet": map[string]interface{}{ + "env": map[string]interface{}{"PARQUET_VAR": "3"}, + }, + })) + require.NoError(t, src.ConvertTo(dst)) + + overrides := dst.Spec.Wandb.LegacyOverrides + require.Contains(t, overrides, "nginx-proxy") + require.NotContains(t, overrides, "nginx") + require.Contains(t, overrides, "weave-trace-evaluate-model-worker") + require.NotContains(t, overrides, "weave-evaluate-model-worker") + require.Contains(t, overrides, "parquet") +} + +func TestConvertTo_LegacyOverridesWithoutLegacyKeyRenamedSectionSkipped(t *testing.T) { + // Manifest predating legacyKey: the nginx section has no reader, so it is + // logged as unmapped and skipped rather than guessed. + withConversionManifestApps(t, "nginx-proxy") + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "nginx": map[string]interface{}{ + "env": map[string]interface{}{"NGINX_VAR": "1"}, + }, + })) + require.NoError(t, src.ConvertTo(dst)) + require.NotContains(t, dst.Spec.Wandb.LegacyOverrides, "nginx-proxy") + require.NotContains(t, dst.Spec.Wandb.LegacyOverrides, "nginx") +} + +func TestConvertTo_LegacyOverridesUnmappedSectionsSkipped(t *testing.T) { + withConversionManifestApps(t, "api") + dst := &appsv2.WeightsAndBiases{} + values := withVersion(map[string]interface{}{ + "console": map[string]interface{}{ + "env": map[string]interface{}{"CONSOLE_VAR": "2"}, + }, + "api": map[string]interface{}{ + "env": map[string]interface{}{"API_VAR": "1"}, + }, + }) + // The monolith section carries env alongside the image tag withVersion set. + values["app"].(map[string]interface{})["env"] = map[string]interface{}{"MONOLITH_VAR": "1"} + src := newV1(values) + require.NoError(t, src.ConvertTo(dst)) + + // Only manifest applications convert; app/console are logged and skipped. + overrides := dst.Spec.Wandb.LegacyOverrides + require.Equal(t, []corev1.EnvVar{{Name: "API_VAR", Value: "1"}}, overrides["api"].Env) + require.NotContains(t, overrides, "app") + require.NotContains(t, overrides, "console") +} + +func TestConvertTo_LegacyOverridesResourcesSizingMerge(t *testing.T) { + withConversionManifestApps(t, "api") + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "global": map[string]interface{}{"size": "medium"}, + "api": map[string]interface{}{ + "sizing": map[string]interface{}{ + "default": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{"cpu": "100m", "memory": "128Mi"}, + }, + }, + "medium": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{"cpu": "2"}, + "limits": map[string]interface{}{"memory": "4Gi"}, + }, + }, + "small": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{"cpu": "should-not-apply"}, + }, + }, + }, + "resources": map[string]interface{}{ + "limits": map[string]interface{}{"cpu": "3"}, + }, + }, + })) + require.NoError(t, src.ConvertTo(dst)) + + resources := dst.Spec.Wandb.LegacyOverrides["api"].Resources + require.NotNil(t, resources) + // medium requests.cpu beat default; default memory request survives; + // flat resources.limits merged over sizing limits. + require.Equal(t, resource.MustParse("2"), resources.Requests[corev1.ResourceCPU]) + require.Equal(t, resource.MustParse("128Mi"), resources.Requests[corev1.ResourceMemory]) + require.Equal(t, resource.MustParse("3"), resources.Limits[corev1.ResourceCPU]) + require.Equal(t, resource.MustParse("4Gi"), resources.Limits[corev1.ResourceMemory]) +} + +func TestConvertTo_LegacyOverridesResourcesPerAppSizeWins(t *testing.T) { + withConversionManifestApps(t, "parquet") + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "global": map[string]interface{}{"size": "small"}, + "parquet": map[string]interface{}{ + "size": "xlarge", + "sizing": map[string]interface{}{ + "small": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{"cpu": "1"}, + }, + }, + "xlarge": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{"cpu": "8"}, + }, + }, + }, + }, + })) + require.NoError(t, src.ConvertTo(dst)) + + resources := dst.Spec.Wandb.LegacyOverrides["parquet"].Resources + require.NotNil(t, resources) + require.Equal(t, resource.MustParse("8"), resources.Requests[corev1.ResourceCPU]) +} + +func TestConvertTo_LegacyOverridesResourcesDefaultSizeIsSmall(t *testing.T) { + withConversionManifestApps(t, "weave") + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "weave": map[string]interface{}{ + "sizing": map[string]interface{}{ + "small": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{"memory": "1Gi"}, + }, + }, + "large": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{"memory": "16Gi"}, + }, + }, + }, + }, + })) + require.NoError(t, src.ConvertTo(dst)) + + resources := dst.Spec.Wandb.LegacyOverrides["weave"].Resources + require.NotNil(t, resources) + require.Equal(t, resource.MustParse("1Gi"), resources.Requests[corev1.ResourceMemory]) +} + +func TestConvertTo_LegacyOverridesManifestUnavailable(t *testing.T) { + SetConversionManifestGetter(func(_ context.Context, _, _ string) (serverManifest.Manifest, error) { + return serverManifest.Manifest{}, errors.New("registry unreachable") + }) + t.Cleanup(disableConversionManifestFetch) + + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "global": map[string]interface{}{ + "env": map[string]interface{}{"HTTP_PROXY": "http://proxy"}, + }, + "api": map[string]interface{}{ + "env": map[string]interface{}{"API_VAR": "1"}, + }, + })) + // A manifest fetch failure must never fail conversion: global env still + // converts, per-app extraction is skipped. + require.NoError(t, src.ConvertTo(dst)) + + overrides := dst.Spec.Wandb.LegacyOverrides + require.Contains(t, overrides, appsv2.LegacyOverridesGlobalKey) + require.NotContains(t, overrides, "api") +} + +func TestConvertTo_LegacyOverridesManifestFailureCooldown(t *testing.T) { + // The cooldown keeps repeat conversions from stalling on an unreachable registry. + var calls atomic.Int32 + SetConversionManifestGetter(func(_ context.Context, _, _ string) (serverManifest.Manifest, error) { + calls.Add(1) + return serverManifest.Manifest{}, errors.New("registry unreachable") + }) + t.Cleanup(disableConversionManifestFetch) + + for i := 0; i < 3; i++ { + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "api": map[string]interface{}{ + "env": map[string]interface{}{"API_VAR": "1"}, + }, + })) + require.NoError(t, src.ConvertTo(dst)) + require.NotContains(t, dst.Spec.Wandb.LegacyOverrides, "api") + } + + require.Equal(t, int32(1), calls.Load(), "repeat conversions within the cooldown must not retry the fetch") +} + +func TestConvertTo_LegacyOverridesPrefersActiveSpecValues(t *testing.T) { + withConversionManifestApps(t, "api") + withConversionReader(t, activeSpecSecret(t, "default", "wandb", withVersion(map[string]interface{}{ + "api": map[string]interface{}{ + "env": map[string]interface{}{"FROM_ACTIVE": "yes"}, + }, + }))) + + dst := &appsv2.WeightsAndBiases{} + src := newV1(withVersion(map[string]interface{}{ + "api": map[string]interface{}{ + "env": map[string]interface{}{"FROM_CR": "yes"}, + }, + })) + require.NoError(t, src.ConvertTo(dst)) + + require.Equal(t, []corev1.EnvVar{{Name: "FROM_ACTIVE", Value: "yes"}}, + dst.Spec.Wandb.LegacyOverrides["api"].Env) +} + +func TestConvertRoundTrip_LegacyOverridesIdempotent(t *testing.T) { + withConversionManifestApps(t, "api") + values := withVersion(map[string]interface{}{ + "global": map[string]interface{}{ + "size": "medium", + "env": map[string]interface{}{"B": "2", "A": "1"}, + "extraEnv": map[string]interface{}{ + "C": true, + }, + }, + "api": map[string]interface{}{ + "env": map[string]interface{}{ + "KEY": map[string]interface{}{ + "valueFrom": map[string]interface{}{ + "secretKeyRef": map[string]interface{}{"name": "s", "key": "k"}, + }, + }, + }, + "sizing": map[string]interface{}{ + "medium": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{"cpu": "2"}, + }, + }, + }, + }, + }) + + first := &appsv2.WeightsAndBiases{} + require.NoError(t, newV1(values).ConvertTo(first)) + + bounced := &WeightsAndBiases{} + require.NoError(t, bounced.ConvertFrom(first)) + + second := &appsv2.WeightsAndBiases{} + require.NoError(t, bounced.ConvertTo(second)) + + require.Equal(t, first.Spec.Wandb.LegacyOverrides, second.Spec.Wandb.LegacyOverrides) + require.Contains(t, first.Spec.Wandb.LegacyOverrides, "api") +} diff --git a/api/v1/weightsandbiases_conversion_test.go b/api/v1/weightsandbiases_conversion_test.go index 3519eecf..014b614c 100644 --- a/api/v1/weightsandbiases_conversion_test.go +++ b/api/v1/weightsandbiases_conversion_test.go @@ -145,6 +145,20 @@ func TestConvertTo_SizeUnrecognized(t *testing.T) { require.Contains(t, err.Error(), `"testing"`) } +func TestConvertTo_CustomCACerts(t *testing.T) { + dst := &appsv2.WeightsAndBiases{} + src := newV1(map[string]interface{}{ + "global": map[string]interface{}{ + "customCACerts": []interface{}{"---cert-one---", "---cert-two---"}, + "caCertsConfigMap": "corp-ca-certs", + }, + }) + require.NoError(t, src.ConvertTo(dst)) + + require.Equal(t, []string{"---cert-one---", "---cert-two---"}, dst.Spec.Global.CustomCACerts) + require.Equal(t, "corp-ca-certs", dst.Spec.Global.CACertsConfigMap) +} + func TestConvertTo_VersionFromAppImageTag(t *testing.T) { dst := &appsv2.WeightsAndBiases{} src := newV1(map[string]interface{}{ @@ -781,9 +795,9 @@ func TestConvertTo_MySQLAllLiterals(t *testing.T) { require.Equal(t, "---cert---", decoded["caCert"]) require.NotContains(t, decoded, "passwordSecret") - require.NotNil(t, dst.Spec.MySQL.ExternalMysql, "externalMysql is always allocated; reconciler fills selectors from the annotation") - require.Empty(t, dst.Spec.MySQL.ExternalMysql.Host.Name) - require.Empty(t, dst.Spec.MySQL.ExternalMysql.Password.Name) + require.NotNil(t, dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql, "externalMysql is always allocated; reconciler fills selectors from the annotation") + require.Empty(t, dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Host.Name) + require.Empty(t, dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Password.Name) } func TestConvertTo_MySQLLegacyPasswordSecret(t *testing.T) { @@ -803,9 +817,9 @@ func TestConvertTo_MySQLLegacyPasswordSecret(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.MySQL.ExternalMysql) - require.Equal(t, "mysql-creds", dst.Spec.MySQL.ExternalMysql.Password.Name) - require.Equal(t, "MYSQL_PASSWORD", dst.Spec.MySQL.ExternalMysql.Password.Key) + require.NotNil(t, dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql) + require.Equal(t, "mysql-creds", dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Password.Name) + require.Equal(t, "MYSQL_PASSWORD", dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Password.Key) raw := dst.Annotations[MySQLPendingAnnotation] var decoded map[string]interface{} @@ -828,9 +842,9 @@ func TestConvertTo_MySQLLegacyPasswordSecretDefaultKey(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.MySQL.ExternalMysql) - require.Equal(t, "mysql-creds", dst.Spec.MySQL.ExternalMysql.Password.Name) - require.Equal(t, "MYSQL_PASSWORD", dst.Spec.MySQL.ExternalMysql.Password.Key) + require.NotNil(t, dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql) + require.Equal(t, "mysql-creds", dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Password.Name) + require.Equal(t, "MYSQL_PASSWORD", dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Password.Key) } func TestConvertTo_MySQLValueFromRef(t *testing.T) { @@ -859,8 +873,8 @@ func TestConvertTo_MySQLValueFromRef(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.MySQL.ExternalMysql) - conn := dst.Spec.MySQL.ExternalMysql + require.NotNil(t, dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql) + conn := dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql require.Equal(t, "mysql-settings", conn.Host.Name) require.Equal(t, "endpoint", conn.Host.Key) require.Equal(t, "mysql-secret", conn.Password.Name) @@ -890,9 +904,9 @@ func TestConvertTo_MySQLMixedLiteralsAndRefs(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.MySQL.ExternalMysql) - require.Equal(t, "mysql-secret", dst.Spec.MySQL.ExternalMysql.Password.Name) - require.Empty(t, dst.Spec.MySQL.ExternalMysql.Host.Name) + require.NotNil(t, dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql) + require.Equal(t, "mysql-secret", dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Password.Name) + require.Empty(t, dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Host.Name) raw := dst.Annotations[MySQLPendingAnnotation] var decoded map[string]interface{} @@ -924,8 +938,8 @@ func TestConvertTo_MySQLValueFromWinsOverPasswordSecret(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.MySQL.ExternalMysql) - require.Equal(t, "valueFrom-secret", dst.Spec.MySQL.ExternalMysql.Password.Name, + require.NotNil(t, dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql) + require.Equal(t, "valueFrom-secret", dst.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Password.Name, "password.valueFrom should win over the legacy passwordSecret block") } @@ -994,8 +1008,8 @@ func TestConvertTo_RedisAllLiterals(t *testing.T) { require.NotContains(t, decoded, "external", "fields outside the known v2 mapping must be dropped") require.NotContains(t, decoded, "secret") - require.NotNil(t, dst.Spec.Redis.ExternalRedis, "externalRedis is always allocated; reconciler fills selectors from the annotation") - require.Empty(t, dst.Spec.Redis.ExternalRedis.Host.Name) + require.NotNil(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis, "externalRedis is always allocated; reconciler fills selectors from the annotation") + require.Empty(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Host.Name) } func TestConvertTo_RedisLegacySecretRef(t *testing.T) { @@ -1014,9 +1028,9 @@ func TestConvertTo_RedisLegacySecretRef(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.Redis.ExternalRedis) - require.Equal(t, "redis-creds", dst.Spec.Redis.ExternalRedis.Password.Name) - require.Equal(t, "REDIS_PASSWORD", dst.Spec.Redis.ExternalRedis.Password.Key) + require.NotNil(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis) + require.Equal(t, "redis-creds", dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Password.Name) + require.Equal(t, "REDIS_PASSWORD", dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Password.Key) raw := dst.Annotations[RedisPendingAnnotation] var decoded map[string]interface{} @@ -1038,9 +1052,9 @@ func TestConvertTo_RedisLegacySecretRefDefaultKey(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.Redis.ExternalRedis) - require.Equal(t, "redis-creds", dst.Spec.Redis.ExternalRedis.Password.Name) - require.Equal(t, "REDIS_PASSWORD", dst.Spec.Redis.ExternalRedis.Password.Key) + require.NotNil(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis) + require.Equal(t, "redis-creds", dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Password.Name) + require.Equal(t, "REDIS_PASSWORD", dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Password.Key) } func TestConvertTo_RedisValueFromRef(t *testing.T) { @@ -1069,8 +1083,8 @@ func TestConvertTo_RedisValueFromRef(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.Redis.ExternalRedis) - conn := dst.Spec.Redis.ExternalRedis + require.NotNil(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis) + conn := dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis require.Equal(t, "redis-settings", conn.Host.Name) require.Equal(t, "endpoint", conn.Host.Key) require.Equal(t, "redis-secret", conn.Password.Name) @@ -1100,9 +1114,9 @@ func TestConvertTo_RedisMixedLiteralsAndRefs(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.Redis.ExternalRedis) - require.Equal(t, "redis-secret", dst.Spec.Redis.ExternalRedis.Password.Name) - require.Empty(t, dst.Spec.Redis.ExternalRedis.Host.Name) + require.NotNil(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis) + require.Equal(t, "redis-secret", dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Password.Name) + require.Empty(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Host.Name) raw := dst.Annotations[RedisPendingAnnotation] var decoded map[string]interface{} @@ -1134,8 +1148,8 @@ func TestConvertTo_RedisValueFromWinsOverLegacySecret(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.Redis.ExternalRedis) - require.Equal(t, "valueFrom-secret", dst.Spec.Redis.ExternalRedis.Password.Name, + require.NotNil(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis) + require.Equal(t, "valueFrom-secret", dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Password.Name, "password.valueFrom should win over the legacy secret block") } @@ -1178,9 +1192,9 @@ func TestConvertTo_RedisTLSValueFromInParams(t *testing.T) { }, }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.Redis.ExternalRedis) - require.Equal(t, "redis-tls", dst.Spec.Redis.ExternalRedis.Tls.Name) - require.Equal(t, "enabled", dst.Spec.Redis.ExternalRedis.Tls.Key) + require.NotNil(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis) + require.Equal(t, "redis-tls", dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Tls.Name) + require.Equal(t, "enabled", dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Tls.Key) } func TestConvertTo_RedisTLSValueFromInParameters(t *testing.T) { @@ -1202,9 +1216,9 @@ func TestConvertTo_RedisTLSValueFromInParameters(t *testing.T) { }, }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.Redis.ExternalRedis) - require.Equal(t, "redis-tls", dst.Spec.Redis.ExternalRedis.Tls.Name) - require.Equal(t, "enabled", dst.Spec.Redis.ExternalRedis.Tls.Key) + require.NotNil(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis) + require.Equal(t, "redis-tls", dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Tls.Name) + require.Equal(t, "enabled", dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Tls.Key) } func TestConvertTo_RedisTLSParamsWinsOverParameters(t *testing.T) { @@ -1236,7 +1250,7 @@ func TestConvertTo_RedisTLSParamsWinsOverParameters(t *testing.T) { }, }) require.NoError(t, src.ConvertTo(dst)) - require.Equal(t, "from-params", dst.Spec.Redis.ExternalRedis.Tls.Name, + require.Equal(t, "from-params", dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Tls.Name, "params should be checked before parameters") } @@ -1260,7 +1274,7 @@ func TestConvertTo_RedisTLSLiteralStashedInAnnotation(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(raw), &decoded)) require.Equal(t, "true", decoded["tls"]) - require.Empty(t, dst.Spec.Redis.ExternalRedis.Tls.Name, + require.Empty(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Tls.Name, "literal tls should not be set on the spec; reconciler materializes it") } @@ -1274,8 +1288,8 @@ func TestConvertTo_RedisTLSAbsent(t *testing.T) { }, }) require.NoError(t, src.ConvertTo(dst)) - require.Empty(t, dst.Spec.Redis.ExternalRedis.Tls.Name) - require.Empty(t, dst.Spec.Redis.ExternalRedis.Tls.Key) + require.Empty(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Tls.Name) + require.Empty(t, dst.Spec.Redis[appsv2.DefaultInstanceName].ExternalRedis.Tls.Key) } // TestConvertTo_RedisTLSBooleanStashedAsString locks in that a YAML boolean @@ -1368,8 +1382,8 @@ func TestConvertTo_BucketSecretRef(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.ObjectStore.ExternalObjectStore) - ext := dst.Spec.ObjectStore.ExternalObjectStore + require.NotNil(t, dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore) + ext := dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore require.Equal(t, "bucket-creds", ext.AccessKey.Name) require.Equal(t, "MY_ACCESS", ext.AccessKey.Key) require.Equal(t, "bucket-creds", ext.SecretKey.Name) @@ -1392,8 +1406,8 @@ func TestConvertTo_BucketSecretRefDefaultKeys(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.ObjectStore.ExternalObjectStore) - ext := dst.Spec.ObjectStore.ExternalObjectStore + require.NotNil(t, dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore) + ext := dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore require.Equal(t, "ACCESS_KEY", ext.AccessKey.Key) require.Equal(t, "SECRET_KEY", ext.SecretKey.Key) } @@ -1412,11 +1426,11 @@ func TestConvertTo_BucketSecretRefEmptyName(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.ObjectStore.ExternalObjectStore, + require.NotNil(t, dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore, "externalObjectStore is always allocated; reconciler fills selectors from the annotation") - require.Empty(t, dst.Spec.ObjectStore.ExternalObjectStore.AccessKey.Name, + require.Empty(t, dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore.AccessKey.Name, "empty secretName should not produce an AccessKey selector") - require.Empty(t, dst.Spec.ObjectStore.ExternalObjectStore.SecretKey.Name) + require.Empty(t, dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore.SecretKey.Name) raw := dst.Annotations[BucketPendingAnnotation] var decoded map[string]interface{} @@ -1439,9 +1453,9 @@ func TestConvertTo_BucketLiteralsOnlyBucket(t *testing.T) { }, }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.ObjectStore.ExternalObjectStore, + require.NotNil(t, dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore, "externalObjectStore is always allocated; literals stay in the annotation") - require.Empty(t, dst.Spec.ObjectStore.ExternalObjectStore.AccessKey.Name) + require.Empty(t, dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore.AccessKey.Name) raw, ok := dst.Annotations[BucketPendingAnnotation] require.True(t, ok) @@ -1545,8 +1559,8 @@ func TestConvertTo_BucketSecretRefAndLiterals(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) - require.NotNil(t, dst.Spec.ObjectStore.ExternalObjectStore) - require.Equal(t, "bucket-creds", dst.Spec.ObjectStore.ExternalObjectStore.AccessKey.Name) + require.NotNil(t, dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore) + require.Equal(t, "bucket-creds", dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore.AccessKey.Name) raw := dst.Annotations[BucketPendingAnnotation] var decoded map[string]interface{} @@ -1564,7 +1578,7 @@ func TestConvertTo_BucketAbsent(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) require.NotContains(t, dst.Annotations, BucketPendingAnnotation) - require.Nil(t, dst.Spec.ObjectStore.ExternalObjectStore) + require.Nil(t, dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore) } func TestConvertTo_BucketEmptyMaps(t *testing.T) { @@ -1577,7 +1591,7 @@ func TestConvertTo_BucketEmptyMaps(t *testing.T) { }) require.NoError(t, src.ConvertTo(dst)) require.NotContains(t, dst.Annotations, BucketPendingAnnotation) - require.Nil(t, dst.Spec.ObjectStore.ExternalObjectStore) + require.Nil(t, dst.Spec.ObjectStore[appsv2.DefaultInstanceName].ExternalObjectStore) } func TestConvertTo_BucketAllEmptyValues(t *testing.T) { @@ -1645,8 +1659,8 @@ func TestConvertRoundTrip(t *testing.T) { firstV2 := &appsv2.WeightsAndBiases{} require.NoError(t, original.ConvertTo(firstV2)) require.Equal(t, "http://wandb.localhost", firstV2.Spec.Wandb.Hostname) - require.NotNil(t, firstV2.Spec.MySQL.ExternalMysql) - require.Equal(t, "mysql-creds", firstV2.Spec.MySQL.ExternalMysql.Host.Name) + require.NotNil(t, firstV2.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql) + require.Equal(t, "mysql-creds", firstV2.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Host.Name) // Apiserver bounces through ConvertFrom internally. roundTripped := &WeightsAndBiases{} @@ -1658,8 +1672,8 @@ func TestConvertRoundTrip(t *testing.T) { require.NoError(t, roundTripped.ConvertTo(secondV2)) require.Equal(t, firstV2.Spec.Wandb.Hostname, secondV2.Spec.Wandb.Hostname) - require.NotNil(t, secondV2.Spec.MySQL.ExternalMysql) - require.Equal(t, firstV2.Spec.MySQL.ExternalMysql.Host, secondV2.Spec.MySQL.ExternalMysql.Host) + require.NotNil(t, secondV2.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql) + require.Equal(t, firstV2.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Host, secondV2.Spec.MySQL[appsv2.DefaultInstanceName].ExternalMysql.Host) } func TestConvertFrom_NoAnnotations(t *testing.T) { @@ -1787,3 +1801,205 @@ func TestConvertTo_StashedAnnotationsReflectCRNotActiveSpec(t *testing.T) { require.Equal(t, "http://wandb.from-cr", global["host"], "stashed annotation must preserve the CR's raw values for round-trip") } + +func TestConvertTo_ClickHouseValueFromRef(t *testing.T) { + dst := &appsv2.WeightsAndBiases{} + src := newV1(map[string]interface{}{ + "global": map[string]interface{}{ + "clickhouse": map[string]interface{}{ + "host": map[string]interface{}{ + "valueFrom": map[string]interface{}{ + "secretKeyRef": map[string]interface{}{ + "name": "ch-settings", + "key": "endpoint", + }, + }, + }, + "password": map[string]interface{}{ + "valueFrom": map[string]interface{}{ + "secretKeyRef": map[string]interface{}{ + "name": "ch-secret", + "key": "password", + }, + }, + }, + }, + }, + }) + require.NoError(t, src.ConvertTo(dst)) + + conn := dst.Spec.ClickHouse[appsv2.DefaultInstanceName].ExternalClickHouse + require.NotNil(t, conn) + require.Nil(t, dst.Spec.ClickHouse[appsv2.DefaultInstanceName].ManagedClickHouse, + "external clickhouse must not also be managed") + require.Equal(t, "ch-settings", conn.Host.Name) + require.Equal(t, "endpoint", conn.Host.Key) + require.Equal(t, "ch-secret", conn.Password.Name) + require.Equal(t, "password", conn.Password.Key) + + require.NotContains(t, dst.Annotations, ClickHousePendingAnnotation, + "no literals provided, so no annotation should be created") +} + +func TestConvertTo_ClickHouseLiterals(t *testing.T) { + dst := &appsv2.WeightsAndBiases{} + src := newV1(map[string]interface{}{ + "global": map[string]interface{}{ + "clickhouse": map[string]interface{}{ + "host": "clickhouse.example.com", + "port": int64(8123), + "database": "weave", + "user": "weave", + "password": "shh", + }, + }, + }) + require.NoError(t, src.ConvertTo(dst)) + + raw, ok := dst.Annotations[ClickHousePendingAnnotation] + require.True(t, ok, "expected clickhouse-pending annotation") + + var decoded map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(raw), &decoded)) + require.Equal(t, "clickhouse.example.com", decoded["host"]) + require.Equal(t, "8123", decoded["port"]) + require.Equal(t, "weave", decoded["database"]) + require.Equal(t, "weave", decoded["user"]) + require.Equal(t, "shh", decoded["password"]) + + require.NotNil(t, dst.Spec.ClickHouse[appsv2.DefaultInstanceName].ExternalClickHouse, + "externalClickhouse is always allocated; reconciler fills selectors from the annotation") + require.Empty(t, dst.Spec.ClickHouse[appsv2.DefaultInstanceName].ExternalClickHouse.Host.Name) +} + +func TestConvertTo_ClickHouseMixedLiteralsAndRefs(t *testing.T) { + dst := &appsv2.WeightsAndBiases{} + src := newV1(map[string]interface{}{ + "global": map[string]interface{}{ + "clickhouse": map[string]interface{}{ + "host": "clickhouse.example.com", + "port": int64(8123), + "password": map[string]interface{}{ + "valueFrom": map[string]interface{}{ + "secretKeyRef": map[string]interface{}{ + "name": "ch-secret", + "key": "password", + }, + }, + }, + }, + }, + }) + require.NoError(t, src.ConvertTo(dst)) + + conn := dst.Spec.ClickHouse[appsv2.DefaultInstanceName].ExternalClickHouse + require.NotNil(t, conn) + require.Equal(t, "ch-secret", conn.Password.Name) + require.Equal(t, "password", conn.Password.Key) + + raw := dst.Annotations[ClickHousePendingAnnotation] + var decoded map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(raw), &decoded)) + require.Equal(t, "clickhouse.example.com", decoded["host"]) + require.Equal(t, "8123", decoded["port"]) + require.NotContains(t, decoded, "password", "password came from a ref, not a literal") +} + +func TestConvertTo_ClickHouseLegacyPasswordSecret(t *testing.T) { + dst := &appsv2.WeightsAndBiases{} + src := newV1(map[string]interface{}{ + "global": map[string]interface{}{ + "clickhouse": map[string]interface{}{ + "host": "clickhouse.example.com", + "password": "shh", + "passwordSecret": map[string]interface{}{ + "name": "ch-creds", + "passwordKey": "CLICKHOUSE_PASSWORD", + }, + }, + }, + }) + require.NoError(t, src.ConvertTo(dst)) + + conn := dst.Spec.ClickHouse[appsv2.DefaultInstanceName].ExternalClickHouse + require.NotNil(t, conn) + require.Equal(t, "ch-creds", conn.Password.Name) + require.Equal(t, "CLICKHOUSE_PASSWORD", conn.Password.Key) + + raw := dst.Annotations[ClickHousePendingAnnotation] + var decoded map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(raw), &decoded)) + require.Equal(t, "clickhouse.example.com", decoded["host"]) + require.NotContains(t, decoded, "password", "literal password must not be stashed when passwordSecret took over") +} + +func TestConvertTo_ClickHousePasswordSecretDefaultKey(t *testing.T) { + dst := &appsv2.WeightsAndBiases{} + src := newV1(map[string]interface{}{ + "global": map[string]interface{}{ + "clickhouse": map[string]interface{}{ + "passwordSecret": map[string]interface{}{ + "name": "ch-creds", + }, + }, + }, + }) + require.NoError(t, src.ConvertTo(dst)) + + conn := dst.Spec.ClickHouse[appsv2.DefaultInstanceName].ExternalClickHouse + require.NotNil(t, conn) + require.Equal(t, "ch-creds", conn.Password.Name) + require.Equal(t, "CLICKHOUSE_PASSWORD", conn.Password.Key) +} + +// TestConvertTo_ClickHousePasswordSecretMalformed: a non-string name must +// surface an error instead of being silently skipped. +func TestConvertTo_ClickHousePasswordSecretMalformed(t *testing.T) { + dst := &appsv2.WeightsAndBiases{} + src := newV1(map[string]interface{}{ + "global": map[string]interface{}{ + "clickhouse": map[string]interface{}{ + "passwordSecret": map[string]interface{}{ + "name": float64(123), + }, + }, + }, + }) + err := src.ConvertTo(dst) + require.Error(t, err) + require.Contains(t, err.Error(), "clickhouse.passwordSecret.name") +} + +// TestConvertTo_NoClickHouseLeavesEmpty: no clickhouse block means conversion +// leaves it empty for the defaulter to manage. +func TestConvertTo_NoClickHouseLeavesEmpty(t *testing.T) { + dst := &appsv2.WeightsAndBiases{} + src := newV1(map[string]interface{}{ + "global": map[string]interface{}{ + "host": "http://wandb.example.com", + }, + }) + require.NoError(t, src.ConvertTo(dst)) + + require.Empty(t, dst.Spec.ClickHouse, + "no global.clickhouse means conversion leaves ClickHouse empty for the defaulter to manage") + require.NotContains(t, dst.Annotations, ClickHousePendingAnnotation) +} + +// TestConvertTo_ClickHouseOnlyNonConnectionKeys: keys like replicated/install +// must not be misread as an external connection. +func TestConvertTo_ClickHouseOnlyNonConnectionKeys(t *testing.T) { + dst := &appsv2.WeightsAndBiases{} + src := newV1(map[string]interface{}{ + "global": map[string]interface{}{ + "clickhouse": map[string]interface{}{ + "replicated": true, + }, + }, + }) + require.NoError(t, src.ConvertTo(dst)) + + require.Empty(t, dst.Spec.ClickHouse, + "only non-connection keys must not assert an external clickhouse") + require.NotContains(t, dst.Annotations, ClickHousePendingAnnotation) +} diff --git a/api/v2/weightsandbiases_types.go b/api/v2/weightsandbiases_types.go index 0bfdf54e..1655793f 100644 --- a/api/v2/weightsandbiases_types.go +++ b/api/v2/weightsandbiases_types.go @@ -29,11 +29,12 @@ import ( //+kubebuilder:subresource:status //+kubebuilder:resource:shortName=wandb //+kubebuilder:printcolumn:name="Ready",type=boolean,JSONPath=`.status.ready` -//+kubebuilder:printcolumn:name="MySQL",type=string,JSONPath=`.status.mysqlStatus.state` -//+kubebuilder:printcolumn:name="Redis",type=string,JSONPath=`.status.redisStatus.state` +//+kubebuilder:printcolumn:name="MySQL",type=string,JSONPath=`.status.mysqlStatus.default.state` +//+kubebuilder:printcolumn:name="Redis",type=string,JSONPath=`.status.redisStatus.default.state` //+kubebuilder:printcolumn:name="Kafka",type=string,JSONPath=`.status.kafkaStatus.state` -//+kubebuilder:printcolumn:name="ObjectStore",type=string,JSONPath=`.status.objectStoreStatus.state` -//+kubebuilder:printcolumn:name="ClickHouse",type=string,JSONPath=`.status.clickhouseStatus.state` +//+kubebuilder:printcolumn:name="ObjectStore",type=string,JSONPath=`.status.objectStoreStatus.default.state` +//+kubebuilder:printcolumn:name="ClickHouse",type=string,JSONPath=`.status.clickhouseStatus.default.state` +//+kubebuilder:printcolumn:name="Migration",type=string,JSONPath=`.status.wandb.migration.phase` // WeightsAndBiases is the Schema for the weightsandbiases API. type WeightsAndBiases struct { @@ -57,6 +58,29 @@ func init() { SchemeBuilder.Register(&WeightsAndBiases{}, &WeightsAndBiasesList{}) } +// DefaultInstanceName is the reserved map key identifying the fallback instance +// for each multi-instance infrastructure type (MySQL, Redis, ObjectStore, +// ClickHouse). When an application requests an instance that is not provisioned, +// the operator resolves to this instance instead. +const DefaultInstanceName = "default" + +// ResolveInstance returns the entry for key, falling back to the +// DefaultInstanceName entry when key is empty or absent. The boolean reports +// whether a value was found. +func ResolveInstance[T any](m map[string]T, key string) (T, bool) { + if key == "" { + key = DefaultInstanceName + } + if v, ok := m[key]; ok { + return v, true + } + if v, ok := m[DefaultInstanceName]; ok { + return v, true + } + var zero T + return zero, false +} + type Size string const ( @@ -102,11 +126,14 @@ type WeightsAndBiasesSpec struct { Affinity *corev1.Affinity `json:"affinity,omitempty"` Tolerations *[]corev1.Toleration `json:"tolerations,omitempty"` - MySQL MySQLSpec `json:"mysql,omitempty"` - Redis RedisSpec `json:"redis,omitempty"` - Kafka KafkaSpec `json:"kafka,omitempty"` - ObjectStore ObjectStoreSpec `json:"objectStore,omitempty"` - ClickHouse ClickHouseSpec `json:"clickhouse,omitempty"` + // MySQL, Redis, ObjectStore and ClickHouse are keyed by instance name. The + // reserved DefaultInstanceName key identifies the fallback instance used when + // an application requests an instance that is not provisioned. + MySQL map[string]MySQLSpec `json:"mysql,omitempty"` + Redis map[string]RedisSpec `json:"redis,omitempty"` + Kafka KafkaSpec `json:"kafka,omitempty"` + ObjectStore map[string]ObjectStoreSpec `json:"objectStore,omitempty"` + ClickHouse map[string]ClickHouseSpec `json:"clickhouse,omitempty"` // Networking configures how the W&B application is exposed externally. // +optional @@ -119,6 +146,67 @@ type GlobalSpec struct { // Intended for air-gapped installs whose nodes cannot reach public registries; pair it // with a registry pre-populated by `wsm registry mirror`. ImageRegistry string `json:"imageRegistry,omitempty"` + + // CustomCACerts contains PEM-encoded CA certificates that should be trusted + // by W&B application workloads. + // +optional + CustomCACerts []string `json:"customCACerts,omitempty"` + + // CACertsConfigMap references a ConfigMap in the W&B namespace whose keys + // contain CA certificates. Keys should use a .crt suffix so standard CA + // update tooling can discover them. + // +optional + CACertsConfigMap string `json:"caCertsConfigMap,omitempty"` + + // Proxy configures the forward-proxy egress settings injected into the + // application workloads (app Deployments, their init containers, and + // migration Jobs). The operator emits HTTP_PROXY/HTTPS_PROXY/NO_PROXY and + // their lowercase variants; NO_PROXY is always the operator-computed + // in-cluster exclusions merged over the user-supplied noProxy entries, so + // in-cluster datastore/service traffic never hairpins through the proxy. + // Pair with CustomCACerts for a TLS-intercepting proxy. + // +optional + Proxy *ProxySpec `json:"proxy,omitempty"` +} + +// ProxySpec is the forward-proxy configuration under spec.global.proxy. +type ProxySpec struct { + // HTTPProxy is the proxy URL for plain HTTP egress (HTTP_PROXY/http_proxy). + // +optional + HTTPProxy *ProxyValue `json:"httpProxy,omitempty"` + + // HTTPSProxy is the proxy URL for HTTPS egress (HTTPS_PROXY/https_proxy). + // +optional + HTTPSProxy *ProxyValue `json:"httpsProxy,omitempty"` + + // NoProxy holds EXTRA no-proxy entries appended to the operator-computed + // in-cluster exclusions. Use it for external endpoints (e.g. a BYOB object + // store) that must bypass the proxy. Entries must be comma-free; the + // operator owns the join. + // +optional + NoProxy []string `json:"noProxy,omitempty"` +} + +// ProxyValue is a value-or-secret union mirroring corev1.EnvVar semantics: +// exactly one of Value or ValueFrom must be set. Credential-bearing proxy URLs +// (http://user:pass@host:port) MUST use ValueFrom; the webhook rejects userinfo +// in a literal Value so credentials never land in the CR / etcd / kubectl output. +type ProxyValue struct { + // Value is a literal proxy URL. Must not contain userinfo (credentials). + // +optional + Value string `json:"value,omitempty"` + + // ValueFrom sources the proxy URL from a Secret key (may embed credentials). + // +optional + ValueFrom *ProxyValueSource `json:"valueFrom,omitempty"` +} + +// ProxyValueSource mirrors corev1.EnvVarSource (the secret case): the proxy URL +// is read from a Secret key. +type ProxyValueSource struct { + // SecretKeyRef selects a key of a Secret in the W&B namespace. + // +optional + SecretKeyRef *corev1.SecretKeySelector `json:"secretKeyRef,omitempty"` } type NetworkingMode string @@ -268,6 +356,7 @@ type WandbAppSpec struct { Version string `json:"version"` Features map[string]bool `json:"features"` InternalServiceAuth InternalServiceAuth `json:"internalServiceAuth,omitempty"` + BucketProxy bool `json:"bucketProxy"` ServiceAccount ServiceAccountSpec `json:"serviceAccount,omitempty"` @@ -284,6 +373,34 @@ type WandbAppSpec struct { // +optional OIDC OidcSpec `json:"oidc,omitempty"` + + // LegacyOverrides holds env/resource overrides extracted from v1 + // spec.values, keyed by manifest application name plus the reserved + // "global" key (env only, applied to every application). Unknown keys are + // logged and ignored. Conversion-owned; prefer first-class fields over + // hand-editing. + // +optional + LegacyOverrides map[string]LegacyOverrides `json:"legacyOverrides,omitempty"` +} + +// LegacyOverridesGlobalKey is the reserved LegacyOverrides key whose env +// applies to every application and migration job. +const LegacyOverridesGlobalKey = "global" + +// DefaultManifestRepository is used when spec.wandb.manifestRepository is +// unset — by the defaulting webhook and by v1 conversion (which runs first). +const DefaultManifestRepository = "oci://us-docker.pkg.dev/wandb-production/public/wandb/server-manifest" + +// LegacyOverrides holds v1-derived overrides for one application (or "global"). +type LegacyOverrides struct { + // Env is applied last, replacing same-named manifest or injected vars. + // +optional + Env []corev1.EnvVar `json:"env,omitempty"` + + // Resources overlays sizing-derived resources per field; limits are still + // gated by spec.requireLimits. + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` } type WandbProbeDefaults struct { @@ -312,6 +429,18 @@ type ServiceAccountSpec struct { Annotations map[string]string `json:"annotations,omitempty"` } +// ManagedServiceAccountSpec configures the Kubernetes identity used by a +// managed infrastructure workload. +type ManagedServiceAccountSpec struct { + // Create controls whether the operator reconciles the ServiceAccount. It + // defaults to true; set it to false to reference an existing identity. + Create *bool `json:"create,omitempty"` + // ServiceAccountName defaults to the managed infrastructure resource name. + ServiceAccountName string `json:"serviceAccountName,omitempty"` + // Annotations supports cloud workload identity integrations such as IRSA. + Annotations map[string]string `json:"annotations,omitempty"` +} + type InternalServiceAuth struct { Enabled *bool `json:"enabled,omitempty"` OIDCIssuer string `json:"oidcIssuer,omitempty"` @@ -430,13 +559,15 @@ type KafkaSpec struct { type ManagedKafkaSpec struct { ManagedInfraSpec `json:",inline"` - StorageSize string `json:"storageSize,omitempty"` - Replicas int32 `json:"replicas,omitempty"` - Config KafkaConfig `json:"config,omitempty"` - Namespace string `json:"namespace,omitempty"` - Name string `json:"name,omitempty"` - Telemetry Telemetry `json:"telemetry,omitempty"` - SkipDataRecovery bool `json:"skipDataRecovery,omitempty"` + StorageSize string `json:"storageSize,omitempty"` + Replicas int32 `json:"replicas,omitempty"` + Config KafkaConfig `json:"config,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Telemetry Telemetry `json:"telemetry,omitempty"` + // ServiceAccount configures the identity used by the Bufstream broker. + ServiceAccount ManagedServiceAccountSpec `json:"serviceAccount,omitempty"` + SkipDataRecovery bool `json:"skipDataRecovery,omitempty"` } type KafkaConnection struct { @@ -472,6 +603,7 @@ type ManagedObjectStoreSpec struct { SeaweedObjectStoreSpec SeaweedObjectStoreSpec `json:"SeaweedObjectStoreSpec,omitempty"` StorageSize string `json:"storageSize,omitempty"` Replicas int32 `json:"replicas,omitempty"` + Copies int32 `json:"copies,omitempty"` Config ObjectStoreConfig `json:"config,omitempty"` Namespace string `json:"namespace,omitempty"` Name string `json:"name,omitempty"` @@ -480,6 +612,10 @@ type ManagedObjectStoreSpec struct { type SeaweedObjectStoreSpec struct { TlsEnabled bool `json:"tlsEnabled,omitempty"` + // FilerStorageSize sizes the filer's metadata index disk. It grows with the + // number of objects, not their total size, so bump it for large object counts. + // Defaults to 20Gi when unset. + FilerStorageSize string `json:"filerStorageSize,omitempty"` } // ObjectStoreProvider selects the object store backend for an external object store. @@ -500,9 +636,12 @@ type ObjectStoreConnection struct { AccessKey corev1.SecretKeySelector `json:"accessKey,omitempty"` SecretKey corev1.SecretKeySelector `json:"secretKey,omitempty"` Bucket corev1.SecretKeySelector `json:"bucket,omitempty"` - Region corev1.SecretKeySelector `json:"region,omitempty"` - - URL corev1.SecretKeySelector `json:"url,omitempty"` + // Path is an optional key prefix within the bucket under which W&B stores its data. + Path corev1.SecretKeySelector `json:"path,omitempty"` + Region corev1.SecretKeySelector `json:"region,omitempty"` + TlsEnabled corev1.SecretKeySelector `json:"tlsEnabled,omitempty"` + ForcePathStyle corev1.SecretKeySelector `json:"forcePathStyle,omitempty"` + URL corev1.SecretKeySelector `json:"url,omitempty"` } type ObjectStoreConfig struct { @@ -531,11 +670,54 @@ type ManagedClickHouseSpec struct { Namespace string `json:"namespace,omitempty"` Name string `json:"name,omitempty"` Telemetry Telemetry `json:"telemetry,omitempty"` + // ServiceAccount configures the identity used by ClickHouse server pods. + ServiceAccount ManagedServiceAccountSpec `json:"serviceAccount,omitempty"` + + // ObjectStorage configures the S3-backed disk that holds ClickHouse table + // data in the configured W&B object store (managed SeaweedFS or external + // bucket). Managed ClickHouse always stores table data in object storage; + // StorageSize sizes the local PV used only for metadata, system tables, and + // the S3 read cache. + ObjectStorage ClickHouseObjectStorageSpec `json:"objectStorage,omitempty"` + + // Keeper configures the ClickHouse Keeper ensemble that coordinates + // ReplicatedMergeTree replication across ClickHouse replicas. + Keeper ClickHouseKeeperSpec `json:"keeper,omitempty"` +} + +// ClickHouseObjectStorageSpec configures object-store-backed storage for managed +// ClickHouse. +type ClickHouseObjectStorageSpec struct { + // Prefix is the key prefix within the bucket under which ClickHouse stores + // its data. Lets multiple consumers share a single bucket. Defaults to + // "clickhouse/". + Prefix string `json:"prefix,omitempty"` + + // Insecure connects to the object store over HTTP instead of HTTPS. It only + // applies to external object stores that do not advertise a scheme; the + // managed object store's scheme is taken from its connection. Defaults to + // false (HTTPS). + Insecure bool `json:"insecure,omitempty"` +} + +// ClickHouseKeeperSpec configures the managed ClickHouse Keeper ensemble. +type ClickHouseKeeperSpec struct { + // Replicas is the number of Keeper nodes. Use an odd number (1, 3, 5) so the + // ensemble can form a quorum. Defaults to 3. + Replicas int32 `json:"replicas,omitempty"` + + // StorageSize is the persistent volume size for each Keeper node's raft log + // and snapshots. Keeper state is small; defaults to a modest value. + StorageSize string `json:"storageSize,omitempty"` + + // Config holds resource requirements for the Keeper pods. + Config ClickHouseConfig `json:"config,omitempty"` } type ClickHouseConnection struct { Host corev1.SecretKeySelector `json:"host,omitempty"` - Port corev1.SecretKeySelector `json:"port,omitempty"` + TCPPort corev1.SecretKeySelector `json:"tcpPort,omitempty"` + HTTPPort corev1.SecretKeySelector `json:"httpPort,omitempty"` Database corev1.SecretKeySelector `json:"database,omitempty"` Username corev1.SecretKeySelector `json:"username,omitempty"` Password corev1.SecretKeySelector `json:"password,omitempty"` @@ -549,14 +731,20 @@ type ClickHouseConfig struct { // WeightsAndBiasesStatus defines the observed state of WeightsAndBiases. type WeightsAndBiasesStatus struct { - Ready bool `json:"ready"` - Wandb WandbStatus `json:"wandb,omitempty"` - MySQLStatus MysqlInfraStatus `json:"mysqlStatus,omitempty"` - RedisStatus RedisInfraStatus `json:"redisStatus,omitempty"` - KafkaStatus KafkaInfraStatus `json:"kafkaStatus,omitempty"` - ObjectStoreStatus ObjectStoreInfraStatus `json:"objectStoreStatus,omitempty"` - ClickHouseStatus ClickHouseInfraStatus `json:"clickhouseStatus,omitempty"` - TelemetryStatus TelemetryInfraStatus `json:"telemetryStatus,omitempty"` + Ready bool `json:"ready"` + // Conditions includes the standard Ready condition for the current generation. + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` + Wandb WandbStatus `json:"wandb,omitempty"` + // MySQLStatus, RedisStatus, ObjectStoreStatus and ClickHouseStatus are keyed + // by instance name, mirroring the corresponding spec maps. + MySQLStatus map[string]MysqlInfraStatus `json:"mysqlStatus,omitempty"` + RedisStatus map[string]RedisInfraStatus `json:"redisStatus,omitempty"` + KafkaStatus KafkaInfraStatus `json:"kafkaStatus,omitempty"` + ObjectStoreStatus map[string]ObjectStoreInfraStatus `json:"objectStoreStatus,omitempty"` + ClickHouseStatus map[string]ClickHouseInfraStatus `json:"clickhouseStatus,omitempty"` + TelemetryStatus TelemetryInfraStatus `json:"telemetryStatus,omitempty"` // GeneratedSecrets stores references to secrets generated by the operator // from the server manifest's generatedSecrets section. The key is the // logical secret name from the manifest, and the value is a SecretKeySelector @@ -590,23 +778,31 @@ type WandbStatus struct { Migration WandbMigrationStatus `json:"migration,omitempty"` + // MySQLInit tracks the per-instance database-initialization job, keyed by + // managed MySQL instance name. // +kubebuilder:default:={} - MySQLInit MigrationJobStatus `json:"mysqlInit,omitempty"` + MySQLInit map[string]MigrationJobStatus `json:"mysqlInit,omitempty"` } type WandbMigrationStatus struct { - Version string `json:"version,omitempty"` - LastSuccessVersion string `json:"lastSuccessVersion,omitempty"` - Ready bool `json:"ready,omitempty"` - Reason string `json:"reason,omitempty"` - Jobs map[string]MigrationJobStatus `json:"jobs,omitempty"` + Version string `json:"version,omitempty"` + LastSuccessVersion string `json:"lastSuccessVersion,omitempty"` + Ready bool `json:"ready,omitempty"` + // Phase is Running, Failed, Succeeded, or Unknown. + Phase string `json:"phase,omitempty"` + Reason string `json:"reason,omitempty"` + Jobs map[string]MigrationJobStatus `json:"jobs,omitempty"` } type MigrationJobStatus struct { Name string `json:"name,omitempty"` Succeeded bool `json:"succeeded,omitempty"` Failed bool `json:"failed,omitempty"` - Message string `json:"message,omitempty"` + // Phase is Running, Failed, Succeeded, or Unknown. + Phase string `json:"phase,omitempty"` + // Reason is copied from the terminal Kubernetes Job condition when present. + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` } type WBInfraStatus struct { diff --git a/api/v2/zz_generated.deepcopy.go b/api/v2/zz_generated.deepcopy.go index 8b2409f6..76ab543b 100644 --- a/api/v2/zz_generated.deepcopy.go +++ b/api/v2/zz_generated.deepcopy.go @@ -265,7 +265,8 @@ func (in *ClickHouseConfig) DeepCopy() *ClickHouseConfig { func (in *ClickHouseConnection) DeepCopyInto(out *ClickHouseConnection) { *out = *in in.Host.DeepCopyInto(&out.Host) - in.Port.DeepCopyInto(&out.Port) + in.TCPPort.DeepCopyInto(&out.TCPPort) + in.HTTPPort.DeepCopyInto(&out.HTTPPort) in.Database.DeepCopyInto(&out.Database) in.Username.DeepCopyInto(&out.Username) in.Password.DeepCopyInto(&out.Password) @@ -299,6 +300,37 @@ func (in *ClickHouseInfraStatus) DeepCopy() *ClickHouseInfraStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClickHouseKeeperSpec) DeepCopyInto(out *ClickHouseKeeperSpec) { + *out = *in + in.Config.DeepCopyInto(&out.Config) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClickHouseKeeperSpec. +func (in *ClickHouseKeeperSpec) DeepCopy() *ClickHouseKeeperSpec { + if in == nil { + return nil + } + out := new(ClickHouseKeeperSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClickHouseObjectStorageSpec) DeepCopyInto(out *ClickHouseObjectStorageSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClickHouseObjectStorageSpec. +func (in *ClickHouseObjectStorageSpec) DeepCopy() *ClickHouseObjectStorageSpec { + if in == nil { + return nil + } + out := new(ClickHouseObjectStorageSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClickHouseSpec) DeepCopyInto(out *ClickHouseSpec) { *out = *in @@ -452,6 +484,16 @@ func (in *GatewayStatusSummary) DeepCopy() *GatewayStatusSummary { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GlobalSpec) DeepCopyInto(out *GlobalSpec) { *out = *in + if in.CustomCACerts != nil { + in, out := &in.CustomCACerts, &out.CustomCACerts + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Proxy != nil { + in, out := &in.Proxy, &out.Proxy + *out = new(ProxySpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalSpec. @@ -667,6 +709,33 @@ func (in *KafkaSpec) DeepCopy() *KafkaSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LegacyOverrides) DeepCopyInto(out *LegacyOverrides) { + *out = *in + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LegacyOverrides. +func (in *LegacyOverrides) DeepCopy() *LegacyOverrides { + if in == nil { + return nil + } + out := new(LegacyOverrides) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ListenerTLSConfig) DeepCopyInto(out *ListenerTLSConfig) { *out = *in @@ -698,6 +767,9 @@ func (in *ManagedClickHouseSpec) DeepCopyInto(out *ManagedClickHouseSpec) { in.ManagedInfraSpec.DeepCopyInto(&out.ManagedInfraSpec) in.Config.DeepCopyInto(&out.Config) out.Telemetry = in.Telemetry + in.ServiceAccount.DeepCopyInto(&out.ServiceAccount) + out.ObjectStorage = in.ObjectStorage + in.Keeper.DeepCopyInto(&out.Keeper) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedClickHouseSpec. @@ -752,6 +824,7 @@ func (in *ManagedKafkaSpec) DeepCopyInto(out *ManagedKafkaSpec) { in.ManagedInfraSpec.DeepCopyInto(&out.ManagedInfraSpec) in.Config.DeepCopyInto(&out.Config) out.Telemetry = in.Telemetry + in.ServiceAccount.DeepCopyInto(&out.ServiceAccount) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedKafkaSpec. @@ -820,6 +893,33 @@ func (in *ManagedRedisSpec) DeepCopy() *ManagedRedisSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ManagedServiceAccountSpec) DeepCopyInto(out *ManagedServiceAccountSpec) { + *out = *in + if in.Create != nil { + in, out := &in.Create, &out.Create + *out = new(bool) + **out = **in + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedServiceAccountSpec. +func (in *ManagedServiceAccountSpec) DeepCopy() *ManagedServiceAccountSpec { + if in == nil { + return nil + } + out := new(ManagedServiceAccountSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MigrationJobStatus) DeepCopyInto(out *MigrationJobStatus) { *out = *in @@ -980,7 +1080,10 @@ func (in *ObjectStoreConnection) DeepCopyInto(out *ObjectStoreConnection) { in.AccessKey.DeepCopyInto(&out.AccessKey) in.SecretKey.DeepCopyInto(&out.SecretKey) in.Bucket.DeepCopyInto(&out.Bucket) + in.Path.DeepCopyInto(&out.Path) in.Region.DeepCopyInto(&out.Region) + in.TlsEnabled.DeepCopyInto(&out.TlsEnabled) + in.ForcePathStyle.DeepCopyInto(&out.ForcePathStyle) in.URL.DeepCopyInto(&out.URL) } @@ -1055,6 +1158,76 @@ func (in *OidcSpec) DeepCopy() *OidcSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxySpec) DeepCopyInto(out *ProxySpec) { + *out = *in + if in.HTTPProxy != nil { + in, out := &in.HTTPProxy, &out.HTTPProxy + *out = new(ProxyValue) + (*in).DeepCopyInto(*out) + } + if in.HTTPSProxy != nil { + in, out := &in.HTTPSProxy, &out.HTTPSProxy + *out = new(ProxyValue) + (*in).DeepCopyInto(*out) + } + if in.NoProxy != nil { + in, out := &in.NoProxy, &out.NoProxy + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxySpec. +func (in *ProxySpec) DeepCopy() *ProxySpec { + if in == nil { + return nil + } + out := new(ProxySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyValue) DeepCopyInto(out *ProxyValue) { + *out = *in + if in.ValueFrom != nil { + in, out := &in.ValueFrom, &out.ValueFrom + *out = new(ProxyValueSource) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyValue. +func (in *ProxyValue) DeepCopy() *ProxyValue { + if in == nil { + return nil + } + out := new(ProxyValue) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyValueSource) DeepCopyInto(out *ProxyValueSource) { + *out = *in + if in.SecretKeyRef != nil { + in, out := &in.SecretKeyRef, &out.SecretKeyRef + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyValueSource. +func (in *ProxyValueSource) DeepCopy() *ProxyValueSource { + if in == nil { + return nil + } + out := new(ProxyValueSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RedisConfig) DeepCopyInto(out *RedisConfig) { *out = *in @@ -1346,6 +1519,13 @@ func (in *WandbAppSpec) DeepCopyInto(out *WandbAppSpec) { copy(*out, *in) } in.OIDC.DeepCopyInto(&out.OIDC) + if in.LegacyOverrides != nil { + in, out := &in.LegacyOverrides, &out.LegacyOverrides + *out = make(map[string]LegacyOverrides, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WandbAppSpec. @@ -1421,7 +1601,13 @@ func (in *WandbStatus) DeepCopyInto(out *WandbStatus) { } } in.Migration.DeepCopyInto(&out.Migration) - out.MySQLInit = in.MySQLInit + if in.MySQLInit != nil { + in, out := &in.MySQLInit, &out.MySQLInit + *out = make(map[string]MigrationJobStatus, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WandbStatus. @@ -1497,7 +1683,7 @@ func (in *WeightsAndBiasesList) DeepCopyObject() runtime.Object { func (in *WeightsAndBiasesSpec) DeepCopyInto(out *WeightsAndBiasesSpec) { *out = *in out.RetentionPolicy = in.RetentionPolicy - out.Global = in.Global + in.Global.DeepCopyInto(&out.Global) in.Wandb.DeepCopyInto(&out.Wandb) if in.Affinity != nil { in, out := &in.Affinity, &out.Affinity @@ -1515,11 +1701,35 @@ func (in *WeightsAndBiasesSpec) DeepCopyInto(out *WeightsAndBiasesSpec) { } } } - in.MySQL.DeepCopyInto(&out.MySQL) - in.Redis.DeepCopyInto(&out.Redis) + if in.MySQL != nil { + in, out := &in.MySQL, &out.MySQL + *out = make(map[string]MySQLSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.Redis != nil { + in, out := &in.Redis, &out.Redis + *out = make(map[string]RedisSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } in.Kafka.DeepCopyInto(&out.Kafka) - in.ObjectStore.DeepCopyInto(&out.ObjectStore) - in.ClickHouse.DeepCopyInto(&out.ClickHouse) + if in.ObjectStore != nil { + in, out := &in.ObjectStore, &out.ObjectStore + *out = make(map[string]ObjectStoreSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.ClickHouse != nil { + in, out := &in.ClickHouse, &out.ClickHouse + *out = make(map[string]ClickHouseSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } in.Networking.DeepCopyInto(&out.Networking) } @@ -1536,12 +1746,43 @@ func (in *WeightsAndBiasesSpec) DeepCopy() *WeightsAndBiasesSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WeightsAndBiasesStatus) DeepCopyInto(out *WeightsAndBiasesStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.Wandb.DeepCopyInto(&out.Wandb) - in.MySQLStatus.DeepCopyInto(&out.MySQLStatus) - in.RedisStatus.DeepCopyInto(&out.RedisStatus) + if in.MySQLStatus != nil { + in, out := &in.MySQLStatus, &out.MySQLStatus + *out = make(map[string]MysqlInfraStatus, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.RedisStatus != nil { + in, out := &in.RedisStatus, &out.RedisStatus + *out = make(map[string]RedisInfraStatus, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } in.KafkaStatus.DeepCopyInto(&out.KafkaStatus) - in.ObjectStoreStatus.DeepCopyInto(&out.ObjectStoreStatus) - in.ClickHouseStatus.DeepCopyInto(&out.ClickHouseStatus) + if in.ObjectStoreStatus != nil { + in, out := &in.ObjectStoreStatus, &out.ObjectStoreStatus + *out = make(map[string]ObjectStoreInfraStatus, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.ClickHouseStatus != nil { + in, out := &in.ClickHouseStatus, &out.ClickHouseStatus + *out = make(map[string]ClickHouseInfraStatus, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } in.TelemetryStatus.DeepCopyInto(&out.TelemetryStatus) if in.GeneratedSecrets != nil { in, out := &in.GeneratedSecrets, &out.GeneratedSecrets diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 2a1924c3..7edc029a 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -30,6 +30,7 @@ import ( nginxGatewayv1alpha1 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1" "github.com/wandb/operator/internal/logx" "github.com/wandb/operator/pkg/utils" + chkv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1" chiv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" argov1alpha1 "github.com/wandb/operator/pkg/vendored/argo-rollouts/argoproj.io.rollouts/v1alpha1" redisv1beta2 "github.com/wandb/operator/pkg/vendored/redis-operator/redis/v1beta2" @@ -85,6 +86,7 @@ func init() { utilruntime.Must(redissentinelv1beta2.AddToScheme(scheme)) utilruntime.Must(seaweedv1.AddToScheme(scheme)) utilruntime.Must(chiv1.AddToScheme(scheme)) + utilruntime.Must(chkv1.AddToScheme(scheme)) utilruntime.Must(mocov1beta2.AddToScheme(scheme)) utilruntime.Must(gatewayv1.Install(scheme)) utilruntime.Must(nginxGatewayv1alpha1.AddToScheme(scheme)) diff --git a/config/crd/bases/apps.wandb.com_weightsandbiases.yaml b/config/crd/bases/apps.wandb.com_weightsandbiases.yaml index b6897d27..4ee11815 100644 --- a/config/crd/bases/apps.wandb.com_weightsandbiases.yaml +++ b/config/crd/bases/apps.wandb.com_weightsandbiases.yaml @@ -51,21 +51,24 @@ spec: - jsonPath: .status.ready name: Ready type: boolean - - jsonPath: .status.mysqlStatus.state + - jsonPath: .status.mysqlStatus.default.state name: MySQL type: string - - jsonPath: .status.redisStatus.state + - jsonPath: .status.redisStatus.default.state name: Redis type: string - jsonPath: .status.kafkaStatus.state name: Kafka type: string - - jsonPath: .status.objectStoreStatus.state + - jsonPath: .status.objectStoreStatus.default.state name: ObjectStore type: string - - jsonPath: .status.clickhouseStatus.state + - jsonPath: .status.clickhouseStatus.default.state name: ClickHouse type: string + - jsonPath: .status.wandb.migration.phase + name: Migration + type: string name: v2 schema: openAPIV3Schema: @@ -518,324 +521,759 @@ spec: type: object type: object clickhouse: - properties: - externalClickhouse: - properties: - database: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - host: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - username: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - managedClickhouse: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: + additionalProperties: + properties: + externalClickhouse: + properties: + database: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + host: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + httpPort: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tcpPort: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + managedClickhouse: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: properties: - preference: - properties: - matchExpressions: - items: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + matchLabels: + additionalProperties: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - operator: + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + config: + properties: + resources: + properties: + claims: + items: + properties: + name: + type: string + request: type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + keeper: + properties: + config: + properties: + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object type: array - x-kubernetes-list-type: atomic - namespaceSelector: + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + replicas: + format: int32 + type: integer + storageSize: + type: string + type: object + name: + type: string + namespace: + type: string + objectStorage: + properties: + insecure: + type: boolean + prefix: + type: string + type: object + replicas: + format: int32 + type: integer + retentionPolicy: + properties: + onDelete: + default: detach + type: string + required: + - onDelete + type: object + serviceAccount: + properties: + annotations: + additionalProperties: + type: string + type: object + create: + type: boolean + serviceAccountName: + type: string + type: object + storageSize: + type: string + telemetry: + properties: + enabled: + default: true + type: boolean + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + version: + type: string + type: object + type: object + type: object + global: + properties: + caCertsConfigMap: + type: string + customCACerts: + items: + type: string + type: array + imageRegistry: + type: string + proxy: + properties: + httpProxy: + properties: + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + httpsProxy: + properties: + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + noProxy: + items: + type: string + type: array + type: object + type: object + kafka: + properties: + managedKafka: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: properties: matchExpressions: items: @@ -855,26 +1293,86 @@ spec: type: object type: array x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string + weight: + format: int32 + type: integer required: - - topologyKey + - preference + - weight type: object type: array x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic type: object - podAntiAffinity: + podAffinity: properties: preferredDuringSchedulingIgnoredDuringExecution: items: @@ -1040,370 +1538,7 @@ spec: type: array x-kubernetes-list-type: atomic type: object - type: object - config: - properties: - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - type: object - name: - type: string - namespace: - type: string - replicas: - format: int32 - type: integer - retentionPolicy: - properties: - onDelete: - default: detach - type: string - required: - - onDelete - type: object - storageSize: - type: string - telemetry: - properties: - enabled: - default: true - type: boolean - required: - - enabled - type: object - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - version: - type: string - type: object - type: object - global: - properties: - imageRegistry: - type: string - type: object - kafka: - properties: - managedKafka: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: + podAntiAffinity: properties: preferredDuringSchedulingIgnoredDuringExecution: items: @@ -1639,6 +1774,17 @@ spec: required: - onDelete type: object + serviceAccount: + properties: + annotations: + additionalProperties: + type: string + type: object + create: + type: boolean + serviceAccountName: + type: string + type: object skipDataRecovery: type: boolean storageSize: @@ -1670,659 +1816,661 @@ spec: type: object type: object mysql: - properties: - externalMysql: - properties: - database: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - host: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslCa: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslCert: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslKey: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - tls: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - username: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - managedMysql: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: + additionalProperties: + properties: + externalMysql: + properties: + database: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + host: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCa: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCert: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tls: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + managedMysql: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: properties: - preference: - properties: - matchExpressions: - items: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + matchLabels: + additionalProperties: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + operator: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - config: - properties: - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + config: + properties: + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - type: object - name: - type: string - namespace: - type: string - replicas: - format: int32 - type: integer - retentionPolicy: - properties: - onDelete: - default: detach - type: string - required: - - onDelete - type: object - storageSize: - type: string - telemetry: - properties: - enabled: - default: true - type: boolean - required: - - enabled - type: object - tolerations: - items: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + name: + type: string + namespace: + type: string + replicas: + format: int32 + type: integer + retentionPolicy: properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: + onDelete: + default: detach type: string + required: + - onDelete type: object - type: array - type: object + storageSize: + type: string + telemetry: + properties: + enabled: + default: true + type: boolean + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object type: object networking: properties: @@ -2417,1929 +2565,1523 @@ spec: type: object type: object objectStore: - properties: - externalObjectStore: - properties: - accessKey: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - bucket: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - endpoint: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - provider: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - region: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secretKey: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - managedObjectStore: - properties: - SeaweedObjectStoreSpec: - properties: - tlsEnabled: - type: boolean - type: object - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + additionalProperties: + properties: + externalObjectStore: + properties: + accessKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + bucket: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpoint: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + forcePathStyle: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + path: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + provider: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + region: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secretKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tlsEnabled: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + managedObjectStore: + properties: + SeaweedObjectStoreSpec: + properties: + filerStorageSize: + type: string + tlsEnabled: + type: boolean + type: object + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic required: - - preference - - weight + - nodeSelectorTerms type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - operator: + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + config: + properties: + accessKey: + type: string + minioBrowserSetting: + type: string + resources: + properties: + claims: + items: + properties: + name: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: + request: type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - config: - properties: - accessKey: - type: string - minioBrowserSetting: - type: string - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - rootUser: - type: string - type: object - name: - type: string - namespace: - type: string - replicas: - format: int32 - type: integer - retentionPolicy: - properties: - onDelete: - default: detach - type: string - required: - - onDelete - type: object - storageSize: - type: string - telemetry: - properties: - enabled: - default: true - type: boolean - required: - - enabled - type: object - tolerations: - items: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + rootUser: + type: string + type: object + copies: + format: int32 + type: integer + name: + type: string + namespace: + type: string + replicas: + format: int32 + type: integer + retentionPolicy: properties: - effect: + onDelete: + default: detach type: string + required: + - onDelete + type: object + storageSize: + type: string + telemetry: + properties: + enabled: + default: true + type: boolean + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + type: object + redis: + additionalProperties: + properties: + externalRedis: + properties: + host: + properties: key: type: string - operator: + name: + default: "" type: string - tolerationSeconds: - format: int64 - type: integer - value: + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" type: string + optional: + type: boolean + required: + - key type: object - type: array - type: object - type: object - redis: - properties: - externalRedis: - properties: - host: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslCa: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - tls: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - managedRedis: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCa: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tls: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + managedRedis: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + operator: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic required: - - preference - - weight + - nodeSelectorTerms type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + config: + properties: + resources: + properties: + claims: + items: + properties: + name: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: + request: type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - config: - properties: - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - type: object - name: - type: string - namespace: - type: string - retentionPolicy: - properties: - onDelete: - default: detach - type: string - required: - - onDelete - type: object - sentinel: - properties: - config: - properties: - masterName: - type: string - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: + required: - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true type: object - type: object - type: object - enabled: - type: boolean - required: - - enabled - type: object - storageSize: - type: string - telemetry: - properties: - enabled: - default: true - type: boolean - required: - - enabled - type: object - tolerations: - items: + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + name: + type: string + namespace: + type: string + retentionPolicy: properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: + onDelete: + default: detach type: string + required: + - onDelete type: object - type: array - type: object - type: object - requireLimits: - type: boolean - retentionPolicy: - properties: - onDelete: - default: detach - type: string - required: - - onDelete - type: object - size: - enum: - - dev - - micro - - small - - medium - - large - - xlarge - - xxlarge - type: string - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - wandb: - properties: - additionalHostnames: - items: - type: string - type: array - features: - additionalProperties: - type: boolean - type: object - hostname: - type: string - internalServiceAuth: - properties: - enabled: - type: boolean - oidcIssuer: - type: string - type: object - license: - type: string - manifestRepository: - type: string - oidc: - properties: - authMethod: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - clientId: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - clientSecret: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - issuerUrl: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sessionLength: - type: string - type: object - probes: - properties: - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - startupProbe: - properties: - exec: - properties: - command: - items: + sentinel: + properties: + config: + properties: + masterName: type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - type: object - serviceAccount: - properties: - annotations: - additionalProperties: - type: string - type: object - create: - default: true - type: boolean - serviceAccountName: - default: wandb - type: string - required: - - create - type: object - version: - type: string - required: - - features - - hostname - - version - type: object - required: - - retentionPolicy - type: object - status: - properties: - clickhouseStatus: - properties: - conditions: - items: - properties: - lastTransitionTime: - format: date-time - type: string - message: - maxLength: 32768 - type: string - observedGeneration: - format: int64 - minimum: 0 - type: integer - reason: - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - enum: - - "True" - - "False" - - Unknown - type: string - type: - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - connection: - properties: - database: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - host: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - username: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - ready: - type: boolean - state: - type: string - required: - - ready + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + enabled: + type: boolean + required: + - enabled + type: object + storageSize: + type: string + telemetry: + properties: + enabled: + default: true + type: boolean + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object type: object - gatewayStatus: + requireLimits: + type: boolean + retentionPolicy: properties: - addresses: - items: - type: string - type: array - gatewayRef: - properties: - name: - type: string - namespace: - type: string - required: - - name - type: object - name: + onDelete: + default: detach type: string - ready: - type: boolean + required: + - onDelete type: object - generatedSecrets: - additionalProperties: + size: + enum: + - dev + - micro + - small + - medium + - large + - xlarge + - xxlarge + type: string + tolerations: + items: properties: + effect: + type: string key: type: string - name: - default: "" + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: type: string - optional: - type: boolean - required: - - key type: object - x-kubernetes-map-type: atomic - type: object - ingressStatus: + type: array + wandb: properties: - loadBalancerIngress: + additionalHostnames: items: + type: string + type: array + bucketProxy: + type: boolean + features: + additionalProperties: + type: boolean + type: object + hostname: + type: string + internalServiceAuth: + properties: + enabled: + type: boolean + oidcIssuer: + type: string + type: object + legacyOverrides: + additionalProperties: properties: - hostname: - type: string - ip: - type: string - ipMode: - type: string - ports: + env: items: properties: - error: - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + name: type: string - port: - format: int32 - type: integer - protocol: + value: type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object required: - - error - - port - - protocol + - name type: object type: array - x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object type: object - type: array - name: + type: object + license: type: string - type: object - kafkaStatus: - properties: - conditions: - items: - properties: - lastTransitionTime: - format: date-time - type: string - message: - maxLength: 32768 - type: string - observedGeneration: - format: int64 - minimum: 0 - type: integer - reason: - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - enum: - - "True" - - "False" - - Unknown - type: string - type: - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - connection: - properties: - brokerEndpoint: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - clusterID: + manifestRepository: + type: string + oidc: + properties: + authMethod: properties: key: type: string @@ -4352,7 +4094,7 @@ spec: - key type: object x-kubernetes-map-type: atomic - host: + clientId: properties: key: type: string @@ -4365,7 +4107,7 @@ spec: - key type: object x-kubernetes-map-type: atomic - port: + clientSecret: properties: key: type: string @@ -4378,7 +4120,7 @@ spec: - key type: object x-kubernetes-map-type: atomic - url: + issuerUrl: properties: key: type: string @@ -4390,194 +4132,535 @@ spec: required: - key type: object - x-kubernetes-map-type: atomic + x-kubernetes-map-type: atomic + sessionLength: + type: string + type: object + probes: + properties: + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object type: object - ready: - type: boolean - state: + serviceAccount: + properties: + annotations: + additionalProperties: + type: string + type: object + create: + default: true + type: boolean + serviceAccountName: + default: wandb + type: string + required: + - create + type: object + version: type: string required: - - ready + - bucketProxy + - features + - hostname + - version type: object - mysqlStatus: - properties: - conditions: - items: - properties: - lastTransitionTime: - format: date-time - type: string - message: - maxLength: 32768 - type: string - observedGeneration: - format: int64 - minimum: 0 - type: integer - reason: - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - enum: - - "True" - - "False" - - Unknown - type: string - type: - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - connection: - properties: - database: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - host: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslCa: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslCert: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslKey: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - tls: + required: + - retentionPolicy + type: object + status: + properties: + clickhouseStatus: + additionalProperties: + properties: + conditions: + items: properties: - key: + lastTransitionTime: + format: date-time type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: + message: + maxLength: 32768 type: string - name: - default: "" + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - username: - properties: - key: + status: + enum: + - "True" + - "False" + - Unknown type: string - name: - default: "" + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string - optional: - type: boolean required: - - key + - lastTransitionTime + - message + - reason + - status + - type type: object - x-kubernetes-map-type: atomic + type: array + connection: + properties: + database: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + host: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + httpPort: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tcpPort: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + ready: + type: boolean + state: + type: string + required: + - ready + type: object + type: object + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + gatewayStatus: + properties: + addresses: + items: + type: string + type: array + gatewayRef: + properties: + name: + type: string + namespace: + type: string + required: + - name type: object + name: + type: string ready: type: boolean - state: + type: object + generatedSecrets: + additionalProperties: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + ingressStatus: + properties: + loadBalancerIngress: + items: + properties: + hostname: + type: string + ip: + type: string + ipMode: + type: string + ports: + items: + properties: + error: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - error + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + name: type: string - required: - - ready type: object - objectStoreStatus: + kafkaStatus: properties: conditions: items: @@ -4617,7 +4700,7 @@ spec: type: array connection: properties: - accessKey: + brokerEndpoint: properties: key: type: string @@ -4630,7 +4713,7 @@ spec: - key type: object x-kubernetes-map-type: atomic - bucket: + clusterID: properties: key: type: string @@ -4643,7 +4726,7 @@ spec: - key type: object x-kubernetes-map-type: atomic - endpoint: + host: properties: key: type: string @@ -4669,45 +4752,6 @@ spec: - key type: object x-kubernetes-map-type: atomic - provider: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - region: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secretKey: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic url: properties: key: @@ -4729,136 +4773,511 @@ spec: required: - ready type: object - observedGeneration: - format: int64 - type: integer - ready: - type: boolean - redisStatus: - properties: - conditions: - items: + mysqlStatus: + additionalProperties: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + connection: properties: - lastTransitionTime: - format: date-time - type: string - message: - maxLength: 32768 - type: string - observedGeneration: - format: int64 - minimum: 0 - type: integer - reason: - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - enum: - - "True" - - "False" - - Unknown - type: string - type: - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - connection: - properties: - host: + database: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + host: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCa: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCert: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tls: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + ready: + type: boolean + state: + type: string + required: + - ready + type: object + type: object + objectStoreStatus: + additionalProperties: + properties: + conditions: + items: properties: - key: - type: string - name: - default: "" + lastTransitionTime: + format: date-time type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: + message: + maxLength: 32768 type: string - name: - default: "" + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: + status: + enum: + - "True" + - "False" + - Unknown type: string - name: - default: "" + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string - optional: - type: boolean required: - - key + - lastTransitionTime + - message + - reason + - status + - type type: object - x-kubernetes-map-type: atomic - sslCa: + type: array + connection: + properties: + accessKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + bucket: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpoint: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + forcePathStyle: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + path: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + provider: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + region: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secretKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tlsEnabled: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + ready: + type: boolean + state: + type: string + required: + - ready + type: object + type: object + observedGeneration: + format: int64 + type: integer + ready: + type: boolean + redisStatus: + additionalProperties: + properties: + conditions: + items: properties: - key: + lastTransitionTime: + format: date-time type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - tls: - properties: - key: + message: + maxLength: 32768 type: string - name: - default: "" + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: + status: + enum: + - "True" + - "False" + - Unknown type: string - name: - default: "" + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string - optional: - type: boolean required: - - key + - lastTransitionTime + - message + - reason + - status + - type type: object - x-kubernetes-map-type: atomic - type: object - ready: - type: boolean - state: - type: string - required: - - ready + type: array + connection: + properties: + host: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCa: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tls: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + ready: + type: boolean + state: + type: string + required: + - ready + type: object type: object telemetryStatus: properties: @@ -5897,12 +6316,18 @@ spec: type: string name: type: string + phase: + type: string + reason: + type: string succeeded: type: boolean type: object type: object lastSuccessVersion: type: string + phase: + type: string ready: type: boolean reason: @@ -5911,16 +6336,22 @@ spec: type: string type: object mysqlInit: + additionalProperties: + properties: + failed: + type: boolean + message: + type: string + name: + type: string + phase: + type: string + reason: + type: string + succeeded: + type: boolean + type: object default: {} - properties: - failed: - type: boolean - message: - type: string - name: - type: string - succeeded: - type: boolean type: object required: - hostname diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index acb714d0..22dac7bc 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -125,6 +125,24 @@ rules: - patch - update - watch +- apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/status + verbs: + - get - apiGroups: - clickhouse.altinity.com resources: @@ -370,3 +388,11 @@ rules: - seaweeds/status verbs: - get +- apiGroups: + - security.openshift.io + resourceNames: + - nonroot-v2 + resources: + - securitycontextconstraints + verbs: + - use diff --git a/deploy/operator/Chart.lock b/deploy/operator/Chart.lock index cd83f245..c97534c7 100644 --- a/deploy/operator/Chart.lock +++ b/deploy/operator/Chart.lock @@ -4,13 +4,13 @@ dependencies: version: 0.11.8 - name: moco repository: https://cybozu-go.github.io/moco/ - version: 0.24.0 + version: 0.26.0 - name: redis-operator repository: https://ot-container-kit.github.io/helm-charts version: 0.22.2 - name: seaweedfs-operator repository: https://seaweedfs.github.io/seaweedfs-operator/ - version: 0.1.24 + version: 0.1.35 - name: prometheus-operator-crds repository: https://prometheus-community.github.io/helm-charts version: 29.0.0 @@ -26,5 +26,5 @@ dependencies: - name: telemetry repository: file://../telemetry version: 0.1.0 -digest: sha256:7dd459c8c7d1bc3d27cf9adba1c50765275fe0b459f8c551ab6c25ad5b7a7d67 -generated: "2026-06-22T15:23:59.623415-07:00" +digest: sha256:b03eebae8a867a43e00fe2d3bd3e89c2af552a0c998d0046c56865128090f123 +generated: "2026-07-17T13:39:19.593182-07:00" diff --git a/deploy/operator/Chart.yaml b/deploy/operator/Chart.yaml index 50022c72..f4a538ca 100644 --- a/deploy/operator/Chart.yaml +++ b/deploy/operator/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: operator description: A Helm chart for Weights & Biases operator type: application -version: 2.0.0-alpha.2 -appVersion: "2.0.0-alpha.2" +version: 2.0.0-beta.3 +appVersion: "2.0.0-beta.3" maintainers: - name: wandb email: support@wandb.com @@ -16,7 +16,7 @@ dependencies: repository: https://charts.wandb.ai/ condition: wandb-operator.enabled - name: moco - version: 0.24.0 + version: 0.26.0 repository: https://cybozu-go.github.io/moco/ condition: moco.enabled - name: redis-operator @@ -24,7 +24,7 @@ dependencies: repository: https://ot-container-kit.github.io/helm-charts condition: redis-operator.enabled - name: seaweedfs-operator - version: 0.1.24 + version: 0.1.35 repository: https://seaweedfs.github.io/seaweedfs-operator/ condition: seaweedfs-operator.enabled - name: prometheus-operator-crds diff --git a/deploy/operator/profiles/openshift.yaml b/deploy/operator/profiles/openshift.yaml index b18fe38e..3b289340 100644 --- a/deploy/operator/profiles/openshift.yaml +++ b/deploy/operator/profiles/openshift.yaml @@ -1,3 +1,6 @@ +openshift: + enabled: true + wandb-operator: podSecurityContext: runAsUser: null @@ -11,18 +14,31 @@ wandb-operator: OPENSHIFT: value: "true" -grafana-operator: - isOpenShift: true +# Null these operators' hardcoded IDs so restricted-v2 assigns valid ones. +redis-operator: + podSecurityContext: + runAsUser: null + runAsGroup: null + fsGroup: null + fsGroupChangePolicy: null + +altinity-clickhouse-operator: + podSecurityContext: + runAsUser: null + runAsGroup: null + fsGroup: null + fsGroupChangePolicy: null -# Override moco images with locally-built copies pushed to the CRC internal -# registry. Used to test in-development moco changes (UID/SCC fixes) before -# they ship upstream. Remove once a released chart includes the changes. +seaweedfs-operator: + podSecurityContext: + runAsUser: null + runAsGroup: null + fsGroup: null + +# Disable moco's injected fixed 10000 IDs so restricted-v2 admits MySQL pods. moco: - image: - repository: image-registry.openshift-image-registry.svc:5000/wandb-operators/moco-controller - tag: dev - pullPolicy: Always - fluentbit: - image: - repository: image-registry.openshift-image-registry.svc:5000/wandb-operators/moco-fluent-bit - tag: dev + extraArgs: + - --disable-default-security-context + +grafana-operator: + isOpenShift: true diff --git a/deploy/operator/profiles/telemetry-full.yaml b/deploy/operator/profiles/telemetry-full.yaml index 0659cea2..0df63d81 100644 --- a/deploy/operator/profiles/telemetry-full.yaml +++ b/deploy/operator/profiles/telemetry-full.yaml @@ -1,5 +1,7 @@ telemetry: mode: "full" + scrape: + kubeStateMetrics: true victoria-metrics-operator: enabled: true diff --git a/deploy/operator/templates/_helpers.tpl b/deploy/operator/templates/_helpers.tpl index 8c1e63e5..b39d2ee8 100644 --- a/deploy/operator/templates/_helpers.tpl +++ b/deploy/operator/templates/_helpers.tpl @@ -37,3 +37,45 @@ inside operator-crds/crds.yaml: honor an explicit override on {{- define "wandb-operator.serviceName" -}} {{ include "wandb-base.serviceName" (dict "Release" (dict "Name" .Release.Name) "Chart" (dict "Name" "wandb-operator") "Values" (dict "nameOverride" (dig "wandb-operator" "nameOverride" "" .Values.AsMap) "service" (dict "name" (dig "wandb-operator" "service" "name" "" .Values.AsMap)))) }} {{- end }} + +{{/* +Operator custom-CA trust (wandb-operator.caCerts). Referenced from the +wandb-operator subchart values as volumesTpls / volumeMountsTpls / envTpls +strings, so they render in the subchart context where .Values is the +wandb-operator values. Each is inert unless a CA source is configured. +*/}} +{{- define "wandb-operator.caCertsActive" -}} +{{- $ca := .Values.caCerts | default dict -}} +{{- if or $ca.certs $ca.existingSecret $ca.existingConfigMap -}}true{{- end -}} +{{- end -}} + +{{- define "wandb-operator.caCertsVolume" -}} +{{- $ca := .Values.caCerts | default dict -}} +{{- if include "wandb-operator.caCertsActive" . -}} +- name: wandb-operator-ca-certs + {{- if $ca.existingConfigMap }} + configMap: + name: {{ $ca.existingConfigMap }} + {{- else }} + secret: + secretName: {{ $ca.existingSecret | default (printf "%s-operator-ca-certs" .Release.Name) }} + {{- end }} +{{- end -}} +{{- end -}} + +{{- define "wandb-operator.caCertsVolumeMount" -}} +{{- $ca := .Values.caCerts | default dict -}} +{{- if include "wandb-operator.caCertsActive" . -}} +- name: wandb-operator-ca-certs + mountPath: {{ $ca.mountPath | default "/etc/wandb/ca-certs" }} + readOnly: true +{{- end -}} +{{- end -}} + +{{- define "wandb-operator.caCertsEnv" -}} +{{- $ca := .Values.caCerts | default dict -}} +{{- if include "wandb-operator.caCertsActive" . -}} +- name: SSL_CERT_DIR + value: "{{ $ca.mountPath | default "/etc/wandb/ca-certs" }}:/etc/ssl/certs:/etc/pki/tls/certs" +{{- end -}} +{{- end -}} diff --git a/deploy/operator/templates/hooks/crd-installer-job.yaml b/deploy/operator/templates/hooks/crd-installer-job.yaml index 9cdfacc2..ae859f79 100644 --- a/deploy/operator/templates/hooks/crd-installer-job.yaml +++ b/deploy/operator/templates/hooks/crd-installer-job.yaml @@ -55,7 +55,11 @@ spec: {{- end }} securityContext: runAsNonRoot: {{ dig "wandb-operator" "podSecurityContext" "runAsNonRoot" true .Values.AsMap }} - runAsUser: {{ dig "wandb-operator" "podSecurityContext" "runAsUser" 1000 .Values.AsMap }} + {{- /* runAsUser only if set, else restricted-v2 assigns it. */ -}} + {{- $crdInstallerRunAsUser := dig "wandb-operator" "podSecurityContext" "runAsUser" nil .Values.AsMap }} + {{- if $crdInstallerRunAsUser }} + runAsUser: {{ $crdInstallerRunAsUser }} + {{- end }} seccompProfile: type: {{ dig "wandb-operator" "podSecurityContext" "seccompProfile" "type" "RuntimeDefault" .Values.AsMap }} containers: diff --git a/deploy/operator/templates/openshift-owner-finalizers-rbac.yaml b/deploy/operator/templates/openshift-owner-finalizers-rbac.yaml new file mode 100644 index 00000000..3157bd7e --- /dev/null +++ b/deploy/operator/templates/openshift-owner-finalizers-rbac.yaml @@ -0,0 +1,56 @@ +{{- /* OpenShift-only: finalizers grants for moco + seaweedfs owners. */ -}} +{{- if dig "openshift" "enabled" false .Values.AsMap }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-openshift-owner-finalizers + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/part-of: wandb-operator +rules: + - apiGroups: + - moco.cybozu.com + resources: + - mysqlclusters/finalizers + verbs: + - update + - apiGroups: + - seaweed.seaweedfs.com + resources: + - seaweeds/finalizers + verbs: + - update +--- +# StatefulSet controller creates moco PVCs, so it sets mysqlcluster finalizers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-openshift-mysqlcluster-finalizers + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/part-of: wandb-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-openshift-owner-finalizers +subjects: + - kind: ServiceAccount + name: statefulset-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-openshift-seaweed-finalizers + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/part-of: wandb-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-openshift-owner-finalizers +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-seaweedfs-operator + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/deploy/operator/templates/openshift-scc-rbac.yaml b/deploy/operator/templates/openshift-scc-rbac.yaml new file mode 100644 index 00000000..3a9c118f --- /dev/null +++ b/deploy/operator/templates/openshift-scc-rbac.yaml @@ -0,0 +1,35 @@ +{{- /* OpenShift-only: operator needs `use` on the SCCs it grants. */ -}} +{{- if dig "openshift" "enabled" false .Values.AsMap }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-openshift-scc-nonroot-v2 + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/part-of: wandb-operator +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - nonroot-v2 + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-openshift-scc-nonroot-v2 + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/part-of: wandb-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-openshift-scc-nonroot-v2 +subjects: + - kind: ServiceAccount + name: {{ include "wandb-operator.fullname" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/deploy/operator/templates/operator-ca-certs.yaml b/deploy/operator/templates/operator-ca-certs.yaml new file mode 100644 index 00000000..35666852 --- /dev/null +++ b/deploy/operator/templates/operator-ca-certs.yaml @@ -0,0 +1,25 @@ +{{- /* +Synthesizes a Secret from inline wandb-operator.caCerts.certs (PEM blocks) so the +operator container can trust a private/self-signed registry CA. Skipped when the +user points caCerts at an existing Secret/ConfigMap instead. The name matches the +default secretName referenced by wandb-operator.volumesTpls in values.yaml. +*/ -}} +{{- $op := index .Values "wandb-operator" | default dict -}} +{{- $ca := $op.caCerts | default dict -}} +{{- if and $ca.certs (not $ca.existingSecret) (not $ca.existingConfigMap) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-operator-ca-certs + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/part-of: wandb +type: Opaque +stringData: + {{- range $i, $pem := $ca.certs }} + ca-{{ $i }}.crt: | + {{- $pem | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/operator/templates/wandb-operator-clickhouse-role.yaml b/deploy/operator/templates/wandb-operator-clickhouse-role.yaml index 4d1d5765..23cb949c 100644 --- a/deploy/operator/templates/wandb-operator-clickhouse-role.yaml +++ b/deploy/operator/templates/wandb-operator-clickhouse-role.yaml @@ -22,6 +22,24 @@ rules: - clickhouseinstallations/status verbs: - get + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/status + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/operator/values.yaml b/deploy/operator/values.yaml index ddd16904..61d72dd2 100644 --- a/deploy/operator/values.yaml +++ b/deploy/operator/values.yaml @@ -1,17 +1,21 @@ helmHooks: enabled: true +# Renders OpenShift-only resources; profiles/openshift.yaml turns this on. +openshift: + enabled: false + wandb: install: true size: small namespace: wandb - version: 0.81.0 + version: 0.83.1 internalServiceAuth: false wandb-operator: image: repository: us-docker.pkg.dev/wandb-production/public/wandb/operator - tag: 2.0.0-alpha.2 + tag: 2.0.0-beta.3 containers: operator: command: @@ -61,6 +65,27 @@ wandb-operator: - mountPath: /tmp/wandb-operator/serving-certs name: serving-certs readOnly: true + # Mounts the custom CA bundle (see caCerts below) into the operator + # container. Rendered only when caCerts has a source. + volumeMountsTpls: + - '{{ include "wandb-operator.caCertsVolumeMount" . }}' + + # Custom CA certificates trusted by the operator process. Required when the + # operator must pull the server manifest from a private/self-signed OCI + # registry (air-gapped installs). Activates when any source below is set; + # certs are added to SSL_CERT_DIR additively (Go still loads the ubi system + # bundle, so public roots are preserved). + caCerts: + # Inline PEM blocks; the parent chart synthesizes them into a Secret. + certs: [] + # Or reference an existing Secret / ConfigMap whose keys are PEM certs. + existingSecret: "" + existingConfigMap: "" + mountPath: /etc/wandb/ca-certs + volumesTpls: + - '{{ include "wandb-operator.caCertsVolume" . }}' + envTpls: + - '{{ include "wandb-operator.caCertsEnv" . }}' service: enabled: true @@ -243,6 +268,7 @@ telemetry: operators: true wandbApi: true infrastructure: true + kubeStateMetrics: false alerting: enabled: false evaluationInterval: 30s diff --git a/deploy/telemetry/dashboards/wandb-application.json b/deploy/telemetry/dashboards/wandb-application.json new file mode 100644 index 00000000..f32f803a --- /dev/null +++ b/deploy/telemetry/dashboards/wandb-application.json @@ -0,0 +1,558 @@ +{ + "__inputs": [ + { + "name": "DS_VICTORIAMETRICS", + "label": "VictoriaMetrics", + "type": "datasource", + "pluginId": "victoriametrics-metrics-datasource", + "pluginName": "VictoriaMetrics" + }, + {"name": "DS_VICTORIATRACES", "label": "VictoriaTraces", "type": "datasource", "pluginId": "jaeger", "pluginName": "Jaeger"} + ], + "annotations": {"list": []}, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "targetBlank": true, + "title": "Open Traces in Explore", + "type": "link", + "url": "/explore?panes=%7B%22A%22:%7B%22datasource%22:%22${DS_VICTORIATRACES}%22,%22queries%22:[],%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" + }, + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": true, + "keepTime": true, + "targetBlank": false, + "title": "W&B Field Investigation", + "type": "link", + "url": "/d/wandb-field-investigation" + } + ], + "panels": [ + { + "type": "text", + "title": "", + "gridPos": {"h": 6, "w": 24, "x": 0, "y": 0}, + "options": { + "mode": "markdown", + "content": "# W&B Application\n\nAnswers **\"is the whole W&B app slow or erroring for this customer?\"** Start here when a customer reports the UI is slow, requests fail, or the SDK can't reach the server.\n\nThe top sections cover **overall API health** — request rate, error codes, and latency. The **GraphQL errors** section attributes failures to the specific service/operation responsible. The **Operations & background** section shows read-path, artifact, and export span latencies from traces.\n\nUse the **W&B Namespace** dropdown to pick the customer's install. Use **Open Traces in Explore** (top-right) to drill from any symptom to a real trace." + } + }, + { + "type": "row", + "title": "Request health", + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 6}, + "collapsed": false + }, + { + "type": "timeseries", + "title": "Request rate by status code", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 8, "w": 16, "x": 0, "y": 7}, + "targets": [ + { + "expr": "sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name=\"gorilla\"}[$__rate_interval]))", + "legendFormat": "{{http_response_status_code}}", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "reqps", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi"}} + }, + { + "type": "text", + "title": "About: Request rate by status code", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 7}, + "options": { + "mode": "markdown", + "content": "**What it shows** — every HTTP request the W&B API serves, per second, split by response code. This is the single best \"is the app up?\" view.\n\n**What bad looks like** — a wall of `5xx` appearing, or `2xx` dropping to near-zero (traffic no longer reaching the API: ingress / service / DNS / pod crash).\n\n**Do next** — 5xx climbing → check pod logs. To attribute GraphQL errors to a feature, see the **GraphQL errors** section below." + } + }, + { + "type": "timeseries", + "title": "4xx vs 5xx request rate", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 8, "w": 16, "x": 0, "y": 15}, + "targets": [ + { + "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=\"gorilla\", http_response_status_code=~\"4..\"}[$__rate_interval])) or vector(0)", + "legendFormat": "4xx (client)", + "refId": "A" + }, + { + "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=\"gorilla\", http_response_status_code=~\"5..\"}[$__rate_interval])) or vector(0)", + "legendFormat": "5xx (server)", + "refId": "B" + } + ], + "fieldConfig": {"defaults": {"unit": "reqps", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi"}} + }, + { + "type": "text", + "title": "About: 4xx vs 5xx", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 15}, + "options": { + "mode": "markdown", + "content": "**What it shows** — client errors (`4xx`) vs server errors (`5xx`) as two clean lines, so you can tell whose fault it is.\n\n**What bad looks like** — any sustained `5xx` is a server problem. A `4xx` spike is usually the customer's SDK/auth (version mismatch, bad token, malformed query) and is often *not* an outage.\n\n**Do next** — 5xx → check API pod logs. 4xx → confirm SDK version against **Service Image Versions** on [W&B Field Investigation](/d/wandb-field-investigation).\n\n**Note** — GraphQL errors return HTTP `200` (the error is in the response body), so they don't appear here; use the **GraphQL errors** section below for those. This panel catches HTTP transport/server failures; a flat `0` means healthy." + } + }, + { + "type": "timeseries", + "title": "Error ratio (5xx / all requests)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 8, "w": 16, "x": 0, "y": 23}, + "targets": [ + { + "expr": "(sum(rate(http_server_request_duration_seconds_count{service_name=\"gorilla\", http_response_status_code=~\"5..\"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name=\"gorilla\"}[$__rate_interval]))", + "legendFormat": "5xx ratio", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi"}} + }, + { + "type": "text", + "title": "About: Error ratio", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 23}, + "options": { + "mode": "markdown", + "content": "**What it shows** — the share of requests failing with a server error, as a percentage. Normalizes for traffic so a busy install and a quiet one read the same.\n\n**What bad looks like** — above ~1% sustained is a real incident; a brief blip during a deploy is usually fine.\n\n**Do next** — if the ratio is high, capture a snapshot of this panel and use the **GraphQL errors** section below to see which feature is responsible, then drill to a trace.\n\n**Note** — this is HTTP `5xx` only; GraphQL errors are HTTP `200`, so a `0` ratio doesn't mean error-free — check the **GraphQL errors** section below." + } + }, + { + "type": "row", + "title": "Latency", + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 31}, + "collapsed": false + }, + { + "type": "timeseries", + "title": "Request latency (p50 / p95 / p99)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 8, "w": 16, "x": 0, "y": 32}, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by(le)(rate(http_server_request_duration_seconds_bucket{service_name=\"gorilla\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by(le)(rate(http_server_request_duration_seconds_bucket{service_name=\"gorilla\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by(le)(rate(http_server_request_duration_seconds_bucket{service_name=\"gorilla\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": {"defaults": {"unit": "s", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi"}}, + "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "yellow", "value": 1}, {"color": "red", "value": 3}]} + }, + { + "type": "text", + "title": "About: Request latency", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 32}, + "options": { + "mode": "markdown", + "content": "**What it shows** — how long requests take, across all gorilla requests. p50 is the typical user; p95/p99 are the slowest 5% / 1%.\n\n**What bad looks like** — p95 above ~3s sustained for more than a few minutes means users feel it. Brief spikes under load are tolerable.\n\n**Do next** — if p95/p99 are high, use **p95 by operation** and **Top operations by slow request count** below to find the culprit, then Open Traces in Explore (top-right) and filter `duration>3s`." + } + }, + { + "type": "timeseries", + "title": "p95 latency by operation (top 10)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 8, "w": 16, "x": 0, "y": 40}, + "targets": [ + { + "expr": "topk(10, histogram_quantile(0.95, sum by(le, span_name)(rate(traces_spanmetrics_duration_milliseconds_bucket{service_name=\"gorilla\"}[$__rate_interval]))))", + "legendFormat": "{{span_name}}", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "ms", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "right", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi", "sort": "desc"}} + }, + { + "type": "text", + "title": "About: p95 by operation", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 40}, + "options": { + "mode": "markdown", + "content": "**What it shows** — the 10 operations (trace span names) with the highest p95 latency, over time. Tells you *which operation* is slow, not just that something is. Sourced from spanmetrics (`traces_spanmetrics_*`, milliseconds) since gorilla's HTTP metrics don't carry a route label.\n\n**What bad looks like** — one operation sitting well above the rest, especially graphql spans climbing after a deploy.\n\n**Do next** — a slow graphql span → see the **GraphQL errors** section below; a slow read/export span → see **Operations & background** below. Then drill to a trace for that operation." + } + }, + { + "type": "table", + "title": "Top operations by slow request count (>2.5s)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 10, "w": 16, "x": 0, "y": 48}, + "targets": [ + { + "expr": "topk(20, sum by (span_name) (increase(traces_spanmetrics_duration_milliseconds_count{service_name=\"gorilla\"}[$__range]) - increase(traces_spanmetrics_duration_milliseconds_bucket{service_name=\"gorilla\", le=\"2500\"}[$__range])))", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {"Time": true}, + "indexByName": {"span_name": 0, "Value": 1}, + "renameByName": {"span_name": "Operation", "Value": "Slow requests (>2.5s) in range"} + } + } + ], + "options": { + "showHeader": true, + "cellHeight": "md", + "footer": {"show": true, "reducer": ["sum"], "fields": ["Value"]}, + "sortBy": [{"displayName": "Slow requests (>2.5s) in range", "desc": true}] + }, + "fieldConfig": { + "defaults": {"custom": {"align": "left", "filterable": true}}, + "overrides": [ + { + "matcher": {"id": "byName", "options": "Slow requests (>2.5s) in range"}, + "properties": [ + {"id": "custom.cellOptions", "value": {"type": "gauge", "mode": "gradient"}}, + {"id": "custom.align", "value": "right"} + ] + } + ] + } + }, + { + "type": "text", + "title": "About: Top operations by slow request count", + "gridPos": {"h": 10, "w": 8, "x": 16, "y": 48}, + "options": { + "mode": "markdown", + "content": "**What it shows** — the operations that produced the most requests slower than 2.5 seconds over the selected range (counted from the latency histogram). This is the volume view of slowness: which operations hurt the most users.\n\n**What bad looks like** — an operation with a large, growing count of >2.5s requests, especially the workspace, run-table, or artifact operations.\n\n**Do next** — note the top operation, Open Traces in Explore (top-right), filter `service.name=gorilla` and `duration>2.5s` on that operation to read a real slow trace." + } + }, + { + "type": "row", + "title": "GraphQL operations", + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 58}, + "collapsed": false + }, + { + "type": "timeseries", + "title": "GraphQL operation rate by span (top, sampled)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 8, "w": 16, "x": 0, "y": 59}, + "targets": [ + { + "expr": "topk(10, sum by (span_name) (rate(traces_spanmetrics_calls_milliseconds_total{graphql_service!=\"\"}[$__rate_interval])))", + "legendFormat": "{{span_name}}", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "reqps", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "right", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "description": "From spanmetrics (traces), which are sampled — trust the shape, not the absolute count. Spanmetrics have NO namespace label, so this is not filtered by $namespace; it reflects the whole trace stream." + }, + { + "type": "text", + "title": "About: GraphQL operation rate", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 59}, + "options": { + "mode": "markdown", + "content": "**What it shows** — the busiest GraphQL operations by request rate, from sampled traces. Good for seeing *what the customer's app is doing* and whether a particular operation surged.\n\n**What bad looks like** — one operation dominating far above normal (a client hot-loop or retry storm), or the whole stream going flat (traffic stopped).\n\n**Do next** — for *errors* by GraphQL service/operation, see the **GraphQL errors** section below. Note: spanmetrics have no namespace label, so this is not scoped to the selected install." + } + }, + { + "type": "timeseries", + "title": "GraphQL p95 latency by span (sampled)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 8, "w": 16, "x": 0, "y": 67}, + "targets": [ + { + "expr": "topk(10, histogram_quantile(0.95, sum by(le, span_name)(rate(traces_spanmetrics_duration_milliseconds_bucket{graphql_service!=\"\"}[$__rate_interval]))))", + "legendFormat": "{{span_name}}", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "ms", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "right", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "description": "From spanmetrics (traces), which are sampled. Spanmetrics have NO namespace label, so this is not filtered by $namespace." + }, + { + "type": "text", + "title": "About: GraphQL p95 latency", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 67}, + "options": { + "mode": "markdown", + "content": "**What it shows** — the slowest GraphQL operations at p95, in milliseconds, from sampled traces.\n\n**What bad looks like** — an operation's p95 climbing into the seconds, especially if it lines up with the customer's complaint.\n\n**Do next** — Open Traces in Explore (top-right), filter to that operation, and read the resolver chain. For error attribution by service, see the **GraphQL errors** section below." + } + }, + { + "type": "row", + "title": "GraphQL errors — attribution (traces)", + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 75}, + "collapsed": false + }, + { + "type": "stat", + "title": "GraphQL errors in range (sampled)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 4, "w": 12, "x": 0, "y": 76}, + "targets": [ + { + "expr": "sum(increase(traces_spanmetrics_calls_milliseconds_total{graphql_service!=\"\", status_code=\"STATUS_CODE_ERROR\"}[$__range]))", + "instant": true, + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "short", "decimals": 0}, "overrides": []}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "graphMode": "area"} + }, + { + "type": "stat", + "title": "Distinct erroring services", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 4, "w": 12, "x": 12, "y": 76}, + "targets": [ + { + "expr": "count(count by (graphql_service) (increase(traces_spanmetrics_calls_milliseconds_total{graphql_service!=\"\", status_code=\"STATUS_CODE_ERROR\"}[$__range]) > 0))", + "instant": true, + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "short", "decimals": 0}, "overrides": []}, + "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "graphMode": "none"} + }, + { + "type": "text", + "title": "About: Error overview", + "gridPos": {"h": 4, "w": 24, "x": 0, "y": 80}, + "options": { + "mode": "markdown", + "content": "**What it shows** — top-line GraphQL error health, from sampled traces (trust the trend, not the exact count).\n\n**What bad looks like** — *distinct erroring services* jumping from ~1 to many (a broad outage vs. one bad dependency), or the error count climbing off a flat baseline.\n\n**Do next** — one service dominating → look at **Attribution by service** below and open its trace via **Open Traces in Explore** (top-right)." + } + }, + { + "type": "bargauge", + "title": "Top services by error count", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 9, "w": 12, "x": 0, "y": 84}, + "targets": [ + { + "expr": "topk(15, sum by (graphql_service) (increase(traces_spanmetrics_calls_milliseconds_total{graphql_service!=\"\", status_code=\"STATUS_CODE_ERROR\"}[$__range])))", + "legendFormat": "{{graphql_service}}", + "instant": true, + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "short", "decimals": 0}, "overrides": []}, + "options": {"orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true} + }, + { + "type": "table", + "title": "Top service × operation by errors", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 9, "w": 12, "x": 12, "y": 84}, + "targets": [ + { + "expr": "topk(50, sum by (graphql_service, graphql_operationName) (increase(traces_spanmetrics_calls_milliseconds_total{graphql_service!=\"\", status_code=\"STATUS_CODE_ERROR\"}[$__range])))", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {"Time": true}, + "indexByName": {"graphql_service": 0, "graphql_operationName": 1, "Value": 2}, + "renameByName": {"graphql_service": "Service", "graphql_operationName": "Operation", "Value": "Errors in range"} + } + } + ], + "options": {"showHeader": true, "cellHeight": "sm", "footer": {"show": false}, "sortBy": [{"displayName": "Errors in range", "desc": true}]}, + "fieldConfig": { + "defaults": {"custom": {"align": "left", "filterable": true}}, + "overrides": [ + { + "matcher": {"id": "byName", "options": "Errors in range"}, + "properties": [{"id": "custom.cellOptions", "value": {"type": "gauge", "mode": "gradient"}}, {"id": "custom.align", "value": "right"}] + } + ] + } + }, + { + "type": "text", + "title": "About: Attribution by service", + "gridPos": {"h": 5, "w": 24, "x": 0, "y": 93}, + "options": { + "mode": "markdown", + "content": "**What it shows** — which GraphQL **service** (schema source file) and which **operation** are producing errors, from sampled traces. The `graphql.service` label rides on gorilla's spans (core PR #43519).\n\n**What bad looks like** — one service far above the rest = a single dependency/resolver in trouble; errors spread evenly across many services = something shared (DB, auth, rate-limiter).\n\n**Do next** — note the top service/operation, then **Open Traces in Explore** (top-right), filter to gorilla + `error=true` and that operation name, and read the resolver chain / error message on the actual trace. If this panel is empty, the spanmetrics `graphql.service` dimension may not be flowing yet — confirm traces reach the gateway and the collector was restarted after the dimension was added." + } + }, + { + "type": "timeseries", + "title": "GraphQL errors per minute, by service (sampled)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 9, "w": 16, "x": 0, "y": 98}, + "targets": [ + { + "expr": "sum by (graphql_service) (rate(traces_spanmetrics_calls_milliseconds_total{graphql_service!=\"\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) * 60", + "legendFormat": "{{graphql_service}}", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "short", "min": 0, "custom": {"fillOpacity": 30, "stacking": {"mode": "normal"}}}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "right", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi", "sort": "desc"}} + }, + { + "type": "text", + "title": "About: Errors/min by service", + "gridPos": {"h": 9, "w": 8, "x": 16, "y": 98}, + "options": { + "mode": "markdown", + "content": "**What it shows** — the same service attribution over time, stacked so you can see *when* a service started erroring and whether it correlates with a deploy.\n\n**What bad looks like** — a service's band appearing or ballooning at a specific minute.\n\n**Do next** — line up the onset time with the **Service Image Versions** panel on [W&B Field Investigation](/d/wandb-field-investigation) (a rollout?) and with the customer's report. Then drill to a trace from that window." + } + }, + { + "type": "row", + "title": "Drill down to traces", + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 107}, + "collapsed": false + }, + { + "type": "text", + "title": "About: Drill-down", + "gridPos": {"h": 4, "w": 24, "x": 0, "y": 108}, + "options": { + "mode": "markdown", + "content": "**Get to the cause in three hops:** (1) identify the top service/operation above → (2) **Open Traces in Explore** (top-right), filter `service.name` to gorilla, `error=true`, and the operation → (3) read the resolver chain and error message on the actual trace. If gorilla's OTEL log export is enabled, follow **Trace → Logs** from there to the correlated log lines." + } + }, + { + "type": "row", + "title": "Operations & background (traces)", + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 112}, + "collapsed": false + }, + { + "type": "timeseries", + "title": "History-reader span latency p95 (traces, sampled)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 9, "w": 16, "x": 0, "y": 113}, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, span_name) (rate(traces_spanmetrics_duration_milliseconds_bucket{span_name=~\"(?i).*(history|clickhouse|read).*\"}[$__rate_interval])))", + "legendFormat": "{{span_name}}", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "ms", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "right", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "description": "Trace-derived (traces_spanmetrics_*, confirmed present) so it is SAMPLED — trust the shape, not exact values. The span_name regex is a best guess; if this panel is empty, run label_values(traces_spanmetrics_duration_milliseconds_bucket, span_name) and adjust the regex to the history-reader / clickhouse span names on your stack." + }, + { + "type": "text", + "title": "About: History-reader span latency", + "gridPos": {"h": 9, "w": 8, "x": 16, "y": 113}, + "options": { + "mode": "markdown", + "content": "**What it shows** — p95 duration of the read-path spans (history reader / ClickHouse query) from sampled traces, split by span. Answers \"why won't the customer's charts load?\"\n\n**What bad looks like** — one span's p95 climbing into seconds, or a new span appearing at a specific minute.\n\n**Do next** — note the slow span and the minute it started, then **Open Traces in Explore** (top-right) and read the actual span chain. If the panel is empty, the span-name regex needs adjusting (see the panel description)." + } + }, + { + "type": "timeseries", + "title": "Artifact operation latency p95 (traces, sampled)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 9, "w": 16, "x": 0, "y": 122}, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, span_name) (rate(traces_spanmetrics_duration_milliseconds_bucket{span_name=~\"(?i).*artifact.*\"}[$__rate_interval])))", + "legendFormat": "{{span_name}}", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "ms", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "right", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "description": "Trace-derived (traces_spanmetrics_*, confirmed present) so it is SAMPLED — trust the shape, not exact values. The span_name regex matches any span containing 'artifact'; if this panel is empty, run label_values(traces_spanmetrics_duration_milliseconds_bucket, span_name) and adjust the regex to the artifact span names on your stack." + }, + { + "type": "text", + "title": "About: Artifact traces", + "gridPos": {"h": 9, "w": 8, "x": 16, "y": 122}, + "options": { + "mode": "markdown", + "content": "**What it shows** — p95 latency of artifact-related spans from sampled traces, split by span. Answers \"why is the customer's artifact upload/download slow?\"\n\n**What bad looks like** — one artifact span's p95 climbing into seconds.\n\n**Do next** — to see an individual slow artifact request end to end, use the **Open Traces in Explore** link (top-right), set the service to gorilla and filter for `duration > 3s` and an artifact operation, then read the span chain. If this panel is empty, adjust the span-name regex (see the panel description)." + } + }, + { + "type": "bargauge", + "title": "Top parquet/glue spans by call count (sampled)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 9, "w": 16, "x": 0, "y": 131}, + "targets": [ + { + "expr": "topk(15, sum by (span_name) (increase(traces_spanmetrics_calls_milliseconds_total{span_name=~\".*(parquet|glue).*\"}[$__range])))", + "legendFormat": "{{span_name}}", + "instant": true, + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "short", "min": 0, "decimals": 0}, "overrides": []}, + "options": {"orientation": "horizontal", "displayMode": "gradient", "showUnfilled": true} + }, + { + "type": "text", + "title": "About: Parquet & glue spans", + "gridPos": {"h": 9, "w": 8, "x": 16, "y": 131}, + "options": { + "mode": "markdown", + "content": "**What it shows** — trace-derived (spanmetrics) call counts for spans whose name mentions parquet or glue. This is the background export/compaction work as seen from tracing.\n\n**Note** — spanmetrics are **not** namespace-scoped and are **sampled**: trust the *shape* (which span dominates), not exact counts.\n\n**Do next** — to see an actual slow/failed export, use **Open Traces in Explore** (top-right) and filter by the span name shown here. If this panel is empty, the span name pattern may differ on your stack — check `label_values(traces_spanmetrics_calls_milliseconds_total, span_name)`." + } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": ["wandb", "application", "api", "graphql", "observability"], + "templating": { + "list": [ + { + "name": "namespace", + "label": "W&B Namespace", + "type": "query", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "definition": "label_values(wandb_application_info, namespace)", + "query": { + "query": "label_values(wandb_application_info, namespace)", + "refId": "namespace-variable-query" + }, + "regex": "/^(?!kube-|cert-manager$|monitoring$|wandb-operators$|operator-system$|grafana$|victoria.*$|telemetry$).*$/", + "refresh": 1, + "sort": 1, + "includeAll": false, + "multi": false, + "current": {"text": "wandb", "value": "wandb"} + } + ] + }, + "time": {"from": "now-3h", "to": "now"}, + "timezone": "browser", + "title": "W&B Application", + "uid": "wandb-application", + "version": 1 +} diff --git a/deploy/telemetry/dashboards/wandb-field-investigation.json b/deploy/telemetry/dashboards/wandb-field-investigation.json index 166cc9f6..a85c6553 100644 --- a/deploy/telemetry/dashboards/wandb-field-investigation.json +++ b/deploy/telemetry/dashboards/wandb-field-investigation.json @@ -41,6 +41,16 @@ "title": "W&B Managed Install Performance", "type": "link", "url": "/d/wandb-managed-install-performance" + }, + { + "asDropdown": true, + "icon": "dashboard", + "includeVars": true, + "keepTime": true, + "targetBlank": false, + "title": "Deep-dives", + "type": "dashboards", + "tags": ["observability"] } ], "panels": [ @@ -48,23 +58,23 @@ "type": "text", "title": "", "transparent": false, - "gridPos": {"h": 6, "w": 24, "x": 0, "y": 0}, + "gridPos": {"h": 9, "w": 24, "x": 0, "y": 0}, "options": { "mode": "markdown", - "content": "# W&B Field Investigation\n\nFirst-look diagnostic for the W&B application stack. Use this dashboard to answer **\"are users seeing problems, and where in the W&B app stack?\"**\n\nFor underlying infrastructure (MySQL, Redis, Kafka, SeaweedFS, ClickHouse, container resources) see the [W&B Managed Install Performance](/d/wandb-managed-install-performance) dashboard.\n\n**Quick lookups from the shell:**\n\n- Install size & version: `kubectl get wandb -n -o yaml`\n- Per-pod images: `kubectl get pods -n -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{.spec.containers[*].image}{\\\"\\n\\\"}{end}'`\n- Operator logs: `kubectl logs -n wandb-operators -l control-plane=controller-manager --tail=200`\n\nUse the **W&B Namespace** dropdown above to switch installs." + "content": "# W&B Field Investigation — start here\n\nFirst-look triage for the W&B app. Below is the golden-signal view; the panels answer **\"are users seeing problems?\"** Once you know the symptom, jump to the matching deep-dive (also in the **Deep-dives** menu, top-right).\n\n**Symptom → dashboard**\n\n| Customer says… | Open |\n|---|---|\n| Whole app slow or erroring | [Application](/d/wandb-application) |\n| Specific queries fail / errors | [Application](/d/wandb-application) · *GraphQL errors* section |\n| Charts won't load / data slow | [Application](/d/wandb-application) · *Operations & background* section |\n| Artifact won't upload/download | [Application](/d/wandb-application) · *Operations & background* section |\n| DB/cache is the bottleneck | [Managed Install](/d/wandb-managed-install-performance) |\n| Infra host health (CPU/mem/disk) | [Managed Install](/d/wandb-managed-install-performance) |\n\n**Handoff to engineering:** capture a **Grafana snapshot** of what you're seeing (Share → Snapshot) — data ages out fast (short retention). A plain dashboard export has no data.\n\n**Shell lookups:** version `kubectl get wandb -n -o yaml` · operator logs `kubectl logs -n wandb-operators -l control-plane=controller-manager --tail=200`. Switch installs with the **W&B Namespace** dropdown." } }, { "type": "row", "title": "Service & Versions", - "gridPos": {"h": 1, "w": 24, "x": 0, "y": 6}, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 9}, "collapsed": false }, { "type": "table", "title": "Service Image Versions", "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, - "gridPos": {"h": 12, "w": 16, "x": 0, "y": 7}, + "gridPos": {"h": 12, "w": 16, "x": 0, "y": 10}, "targets": [ { "expr": "wandb_application_info{namespace=\"$namespace\"}", @@ -96,7 +106,7 @@ { "type": "text", "title": "About: Service Image Versions", - "gridPos": {"h": 12, "w": 8, "x": 16, "y": 7}, + "gridPos": {"h": 12, "w": 8, "x": 16, "y": 10}, "options": { "mode": "markdown", "content": "### What is this?\n\nThe currently-running container image for each managed W&B service in this install, split into image / tag / digest columns.\n\n### Why it matters\n\n- **Version lag** — the tag column tells you whether a fix has shipped here yet.\n- **Mid-upgrade drift** — if `api` is on a newer tag than `parquet`, a rollout is in progress (or got stuck) and behavior may be inconsistent.\n- **Internal registry** — the image path shows whether services are pulling from your internal mirror.\n\n### How to read it\n\nOne row per managed W&B service. If a service appears with two rows, that service is mid-rollout." @@ -105,14 +115,14 @@ { "type": "row", "title": "W&B API Health", - "gridPos": {"h": 1, "w": 24, "x": 0, "y": 19}, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 22}, "collapsed": false }, { "type": "timeseries", "title": "W&B API Request Latency (p50 / p95 / p99)", "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, - "gridPos": {"h": 10, "w": 16, "x": 0, "y": 20}, + "gridPos": {"h": 10, "w": 16, "x": 0, "y": 23}, "targets": [ { "expr": "histogram_quantile(0.50, sum by(le)(rate(http_server_request_duration_seconds_bucket{service_name=\"gorilla\"}[$__rate_interval])))", @@ -150,7 +160,7 @@ { "type": "text", "title": "About: W&B API Latency", - "gridPos": {"h": 10, "w": 8, "x": 16, "y": 20}, + "gridPos": {"h": 10, "w": 8, "x": 16, "y": 23}, "options": { "mode": "markdown", "content": "### What is this?\n\nThe p50/p95/p99 latency of all HTTP requests served by the W&B API. Chart loads, run-table queries, the workspace, and SDK sync all flow through this histogram.\n\n### Why it matters\n\nLatency above 3 seconds sustained for more than ~5 minutes means users are feeling it. Brief spikes under load are tolerable.\n\n### How to read it\n\nThis is a single aggregate across all routes. For a per-route view of which operations are slow, see the **Slow Operations** section below." @@ -160,7 +170,7 @@ "type": "timeseries", "title": "W&B API Request Rate by Status Code", "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, - "gridPos": {"h": 10, "w": 16, "x": 0, "y": 30}, + "gridPos": {"h": 10, "w": 16, "x": 0, "y": 33}, "targets": [ { "expr": "sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name=\"gorilla\"}[$__rate_interval]))", @@ -180,7 +190,7 @@ { "type": "text", "title": "About: W&B API Request Rate", - "gridPos": {"h": 10, "w": 8, "x": 16, "y": 30}, + "gridPos": {"h": 10, "w": 8, "x": 16, "y": 33}, "options": { "mode": "markdown", "content": "### What is this?\n\nW&B API requests per second, broken down by HTTP response status code.\n\n### Why it matters\n\n- A spike in **5xx** means the API is failing — check pod logs and resource saturation.\n- A spike in **4xx** is usually client-side: SDK version mismatch, auth issues, malformed queries.\n- A sudden drop in **2xx** to near-zero usually means traffic is no longer reaching the API (ingress / service / DNS issue).\n\n### What to do\n\nFor 5xx: `kubectl logs -n -l app.kubernetes.io/name=*-api --tail=200`\n\nFor 4xx: confirm the SDK version. The **Service Image Versions** panel above shows the server side." @@ -189,14 +199,15 @@ { "type": "row", "title": "Ingest", - "gridPos": {"h": 1, "w": 24, "x": 0, "y": 40}, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 43}, "collapsed": false }, { "type": "timeseries", "title": "Filestream HTTP Request Rate by Status", + "description": "TODO: empty until gorilla emits an http.route label — only `/ping` is set today, so the `file_stream` filter matches nothing. The unsampled filestream signal is DogStatsD, which needs GORILLA_STATSD_PORT>0 (currently 0).", "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, - "gridPos": {"h": 10, "w": 16, "x": 0, "y": 41}, + "gridPos": {"h": 10, "w": 16, "x": 0, "y": 44}, "targets": [ { "expr": "sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name=\"gorilla\", http_route=~\".*file_stream.*\"}[$__rate_interval]))", @@ -216,7 +227,7 @@ { "type": "text", "title": "About: Filestream", - "gridPos": {"h": 10, "w": 8, "x": 16, "y": 41}, + "gridPos": {"h": 10, "w": 8, "x": 16, "y": 44}, "options": { "mode": "markdown", "content": "### What is this?\n\nRate of HTTP requests to the filestream endpoints, broken down by response status code. Filestream is the SDK's primary ingest path — every `wandb.log()` call from a running training job ends up here.\n\n### Why it matters\n\n- A **5xx spike** means the API is rejecting writes. Active runs will see SDK warnings and data may be delayed or dropped.\n- A **4xx spike** is usually SDK clients sending malformed or unauthorized requests — often a version mismatch.\n- A sudden **drop in 2xx** means traffic stopped reaching the API (ingress, service, DNS, or pod crash).\n\n### What to do\n\nFor 5xx: check API pod logs and resource saturation (see the **W&B Managed Install Performance** dashboard for container resources)." @@ -225,26 +236,26 @@ { "type": "row", "title": "Slow Operations", - "gridPos": {"h": 1, "w": 24, "x": 0, "y": 51}, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 54}, "collapsed": false }, { "type": "text", "title": "About: Slow Operations", - "gridPos": {"h": 4, "w": 24, "x": 0, "y": 52}, + "gridPos": {"h": 4, "w": 24, "x": 0, "y": 55}, "options": { "mode": "markdown", - "content": "Top W&B API routes by **slow request count** (requests taking longer than 3 seconds) over the selected time range. Use this to identify which operations are contributing most to user-visible slowness. To drill into individual slow traces, open the **Open Traces in Explore** link in the dashboard header and filter by `service.name=gorilla` and `duration>3s`." + "content": "Top W&B API operations (trace span names) by **slow request count** (requests taking longer than 2.5s) over the selected time range. Sourced from spanmetrics (`traces_spanmetrics_*`) because gorilla's HTTP metrics carry no route label. Use this to identify which operations are contributing most to user-visible slowness. To drill into individual slow traces, open the **Open Traces in Explore** link in the dashboard header and filter by `service.name=gorilla` and `duration>2.5s`." } }, { "type": "table", - "title": "Top Routes by Slow Request Count (>3s)", + "title": "Top Operations by Slow Request Count (>2.5s)", "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, - "gridPos": {"h": 12, "w": 24, "x": 0, "y": 56}, + "gridPos": {"h": 12, "w": 24, "x": 0, "y": 59}, "targets": [ { - "expr": "topk(20, sum by (http_route) (increase(http_server_request_duration_seconds_count{service_name=\"gorilla\"}[$__range]) - increase(http_server_request_duration_seconds_bucket{service_name=\"gorilla\", le=\"3\"}[$__range])))", + "expr": "topk(20, sum by (span_name) (increase(traces_spanmetrics_duration_milliseconds_count{service_name=\"gorilla\"}[$__range]) - increase(traces_spanmetrics_duration_milliseconds_bucket{service_name=\"gorilla\", le=\"2500\"}[$__range])))", "format": "table", "instant": true, "refId": "A" @@ -255,8 +266,8 @@ "id": "organize", "options": { "excludeByName": {"Time": true}, - "indexByName": {"http_route": 0, "Value": 1}, - "renameByName": {"http_route": "Route", "Value": "Slow requests (>3s) in range"} + "indexByName": {"span_name": 0, "Value": 1}, + "renameByName": {"span_name": "Operation", "Value": "Slow requests (>2.5s) in range"} } } ], diff --git a/deploy/telemetry/dashboards/wandb-managed-install-performance.json b/deploy/telemetry/dashboards/wandb-managed-install-performance.json index 340ff97e..d5023356 100644 --- a/deploy/telemetry/dashboards/wandb-managed-install-performance.json +++ b/deploy/telemetry/dashboards/wandb-managed-install-performance.json @@ -376,50 +376,160 @@ }, { "type": "timeseries", - "title": "Kafka Messages In by Topic", + "title": "Kafka Request Rate by API", "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, "gridPos": {"h": 10, "w": 16, "x": 0, "y": 90}, "targets": [ { - "expr": "sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesinpersec_total[$__rate_interval]))", - "legendFormat": "{{topic}}", + "expr": "sum by (kafka_api_key) (rate(bufstream_kafka_request_count_total{namespace=\"$namespace\"}[$__rate_interval]))", + "legendFormat": "{{kafka_api_key}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {"unit": "reqps", "min": 0}, + "overrides": [] + }, + "options": { + "legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}, + "tooltip": {"mode": "multi"} + } + }, + { + "type": "stat", + "title": "Bufstream Status", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 4, "w": 8, "x": 16, "y": 90}, + "targets": [ + { + "expr": "max(bufstream_status{namespace=\"$namespace\"})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "red", "value": 1}]} + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "value" + } + }, + { + "type": "timeseries", + "title": "Kafka Consumer Lag by Topic/Partition", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 10, "w": 16, "x": 0, "y": 100}, + "targets": [ + { + "expr": "max by (kafka_topic_name, kafka_topic_partition) (bufstream_kafka_consumer_group_offset_lag{namespace=\"$namespace\"})", + "legendFormat": "{{kafka_topic_name}} p{{kafka_topic_partition}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {"unit": "short", "min": 0}, + "overrides": [] + }, + "options": { + "legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}, + "tooltip": {"mode": "multi"} + } + }, + { + "type": "timeseries", + "title": "Kafka Request Latency p95 by API", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 10, "w": 8, "x": 16, "y": 100}, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, kafka_api_key) (rate(bufstream_kafka_request_latency_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])))", + "legendFormat": "{{kafka_api_key}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {"unit": "s", "min": 0}, + "overrides": [] + }, + "options": { + "legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}, + "tooltip": {"mode": "multi"} + } + }, + { + "type": "timeseries", + "title": "Kafka Byte Throughput (in/out)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 10, "w": 16, "x": 0, "y": 110}, + "targets": [ + { + "expr": "sum(rate(bufstream_kafka_request_bytes_sum{namespace=\"$namespace\"}[$__rate_interval]))", + "legendFormat": "in (requests)", "refId": "A" }, { - "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount)", - "legendFormat": "offline partitions (total)", + "expr": "sum(rate(bufstream_kafka_response_bytes_sum{namespace=\"$namespace\"}[$__rate_interval]))", + "legendFormat": "out (responses)", "refId": "B" } ], "fieldConfig": { - "defaults": {"min": 0}, + "defaults": {"unit": "Bps", "min": 0}, "overrides": [] }, "options": { - "legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "min", "max", "sum"]}, + "legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}, + "tooltip": {"mode": "multi"} + } + }, + { + "type": "timeseries", + "title": "Partitions by Topic", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 10, "w": 8, "x": 16, "y": 110}, + "targets": [ + { + "expr": "sum by (kafka_topic_name) (bufstream_kafka_topic_partition_count{namespace=\"$namespace\"})", + "legendFormat": "{{kafka_topic_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {"unit": "short", "min": 0}, + "overrides": [] + }, + "options": { + "legend": {"displayMode": "table", "placement": "bottom", "calcs": ["last"]}, "tooltip": {"mode": "multi"} } }, { "type": "text", "title": "About: Kafka", - "gridPos": {"h": 10, "w": 8, "x": 16, "y": 90}, + "gridPos": {"h": 6, "w": 8, "x": 16, "y": 94}, "options": { "mode": "markdown", - "content": "### What is this?\n\n- **Messages in by topic** — incoming message rate per Kafka topic, in messages/sec.\n- **offline partitions** — partitions that have no leader and can't accept reads or writes. Should always be 0.\n\n### Why it matters\n\nKafka is the asynchronous backbone for run-state events, parquet-export task scheduling, and other internal queues. When Kafka throughput drops or partitions go offline, downstream consumers stall and user-visible operations slow down.\n\n### What to do\n\n- **Offline partitions > 0**: investigate managed Kafka health immediately. `kubectl get pods -n -l weightsandbiases.apps.wandb.com/component=kafka`, inspect logs, and check that Bufstream and etcd pods are Ready.\n- **Sudden drop in messages**: check whether the producer-side services (api, filestream) are healthy in the W&B Field Investigation dashboard." + "content": "### What is this?\n\nManaged Kafka is **Bufstream** — stateless and object-storage-backed, so broker-era metrics like offline / under-replicated partitions don't exist.\n\n- **Bufstream Status** — `max(bufstream_status)` across the health probes (etcd, object storage, metadata, kafka). **0 = healthy**; ≥1 means a probe is failing.\n- **Request rate / latency** — Kafka API throughput and p95 latency by request type (`bufstream_kafka_request_*`).\n- **Consumer lag** — committed-offset lag per topic/partition (`bufstream_kafka_consumer_group_offset_lag`); sustained growth means consumers are falling behind.\n- **Byte throughput** — bytes in/out on the wire.\n\n### What to do\n\n**Status ≥ 1 or lag climbing**: check managed Kafka health — `kubectl get pods -n -l weightsandbiases.apps.wandb.com/component=kafka` and inspect the Bufstream and etcd pods." } }, { "type": "row", "title": "ClickHouse", - "gridPos": {"h": 1, "w": 24, "x": 0, "y": 100}, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 120}, "collapsed": false }, { "type": "timeseries", "title": "ClickHouse Memory, Connections, Merges", "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, - "gridPos": {"h": 10, "w": 16, "x": 0, "y": 101}, + "gridPos": {"h": 10, "w": 16, "x": 0, "y": 121}, "targets": [ { "expr": "sum(ClickHouseMetrics_MemoryTracking)", @@ -451,7 +561,7 @@ { "type": "text", "title": "About: ClickHouse", - "gridPos": {"h": 10, "w": 8, "x": 16, "y": 101}, + "gridPos": {"h": 10, "w": 8, "x": 16, "y": 121}, "options": { "mode": "markdown", "content": "### What is this?\n\n- **memory tracking** — total bytes ClickHouse is currently using across all queries.\n- **HTTP connections** — active HTTP client connections to ClickHouse.\n- **merges in progress** — background MergeTree merge operations currently running.\n\n### Why it matters\n\n- **Memory** climbing toward the configured limit signals query-side pressure. ClickHouse aborts queries when memory is exhausted, which surfaces as errors in dependent services.\n- **HTTP connections** persistently near zero may indicate clients can't reach ClickHouse.\n- **Merges** that climb without coming back down indicate ingest exceeding merge throughput; this leads to part-count limits and eventual write rejections.\n\n### What to do\n\nFor sustained memory pressure or merge backlog, increase the install size, which scales ClickHouse resources. For per-query memory diagnostics, query `system.query_log` directly on the ClickHouse pod." @@ -460,14 +570,14 @@ { "type": "row", "title": "Object Store", - "gridPos": {"h": 1, "w": 24, "x": 0, "y": 111}, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 131}, "collapsed": false }, { "type": "timeseries", "title": "Object Store Capacity Used % & Request Rate", "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, - "gridPos": {"h": 10, "w": 16, "x": 0, "y": 112}, + "gridPos": {"h": 10, "w": 16, "x": 0, "y": 132}, "targets": [ { "expr": "100 * (1 - (sum(SeaweedFS_volumeServer_resource{type=\"free\"}) / sum(SeaweedFS_volumeServer_resource{type=\"all\"})))", @@ -503,7 +613,7 @@ { "type": "text", "title": "About: Object Store", - "gridPos": {"h": 10, "w": 8, "x": 16, "y": 112}, + "gridPos": {"h": 10, "w": 8, "x": 16, "y": 132}, "options": { "mode": "markdown", "content": "### What is this?\n\n- **capacity used %** — percentage of usable volume-server capacity in use, derived from `SeaweedFS_volumeServer_resource{type=\"free\"|\"all\"}`.\n- **requests/sec** — filer HTTP request rate across all operations (`SeaweedFS_filer_request_total`).\n\n### Why it matters\n\nSeaweedFS is the object store for parquet files, artifact contents, and run media.\n\n- **Capacity above 90%** = urgent action required. New writes will fail soon.\n- **Capacity above 75%** = plan a storage increase.\n- **Sudden drop in request rate** = the API or executor services may be unable to reach the filer; check the seaweedfs Service and pod logs.\n\n### What to do\n\nFor capacity: increase storage in the WeightsAndBiases CR (`spec.objectStore.managedObjectStore.storageSize`) and re-apply, or migrate older artifacts off-cluster. For request failures: `kubectl logs -n -l app.kubernetes.io/managed-by=seaweedfs-operator,app.kubernetes.io/component=filer --tail=200`." @@ -512,14 +622,14 @@ { "type": "row", "title": "Storage Path Diagnostics", - "gridPos": {"h": 1, "w": 24, "x": 0, "y": 122}, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 142}, "collapsed": false }, { "type": "timeseries", "title": "Slowest Storage Operations (p95, by span)", "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, - "gridPos": {"h": 10, "w": 16, "x": 0, "y": 123}, + "gridPos": {"h": 10, "w": 16, "x": 0, "y": 143}, "targets": [ { "expr": "topk(10, histogram_quantile(0.95, sum by (service_name, span_name, le) (rate(traces_spanmetrics_duration_milliseconds_bucket{service_name=~\"gorilla.*\", span_name=~\"(?i)(parquet|filestream|filehandler|runstore|historystore|metadatastore|s3|sql).*\"}[$__rate_interval]))))", @@ -539,7 +649,7 @@ { "type": "text", "title": "About: Slow Storage Operations", - "gridPos": {"h": 10, "w": 8, "x": 16, "y": 123}, + "gridPos": {"h": 10, "w": 8, "x": 16, "y": 143}, "options": { "mode": "markdown", "content": "### What is this?\n\np95 latency per storage operation, derived from traces by the OTel `spanmetrics` connector. Covers parquet reads/writes, filestream chunks, S3 calls, and SQL statements.\n\n### Why it matters\n\nSlow chart loads, slow file uploads, and stuck artifact downloads almost always trace back to one storage path being slow. This panel surfaces which one without opening individual traces.\n\n### What to do\n\n- Spikes on `ParquetHistoryStore.*` or `HistoryStore.*` → chart loads will feel slow. Check the parquet pod's CPU/memory in **Container Resource Usage** above, and ClickHouse load if installed.\n- Spikes on `FileStreamStore.*` / `FileHandler.*` → ingest stalls. Check filestream container restarts and object store capacity below.\n- Spikes on `sql.*` → MySQL is the bottleneck. See **MySQL Errors & Slow Queries** above.\n- For per-request detail, [open Traces in Explore](/explore?panes=%7B%22A%22:%7B%22datasource%22:%22${DS_VICTORIATRACES}%22,%22queries%22:[],%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1) and filter by the offending `service.name` + `span.name` with `duration > 1s`." diff --git a/deploy/telemetry/dashboards/wandb-telemetry-overview.json b/deploy/telemetry/dashboards/wandb-telemetry-overview.json index c9702b75..b641567c 100644 --- a/deploy/telemetry/dashboards/wandb-telemetry-overview.json +++ b/deploy/telemetry/dashboards/wandb-telemetry-overview.json @@ -1861,7 +1861,7 @@ }, { "type": "stat", - "title": "Kafka Messages / sec", + "title": "Kafka Requests / sec", "datasource": { "type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}" @@ -1874,7 +1874,7 @@ }, "targets": [ { - "expr": "sum(rate(kafka_server_brokertopicmetrics_messagesinpersec_total[$__rate_interval]))", + "expr": "sum(rate(bufstream_kafka_request_count_total{namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -1895,7 +1895,7 @@ }, { "type": "stat", - "title": "Offline Partitions", + "title": "Bufstream Status", "datasource": { "type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}" @@ -1908,7 +1908,7 @@ }, "targets": [ { - "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount)", + "expr": "max(bufstream_status{namespace=\"$namespace\"})", "refId": "A" } ], @@ -2046,7 +2046,7 @@ }, { "type": "timeseries", - "title": "Kafka Messages by Topic", + "title": "Kafka Consumer Lag by Topic", "datasource": { "type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}" @@ -2059,11 +2059,12 @@ }, "targets": [ { - "expr": "sum(rate(kafka_server_brokertopicmetrics_messagesinpersec_total[$__rate_interval])) by (topic)", - "legendFormat": "{{topic}}", + "expr": "max by (kafka_topic_name) (bufstream_kafka_consumer_group_offset_lag{namespace=\"$namespace\"})", + "legendFormat": "{{kafka_topic_name}}", "refId": "A" } - ] + ], + "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []} }, { "type": "timeseries", @@ -2138,6 +2139,87 @@ "refId": "A" } ] + }, + { + "type": "row", + "title": "Telemetry Self-Health (can I trust this data?)", + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 137}, + "collapsed": false + }, + { + "type": "timeseries", + "title": "Span Export Failure %", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 8, "w": 16, "x": 0, "y": 138}, + "targets": [ + { + "expr": "sum(rate(otelcol_exporter_send_failed_spans[$__rate_interval])) / clamp_min(sum(rate(otelcol_exporter_sent_spans[$__rate_interval])) + sum(rate(otelcol_exporter_send_failed_spans[$__rate_interval])), 1)", + "legendFormat": "failed export ratio", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "yellow", "value": 0.01}, {"color": "red", "value": 0.05}]} + }, + { + "type": "text", + "title": "About: Telemetry Self-Health", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 138}, + "options": { + "mode": "markdown", + "content": "### Read this FIRST when a dashboard looks wrong\n\n**What it shows** — the fraction of trace spans the collector failed to export to storage. If this is non-zero, the trace/RED panels across every dashboard are **undercounting** — the data itself is lossy, not the app.\n\n**What bad looks like** — sustained > ~1%. Above ~5% treat trace-derived numbers as unreliable.\n\n**Do next** — if failures are high, check the `victoria-otlp-gateway` collector pod and the VictoriaTraces backend before trusting any trace panel. Also cross-check **Scrape Success** / **Down Targets** at the top of this dashboard for metric-side gaps." + } + }, + { + "type": "timeseries", + "title": "Trace ingestion by receiver (DataDog / OTLP)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 8, "w": 16, "x": 0, "y": 146}, + "targets": [ + { + "expr": "sum by (receiver) (rate(otelcol_receiver_accepted_spans[$__rate_interval]))", + "legendFormat": "{{receiver}}", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "cps", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi"}}, + "description": "Spans/sec the collector accepts, split by receiver. The `datadog` receiver (port 8126) is the DataDog-APM path — background/SDK services (weave-trace, glue, parquet, filemeta, metric-observer, anaconda2) send here; `otlp` is the main gorilla trace path." + }, + { + "type": "text", + "title": "About: Ingestion by receiver", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 146}, + "options": { + "mode": "markdown", + "content": "**What it shows** — whether each ingestion path on the gateway is receiving data. This is how you confirm **DataDog** traces/metrics are populated.\n\n- **`datadog`** — DataDog-APM traces (port 8126). A non-zero line here means DataDog-protocol tracing is flowing; downstream it appears as `traces_spanmetrics_calls_milliseconds_total` for the SDK services.\n- **`otlp`** — the main gorilla OpenTelemetry path (the bulk of spans/metrics).\n- **`statsd`** — DogStatsD metrics (port 8125). Gorilla emits these via `GORILLA_STATSD_ADDRESS` regardless of the `GORILLA_STATSD_PORT` setting (`GORILLA_STATSD_PORT=0` is a vestigial no-op), so datagrams do reach the collector. This line can look thin on an idle cluster because gorilla emits most app metrics only under traffic — not because DogStatsD is disabled.\n\n**What bad looks like** — a receiver you expect traffic on flatlining at 0, or `otelcol_receiver_refused_*` climbing (data arriving but rejected)." + } + }, + { + "type": "timeseries", + "title": "Metric-point ingestion by receiver (DogStatsD / OTLP)", + "datasource": {"type": "victoriametrics-metrics-datasource", "uid": "${DS_VICTORIAMETRICS}"}, + "gridPos": {"h": 8, "w": 16, "x": 0, "y": 154}, + "targets": [ + { + "expr": "sum by (receiver) (rate(otelcol_receiver_accepted_metric_points[$__rate_interval]))", + "legendFormat": "{{receiver}}", + "refId": "A" + } + ], + "fieldConfig": {"defaults": {"unit": "cps", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]}, "tooltip": {"mode": "multi"}}, + "description": "Metric points/sec the collector accepts, split by receiver. Gorilla emits DogStatsD via `GORILLA_STATSD_ADDRESS`, so a `statsd` line can appear here; it may be sparse on an idle cluster because most app metrics are emitted only under traffic." + }, + { + "type": "text", + "title": "About: Receiver errors", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 154}, + "options": { + "mode": "markdown", + "content": "**Pairs with the panel to the left.** If a receiver shows accepted traffic but data still isn't landing in dashboards, check the refused counters:\n\n- `sum by (receiver) (rate(otelcol_receiver_refused_spans[$__rate_interval]))`\n- `sum by (receiver) (rate(otelcol_receiver_refused_metric_points[$__rate_interval]))`\n\n**Accepted > 0, refused ≈ 0** = healthy ingestion. **Refused climbing** = the gateway is receiving but rejecting (bad payloads, queue full) — check the `victoria-otlp-gateway` collector logs." + } } ], "refresh": "30s", diff --git a/deploy/telemetry/templates/telemetry-otlp-gateway.yaml b/deploy/telemetry/templates/telemetry-otlp-gateway.yaml index 98c95a3f..50533b87 100644 --- a/deploy/telemetry/templates/telemetry-otlp-gateway.yaml +++ b/deploy/telemetry/templates/telemetry-otlp-gateway.yaml @@ -22,6 +22,23 @@ data: statsd/dogstatsd: endpoint: 0.0.0.0:8125 aggregation_interval: 15s + # Map DogStatsD timing/histogram/distribution metrics to histograms so + # percentiles (p50/p95/p99) are queryable. Without this the receiver emits + # summaries (count/sum only) and gorilla's operation.duration, dist_*, and + # *_dist_duration families lose their tails — leaving only averages. + timer_histogram_mapping: + - statsd_type: timing + observer_type: histogram + histogram: + max_size: 100 + - statsd_type: histogram + observer_type: histogram + histogram: + max_size: 100 + - statsd_type: distribution + observer_type: histogram + histogram: + max_size: 100 datadog: endpoint: 0.0.0.0:8126 @@ -30,15 +47,28 @@ data: connectors: # Derives RED-style metrics (request count + duration histogram) from - # incoming traces. service.name and span.name are implicit identity; - # http.route is added so dashboards can break down by HTTP route. - # Cardinality budget per Phase 2.5: ~50k series. user.id / - # project.name / entity.name dimensions are deferred to Wave C.2. + # incoming traces. service.name and span.name are implicit identity. + # Dimensions promote span attributes to metric labels for breakdowns: + # - http.route: HTTP route. + # - graphql.service / graphql.operationName: gorilla sets these on its + # OTEL spans (core PR #43519), enabling per-service GraphQL error + # attribution from metrics. graphql.service is bounded (~schema source + # files); graphql.operationName is higher-cardinality — watch the + # ~50k-series budget and drop it to trace-only if threatened. + # user.id / project.name / entity.name remain deferred (too high-card). spanmetrics: namespace: traces_spanmetrics dimensions: - name: http.route default: "" + - name: graphql.service + default: "" + - name: graphql.operationName + default: "" + # Attach trace IDs to metric datapoints so a latency/error spike links + # straight to an example trace. + exemplars: + enabled: true histogram: explicit: buckets: [5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s] diff --git a/deploy/telemetry/templates/telemetry-ui.yaml b/deploy/telemetry/templates/telemetry-ui.yaml index 5d80854f..58550846 100644 --- a/deploy/telemetry/templates/telemetry-ui.yaml +++ b/deploy/telemetry/templates/telemetry-ui.yaml @@ -44,12 +44,18 @@ spec: dashboards: grafana datasource: name: VictoriaMetrics + uid: victoriametrics type: victoriametrics-metrics-datasource access: proxy url: {{ include "telemetry.metricsEndpoint" . | trimSuffix "/opentelemetry/v1/metrics" | quote }} isDefault: true jsonData: timeInterval: 30s + # Pivot from a metric exemplar (spanmetrics attaches trace IDs) to the trace. + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: victoriatraces + urlDisplayLabel: View trace --- apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaDatasource @@ -65,6 +71,7 @@ spec: dashboards: grafana datasource: name: VictoriaLogs + uid: victorialogs type: victoriametrics-logs-datasource access: proxy url: {{ include "telemetry.logsEndpoint" . | trimSuffix "/insert/opentelemetry/v1/logs" | quote }} @@ -85,9 +92,20 @@ spec: dashboards: grafana datasource: name: VictoriaTraces + uid: victoriatraces type: jaeger access: proxy url: {{ printf "%s/select/jaeger" (include "telemetry.tracesEndpoint" . | trimSuffix "/insert/opentelemetry/v1/traces") | quote }} + jsonData: + # Pivot from a trace to its logs in VictoriaLogs, correlating on service name. + tracesToLogsV2: + datasourceUid: victorialogs + spanStartTimeShift: "-1h" + spanEndTimeShift: "1h" + filterByTraceID: false + tags: + - key: service.name + value: service --- apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaDashboard @@ -154,4 +172,28 @@ spec: datasourceName: VictoriaTraces json: |- {{ .Files.Get "dashboards/wandb-managed-install-performance.json" | nindent 4 }} +--- +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: wandb-application + namespace: {{ include "telemetry.namespace" . }} + labels: + app.kubernetes.io/component: telemetry + app.kubernetes.io/part-of: wandb +spec: + uid: wandb-application + folder: W&B + instanceSelector: + matchLabels: + dashboards: grafana + datasources: + - inputName: DS_VICTORIAMETRICS + datasourceName: VictoriaMetrics + - inputName: DS_VICTORIATRACES + datasourceName: VictoriaTraces + - inputName: DS_VICTORIALOGS + datasourceName: VictoriaLogs + json: |- +{{ .Files.Get "dashboards/wandb-application.json" | nindent 4 }} {{- end }} diff --git a/docs/design/wandb_v2/legacy_overrides.md b/docs/design/wandb_v2/legacy_overrides.md new file mode 100644 index 00000000..c0f7a857 --- /dev/null +++ b/docs/design/wandb_v2/legacy_overrides.md @@ -0,0 +1,379 @@ +# Legacy Overrides: carrying v1 helm values into v2 + +## Problem + +The v1 `WeightsAndBiases` spec is untyped: `spec.values` is an arbitrary map passed to +the `operator-wandb` helm chart. Many v1 configurations have no strongly-typed home in +the v2 spec — most importantly: + +- **global env vars** (`global.env`, `global.extraEnv`) applied to every application, +- **per-application env vars** (`.env`, `.extraEnv`), +- **per-application resource overrides** (`.resources`, `.sizing..resources`). + +Until these get first-class v2 fields, we carry them through conversion in +`spec.wandb.legacyOverrides` (added in commit `8a344a7`) and apply them during +reconcile so a converted install keeps its effective v1 configuration. + +```go +// api/v2/weightsandbiases_types.go +LegacyOverrides map[string]LegacyOverrides `json:"legacyOverrides,omitempty"` // on WandbAppSpec + +type LegacyOverrides struct { + Env []corev1.EnvVar `json:"env,omitempty"` + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` +} +``` + +Map keys are **v2 manifest application names**, plus the reserved key **`global`** +(env only) which applies to every application. The server manifest is the +authority on which keys are valid: sections that don't correspond to a manifest +application are logged at reconcile time and never applied — but for now they are +left in place in the spec (see *Manifest validation*). + +## How v1 (the helm chart) actually behaves + +Facts established from `wandb-base/templates/_containers.tpl` and verified with +`helm template` renders; the conversion and reconcile semantics below are derived +from them. + +- Every app key in `operator-wandb/values.yaml` is an aliased instance of the + `wandb-base` library chart, so all apps support the same env/resources keys. +- Env layers are **maps** keyed by env var name; values are either scalars + (string-coerced) or full `EnvVar` bodies (`valueFrom:` supported). Precedence + (highest → lowest), collapsing to one entry per name: + 1. `.containers..env` + 2. `.env` + 3. `.extraEnv` + 4. `global.env` + 5. `global.extraEnv` + 6. `sizing..env` + 7. chart-computed `envTpls` (an entry is *removed* when any layer above defines the same name) + 8. `envFrom` config maps/secrets +- **User env at any layer beats chart-computed env.** This is the load-bearing + behavior the v2 reconcile must reproduce: overrides must win against + manifest-provided env. +- Resources merge deep, per-field: `containers..resources` > `.resources` + (legacy flat key) > `sizing..resources`, where effective size is + `coalesce(.size, global.size, "small")` and the size entry is overlaid onto + `sizing.default`. +- All env layers reach main containers, init containers, and Job/CronJob pods. + `Values.resources`/sizing resources reach main containers only. + +## Design + +The server manifest is the single authority on which applications exist — +there is no hardcoded list of helm app keys anywhere. Conversion resolves the +manifest itself: `mapVersion` runs first and derives `spec.wandb.version` from +`app.image.tag`/`api.image.tag`, and the manifest for that version is an +immutable artifact fetched through the same `manifest.GetServerManifest` +resolver the reconciler uses (OCI via ORAS, or `file://`). This keeps +conversion stateless in the way that matters: it remains a pure function of +(values, version) — the manifest is just versioned static data. Successful +fetches need no extra caching (the resolver is local-first: once a version is +in its on-disk ORAS store, no network is involved); only *failures* are +remembered in-process for a minute, because the store retries the remote on +every call and a conversion webhook stalling for the fetch timeout (15 s) on +every v1 write would make an unreachable registry very painful. + +Reconcile-time validation still exists as a second line — it guards +hand-edited v2 CRs and version drift — mirroring the existing split for +external-infra literals (`mapMySQL` → pending annotation → +`migrateLegacyAnnotations`). + +### Phase 1: Conversion (v1 → v2) + +A new mapper `mapLegacyOverrides(values, dst)` runs in `applyValueMappings` +(`api/v1/weightsandbiases_conversion_mapping.go`) alongside the other +peer-of-global mappers (it reads top-level app keys, like `mapVersion` does). +It is pure extraction — stateless, no client — over the `resolveValues()` output, +so the spec-active Secret's coalesced values are preferred, same as every other +mapper. + +**Manifest-driven extraction.** Conversion resolves the server manifest for the +converted version and iterates `manifest.Applications`: for each application it +reads the top-level values section named by the application's **`legacyKey`** +when the manifest sets one (renamed helm aliases, e.g. `nginx-proxy` declares +`legacyKey: nginx`, `weave-trace-evaluate-model-worker` declares +`legacyKey: weave-evaluate-model-worker`), else by the application name — and +extracts its env/extraEnv/resources. The operator carries **no rename table**: +even the helm-key knowledge ships with the manifest (`legacyKey` is set +upstream in the wandb/core server-manifest generator; a manifest that predates +the field simply leaves renamed sections unmapped, logged like any other). +Only manifest applications are ever copied, so the spec never carries junk +keys, and a new application in a future manifest needs no conversion change. + +Sections that carry the override shape we extract (`env`/`extraEnv`/`sizing`) +but match no manifest application — the v1 monolith `app`, `console`, +`history-updater`, `mcp-server`, … — are **logged and skipped** at conversion. +(The flat `resources` key is deliberately not part of that detection signature: +infra subchart sections like `mysql` legitimately carry it and would be false +positives.) There is **no `app` → `api` translation**: the monolith's overrides +were tuned for a different binary, and grafting them onto v2's `api` risks more +than it fixes. Skipped sections stay recoverable in the +`legacy.operator.wandb.com/v1-values` annotation. + +**Failure containment.** Manifest resolution is best-effort and never fails +conversion: a fetch error (offline cluster, version with no published manifest) +would otherwise make v1 objects unservable and v1 writes impossible. On error — +or when values yield no version at all — only the per-application extraction is +skipped, with a log; global env still converts, and a later re-apply (once the +manifest is reachable) re-extracts everything. + +**Env extraction.** + +- `legacyOverrides["global"].Env` = `merge(global.env over global.extraEnv)`. +- `legacyOverrides[].Env` = `merge(.env over .extraEnv)`. +- Map-shaped values decode into `corev1.EnvVar` via JSON round-trip (name from the + map key); malformed bodies fail conversion with a `spec.values.` error, + matching existing mapper behavior. Scalars go through the existing + `scalarToString` (bools/numbers become strings, as helm's `toString` did). +- Entries whose string value contains `{{` are **dropped with a log line, never a + conversion failure**: they are helm template expressions we cannot evaluate, + and failing conversion would block serving the object over one env var. + Kubernetes `$(VAR)` interpolation passes through untouched. +- Env slices are **sorted by name** so conversion is deterministic and the + v2 → v1 → v2 round-trip is idempotent (required by `TestConvertRoundTrip`). + +**Resources extraction.** Per candidate key, deep-merge exactly what the user set — +`sizing.default.resources`, then `sizing..resources`, then +`.resources` — into one `ResourceRequirements`. We extract only fragments +present in the resolved values (user values + release-channel defaults; chart +defaults never appear there), so an install that never touched resources converts +with no resource overrides and v2 manifest sizing applies untouched. No resources +are extracted for `global` (the chart has no global resources). + +`ConvertFrom` needs **no reverse mapping** — it already restores v1 purely from +the v1-values annotation. + +### Phase 2: Manifest validation at reconcile (log-only, no pruning for now) + +`validateLegacyOverrides(ctx, wandb, manifest)` runs in the v2 reconcile +immediately after the server manifest is resolved and before +`reconcileApplications`. With conversion already filtering against the +manifest, this is a second line of defense for hand-edited v2 CRs and for +version drift (the spec's overrides were extracted against one manifest +version; the reconciler may be running another): + +- Valid keys are the reserved `"global"` plus any name in + `manifest.Applications`. Validity is judged against the full application map, + **not** the feature-filtered set — an override for a feature-gated app is + valid and takes effect if the feature is enabled. +- Every other key is logged at Warn once per reconcile pass: `legacy override + section %q does not map to any application in server manifest %s; ignoring`. +- **The spec is not modified.** Unmapped keys stay in + `spec.wandb.legacyOverrides`; they are simply never applied, because the apply + path only ever looks up `"global"` and manifest application names. Pruning the + spec was considered and deferred — we can revisit once migration behavior has + been observed in the field. + +### Phase 3: Applying overrides during reconcile + +All changes are on the `WeightsAndBiases` side (`internal/controller/reconciler/`); +the `Application` controller copies pod templates verbatim and needs no changes. +`reconcileApplications`, `resolveContainers`, and `ResolveResources` already +receive the full `*v2.WeightsAndBiases`, so no plumbing is required. The apply +path looks up only `"global"` and the current app's manifest name, so unmapped +keys left in the spec are inert here by construction. + +**Env.** In `reconcileApplications` (`reconcile_v2.go`), after all existing env +construction (manifest env, telemetry injection, custom-CA injection) and +immediately before `resolveContainers`: + +```go +envVars = overrideEnvVars(envVars, wandb.Spec.Wandb.LegacyOverrides["global"].Env) +envVars = overrideEnvVars(envVars, wandb.Spec.Wandb.LegacyOverrides[app.Name].Env) +``` + +`overrideEnvVars(base, overrides)` is a new helper: replace-by-name in place, +append when missing — the inverse of `appendMissingEnvVars` (existing wins), which +cannot be reused. Ordering: global first, per-app second, so per-app beats global — +matching the chart's layer precedence. Applying last means overrides beat +manifest/common env, telemetry defaults, and CA env — exactly as user env beat +chart-computed `envTpls` in v1. Because the env slice is shared, init containers +receive the same overrides, which also matches v1. + +**Resources.** In `ResolveResources` (`sizing.go`), overlay the per-app override as +the final merge step, after the container-level merge: + +```go +if lo, ok := wandb.Spec.Wandb.LegacyOverrides[app.Name]; ok && lo.Resources != nil { + resources = mergeResources(resources, lo.Resources, wandb.Spec.RequireLimits) +} +``` + +The overlay respects `spec.requireLimits`, like every other merge in +`ResolveResources`: when `requireLimits=false` (the default), only the requests +from a legacy override are applied and its limits are stripped. This keeps the +v2 no-limits-by-default policy uniform across sizing- and legacy-derived +resources; converted requests are preserved either way, and setting +`requireLimits: true` re-enables the converted limits. (Deliberate divergence +from v1, where configured limits were always enforced — to be reevaluated.) +`ResolveResources` runs per main container only — init containers get no +`Values.resources`/sizing merge, same as v1. The `global` entry's `Resources` is +ignored (documented; there is no v1 analog). + +**Migrations.** The `global` env entry is also applied (same helper) to migration +task env in `runMigrations`, since v1's global env reached job pods too — the +canonical use case is `HTTP_PROXY`/`NO_PROXY`, which migrations need as much as +the apps do. Per-app entries do not apply to migrations (no v1 analog). + +**Hand-authored content.** Editing `legacyOverrides` directly in a v2 CR is +discouraged but not prevented — and we assume it will happen. So there is no +blocking webhook validation beyond the CRD schema, the field's doc comment (and +`docs/config-api.md`) steer users toward first-class fields, and the apply path +is defensive: entries with an empty name are skipped with a log, +`overrideEnvVars` resolves duplicate names within an override deterministically +(last entry wins), and unknown map keys are already logged and ignored by +manifest validation. Invalid env var names or resource quantities the CRD schema +can't catch surface as Deployment create/update errors on the owned +`Application`, same as any other bad pod-template input. + +### Precedence summary (v2, after this change) + +Highest → lowest for an application's container env: + +1. `legacyOverrides[].env` +2. `legacyOverrides["global"].env` +3. custom-CA / telemetry injected env (append-if-missing passes) +4. manifest `app.Env` +5. manifest `CommonEnvs` groups + +Resources, per main container: `legacyOverrides[].resources` > +manifest container resources > manifest `sizing[spec.size]` > manifest +`sizing.default`. + +### Flow + +```mermaid +flowchart LR + subgraph v1["v1 CR (untyped)"] + V[spec.values
global.env / extraEnv
app.env / resources / sizing] + end + subgraph conv["Conversion webhook (api/v1)"] + RV[resolveValues
spec-active Secret or spec.values] --> MLO[mapLegacyOverrides
fetch manifest for mapVersion's version
iterate manifest apps via legacyKey/name
env map → EnvVar list + sizing merge
log + skip unmapped sections] + end + subgraph rec["v2 reconcile"] + GM[GetServerManifest] --> PR[validateLegacyOverrides
log keys not in
manifest.Applications] + PR --> ENV[resolveEnvvars + telemetry + CA] --> OV[overrideEnvVars
global then per-app] --> RC[resolveContainers] + PR --> RR[ResolveResources
+ legacy overlay] --> RC + end + V --> RV + MLO --> LO["spec.wandb.legacyOverrides"] + LO --> PR +``` + +## Out of scope (dropped, preserved only in the v1-values annotation) + +- `.containers..env|resources` (per-container overrides) +- `envFrom` (configMapRef/secretRef maps), `envTpls`, `global.extraEnvFrom` +- helm-template-valued env entries (`{{ ... }}`) +- `sizing..env` (t-shirt env vars; the v2 manifest owns these) +- infra subchart keys (`mysql`, `redis`, `kafka`, `clickhouse`, …) — already + mapped to typed fields by the other conversion mappers, or genuinely not apps + +Sections for apps with no v2 counterpart (`app`, `console`, `history-updater`, +`mcp-server`, job/hook keys, …) are logged and skipped at conversion — they +never enter the spec. + +## Implementation plan + +### Step 0 — API cleanup (before anything references the field) + +1. `api/v2/weightsandbiases_types.go`: rename Go field `LegacyOveriddes` → + `LegacyOverrides` (JSON tag already correct, so this is API-compatible); + change `Resources` to `*corev1.ResourceRequirements` (`omitempty` is a no-op on + struct values, and presence must be distinguishable); add doc comments — + including an explicit note that the field is populated by v1→v2 conversion + and hand-editing is discouraged in favor of first-class fields. (This repo + generates CRDs with `maxDescLen=0`, so the comments serve godoc/readers of + the types, not the CRD schema.) +2. `make manifests generate sync-crd-embed`. + +### Step 1 — Conversion + +3. New `api/v1/weightsandbiases_conversion_overrides.go`: + - `mapLegacyOverrides(values map[string]interface{}, dst *appsv2.WeightsAndBiases) error`, + registered in `applyValueMappings` after `mapIngress` (so `mapVersion` has + already derived the version). + - `legacyManifestApps`: resolves the manifest via + `manifest.GetServerManifest` (repository = the shared + `appsv2.DefaultManifestRepository` constant, also used by the defaulting + webhook) with a per-(repository, version) failure cooldown — successes are + already cached on disk by the resolver's ORAS store — and a + `SetConversionManifestGetter` test seam; failures skip per-app extraction + with a log. + - `LegacyKey` field on the manifest `Application` type (set upstream by the + wandb/core server-manifest generator; also added to the local dev + manifests under `hack/testing-manifests/server-manifest/`), plus helpers + following existing idioms (`unstructured.Nested*`, errors prefixed + `spec.values.`): env over extraEnv merge, scalar coercion, strict + EnvVar-body decode, `{{` skip, name-sorted output; resources = sizing + default → effective size → flat `resources` merge, using + `coalesce(.size, global.size, "small")`; unmapped-section logging + keyed on the `env`/`extraEnv`/`sizing` shape. +4. Tests in `api/v1/weightsandbiases_conversion_overrides_test.go` (plain Go + + `require`, `newV1(values)` fixtures, fake manifest getter installed by a + package `TestMain` so unit tests never fetch over the network, + `withConversionReader` for spec-active Secret cases): global/per-app + extraction and env/extraEnv precedence, scalar coercion, `valueFrom` bodies, + template-string drop, renames, unmapped sections skipped, resources/size + selection, manifest-unavailable and no-version fallbacks, per-version fetch + caching, round-trip idempotency, spec-active Secret preference. + +### Step 2 — Manifest validation + +5. `validateLegacyOverrides` in `internal/controller/reconciler/` (new file + `legacy_overrides.go`), called from the v2 reconcile after + `GetServerManifest`, before `reconcileApplications`; Warn log per unmapped + key; **no spec mutation, no CR update**. +6. Tests (in-memory manifest, no client needed): valid keys accepted, `global` + always accepted, unmapped keys reported, feature-gated apps accepted, + spec unchanged afterward. + +### Step 3 — Applying overrides + +7. `overrideEnvVars` helper + unit tests (including hand-authored edge cases: + duplicate names within an override → last wins; empty-name entries skipped + with a log). +8. Apply global + per-app env in `reconcileApplications` before + `resolveContainers`; apply global env in `runMigrations`. +9. Legacy overlay in `ResolveResources` (+ `RequireLimits` behavior tests in + `reconcile_v2_sizing_test.go` style). +10. Tests: plain unit tests in package `reconciler` (fake client only where + `resolveEnvvars` is exercised), covering the precedence table above. + +### Step 4 — Verification & docs + +11. `make lint && make test`. +12. Add a v1 fixture with env/resources overrides (including one deliberately + unmapped section, e.g. `console`) under `hack/testing-manifests/wandb/` and + verify end-to-end via Tilt/Kind: create the v1 CR, confirm `legacyOverrides` + on the stored v2 object, confirm the unmapped section is logged and remains + in the spec without affecting any Deployment, confirm env and resources land + on the rendered Deployments, confirm + round-trip (`kubectl get wandb.v1... -o yaml` still shows original values). +13. User-facing documentation lives in the API type's doc comments and this + design doc. (`docs/config-api.md` turned out to be v1 console-API docs — the + wrong venue; revisit if a v2 CR reference doc is added.) + +## Resolved design decisions + +| Decision | Choice | Rationale | +|---|---|---| +| Authority on valid app names | the server manifest, resolved *during conversion* (and re-checked at reconcile) | no hardcoded app list to rot; the manifest is immutable versioned data, so fetching it keeps conversion a pure function of (values, version) | +| Manifest fetch failure / no version | skip per-app extraction with a log, never fail conversion; global env still converts | erroring would make v1 objects unservable for offline clusters or versions with no published manifest | +| Unmapped sections | logged and skipped at conversion; reconcile re-checks spec keys (hand-edits, version drift) and leaves them in place | visibility without destructive spec edits; only manifest apps ever enter the spec | +| helm `app` (monolith) overrides | not translated to `api`; unmapped, so logged and skipped | monolith env/resources were tuned for a different binary; grafting them onto v2 `api` causes more problems than it solves | +| Renamed helm keys | `legacyKey` field on the manifest `Application`, set upstream | no rename table in the operator; the manifest owns all application knowledge, helm aliases included | +| Global env representation | reserved `"global"` map key, merged at reconcile | keeps CR small; apps added by newer manifests still inherit it | +| Override vs manifest env | overrides win (replace-by-name) | mirrors v1, where user env at any layer displaced chart-computed env | +| Legacy limits vs `requireLimits=false` | respect `requireLimits`: limits stripped unless it's true | keeps the v2 no-limits-by-default policy uniform; reevaluate later if migrated installs need their v1 limits back | +| Resources: replace or merge | deep-merge overlay per field | mirrors helm's deep merge (verified: overriding `requests.cpu` kept chart limits) | +| Malformed env bodies | fail conversion | consistent with `classifyValueFromOrLiteral` and other mappers | +| Helm-templated env values (`{{ ... }}`) | log and drop, never fail conversion | can't be evaluated outside helm; failing would block serving the object over one env var | +| Hand-editing `legacyOverrides` | discouraged, not prevented; assumed to happen | no blocking webhook validation; doc comments + docs steer to first-class fields; apply path is defensive (empty names skipped, duplicates last-wins, unknown keys logged + ignored) | +| Reverse (v2→v1) mapping | none | `ConvertFrom` restores from the v1-values annotation already | + +## Open questions + +None — all design decisions are resolved above. diff --git a/docs/openshift.md b/docs/openshift.md new file mode 100644 index 00000000..51312218 --- /dev/null +++ b/docs/openshift.md @@ -0,0 +1,201 @@ +# Deploying on OpenShift + +This guide covers installing the W&B operator on an OpenShift Container Platform +(OCP) cluster, the OpenShift-specific configuration the operator needs, and the +known limitations of running under OpenShift's default `restricted-v2` Security +Context Constraint (SCC). + +For **local** OpenShift development with CRC + Tilt, see +[`config/openshift-dev/README.md`](../config/openshift-dev/README.md) instead — +that path is automated and does not require the manual steps below. + +## Why OpenShift needs special handling + +OpenShift admits every pod through an SCC. The default `restricted-v2` SCC is +stricter than upstream Kubernetes defaults: it assigns each pod an **arbitrary +UID** from the namespace's `openshift.io/sa.scc.uid-range`, forbids running as a +fixed UID, drops all capabilities, disallows privileged ports (`<1024`), and +requires a `runtime/default` seccomp profile. + +Several components the operator manages ship images that assume a fixed UID or a +privileged port, so they must be adapted. The operator does this automatically +when it knows it is running on OpenShift, driven by two switches: + +| Switch | Where | Effect | +| --- | --- | --- | +| `OPENSHIFT=true` env on the operator | `profiles/openshift.yaml` | Makes `utils.IsOpenShift()` true, so managed infra specs omit fixed UID/GID and the Kafka pods get a dedicated SA bound to `nonroot-v2`. | +| `openshift.enabled=true` chart value | `profiles/openshift.yaml` | Renders the OpenShift-only RBAC/SCC templates (`openshift-owner-finalizers-rbac.yaml`, `openshift-scc-rbac.yaml`). | + +Both are set for you by the `profiles/openshift.yaml` values overlay. + +## Prerequisites + +- OpenShift 4.x cluster. +- `cluster-admin` (or equivalent) for the install: the chart creates + cluster-scoped RBAC and SCC grants. +- [`helm`](https://github.com/helm/helm) 3.x and [`oc`](https://formulae.brew.sh/formula/openshift-cli). +- A W&B server version and (for production) a container image registry the + cluster can pull from. + +## Known limitations + +| Component | Limitation | Status / workaround | +| --- | --- | --- | +| **Ingress / Frontend (`frontend-nginx`)** | The bundled frontend image runs as a fixed, non-numeric user (`nginx`) that owns `/usr/share/nginx/html` and rewrites files there at startup. `restricted-v2`'s arbitrary UID cannot write, and `nonroot-v2` rejects the pod because the kubelet cannot verify a non-numeric user is non-root. | **BYO ingress required.** Front W&B with your own ingress/route. | +| **Kafka (bufstream)** | The distroless broker image ships a `0700` binary owned by a fixed UID (65532) that can only be executed as that exact user, so it cannot run under `restricted-v2`. | Currently runs under `nonroot-v2` (via a dedicated ServiceAccount) rather than `restricted-v2`. | +| **Cluster-scoped install** | The chart creates SCC grants and cluster-scoped RBAC. | Requires `cluster-admin` at install time. | + +## Required: bring your own ingress + +On OpenShift you **must** supply your own ingress/edge. The bundled frontend does +not run under OpenShift's `restricted-v2` SCC (see +[Known limitations](#known-limitations)). + +- **Ingress (BYO required).** Front W&B with the cluster's own edge — an + OpenShift `Route` or your ingress controller. +- **Object storage (optional BYO).** Managed SeaweedFS runs under `restricted-v2` + (its S3 gateway binds an unprivileged port, so no root/`anyuid` grant is + needed). You can still point the CR at an external object store (S3, GCS, Azure + Blob, or any S3-compatible endpoint you run) via + `spec.objectStore.externalObjectStore` if you prefer. See + [Infrastructure Connection Settings](infra-connection-settings.md). + +The rest of the managed infra (MySQL, Redis, ClickHouse, Kafka) is supported on +OpenShift via the adaptations described below. + +## Deploying + +### 1. Install the operator with the OpenShift profile + +From a checkout of this repository: + +```bash +helm install wandb-operator ./deploy/operator \ + --namespace wandb-operators --create-namespace \ + -f deploy/operator/profiles/openshift.yaml +``` + +Installing from the published OCI chart works the same way, but you must supply +the OpenShift values yourself (the `-f` file must be a local path). Save the +snippet below as `openshift-values.yaml`: + +```yaml +openshift: + enabled: true +wandb-operator: + podSecurityContext: + runAsUser: null + runAsGroup: null + fsGroup: null + fsGroupChangePolicy: null + containers: + operator: + env: + OPENSHIFT: + value: "true" +redis-operator: + podSecurityContext: { runAsUser: null, runAsGroup: null, fsGroup: null, fsGroupChangePolicy: null } +altinity-clickhouse-operator: + podSecurityContext: { runAsUser: null, runAsGroup: null, fsGroup: null, fsGroupChangePolicy: null } +seaweedfs-operator: + podSecurityContext: { runAsUser: null, runAsGroup: null, fsGroup: null } +moco: + extraArgs: + - --disable-default-security-context +grafana-operator: + isOpenShift: true +``` + +```bash +helm install wandb-operator \ + oci://us-docker.pkg.dev/wandb-production/public/wandb/charts/operator \ + --namespace wandb-operators --create-namespace \ + -f openshift-values.yaml +``` + +### 2. Apply a `WeightsAndBiases` resource + +Object storage can be managed (SeaweedFS runs under `restricted-v2`) or external. +The example below wires an external object store; omit the `objectStore` block to +use the managed default. When bringing your own, provide the connection details +in a Secret and reference its keys: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: wandb-object-store + namespace: wandb +stringData: + bucket: my-wandb-bucket + region: us-east-1 + accessKey: + secretKey: + # endpoint/port only for non-AWS, S3-compatible stores (e.g. MinIO) + # endpoint: minio.example.com + # port: "9000" +--- +apiVersion: apps.wandb.com/v2 +kind: WeightsAndBiases +metadata: + name: wandb + namespace: wandb +spec: + size: small + retentionPolicy: + onDelete: detach + wandb: + version: + networking: + mode: ingress + objectStore: + externalObjectStore: + bucket: { name: wandb-object-store, key: bucket } + region: { name: wandb-object-store, key: region } + accessKey: { name: wandb-object-store, key: accessKey } + secretKey: { name: wandb-object-store, key: secretKey } + # endpoint: { name: wandb-object-store, key: endpoint } + # port: { name: wandb-object-store, key: port } +``` + +```bash +kubectl apply -f wandb.yaml +``` + +On OpenShift you must front the deployment with the cluster's own edge — an +OpenShift `Route` or your ingress controller — not a bundled load balancer or +the bundled frontend. See +[`docs/infra-connection-settings.md`](infra-connection-settings.md) for +networking and object-store connection options. + +### 3. Verify + +```bash +oc get pods -n wandb +``` + +Every managed pod should reach `Running`. You can confirm the SCC each pod was +admitted under with: + +```bash +oc get pods -n wandb \ + -o custom-columns=NAME:.metadata.name,SCC:'.metadata.annotations.openshift\.io/scc' +``` + +Managed infra pods run under `restricted-v2`, except the Kafka broker/etcd pods, +which run under `nonroot-v2` (see below). + +## What the OpenShift profile changes + +- **wandb-operator, crd-installer, and the dependency operators** (redis, + Altinity ClickHouse, SeaweedFS) drop their hardcoded `runAsUser`/`runAsGroup`/ + `fsGroup`, so OpenShift assigns compliant IDs at admission. +- **MySQL (moco)** runs the controller with `--disable-default-security-context` + so it does not inject a fixed UID/GID (10000) that `restricted-v2` rejects. +- **Kafka (bufstream)** gets a dedicated ServiceAccount bound to the + `nonroot-v2` SCC, because its distroless broker image ships a `0700` binary + owned by a fixed UID (65532) that can only be executed as that exact user. +- **OwnerReferencesPermissionEnforcement** RBAC is rendered so the built-in + StatefulSet controller (moco PVCs) can set finalizers on the resources it owns + — OpenShift's admission plugin requires this and it is a no-op on upstream + Kubernetes. diff --git a/docs/pod-labeling-standards.md b/docs/pod-labeling-standards.md new file mode 100644 index 00000000..1146e826 --- /dev/null +++ b/docs/pod-labeling-standards.md @@ -0,0 +1,271 @@ +# Pod Labeling Standards + +This document defines the labeling standards for all pods (and their controlling +workloads) that the W&B operator creates or manages, with a dedicated section for +non-pod resources such as ServiceAccounts. It is the contract that NetworkPolicies, +dashboards, metrics, and `kubectl` selectors depend on, so treat these labels as a +stable, public API. + +## Goals + +- Every operator-managed pod is identifiable by a **single, consistent** set of labels. +- Standard ecosystem tooling (kube-state-metrics, Grafana, Lens, k9s, ArgoCD, + `kubectl`) works out of the box. +- The operator has **stable, immutable, collision-free** selectors for its own + ownership, pruning, and retention logic. +- Users can write portable NetworkPolicies and selectors against a documented label. + +## The two label families + +We deliberately maintain **two** label families, each with a distinct job. Do not +collapse them into one. + +| Family | Prefix | Purpose | Mutable? | +|--------|--------|---------|----------| +| **Standard / descriptive** | `app.kubernetes.io/*` | Interop with ecosystem tooling; human legibility | Yes — informational only | +| **Operator / ownership** | `weightsandbiases.apps.wandb.com/*` | Workload `spec.selector` and the operator's own list/match/retention logic | No — immutable selector anchor | + +**Rule of thumb:** if a human or third-party tool reads it, it's `app.kubernetes.io/*`. +If the operator matches on it or it backs an immutable `spec.selector`, it's +`weightsandbiases.apps.wandb.com/*`. Both families appear on every managed pod. + +## Standard labels (`app.kubernetes.io/*`) + +Apply all of the following to every operator-managed pod template. + +| Label | Value | Example | +|-------|-------|---------| +| `app.kubernetes.io/name` | The **software/service** that runs in the pod | `api`, `executor`, `mysql`, `weave-trace` | +| `app.kubernetes.io/instance` | The **owning `WeightsAndBiases` CR name** (the release) | `wandb` | +| `app.kubernetes.io/component` | The **architectural role** the workload plays | `server`, `worker`, `proxy`, `database`, `cache` | +| `app.kubernetes.io/part-of` | Always `wandb` | `wandb` | +| `app.kubernetes.io/managed-by` | Always `wandb-operator` | `wandb-operator` | +| `app.kubernetes.io/version` | W&B server version (optional but recommended) | `0.79.0` | + +### `name` vs `component` + +These are different axes and MUST NOT be treated as synonyms: + +- `name` answers *"what software is this?"* — the service/binary/image (`api`, + `mysql`, `redis`). +- `component` answers *"what role does it play?"* — its place in the architecture + (`server`, `database`, `cache`, `worker`). + +They coincide only in the degenerate case of a standalone app that is its own single +role. They diverge whenever the software name differs from its role (`mysql` → +`database`), when one binary runs in multiple roles (`weave-trace` server vs worker), +or when you want tier-level grouping (`component: server` matches every stateless web +service at once). Use `name` for per-service targeting and `component` for per-tier +targeting. + +### Rules + +- `app.kubernetes.io/part-of: wandb` MUST be present on **every** managed pod. It + is the single anchor that matches the entire deployment and the documented + NetworkPolicy selector. +- `app.kubernetes.io/instance` MUST be the CR/release name, **not** the namespace. + (This corrects the current app-pod behavior where `instance` is set to the + namespace.) +- `app.kubernetes.io/name` MUST come from the service vocabulary and + `app.kubernetes.io/component` from the role vocabulary (see the Vocabularies + section). No free-form values. +- These labels are **descriptive**. Never use them as a `Deployment`/`StatefulSet` + `spec.selector`, because Helm and users routinely override them and selectors are + immutable. + +## Operator labels (`weightsandbiases.apps.wandb.com/*`) + +These back the immutable `spec.selector` and the operator's ownership queries. Keep +the set **minimal and low-cardinality**. + +| Label | Value | +|-------|-------| +| `weightsandbiases.apps.wandb.com/name` | The owning CR name | +| `weightsandbiases.apps.wandb.com/namespace` | The owning CR namespace | +| `weightsandbiases.apps.wandb.com/component` | The **service identity** (equivalent to `app.kubernetes.io/name`, e.g. `mysql`) | + +### Naming caveat + +The operator family's `/component` key does **not** hold the role-based value from +`app.kubernetes.io/component`. For historical reasons (`common.BuildWandbLabels` +populates it from the module/service name), it carries the **service identity** — +i.e. it lines up with `app.kubernetes.io/name`, not the role. This is intentional: +the selector needs to be unique *per service*, and the service name is the stable, +collision-free key for that. Do not "fix" this to match the role vocabulary; doing +so would change immutable selectors. + +### Rules + +- These are produced by `common.BuildWandbLabels(wandb, component)` — use that + helper, do not hand-roll the keys. +- The subset used in a workload `spec.selector` is **immutable**. Once a workload + exists you cannot change its selector; altering it requires deleting and + recreating the workload. Treat any change here as a breaking migration. +- Never put high-cardinality or user-mutable values here. + +## Non-pod resources + +This standard is written for **workload pods**, but the operator also creates +ServiceAccounts, Services, Roles/RoleBindings, Secrets, and ConfigMaps. Apply the +labels to those as follows. + +- **Descriptive identity labels — apply everywhere.** Every operator-created object + MUST carry `app.kubernetes.io/part-of: wandb`, + `app.kubernetes.io/managed-by: wandb-operator`, + `app.kubernetes.io/instance: `, and (where known) + `app.kubernetes.io/version`. This keeps the whole release queryable and + attributable regardless of resource kind. + +- **`name` / `component` role model — pods only, with judgement for others.** The + service/role split is meaningful for a workload that runs a specific service in a + specific role. For a resource dedicated to one service (e.g. that service's own + `Service` object), set `name`/`component` to match its pods. For shared or + role-less resources, omit them rather than inventing a value. + +- **Operator / selector family — pods and their workloads only.** The + `weightsandbiases.apps.wandb.com/*` family exists to back immutable `spec.selector` + fields and pod-ownership queries. Non-pod resources have no `spec.selector`, so it + is not required on them (the operator already tracks them via owner references). + +### The shared ServiceAccount + +There is a single ServiceAccount (default `wandb`) **shared by every application +pod**, so it has no single service identity or architectural role. It is explicitly +**exempt from `name` and `component`**. It MUST still carry the descriptive identity +labels (`part-of`, `managed-by`, `instance`, and `version` when available), and user +annotations continue to flow from `spec.wandb.serviceAccount.annotations`. + +## Vocabularies + +Two closed vocabularies feed the labels above. If a new workload type is introduced, +extend both lists in the same PR. + +### Service names (`app.kubernetes.io/name`) + +The service/software identity. Sourced from the manifest application name or infra +module name. + +- Applications: `api`, `executor`, `filestream`, `filemeta`, `glue`, `parquet`, + `weave`, `weave-trace`, `weave-trace-worker`, `nginx-proxy`, + `flat-run-fields-updater`, `metric-observer`. +- Infrastructure: `mysql`, `redis`, `clickhouse`, `kafka`, `seaweedfs`. +- Operational: `migration`. + +### Component roles (`app.kubernetes.io/component`) + +The architectural role. Keep this list small and generic. + +- `server` — stateless request-serving apps. +- `worker` — async/background processors. +- `proxy` — ingress/edge proxies. +- `database` — relational stores. +- `cache` — in-memory caches. +- `analytics-db` — columnar/analytics stores. +- `queue` — message/streaming brokers. +- `object-storage` — blob/object stores. +- `migration` — one-shot migration/init jobs. + +### Mapping + +| Workload | `name` | `component` | +|----------|--------|-------------| +| api | `api` | `server` | +| executor | `executor` | `worker` | +| filestream | `filestream` | `server` | +| parquet | `parquet` | `worker` | +| weave-trace | `weave-trace` | `server` | +| weave-trace-worker | `weave-trace` | `worker` | +| nginx-proxy | `nginx-proxy` | `proxy` | +| MySQL | `mysql` | `database` | +| Redis | `redis` | `cache` | +| ClickHouse | `clickhouse` | `analytics-db` | +| Kafka | `kafka` | `queue` | +| SeaweedFS | `seaweedfs` | `object-storage` | +| migration/init job | `migration` | `migration` | + +Note that `name` and `component` differ for most workloads. They coincide only where +the service *is* its own single role (e.g. the `migration` job). + +## Worked example + +A pod for the `api` application in a CR named `wandb`, running server version +`0.79.0`, should carry: + +```yaml +metadata: + labels: + # Standard / descriptive + app.kubernetes.io/name: api # the service + app.kubernetes.io/instance: wandb + app.kubernetes.io/component: server # the role + app.kubernetes.io/part-of: wandb + app.kubernetes.io/managed-by: wandb-operator + app.kubernetes.io/version: 0.79.0 + # Operator / ownership (selector anchor) + weightsandbiases.apps.wandb.com/name: wandb + weightsandbiases.apps.wandb.com/namespace: wandb-system + weightsandbiases.apps.wandb.com/component: api # service identity (see Naming caveat) +``` + +The workload `spec.selector` matches only on the operator family, e.g.: + +```yaml +spec: + selector: + matchLabels: + weightsandbiases.apps.wandb.com/name: wandb + weightsandbiases.apps.wandb.com/component: api +``` + +## Using the labels + +### NetworkPolicies + +Document `app.kubernetes.io/part-of: wandb` as the anchor for the whole deployment. +Use `app.kubernetes.io/name` to target a **specific service** and +`app.kubernetes.io/component` to target a **whole tier** (e.g. every `database`). A +`NetworkPolicy` `podSelector` is independent of the workload's immutable +`spec.selector`, so it is safe to select on the descriptive labels here. + +```yaml +# Restrict the MySQL service's ingress to W&B pods only +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: wandb-mysql-restrict +spec: + podSelector: + matchLabels: + app.kubernetes.io/part-of: wandb + app.kubernetes.io/name: mysql # this specific service + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/part-of: wandb + ports: + - { protocol: TCP, port: 3306 } +``` + +To apply a rule to every storage tier at once, select on the role instead — e.g. +`app.kubernetes.io/component: database` for all relational stores. + +### Metrics and dashboards + +kube-state-metrics exposes `app.kubernetes.io/*` as metric labels. Scope to a release +with `app.kubernetes.io/instance`, break down a single service with +`app.kubernetes.io/name`, and roll up a tier with `app.kubernetes.io/component`. + +### Ad-hoc queries + +```bash +# Everything in a release +kubectl get pods -l app.kubernetes.io/part-of=wandb,app.kubernetes.io/instance=wandb + +# One specific service +kubectl get pods -l app.kubernetes.io/name=clickhouse + +# A whole tier (all databases) +kubectl get pods -l app.kubernetes.io/component=database +``` diff --git a/docs/releasing-v1.md b/docs/releasing-v1.md new file mode 100644 index 00000000..e16bb148 --- /dev/null +++ b/docs/releasing-v1.md @@ -0,0 +1,35 @@ +# Releasing Operator v1 + +Operator v1 releases are prepared through a reviewed pull request and published +from an annotated `v1.x.y` tag. The release workflow never writes to the `v1` +branch. + +1. Open a release pull request against `v1` containing the reviewed changelog + entry and all intended release changes. +2. Merge the pull request after its required checks and approvals pass. +3. Update the local branch and confirm it exactly matches the remote branch: + + ```bash + git switch v1 + git pull --ff-only origin v1 + test "$(git rev-parse HEAD)" = "$(git rev-parse origin/v1)" + ``` + +4. Create and push an annotated release tag at that commit: + + ```bash + version=v1.22.1 + git tag -a "${version}" -m "Operator ${version}" + git push origin "${version}" + ``` + +5. Monitor the `Release v1` GitHub Actions workflow. When it succeeds, record + the Docker Hub, Quay.io, and GitHub Release URLs and image digests in the + release record. +6. Never move, delete, or reuse an exact `v1.x.y` release tag. If a published + release is incorrect, fix it with a new patch version. + +The workflow publishes `1.x.y`, `1.x`, `1`, and `latest` to +`docker.io/wandb/controller`. It publishes `1.x.y`, `1.x`, and `1` to +`quay.io/wandb_tools/wandb-k8s-operator`; Quay.io does not receive a `latest` +tag. diff --git a/docs/releasing.md b/docs/releasing.md index 8ddefc5f..1c7d7f17 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,24 +1,61 @@ -# Release +# Releasing Operator v2 -## Description +Production v2 releases are prepared through a reviewed pull request and +published from an annotated `v2.x.y` tag. A single workflow publishes the +operator image and Helm chart from the same commit. -In this document, we'll go over on how to properly create a release, and push it. +## Production release -## Creating a Release +1. Open a release pull request against `main`. +2. Update `wandb.version` in `deploy/operator/values.yaml` to the intended W&B + server release. +3. Set all three operator versions to the release number without a leading + `v`: + - `version` in `deploy/operator/Chart.yaml` + - `appVersion` in `deploy/operator/Chart.yaml` + - `wandb-operator.image.tag` in `deploy/operator/values.yaml` +4. Run the chart validation commands used by CI: -1. Create a branch off `v2` -- it's irrelevant of what the branch name is. -2. Think of the tag you'd like to update to, and we'll continue to use **tag** to reference this. For example, `2.0.0-alpha.3`. -3. In the [operator values](../deploy/operator/values.yaml), update the following: - 1. The version at the top under `wandb:` is pointing to the latest server release. This server release should be the latest that was cut specifically for On-Prem. - 2. The version under `wandb-operator:` needs to update the tag version to the one desired on point 2. -4. In the [operator chart](../deploy/operator/Chart.yaml), update the `version` and `appVersion` to the desired tag. -5. Ensure the [chart repos](..deploy/ct.yaml) match to that of the [chart](..deploy/operator/Chart.yaml). If there's missing ones, please proceed to add them, or update it. -6. Alas, run `ct lint --config deploy/ct.yaml` from root directory of this repository to ensure things will pass -7. Create a PR and get manager approval to merge and create the release + ```bash + helm dependency build deploy/operator + ct lint --all --config deploy/ct.yaml + helm lint --strict deploy/operator + ``` -## Pushing the Release +5. Merge the pull request after all required checks and approvals pass. +6. Update the local branch and confirm it exactly matches the remote branch: -Ensure the PR created has been merged onto `v2` branch. + ```bash + git switch main + git pull --ff-only origin main + test "$(git rev-parse HEAD)" = "$(git rev-parse origin/main)" + ``` -1. Run the following GitHub Action: [Internal Image Publish](https://github.com/wandb/operator/actions/workflows/internal-image-publish.yaml) off the `v2` branch and use the desired tag that was used in part 1. -2. Run the following GitHub Action: [Internal Chart Publish](https://github.com/wandb/operator/actions/workflows/internal-chart-publish.yaml) off the `v2` branch +7. Create and push an annotated release tag at that commit: + + ```bash + version=v2.0.0 + git tag -a "${version}" -m "Operator ${version}" + git push origin "${version}" + ``` + +8. Monitor the `Release v2` workflow. It publishes the versioned GAR image + first, then the matching OCI Helm chart, and creates the GitHub Release only + after both artifacts succeed. +9. Record the source commit, image digest, chart digest, and GitHub Release URL + in the release record. + +Production versions are immutable. Never move, delete, reuse, or overwrite a +`v2.x.y` tag or its `2.x.y` image/chart tags. If a release is incorrect or only +partially publishes, fix it with a new patch version. The production workflow +does not publish a `latest` tag. + +## Development artifacts + +The `Internal Image Publish` workflow accepts only tags in the form +`dev--<7-to-40-character-sha>`, for example +`dev-bucket-proxy-1106901`. It cannot publish production-style tags. + +The `Internal Chart Publish` workflow accepts the prerelease version already +declared in `deploy/operator/Chart.yaml`, such as `2.0.0-rc.1`. It rejects +stable versions and refuses to overwrite an existing prerelease chart tag. diff --git a/go.mod b/go.mod index b4adc4c5..22fb427a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/wandb/operator -go 1.26.3 +go 1.26.5 godebug default=go1.26 @@ -10,25 +10,27 @@ require ( github.com/Masterminds/semver/v3 v3.4.0 github.com/cybozu-go/moco v0.34.0 github.com/go-logr/logr v1.4.3 - github.com/go-playground/validator/v10 v10.28.0 - github.com/golang-jwt/jwt/v4 v4.5.2 + github.com/go-playground/validator/v10 v10.30.3 github.com/golang/glog v1.2.5 github.com/google/uuid v1.6.0 github.com/imdario/mergo v0.3.16 github.com/kedacore/keda/v2 v2.18.3 github.com/lmittmann/tint v1.1.2 - github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 - github.com/miekg/dns v1.1.65 + github.com/maxbrunsfeld/counterfeiter/v6 v6.12.2 github.com/nginx/nginx-gateway-fabric v1.6.2 github.com/onsi/ginkgo/v2 v2.28.1 - github.com/onsi/gomega v1.39.1 + github.com/onsi/gomega v1.42.1 github.com/opencontainers/image-spec v1.1.1 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 github.com/samber/lo v1.52.0 github.com/sanity-io/litter v1.3.0 github.com/stretchr/testify v1.11.1 + github.com/twmb/franz-go v1.21.3 + github.com/twmb/franz-go/pkg/kadm v1.18.0 golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 - golang.org/x/text v0.37.0 + golang.org/x/text v0.38.0 gopkg.in/d4l3k/messagediff.v1 v1.2.1 gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v3 v3.19.2 @@ -36,10 +38,9 @@ require ( k8s.io/apiextensions-apiserver v0.35.3 k8s.io/apimachinery v0.35.3 k8s.io/client-go v0.35.3 - k8s.io/klog/v2 v2.140.0 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 knative.dev/pkg v0.0.0-20250326102644-9f3e60a9244c - oras.land/oras-go/v2 v2.6.0 + oras.land/oras-go/v2 v2.6.2 sigs.k8s.io/controller-runtime v0.22.4 sigs.k8s.io/gateway-api v1.5.0 sigs.k8s.io/yaml v1.6.0 @@ -61,7 +62,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.3 // indirect - github.com/containerd/containerd v1.7.29 // indirect + github.com/containerd/containerd v1.7.33 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect @@ -69,23 +70,20 @@ require ( github.com/d4l3k/messagediff v1.2.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/docker-credential-helpers v0.9.4 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/evanphx/json-patch v5.9.11+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect - github.com/expr-lang/expr v1.17.6 // indirect + github.com/expr-lang/expr v1.17.7 // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.1 // indirect - github.com/gabriel-vasile/mimetype v1.4.10 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect - github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect - github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.22.5 // indirect github.com/go-openapi/jsonreference v0.21.5 // indirect github.com/go-openapi/swag v0.25.5 // indirect @@ -104,7 +102,6 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gobwas/glob v0.2.3 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/cel-go v0.26.0 // indirect github.com/google/gnostic-models v0.7.1 // indirect @@ -112,7 +109,7 @@ require ( github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect @@ -121,19 +118,16 @@ require ( github.com/jmoiron/sqlx v1.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.6 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect - github.com/klauspost/crc32 v1.3.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-sqlite3 v1.14.32 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/miekg/dns v1.1.65 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect @@ -144,73 +138,59 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect - github.com/prometheus/prom2json v1.4.1 // indirect - github.com/prometheus/prometheus v0.304.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect - github.com/rs/xid v1.6.0 // indirect github.com/rubenv/sql-migrate v1.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/safchain/ethtool v0.5.10 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect - github.com/secure-io/sio-go v0.3.1 // indirect - github.com/shirou/gopsutil/v3 v3.24.5 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect - github.com/tinylib/msgp v1.3.0 // indirect - github.com/tklauser/go-sysconf v0.3.15 // indirect - github.com/tklauser/numcpus v0.10.0 // indirect - github.com/twmb/franz-go v1.21.3 // indirect - github.com/twmb/franz-go/pkg/kadm v1.18.0 // indirect github.com/twmb/franz-go/pkg/kmsg v1.13.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xlab/treeprint v1.2.0 // indirect - github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect + go.opentelemetry.io/otel/log v0.11.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect - go.opentelemetry.io/proto/otlp v1.7.1 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.51.0 // indirect - golang.org/x/mod v0.35.0 // indirect - golang.org/x/net v0.53.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.44.0 // indirect - golang.org/x/term v0.43.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.44.0 // indirect + golang.org/x/tools v0.45.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect - google.golang.org/grpc v1.80.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/apiserver v0.35.3 // indirect k8s.io/cli-runtime v0.35.3 // indirect k8s.io/component-base v0.35.3 // indirect + k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 // indirect k8s.io/kubectl v0.35.3 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect diff --git a/go.sum b/go.sum index 6de3c083..615f09c7 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.3 h1:9liNh8t+u26xl5ddmWLmsOsdNLwkdRTg5AG+JnTiM80= github.com/chai2010/gettext-go v1.0.3/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= -github.com/containerd/containerd v1.7.29 h1:90fWABQsaN9mJhGkoVnuzEY+o1XDPbg9BTC9QTAHnuE= -github.com/containerd/containerd v1.7.29/go.mod h1:azUkWcOvHrWvaiUjSQH0fjzuHIwSPg1WL5PshGP4Szs= +github.com/containerd/containerd v1.7.33 h1:iAkYGC/ifR/V+0eR4iXWHNGYUF0DF2PmGV5iz4Irj5M= +github.com/containerd/containerd v1.7.33/go.mod h1:gSbSCVjPCdkfJCjyrzz7aRC+xFlqVbatNpfHfVCYGUM= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= @@ -80,8 +80,6 @@ github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= @@ -90,8 +88,8 @@ github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjT github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= -github.com/expr-lang/expr v1.17.6 h1:1h6i8ONk9cexhDmowO/A64VPxHScu7qfSl2k8OlINec= -github.com/expr-lang/expr v1.17.6/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= +github.com/expr-lang/expr v1.17.7 h1:Q0xY/e/2aCIp8g9s/LGvMDCC5PxYlvHgDZRQ4y16JX8= +github.com/expr-lang/expr v1.17.7/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -104,8 +102,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/gabriel-vasile/mimetype v1.4.10 h1:zyueNbySn/z8mJZHLt6IPw0KoZsiQNszIpU+bX4+ZK0= -github.com/gabriel-vasile/mimetype v1.4.10/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= @@ -116,8 +114,6 @@ github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8b github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -125,9 +121,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= -github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE= @@ -168,8 +161,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688= -github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU= +github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8= +github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= @@ -179,11 +172,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= -github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= @@ -209,8 +199,8 @@ github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -233,15 +223,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kedacore/keda/v2 v2.18.3 h1:PY3o80tzBAzCffS8J6eBZctT4B+g/r26zErNSMleC50= github.com/kedacore/keda/v2 v2.18.3/go.mod h1:gaFzDtqtXg6KPmMcwzeCjv7aoK8Z1I2NaUkqHkREbDk= -github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= -github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= -github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -260,8 +243,6 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lmittmann/tint v1.1.2 h1:2CQzrL6rslrsyjqLDwD11bZ5OpLBPU+g3G/r5LSfS8w= github.com/lmittmann/tint v1.1.2/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= -github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= -github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -273,10 +254,8 @@ github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlzz2rTuu18Ra7M8sU= -github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ= +github.com/maxbrunsfeld/counterfeiter/v6 v6.12.2 h1:V23nK2R2B63g2GhygF9zVGpnigmhvoZoH8d0hrZwMGY= +github.com/maxbrunsfeld/counterfeiter/v6 v6.12.2/go.mod h1:Mr897yU9FmyKaQDPtRlVKibrjz40XXyOHUfyZBPSyZU= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc= @@ -303,8 +282,8 @@ github.com/nginx/nginx-gateway-fabric v1.6.2 h1:ktdShWxT/Drh/5/u8S5QMRgnnBGvVuFD github.com/nginx/nginx-gateway-fabric v1.6.2/go.mod h1:Fi2hdmoNj9nQRX9YQDju+ntMPG4Fgcw+irfl/GYWSEk= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= -github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= -github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= +github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= +github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -313,8 +292,6 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= -github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -323,8 +300,6 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= -github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= @@ -335,10 +310,6 @@ github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTU github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/prometheus/prom2json v1.4.1 h1:7McxdrHgPEOtMwWjkKtd0v5AhpR2Q6QAnlHKVxq0+tQ= -github.com/prometheus/prom2json v1.4.1/go.mod h1:CzOQykSKFxXuC7ELUZHOHQvwKesQ3eN0p2PWLhFitQM= -github.com/prometheus/prometheus v0.304.2 h1:HhjbaAwet87x8Be19PFI/5W96UMubGy3zt24kayEuh4= -github.com/prometheus/prometheus v0.304.2/go.mod h1:ioGx2SGKTY+fLnJSQCdTHqARVldGNS8OlIe3kvp98so= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U= github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc= @@ -352,14 +323,10 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rubenv/sql-migrate v1.8.0 h1:dXnYiJk9k3wetp7GfQbKJcPHjVJL6YK19tKj8t2Ns0o= github.com/rubenv/sql-migrate v1.8.0/go.mod h1:F2bGFBwCU+pnmbtNYDeKvSuvL6lBVtXDXUUv5t+u1qw= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/safchain/ethtool v0.5.10 h1:Im294gZtuf4pSGJRAOGKaASNi3wMeFaGaWuSaomedpc= -github.com/safchain/ethtool v0.5.10/go.mod h1:w9jh2Lx7YBR4UwzLkzCmWl85UY0W2uZdd7/DckVE5+c= github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw= github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= github.com/sanity-io/litter v1.3.0 h1:5ZO+weUsqdSWMUng5JnpkW/Oz8iTXiIdeumhQr1sSjs= @@ -368,16 +335,8 @@ github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEV github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= -github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc= -github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= -github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= @@ -416,12 +375,6 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= -github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= -github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= -github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= -github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= -github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= github.com/twmb/franz-go v1.21.3 h1:q9Mo8ri+OwBQBjKqrerNCqNWJlJnUDe2qnYsj2V3hdI= github.com/twmb/franz-go v1.21.3/go.mod h1:rfoMTnVk7107fhTGxfEKIHP/e7tPe6oyij/ywzO0czk= github.com/twmb/franz-go/pkg/kadm v1.18.0 h1:WRf/LZmDdcDXwX7WMbtDU++v+b3NzYh2bCGoPMmzirw= @@ -432,8 +385,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= -github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= -github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 h1:UW0+QyeyBVhn+COBec3nGhfnFe5lwB0ic1JBVjzhk0w= @@ -452,12 +403,12 @@ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 h1:QcF go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0/go.mod h1:CXIWhUomyWBG/oY2/r/kLp6K/cmx9e/7DLpBuuGdLCA= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0/go.mod h1:ChZSJbbfbl/DcRZNc9Gqh6DYGlfjw4PvO1pEOZH1ZsE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= go.opentelemetry.io/otel/exporters/prometheus v0.54.0 h1:rFwzp68QMgtzu9PgP3jm9XaMICI6TsofWWPcBDKwlsU= go.opentelemetry.io/otel/exporters/prometheus v0.54.0/go.mod h1:QyjcV9qDP6VeK5qPyKETvNjmaaEc7+gqjh4SS0ZYzDU= go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.8.0 h1:CHXNXwfKWfzS65yrlB2PVds1IBZcdsX8Vepy9of0iRU= @@ -478,8 +429,8 @@ go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfC go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= -go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= -go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -490,66 +441,40 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= -golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= -golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= -golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d h1:/aDRtSZJjyLQzm75d+a1wOJaqyKBMvIAfeQmoa3ORiI= -google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:etfGUgejTiadZAUaEP14NP97xi1RGeawqkjDARA/UOs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -592,8 +517,8 @@ k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbe k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= knative.dev/pkg v0.0.0-20250326102644-9f3e60a9244c h1:6IZwH1QHGfWlmfdy7svgDCPhRqWpisWK/Gcp8wdAwE0= knative.dev/pkg v0.0.0-20250326102644-9f3e60a9244c/go.mod h1:gx7Pp9NPcKYApNhR8m0KSOeg71pqhwPWhuhUJ6xCa2g= -oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc= -oras.land/oras-go/v2 v2.6.0/go.mod h1:magiQDfG6H1O9APp+rOsvCPcW1GD2MM7vgnKY0Y+u1o= +oras.land/oras-go/v2 v2.6.2 h1:N04RXngAp1LJKTG6ifz3xHPipasEkWr+hFmInja5YKo= +oras.land/oras-go/v2 v2.6.2/go.mod h1:PlTtg4JTDJkDe8yVHpM2wz7/YDc00GVas+i4jAW2TZ4= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= diff --git a/hack/scripts/pull-server-manifest.sh b/hack/scripts/pull-server-manifest.sh new file mode 100755 index 00000000..acec9b15 --- /dev/null +++ b/hack/scripts/pull-server-manifest.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPOSITORY="us-docker.pkg.dev/wandb-production/public/wandb/server-manifest" +SCRIPT_NAME="$(basename "$0")" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +DEST_ROOT="${REPO_ROOT}/hack/testing-manifests/server-manifest" + +log() { + printf '[%s] %s\n' "${SCRIPT_NAME}" "$*" +} + +usage() { + cat < + +Downloads the published server manifest OCI artifact for from +${REPOSITORY} and unpacks its manifest yaml files into +${DEST_ROOT}/, replacing any existing manifest already checked in +for that version. +EOF +} + +if ! command -v oras >/dev/null 2>&1; then + echo "error: oras is required but not installed (https://oras.land/docs/installation)" >&2 + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "error: jq is required but not installed" >&2 + exit 1 +fi + +if [[ $# -ne 1 || "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 1 +fi + +TAG="$1" +IMAGE_REF="${REPOSITORY}:${TAG}" +DEST_DIR="${DEST_ROOT}/${TAG}" + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "${WORKDIR}"' EXIT + +log "Fetching manifest for ${IMAGE_REF}" +MANIFEST_JSON="$(oras manifest fetch "${IMAGE_REF}")" + +# Multi-platform tags resolve to an image index; follow the first entry, same +# as pkg/wandb/manifest.processManifest does for the operator itself. +IS_INDEX="$(jq -r 'if .manifests then "true" else "false" end' <<<"${MANIFEST_JSON}")" +if [[ "${IS_INDEX}" == "true" ]]; then + CHILD_DIGEST="$(jq -r '.manifests[0].digest' <<<"${MANIFEST_JSON}")" + log "Tag resolved to an image index; following ${CHILD_DIGEST}" + MANIFEST_JSON="$(oras manifest fetch "${REPOSITORY}@${CHILD_DIGEST}")" +fi + +LAYER_DIGESTS="$(jq -r '.layers[]?.digest' <<<"${MANIFEST_JSON}")" +if [[ -z "${LAYER_DIGESTS}" ]]; then + echo "error: no layers found in manifest for ${IMAGE_REF}" >&2 + exit 1 +fi + +EXTRACT_DIR="${WORKDIR}/extracted" +mkdir -p "${EXTRACT_DIR}" + +while IFS= read -r digest; do + [[ -z "${digest}" ]] && continue + log "Extracting layer ${digest}" + oras blob fetch "${REPOSITORY}@${digest}" --output - | tar -xf - -C "${EXTRACT_DIR}" +done <<<"${LAYER_DIGESTS}" + +YAML_FILES=() +while IFS= read -r f; do + [[ -z "${f}" ]] && continue + YAML_FILES+=("${f}") +done < <(find "${EXTRACT_DIR}" -type f -name '*.yaml' | sort) +if [[ ${#YAML_FILES[@]} -eq 0 ]]; then + echo "error: no .yaml files found in manifest layers for ${IMAGE_REF}" >&2 + exit 1 +fi + +rm -rf "${DEST_DIR}" +mkdir -p "${DEST_DIR}" + +for f in "${YAML_FILES[@]}"; do + name="$(basename "${f}")" + if [[ -e "${DEST_DIR}/${name}" ]]; then + log "warning: duplicate manifest file name '${name}' found across layers; last one wins" + fi + cp "${f}" "${DEST_DIR}/${name}" +done + +log "Wrote ${#YAML_FILES[@]} manifest file(s) to ${DEST_DIR}" diff --git a/hack/scripts/verify-custom-ca-e2e.sh b/hack/scripts/verify-custom-ca-e2e.sh new file mode 100755 index 00000000..66819016 --- /dev/null +++ b/hack/scripts/verify-custom-ca-e2e.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash + +set -euo pipefail + +NAMESPACE="wandb-ca-e2e" +NAME="wandb" +API_APP="api" +TIMEOUT="20m" +POLL_SECONDS=10 + +usage() { + cat <] [--name ] [--api-app ] [--timeout ] + +Verifies the Tilt custom CA e2e path: + - inline and user-provided CA ConfigMaps + - generated MySQL/Redis connection URL CA parameters when configured + - W&B workload pod template env, volumes, mounts, and checksum annotation + - live workload pod CA files when a ready workload pod is available + - recent workload logs for TLS trust failures +EOF +} + +log() { + printf '[custom-ca-e2e] %s\n' "$*" +} + +fail() { + printf '[custom-ca-e2e] ERROR: %s\n' "$*" >&2 + exit 1 +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1" +} + +duration_seconds() { + case "$1" in + *m) echo $((${1%m} * 60)) ;; + *s) echo "${1%s}" ;; + *) echo "$1" ;; + esac +} + +wait_until() { + local description="$1" + shift + local deadline + deadline=$(($(date +%s) + $(duration_seconds "${TIMEOUT}"))) + + log "Waiting for ${description}" + until "$@"; do + if [[ "$(date +%s)" -ge "${deadline}" ]]; then + fail "timed out waiting for ${description}" + fi + sleep "${POLL_SECONDS}" + done +} + +secret_value() { + local secret="$1" + local key="$2" + kubectl -n "${NAMESPACE}" get secret "${secret}" -o json | + python3 -c 'import base64,json,sys; obj=json.load(sys.stdin); print(base64.b64decode(obj["data"][sys.argv[1]]).decode())' "${key}" +} + +assert_url_param() { + local url="$1" + local key="$2" + local expected="$3" + python3 - "${url}" "${key}" "${expected}" <<'PY' +import sys +from urllib.parse import parse_qs, urlparse + +url, key, expected = sys.argv[1:] +actual = parse_qs(urlparse(url).query).get(key, [""])[0] +if actual != expected: + raise SystemExit(f"{key}={actual!r}, expected {expected!r} in {url}") +PY +} + +json_has_env() { + local name="$1" + jq -e --arg name "${name}" '[.spec.containers[]?.env[]? | select(.name == $name)] | length > 0' >/dev/null +} + +json_has_volume() { + local name="$1" + jq -e --arg name "${name}" '[.spec.volumes[]? | select(.name == $name)] | length > 0' >/dev/null +} + +json_has_mount() { + local name="$1" + local path="$2" + jq -e --arg name "${name}" --arg path "${path}" \ + '[.spec.containers[]?.volumeMounts[]? | select(.name == $name and .mountPath == $path)] | length > 0' >/dev/null +} + +check_wandb_ready() { + kubectl -n "${NAMESPACE}" get weightsandbiases.apps.wandb.com "${NAME}" -o json | + jq -e '.status.ready == true' >/dev/null +} + +check_application_ready() { + kubectl -n "${NAMESPACE}" get application "${API_APP}" -o json | + jq -e '.status.ready == true' >/dev/null +} + +selected_migration_job_json() { + kubectl -n "${NAMESPACE}" get jobs \ + -l "app.kubernetes.io/component=migration,app.kubernetes.io/instance=${NAME},app.kubernetes.io/managed-by=wandb-operator" \ + -o json | + jq -er ' + [ + .items[] + | select(.spec.template.metadata.annotations["weightsandbiases.apps.wandb.com/ca-certs-checksum"]? != null) + ] + | sort_by(.status.succeeded // 0) + | reverse + | .[0] + ' +} + +check_migration_workload_exists() { + selected_migration_job_json >/dev/null +} + +select_workload_template() { + if kubectl -n "${NAMESPACE}" get application "${API_APP}" >/dev/null 2>&1; then + wait_until "Application ${NAMESPACE}/${API_APP} status.ready=true" check_application_ready + WORKLOAD_KIND="Application" + WORKLOAD_NAME="${API_APP}" + WORKLOAD_POD_SELECTOR="app.kubernetes.io/name=${API_APP},app.kubernetes.io/instance=${NAME}" + WORKLOAD_TEMPLATE_JSON="$(kubectl -n "${NAMESPACE}" get application "${API_APP}" -o json | jq -c '.spec.podTemplate')" + return + fi + + log "Application ${NAMESPACE}/${API_APP} not found; falling back to a custom-CA-injected W&B migration job" + wait_until "custom-CA-injected W&B migration job" check_migration_workload_exists + local job_json + job_json="$(selected_migration_job_json)" + WORKLOAD_KIND="Job" + WORKLOAD_NAME="$(echo "${job_json}" | jq -r '.metadata.name')" + WORKLOAD_POD_SELECTOR="job-name=${WORKLOAD_NAME}" + WORKLOAD_TEMPLATE_JSON="$(echo "${job_json}" | jq -c '.spec.template')" +} + +ready_workload_pod_name() { + kubectl -n "${NAMESPACE}" get pods -l "${WORKLOAD_POD_SELECTOR}" -o json | + jq -r ' + .items[] + | select(.status.phase == "Running") + | select(any(.status.containerStatuses[]?; .ready == true)) + | .metadata.name + ' | + head -n 1 +} + +check_inline_configmap() { + kubectl -n "${NAMESPACE}" get configmap "${NAME}-ca-certs" -o json | + jq -e '.data["customCA0.crt"] | contains("BEGIN CERTIFICATE")' >/dev/null +} + +check_user_configmap() { + kubectl -n "${NAMESPACE}" get configmap "${USER_CONFIGMAP}" -o json | + jq -e '.data | to_entries | any(.value | contains("BEGIN CERTIFICATE"))' >/dev/null +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --namespace) + NAMESPACE="${2:?missing namespace}" + shift 2 + ;; + --name) + NAME="${2:?missing name}" + shift 2 + ;; + --api-app) + API_APP="${2:?missing api app name}" + shift 2 + ;; + --timeout) + TIMEOUT="${2:?missing timeout}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac +done + +need_cmd kubectl +need_cmd jq +need_cmd python3 + +wait_until "WeightsAndBiases ${NAMESPACE}/${NAME} status.ready=true" check_wandb_ready + +wandb_json="$(kubectl -n "${NAMESPACE}" get weightsandbiases.apps.wandb.com "${NAME}" -o json)" +inline_ca_count="$(echo "${wandb_json}" | jq -r '(.spec.global.customCACerts // []) | length')" +USER_CONFIGMAP="$(echo "${wandb_json}" | jq -r '.spec.global.caCertsConfigMap // ""')" +mysql_ca_enabled="$(echo "${wandb_json}" | jq -r '(((.spec.mysql.externalMysql.sslCa.name // "") | length) > 0 and ((.spec.mysql.externalMysql.sslCa.key // "") | length) > 0)')" +redis_ca_enabled="$(echo "${wandb_json}" | jq -r '(((.spec.redis.externalRedis.sslCa.name // "") | length) > 0 and ((.spec.redis.externalRedis.sslCa.key // "") | length) > 0)')" + +if [[ "${inline_ca_count}" == "0" && -z "${USER_CONFIGMAP}" ]]; then + fail "WeightsAndBiases ${NAMESPACE}/${NAME} does not configure global custom CA material" +fi + +if [[ "${inline_ca_count}" != "0" ]]; then + wait_until "inline custom CA ConfigMap" check_inline_configmap +fi + +if [[ -n "${USER_CONFIGMAP}" ]]; then + wait_until "user custom CA ConfigMap ${USER_CONFIGMAP}" check_user_configmap +fi + +if [[ "${mysql_ca_enabled}" == "true" ]]; then + mysql_url="$(secret_value wandb-mysql-connection url)" + assert_url_param "${mysql_url}" "tls" "custom" + assert_url_param "${mysql_url}" "ssl-ca" "/etc/ssl/certs/mysql_ca.pem" + log "MySQL connection URL includes expected CA parameters" +fi + +if [[ "${redis_ca_enabled}" == "true" ]]; then + redis_url="$(secret_value wandb-redis-connection url)" + assert_url_param "${redis_url}" "tls" "true" + assert_url_param "${redis_url}" "caCertPath" "/etc/ssl/certs/redis_ca.pem" + log "Redis connection URL includes expected CA parameters" +fi + +select_workload_template +workload_ref="${WORKLOAD_KIND} ${WORKLOAD_NAME}" + +for env_name in SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE; do + echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_env "${env_name}" || fail "missing env ${env_name} on ${workload_ref}" +done +if [[ "${mysql_ca_enabled}" == "true" ]]; then + echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_env MYSQL_CA_CERT_PATH || fail "missing env MYSQL_CA_CERT_PATH on ${workload_ref}" +fi + +echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_volume wandb-ca-certs-root || fail "missing volume wandb-ca-certs-root on ${workload_ref}" +echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_mount wandb-ca-certs-root /usr/local/share/ca-certificates/ || + fail "missing root CA mount" + +if [[ "${inline_ca_count}" != "0" ]]; then + echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_volume wandb-ca-certs || fail "missing volume wandb-ca-certs on ${workload_ref}" + echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_mount wandb-ca-certs /usr/local/share/ca-certificates/inline || + fail "missing inline CA mount" +fi +if [[ -n "${USER_CONFIGMAP}" ]]; then + echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_volume wandb-ca-certs-user || fail "missing volume wandb-ca-certs-user on ${workload_ref}" + echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_mount wandb-ca-certs-user /usr/local/share/ca-certificates/configmap || + fail "missing user CA ConfigMap mount" +fi +if [[ "${mysql_ca_enabled}" == "true" ]]; then + echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_volume mysql-ca || fail "missing volume mysql-ca on ${workload_ref}" + echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_mount mysql-ca /etc/ssl/certs/mysql_ca.pem || + fail "missing MySQL CA mount" +fi +if [[ "${redis_ca_enabled}" == "true" ]]; then + echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_volume redis-ca || fail "missing volume redis-ca on ${workload_ref}" + echo "${WORKLOAD_TEMPLATE_JSON}" | json_has_mount redis-ca /etc/ssl/certs/redis_ca.pem || + fail "missing Redis CA mount" +fi +echo "${WORKLOAD_TEMPLATE_JSON}" | + jq -e '.metadata.annotations["weightsandbiases.apps.wandb.com/ca-certs-checksum"] | length > 0' >/dev/null || + fail "missing CA checksum annotation" +log "${workload_ref} pod template contains expected CA env, mounts, volumes, and checksum" + +workload_pod="$(ready_workload_pod_name)" +if [[ -n "${workload_pod}" ]]; then + pod_checks=("test -d /usr/local/share/ca-certificates/") + if [[ "${inline_ca_count}" != "0" ]]; then + pod_checks+=("test -d /usr/local/share/ca-certificates/inline") + fi + if [[ -n "${USER_CONFIGMAP}" ]]; then + pod_checks+=("test -d /usr/local/share/ca-certificates/configmap") + fi + if [[ "${mysql_ca_enabled}" == "true" ]]; then + pod_checks+=("test -s /etc/ssl/certs/mysql_ca.pem") + fi + if [[ "${redis_ca_enabled}" == "true" ]]; then + pod_checks+=("test -s /etc/ssl/certs/redis_ca.pem") + fi + pod_check_cmd="$(printf ' && %s' "${pod_checks[@]}")" + kubectl -n "${NAMESPACE}" exec "${workload_pod}" -- sh -c "${pod_check_cmd# && }" || + fail "live ${workload_ref} pod does not have expected CA files" + log "Live ${workload_ref} pod has expected CA files" +else + log "No ready pod found for ${workload_ref}; verified the workload pod template and skipped live filesystem checks" +fi + +workload_logs="$(kubectl -n "${NAMESPACE}" logs -l "${WORKLOAD_POD_SELECTOR}" --all-containers --tail=500 --prefix=true 2>/dev/null || true)" +if echo "${workload_logs}" | grep -Eiq 'x509:|certificate signed by unknown authority|unknown authority|tls: failed to verify'; then + echo "${workload_logs}" >&2 + fail "recent ${workload_ref} logs contain TLS trust failures" +fi + +log "Custom CA e2e verification passed" diff --git a/hack/testing-manifests/server-manifest/0.78.0-single-file.yaml b/hack/testing-manifests/server-manifest/0.78.0-single-file.yaml deleted file mode 100644 index 9d78cdb5..00000000 --- a/hack/testing-manifests/server-manifest/0.78.0-single-file.yaml +++ /dev/null @@ -1,1476 +0,0 @@ ---- -requiredOperatorVersion: ^2.0.0 -manifestVersion: v1alpha1 - -features: - filestreamQueue: false - proxy: false - -bucket: - default: - ingress: - paths: - - /bucket - servicePort: "http-minio" - pathType: Prefix - sizing: - default: - replicas: 1 - pools: 1 # Not implemented yet, but will be needed if we commit to minio - volumeSize: 10Gi - micro: - replicas: 3 - pools: 1 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - -clickhouse: - default: - sizing: - default: - shards: 1 - replicas: 1 - volumeSize: 10Gi - micro: - replicas: 3 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - shards: 1 - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - -generatedSecrets: - - name: session-key - length: 32 - type: password - - name: weave-worker-auth - length: 32 - type: password - useExactName: true - -kafka: - sizing: - default: - replicas: 1 - volumeSize: 10Gi - replicationFactor: 1 - minInSyncReplicas: 1 - offsetsTopicRF: 1 - transactionStateRF: 1 - transactionStateISR: 1 - micro: - replicas: 3 - volumeSize: 10Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - replicas: 3 - volumeSize: 100Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - topics: - - name: filestream - features: - - filestreamQueue - topic: filestream - partitionCount: 48 - - name: flat-run-fields-updater - topic: flat-run-fields-updater - partitionCount: 48 - - name: weave-worker - topic: weave.call_ended - partitionCount: 48 - - name: weave-evaluate-model-worker - topic: weave.evaluate_model - partitionCount: 48 - -mysql: - default: - sizing: - default: - replicas: 1 - volumeSize: 10Gi - micro: - replicas: 3 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - metadata: {} - runs: {} - usage: {} - -redis: - default: - sizing: - default: - shards: 1 - replicas: 1 - volumeSize: 10Gi - micro: - replicas: 2 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - shards: 1 - replicas: 3 - volumeSize: 10Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - -commonEnvvars: - gorillaMysql: - - name: MYSQL - sources: - - name: default - type: mysql - - name: GORILLA_ANALYTICS_SINK - sources: - - name: default - type: mysql - - name: GORILLA_FILE_STREAM_STORE_ADDRESS - sources: - - name: default - type: mysql - - name: GORILLA_METADATA_STORE - sources: - - name: metadata - type: mysql - - name: GORILLA_RUN_STORE - sources: - - name: runs - type: mysql - - name: GORILLA_USAGE_STORE - sources: - - name: usage - type: mysql - gorillaBucket: - - name: AWS_REGION - sources: - - name: default - type: bucket - field: region - defaultValue: "us-east-1" - - name: BUCKET - sources: - - name: default - type: bucket - - name: GORILLA_FILE_STORE - sources: - - name: default - type: bucket - - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE_OVERFLOW_BUCKET_STORE - sources: - - name: default - type: bucket - - name: GORILLA_STORAGE_BUCKET - sources: - - name: default - type: bucket - gorillaRedis: - - name: REDIS - sources: - - name: default - type: redis - - name: GORILLA_ACTIVITY_STORE_CACHE_ADDRESS - sources: - - name: default - type: redis - - name: GORILLA_AUDITOR_CACHE - sources: - - name: default - type: redis - - name: GORILLA_CACHE - sources: - - name: default - type: redis - - name: GORILLA_FILE_METADATA_SOURCE - sources: - - name: default - type: redis - - name: GORILLA_LOCKER - sources: - - name: default - type: redis - - name: GORILLA_METADATA_CACHE - sources: - - name: default - type: redis - - name: GORILLA_SETTINGS_CACHE - sources: - - name: default - type: redis - - name: GORILLA_USAGE_METRICS_CACHE - sources: - - name: default - type: redis - gorillaService: - - name: GORILLA_SWEEP_PROVIDER - sources: - - name: anaconda2 - type: service - proto: "http" - path: "" - gorillaTaskQueueConsumer: - - name: GORILLA_TASK_QUEUE - sources: - - name: taskQueue - type: redis - params: - concurrency: 10 - - name: GORILLA_TASK_QUEUE_WORKER_ENABLED - value: "true" - gorillaTaskQueueProducer: - - name: GORILLA_TASK_QUEUE - sources: - - name: taskQueue - type: redis - - name: GORILLA_TASK_QUEUE_WORKER_ENABLED - value: "false" - gorillaHistoryStore: - - name: GORILLA_HISTORY_STORE - sources: - - name: parquet - type: service - port: parquet - proto: "http" - path: "/_goRPC_" - - name: default - type: mysql - - name: GORILLA_PARQUET_LIVE_HISTORY_STORE - sources: - - name: default - type: mysql - gorillaOnprem: - - name: GORILLA_LOCAL_SERVICE_BYPASS - value: "true" - - name: GORILLA_DEFAULT_REGION - value: "minio-local" - - name: GORILLA_EMAIL_SINK - sources: - - name: email - type: custom-resource - field: status.emailSink - defaultValue: "https://api.wandb.ai/email/dispatch" - - name: GORILLA_FILE_METADATA_SOURCE_IS_INTERNAL - value: "true" - - name: GORILLA_ONPREM - value: "true" - - name: GORILLA_ONPREM_API_KEY_PREFIX - value: "local" - - name: GORILLA_STATSD_PORT - value: "0" - - name: GORILLA_SESSION_KEY - sources: - - name: session-key - type: generatedSecret - - name: BUCKET_PROXY - value: "true" - - name: GORILLA_GLUE_FILE_STORE_IS_PROXIED - value: "true" - - name: GORILLA_FILE_STORE_IS_PROXIED - value: "true" - - name: GORILLA_FILE_HOST - sources: - - name: hostname - type: custom-resource - field: status.wandb.hostname - - name: GORILLA_FRONTEND_HOST - sources: - - name: hostname - type: custom-resource - field: status.wandb.hostname - - name: LICENSE - sources: - - name: license - type: custom-resource - field: spec.wandb.license - - name: GORILLA_LICENSE - sources: - - name: license - type: custom-resource - field: spec.wandb.license - gorillaCustomerSecrets: - - name: GORILLA_CUSTOMER_SECRET_STORE_SOURCE - value: "k8s-secretmanager://" - - name: GORILLA_CUSTOMER_SECRET_STORE_K8S_CONFIG_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: GORILLA_INTERNAL_JWT_SUBJECTS_TO_ISSUERS - sources: - - name: internal - type: jwt-issuer-map - clickhouse: - - name: WF_CLICKHOUSE_HOST - sources: - - name: default - type: clickhouse - field: host - - name: WF_CLICKHOUSE_PORT - sources: - - name: default - type: clickhouse - field: port - - name: WF_CLICKHOUSE_USER - sources: - - name: default - type: clickhouse - field: user - - name: WF_CLICKHOUSE_PASS - sources: - - name: default - type: clickhouse - field: password - - name: WF_CLICKHOUSE_DATABASE - sources: - - name: default - type: clickhouse - field: database - kafka: - - name: KAFKA_BROKER_HOST - sources: - - name: default - type: kafka - field: host - - name: KAFKA_BROKER_PORT - sources: - - name: default - type: kafka - field: port - - name: KAFKA_URL - sources: - - name: default - type: kafka - field: url - weaveTrace: - - name: WANDB_PUBLIC_BASE_URL - sources: - - name: hostname - type: custom-resource - field: status.wandb.hostname - - name: WANDB_BASE_URL - sources: - - name: api - type: service - proto: "http" - path: "/" - - name: WF_TRACE_SERVER_URL - sources: - - name: weave-trace - type: service - proto: "http" - path: "/traces" - - name: WEAVE_TRACE_SERVER_BASE_URL - sources: - - name: weave-trace - type: service - proto: "http" - path: "/traces" - - name: API_PATH_PREFIX - value: "/traces" - - name: WEAVE_ENABLE_ONLINE_EVAL - value: "true" - - name: WEAVE_ENABLE_EVALUATE_MODEL_WORKER - value: "true" - - name: WANDB_INTERNAL_SERVICE_TOKEN - sources: - - name: weave-worker-auth - type: generatedSecret - - name: WANDB_INTERNAL_SERVICE_TOKEN_SECRET_NAME - value: "weave-worker-auth" - frontend: - - name: REACT_APP_HOST - source: - - name: hostname - type: custom-resource - field: status.wandb.hostname - - name: REACT_APP_ENVIRONMENT_NAME - value: "local" - - name: REACT_APP_ENVIRONMENT_IS_PRIVATE - value: "true" - - name: REACT_APP_ANALYTICS_DISABLED - value: "true" - - name: WEAVE_TRACES_ENABLED - value: "true" - - name: SERVER_FLAG_WEAVE_1_PERCENTAGE - value: "100" - flatRunsV2Producer: - - name: KAFKA_RUNS_V2_TOPIC_NAME - value: "flat-run-fields-updater" - - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE_ADDR - value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)" - - name: GORILLA_RUN_STORE_ONPREM_MIGRATE_CREATE_RUN_TABLES - value: "true" - - name: GORILLA_RUN_STORE_ONPREM_MIGRATE_CREATE_RUN_STORE - value: "true" - - name: GORILLA_RUN_STORE_ONPREM_MIGRATE_SHADOW_RUN_UPDATES - value: "true" - - name: GORILLA_RUN_STORE_ONPREM_MIGRATE_DISABLE_READS - value: "false" - - name: GORILLA_RUN_STORE_ONPREM_MIGRATE_FLAT_RUNS_MIGRATOR - value: "true" - flatRunsV2Consumer: - - name: KAFKA_RUNS_V2_TOPIC_NAME - value: "flat-run-fields-updater" - telemetryOtel: - - name: OTEL_EXPORTER_OTLP_PROTOCOL - sources: - - type: telemetry - field: protocol - - name: OTEL_TRACES_EXPORTER - sources: - - type: telemetry - field: tracesExporter - - name: OTEL_METRICS_EXPORTER - sources: - - type: telemetry - field: metricsExporter - - name: OTEL_LOGS_EXPORTER - sources: - - type: telemetry - field: logsExporter - - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT - sources: - - type: telemetry - field: metricsEndpoint - - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT - sources: - - type: telemetry - field: logsEndpoint - - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT - sources: - - type: telemetry - field: tracesEndpoint - - name: OTEL_SERVICE_NAME - sources: - - type: telemetry - field: serviceName - - name: OTEL_RESOURCE_ATTRIBUTES - sources: - - type: telemetry - field: resourceAttributes - - name: GORILLA_TRACER - sources: - - type: telemetry - field: gorillaTracer - -commonVolumeMounts: - internalSigner: - - mountPath: /vol/env - name: wandb-internal-signer-root - source: - name: wandb-internal-signer - type: secret - -applications: - anaconda2: - name: anaconda2 - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/anaconda2 - tag: 0.78.0 - containers: - - name: anaconda2 - ports: - - containerPort: 8080 - name: anaconda2 - protocol: TCP - livenessProbe: - httpGet: - path: /ping - readinessProbe: - httpGet: - path: /ping - service: - ports: - - port: 8080 - protocol: TCP - name: anaconda2 - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 - api: - name: api - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaHistoryStore - - gorillaTaskQueueProducer - - gorillaService - - gorillaOnprem - - gorillaCustomerSecrets - - kafka - - flatRunsV2Producer - - telemetryOtel - env: - - name: GORILLA_LICENSE_CERT_PATH - value: "/jwks.json" - - name: GORILLA_VIEW_SPEC_UPDATER_EXECUTABLE - value: "/view-spec-updater-linux" - - name: INTERNAL_SIGNER_KEY_PATH - value: "/vol/env" - - name: MIGRATE_RUNS_DB - sources: - - name: default - type: mysql - - name: MIGRATE_USAGE_DB - sources: - - name: default - type: mysql - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: api - args: - - gorilla - ports: - - containerPort: 8080 - name: api - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - readinessProbe: - httpGet: - path: /ready - service: - ports: - - port: 8080 - protocol: TCP - name: api - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "4" - memory: 8Gi - requests: - cpu: "4" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 - executor: - name: executor - args: - - executor - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaHistoryStore - - gorillaTaskQueueConsumer - - gorillaService - - gorillaOnprem - - telemetryOtel - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: executor - sizing: - small: - resources: - limits: - cpu: "4" - memory: 16Gi - requests: - cpu: "4" - memory: 16Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - filemeta: - name: filemeta - args: - - filemeta - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaService - - gorillaOnprem - - telemetryOtel - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - filestream: - name: filestream - features: - - filestreamQueue - args: - - filestream - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaService - - gorillaOnprem - - telemetryOtel - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: filestream - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - flat-run-fields-updater: - name: flat-run-fields-updater - args: - - flat-run-fields-updater - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaService - - gorillaOnprem - - kafka - - flatRunsV2Consumer - - telemetryOtel - env: - - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE_SUBSCRIPTIONS_FLAT_RUN_FIELDS_UPDATER - value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)?consumer_group_id=flat-run-fields-updater" - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: flat-run-fields-updater - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 5 - minReplicas: 1 - frontend: - name: frontend - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/frontend-nginx - tag: 0.78.0 - commonEnvs: - - frontend - env: - - name: FRONTEND_APP_BACKEND - sources: - - name: api - type: service - proto: "" - path: "" - - name: FRONTEND_AUTH_BACKEND - sources: - - name: api - type: service - proto: "" - path: "" - - name: FRONTEND_LOCAL_BACKEND - sources: - - name: api - type: service - proto: "" - path: "" - - name: FRONTEND_WEAVE_BACKEND - sources: - - name: weave - type: service - proto: "" - path: "" - - name: WEAVE_ENABLED - value: "true" - - name: OPERATOR_ENABLED - value: "true" - containers: - - name: frontend - ports: - - containerPort: 8080 - name: frontend - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - readinessProbe: - httpGet: - path: /ready - service: - ports: - - port: 8080 - protocol: TCP - name: frontend - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 - glue: - name: glue - args: - - glue - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaHistoryStore - - gorillaService - - gorillaOnprem - - telemetryOtel - env: - - name: GORILLA_LICENSE_CERT_PATH - value: "/jwks.json" - - name: GORILLA_VIEW_SPEC_UPDATER_EXECUTABLE - value: "/view-spec-updater-linux" - - name: GORILLA_GLUE_TASK_PROVIDER - value: "memory://" - - name: GORILLA_GLUE_TASK_CONFIG_PATH - value: "/gorilla_glue_tasks_local.yaml" - - name: GORILLA_GLUE_TASK_STORE - value: "memory://" - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: glue - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 3Gi - requests: - cpu: "1" - memory: 3Gi - small: - resources: - limits: - cpu: "2" - memory: 6Gi - requests: - cpu: "2" - memory: 6Gi - metric-observer: - name: metric-observer - args: - - metric-observer - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaService - - gorillaOnprem - - kafka - - flatRunsV2Consumer - - telemetryOtel - env: - - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE_SUBSCRIPTIONS_METRIC_OBSERVER - value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)?consumer_group_id=metric-observer" - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: metric-observer - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 4 - minReplicas: 1 - parquet: - name: parquet - args: - - parquet - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaHistoryStore - - gorillaTaskQueueProducer - - gorillaService - - gorillaOnprem - - telemetryOtel - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: parquet - ports: - - containerPort: 8080 - name: parquet - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - readinessProbe: - httpGet: - path: /ready - service: - ports: - - port: 8080 - protocol: TCP - name: parquet - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "4" - memory: 16Gi - requests: - cpu: "4" - memory: 16Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 2 - weave: - name: weave - image: - repository: wandb/weave-python - tag: 0.76.1 - commonEnvs: - - telemetryOtel - env: - - name: DATADOG_TRACE_ENABLED - value: "false" - service: - ports: - - port: 9239 - protocol: TCP - name: weave - volumeMounts: - - name: temp-dir - mountPath: /tmp/ - source: - type: emptyDir - name: temp-dir - - name: cache - mountPath: /vol/weave/cache - source: - type: emptyDir - name: cache - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 4Gi - requests: - cpu: "1" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "4" - memory: 16Gi - requests: - cpu: "4" - memory: 16Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 2 - weave-trace: - name: weave-trace - commonEnvs: - - clickhouse - - kafka - - weaveTrace - - telemetryOtel - ports: - - containerPort: 8080 - name: weave-trace - protocol: TCP - livenessProbe: - httpGet: - path: /traces/health - readinessProbe: - httpGet: - path: /traces/health - service: - ports: - - port: 8080 - protocol: TCP - name: weave-trace - jwtTokens: - - name: internal-jwt - mountPath: /tmp/weave-trace/internal-jwt - source: - kubernetesServiceAccount: - audience: internal-service - expirationSeconds: 600 - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 - weave-trace-worker: - name: weave-trace-worker - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.78.0 - env: - - name: DD_TRACE_ENABLED - value: "false" - commonEnvs: - - clickhouse - - kafka - - weaveTrace - - telemetryOtel - containers: - - name: weave-trace-worker - args: - - "python" - - "-m" - - "src.workers.scoring_worker" - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 4 - minReplicas: 1 - weave-trace-evaluate-model-worker: - name: weave-trace-evaluate-model-worker - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.78.0 - env: - - name: DD_TRACE_ENABLED - value: "false" - commonEnvs: - - clickhouse - - kafka - - weaveTrace - - telemetryOtel - containers: - - name: weave-trace-evaluate-model-worker - args: - - "python" - - "-m" - - "src.workers.evaluate_model_worker.evaluate_model_worker" - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - nginx-proxy: - name: nginx-proxy - features: - - proxy - image: - repository: nginxinc/nginx-unprivileged - tag: latest - commonEnvs: - - telemetryOtel - env: - - name: UPSTREAM_FRONTEND - sources: - - name: frontend - type: service - proto: "" - path: "" - - name: UPSTREAM_API - sources: - - name: api - type: service - proto: "" - path: "" - - name: UPSTREAM_WEAVE_TRACE - sources: - - name: weave-trace - type: service - proto: "" - path: "" - - name: UPSTREAM_WEAVE - sources: - - name: weave - type: service - proto: "" - path: "" - - name: BUCKET_HOST - sources: - - name: default - type: bucket - field: host - - name: BUCKET_PORT - sources: - - name: default - type: bucket - field: port - - name: UPSTREAM_BUCKET - value: "$(BUCKET_HOST):$(BUCKET_PORT)" - files: - - name: envvar.conf.template - mountPath: /etc/nginx/templates - fileName: envvar.conf.template - inline: | - upstream frontend { - server $UPSTREAM_FRONTEND; - } - - upstream api { - server $UPSTREAM_API; - } - - upstream weave_trace { - server $UPSTREAM_WEAVE_TRACE; - } - - upstream weave { - server $UPSTREAM_WEAVE; - } - - upstream bucket { - server $UPSTREAM_BUCKET; - } - - map $host $bucket{ - default $UPSTREAM_BUCKET; - } - - name: nginx.conf - mountPath: /etc/nginx - fileName: nginx.conf - inline: | - worker_processes auto; - - error_log /var/log/nginx/error.log notice; - pid /tmp/nginx.pid; - - events { - worker_connections 1024; - } - - http { - - include /etc/nginx/conf.d/envvar.conf; - - server { - listen 8080; - proxy_set_header Host $http_host; - client_max_body_size 0; - location / { - proxy_pass http://frontend; - } - - location /api { - proxy_pass http://api; - } - location /artifacts { - proxy_pass http://api; - } - location /artifactsV2 { - proxy_pass http://api; - } - - location /files { - proxy_pass http://api; - } - - location /graphql { - proxy_pass http://api; - } - - location /graphql2 { - proxy_pass http://api; - } - - location /oidc { - proxy_pass http://api; - } - - location /traces { - proxy_pass http://weave_trace; - } - - location /weave/ { - proxy_pass http://weave/; - } - - location /bucket { - proxy_ssl_verify off; - proxy_set_header Host $bucket; - proxy_pass https://bucket; - } - } - } - service: - type: NodePort - ports: - - name: http - port: 8080 - protocol: TCP - sizing: - default: - autoscaling: - enabled: true - minReplicas: 2 - maxReplicas: 4 - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 -migrations: - gorilla: - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - args: - - "migrate" - - "--db=$(GORILLA_METADATA_STORE)" - - "--runs-db=$(GORILLA_RUN_STORE)" - - "--usage-db=$(GORILLA_USAGE_STORE)" - - "--squash" - - "true" - commonEnvs: - - gorillaMysql - internal-signer: - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - args: - - "secret-generation-job" - env: - - name: INTERNAL_SIGNER_K8S_SECRET_NAME - value: "wandb-internal-signer" - - name: INTERNAL_SIGNER_K8S_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - weave-trace: - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.78.0 - args: - - python - - migrator.py - commonEnvs: - - clickhouse diff --git a/hack/testing-manifests/server-manifest/0.78.0/manifest.yaml b/hack/testing-manifests/server-manifest/0.78.0/manifest.yaml deleted file mode 100644 index 6e500f53..00000000 --- a/hack/testing-manifests/server-manifest/0.78.0/manifest.yaml +++ /dev/null @@ -1,1487 +0,0 @@ ---- -requiredOperatorVersion: ^2.0.0 -manifestVersion: v1alpha1 - -features: - filestreamQueue: false - proxy: false - -bucket: - default: - ingress: - paths: - - /bucket - servicePort: "http-minio" - pathType: Prefix - sizing: - default: - replicas: 1 - pools: 1 # Not implemented yet, but will be needed if we commit to minio - volumeSize: 10Gi - micro: - replicas: 3 - pools: 1 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - -clickhouse: - default: - sizing: - default: - shards: 1 - replicas: 1 - volumeSize: 10Gi - micro: - replicas: 3 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - shards: 1 - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - -generatedSecrets: - - name: session-key - length: 32 - type: password - - name: weave-worker-auth - length: 32 - type: password - useExactName: true - -kafka: - sizing: - default: - replicas: 1 - volumeSize: 10Gi - replicationFactor: 1 - minInSyncReplicas: 1 - offsetsTopicRF: 1 - transactionStateRF: 1 - transactionStateISR: 1 - micro: - replicas: 3 - volumeSize: 10Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - replicas: 3 - volumeSize: 100Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - topics: - - name: filestream - features: - - filestreamQueue - topic: filestream - partitionCount: 48 - - name: flat-run-fields-updater - topic: flat-run-fields-updater - partitionCount: 48 - - name: weave-worker - topic: weave.call_ended - partitionCount: 48 - - name: weave-evaluate-model-worker - topic: weave.evaluate_model - partitionCount: 48 - -mysql: - default: - sizing: - default: - replicas: 1 - volumeSize: 10Gi - micro: - replicas: 3 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - metadata: {} - runs: {} - usage: {} - -redis: - default: - sizing: - default: - shards: 1 - replicas: 1 - volumeSize: 10Gi - micro: - replicas: 2 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - shards: 1 - replicas: 3 - volumeSize: 10Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - -commonEnvvars: - gorillaMysql: - - name: MYSQL - sources: - - name: default - type: mysql - - name: GORILLA_ANALYTICS_SINK - sources: - - name: default - type: mysql - - name: GORILLA_FILE_STREAM_STORE_ADDRESS - sources: - - name: default - type: mysql - - name: GORILLA_METADATA_STORE - sources: - - name: metadata - type: mysql - - name: GORILLA_RUN_STORE - sources: - - name: runs - type: mysql - - name: GORILLA_USAGE_STORE - sources: - - name: usage - type: mysql - gorillaBucket: - - name: AWS_REGION - sources: - - name: default - type: bucket - field: region - defaultValue: "us-east-1" - - name: BUCKET - sources: - - name: default - type: bucket - - name: GORILLA_FILE_STORE - sources: - - name: default - type: bucket - - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE_OVERFLOW_BUCKET_STORE - sources: - - name: default - type: bucket - - name: GORILLA_STORAGE_BUCKET - sources: - - name: default - type: bucket - gorillaRedis: - - name: REDIS - sources: - - name: default - type: redis - - name: GORILLA_ACTIVITY_STORE_CACHE_ADDRESS - sources: - - name: default - type: redis - - name: GORILLA_AUDITOR_CACHE - sources: - - name: default - type: redis - - name: GORILLA_CACHE - sources: - - name: default - type: redis - - name: GORILLA_FILE_METADATA_SOURCE - sources: - - name: default - type: redis - - name: GORILLA_LOCKER - sources: - - name: default - type: redis - - name: GORILLA_METADATA_CACHE - sources: - - name: default - type: redis - - name: GORILLA_SETTINGS_CACHE - sources: - - name: default - type: redis - - name: GORILLA_USAGE_METRICS_CACHE - sources: - - name: default - type: redis - gorillaService: - - name: GORILLA_SWEEP_PROVIDER - sources: - - name: anaconda2 - type: service - proto: "http" - path: "" - gorillaTaskQueueConsumer: - - name: GORILLA_TASK_QUEUE - sources: - - name: taskQueue - type: redis - params: - concurrency: 10 - - name: GORILLA_TASK_QUEUE_WORKER_ENABLED - value: "true" - gorillaTaskQueueProducer: - - name: GORILLA_TASK_QUEUE - sources: - - name: taskQueue - type: redis - - name: GORILLA_TASK_QUEUE_WORKER_ENABLED - value: "false" - gorillaHistoryStore: - - name: GORILLA_HISTORY_STORE - sources: - - name: parquet - type: service - port: parquet - proto: "http" - path: "/_goRPC_" - - name: default - type: mysql - - name: GORILLA_PARQUET_LIVE_HISTORY_STORE - sources: - - name: default - type: mysql - gorillaOnprem: - - name: GORILLA_LOCAL_SERVICE_BYPASS - value: "true" - - name: GORILLA_DEFAULT_REGION - value: "minio-local" - - name: GORILLA_EMAIL_SINK - sources: - - name: email - type: custom-resource - field: status.emailSink - defaultValue: "https://api.wandb.ai/email/dispatch" - - name: GORILLA_FILE_METADATA_SOURCE_IS_INTERNAL - value: "true" - - name: GORILLA_ONPREM - value: "true" - - name: GORILLA_ONPREM_API_KEY_PREFIX - value: "local" - - name: GORILLA_STATSD_PORT - value: "0" - - name: GORILLA_SESSION_KEY - sources: - - name: session-key - type: generatedSecret - - name: BUCKET_PROXY - value: "true" - - name: GORILLA_GLUE_FILE_STORE_IS_PROXIED - value: "true" - - name: GORILLA_FILE_STORE_IS_PROXIED - value: "true" - - name: GORILLA_FILE_HOST - sources: - - name: hostname - type: custom-resource - field: status.wandb.hostname - - name: GORILLA_FRONTEND_HOST - sources: - - name: hostname - type: custom-resource - field: status.wandb.hostname - - name: LICENSE - sources: - - name: license - type: custom-resource - field: spec.wandb.license - - name: GORILLA_LICENSE - sources: - - name: license - type: custom-resource - field: spec.wandb.license - gorillaCustomerSecrets: - - name: GORILLA_CUSTOMER_SECRET_STORE_SOURCE - value: "k8s-secretmanager://" - - name: GORILLA_CUSTOMER_SECRET_STORE_K8S_CONFIG_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: GORILLA_INTERNAL_JWT_SUBJECTS_TO_ISSUERS - sources: - - name: internal - type: jwt-issuer-map - clickhouse: - - name: WF_CLICKHOUSE_HOST - sources: - - name: default - type: clickhouse - field: host - - name: WF_CLICKHOUSE_PORT - sources: - - name: default - type: clickhouse - field: port - - name: WF_CLICKHOUSE_USER - sources: - - name: default - type: clickhouse - field: user - - name: WF_CLICKHOUSE_PASS - sources: - - name: default - type: clickhouse - field: password - - name: WF_CLICKHOUSE_DATABASE - sources: - - name: default - type: clickhouse - field: database - kafka: - - name: KAFKA_BROKER_HOST - sources: - - name: default - type: kafka - field: host - - name: KAFKA_BROKER_PORT - sources: - - name: default - type: kafka - field: port - - name: KAFKA_URL - sources: - - name: default - type: kafka - field: url - weaveTrace: - - name: WANDB_PUBLIC_BASE_URL - sources: - - name: hostname - type: custom-resource - field: status.wandb.hostname - - name: WANDB_BASE_URL - sources: - - name: api - type: service - proto: "http" - path: "/" - - name: WF_TRACE_SERVER_URL - sources: - - name: weave-trace - type: service - proto: "http" - path: "/traces" - - name: WEAVE_TRACE_SERVER_BASE_URL - sources: - - name: weave-trace - type: service - proto: "http" - path: "/traces" - - name: API_PATH_PREFIX - value: "/traces" - - name: WEAVE_ENABLE_ONLINE_EVAL - value: "true" - - name: WEAVE_ENABLE_EVALUATE_MODEL_WORKER - value: "true" - - name: WANDB_INTERNAL_SERVICE_TOKEN - sources: - - name: weave-worker-auth - type: generatedSecret - - name: WANDB_INTERNAL_SERVICE_TOKEN_SECRET_NAME - value: "weave-worker-auth" - frontend: - - name: REACT_APP_HOST - source: - - name: hostname - type: custom-resource - field: status.wandb.hostname - - name: REACT_APP_ENVIRONMENT_NAME - value: "local" - - name: REACT_APP_ENVIRONMENT_IS_PRIVATE - value: "true" - - name: REACT_APP_ANALYTICS_DISABLED - value: "true" - - name: WEAVE_TRACES_ENABLED - value: "true" - - name: SERVER_FLAG_WEAVE_1_PERCENTAGE - value: "100" - flatRunsV2Producer: - - name: KAFKA_RUNS_V2_TOPIC_NAME - value: "flat-run-fields-updater" - - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE_ADDR - value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)" - - name: GORILLA_RUN_STORE_ONPREM_MIGRATE_CREATE_RUN_TABLES - value: "true" - - name: GORILLA_RUN_STORE_ONPREM_MIGRATE_CREATE_RUN_STORE - value: "true" - - name: GORILLA_RUN_STORE_ONPREM_MIGRATE_SHADOW_RUN_UPDATES - value: "true" - - name: GORILLA_RUN_STORE_ONPREM_MIGRATE_DISABLE_READS - value: "false" - - name: GORILLA_RUN_STORE_ONPREM_MIGRATE_FLAT_RUNS_MIGRATOR - value: "true" - flatRunsV2Consumer: - - name: KAFKA_RUNS_V2_TOPIC_NAME - value: "flat-run-fields-updater" - -commonVolumeMounts: - internalSigner: - - mountPath: /vol/env - name: wandb-internal-signer-root - source: - name: wandb-internal-signer - type: secret - -applications: - anaconda2: - name: anaconda2 - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/anaconda2 - tag: 0.78.0 - containers: - - name: anaconda2 - ports: - - containerPort: 8080 - name: anaconda2 - protocol: TCP - livenessProbe: - httpGet: - path: /ping - readinessProbe: - httpGet: - path: /ping - service: - ports: - - port: 8080 - protocol: TCP - name: anaconda2 - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 - api: - name: api - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaHistoryStore - - gorillaTaskQueueProducer - - gorillaService - - gorillaOnprem - - gorillaCustomerSecrets - - kafka - - flatRunsV2Producer - commonVolumeMounts: - - internalSigner - env: - - name: GORILLA_LICENSE_CERT_PATH - value: "/jwks.json" - - name: GORILLA_VIEW_SPEC_UPDATER_EXECUTABLE - value: "/view-spec-updater-linux" - - name: INTERNAL_SIGNER_KEY_PATH - value: "/vol/env" - - name: MIGRATE_RUNS_DB - sources: - - name: default - type: mysql - - name: MIGRATE_USAGE_DB - sources: - - name: default - type: mysql - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: api - args: - - gorilla - ports: - - containerPort: 8080 - name: api - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - readinessProbe: - httpGet: - path: /ready - service: - ports: - - port: 8080 - protocol: TCP - name: api - ingress: - paths: - - /api - - /artifacts - - /artifactsV2 - - /files - - /graphql - - /graphql2 - - /oidc - servicePort: "8080" - pathType: Prefix - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "4" - memory: 8Gi - requests: - cpu: "4" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 - executor: - name: executor - args: - - executor - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaHistoryStore - - gorillaTaskQueueConsumer - - gorillaService - - gorillaOnprem - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: executor - sizing: - small: - resources: - limits: - cpu: "4" - memory: 16Gi - requests: - cpu: "4" - memory: 16Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - filemeta: - name: filemeta - args: - - filemeta - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaService - - gorillaOnprem - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - filestream: - name: filestream - features: - - filestreamQueue - args: - - filestream - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaService - - gorillaOnprem - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: filestream - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - flat-run-fields-updater: - name: flat-run-fields-updater - args: - - flat-run-fields-updater - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaService - - gorillaOnprem - - kafka - - flatRunsV2Consumer - env: - - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE_SUBSCRIPTIONS_FLAT_RUN_FIELDS_UPDATER - value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)?consumer_group_id=flat-run-fields-updater" - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: flat-run-fields-updater - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 5 - minReplicas: 1 - frontend: - name: frontend - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/frontend-nginx - tag: 0.78.0 - commonEnvs: - - frontend - env: - - name: FRONTEND_APP_BACKEND - sources: - - name: api - type: service - proto: "" - path: "" - - name: FRONTEND_AUTH_BACKEND - sources: - - name: api - type: service - proto: "" - path: "" - - name: FRONTEND_LOCAL_BACKEND - sources: - - name: api - type: service - proto: "" - path: "" - - name: FRONTEND_WEAVE_BACKEND - sources: - - name: weave - type: service - proto: "" - path: "" - - name: WEAVE_ENABLED - value: "true" - - name: OPERATOR_ENABLED - value: "true" - containers: - - name: frontend - ports: - - containerPort: 8080 - name: frontend - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - readinessProbe: - httpGet: - path: /ready - service: - ports: - - port: 8080 - protocol: TCP - name: frontend - ingress: - paths: - - / - servicePort: "8080" - pathType: Prefix - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 - glue: - name: glue - args: - - glue - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaHistoryStore - - gorillaService - - gorillaOnprem - commonVolumeMounts: - - internalSigner - env: - - name: GORILLA_LICENSE_CERT_PATH - value: "/jwks.json" - - name: GORILLA_VIEW_SPEC_UPDATER_EXECUTABLE - value: "/view-spec-updater-linux" - - name: GORILLA_GLUE_TASK_PROVIDER - value: "memory://" - - name: GORILLA_GLUE_TASK_CONFIG_PATH - value: "/gorilla_glue_tasks_local.yaml" - - name: GORILLA_GLUE_TASK_STORE - value: "memory://" - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: glue - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 3Gi - requests: - cpu: "1" - memory: 3Gi - small: - resources: - limits: - cpu: "2" - memory: 6Gi - requests: - cpu: "2" - memory: 6Gi - metric-observer: - name: metric-observer - args: - - metric-observer - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaService - - gorillaOnprem - - kafka - - flatRunsV2Consumer - env: - - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE_SUBSCRIPTIONS_METRIC_OBSERVER - value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)?consumer_group_id=metric-observer" - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: metric-observer - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 4 - minReplicas: 1 - parquet: - name: parquet - args: - - parquet - commonEnvs: - - gorillaMysql - - gorillaBucket - - gorillaRedis - - gorillaHistoryStore - - gorillaTaskQueueProducer - - gorillaService - - gorillaOnprem - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - containers: - - name: parquet - ports: - - containerPort: 8080 - name: parquet - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - readinessProbe: - httpGet: - path: /ready - service: - ports: - - port: 8080 - protocol: TCP - name: parquet - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "4" - memory: 16Gi - requests: - cpu: "4" - memory: 16Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 2 - weave: - name: weave - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-python - tag: 0.78.0 - containers: - - name: weave - ports: - - containerPort: 9239 - name: weave - protocol: TCP - livenessProbe: - httpGet: - path: /__weave/hello - readinessProbe: - httpGet: - path: /__weave/hello - - name: weave-cache-clear - command: - - python - - weave-public/weave_query/scripts/clear_cache.py - resources: - limits: - cpu: 2 - memory: 2Gi - requests: - cpu: 100m - memory: 128Mi - env: - - name: DATADOG_TRACE_ENABLED - value: "false" - service: - ports: - - port: 9239 - protocol: TCP - name: weave - ingress: - paths: - - /weave/ - servicePort: "9239" - pathType: Prefix - volumeMounts: - - name: temp-dir - mountPath: /tmp/ - source: - type: emptyDir - name: temp-dir - - name: cache - mountPath: /vol/weave/cache - source: - type: emptyDir - name: cache - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 4Gi - requests: - cpu: "1" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "4" - memory: 16Gi - requests: - cpu: "4" - memory: 16Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 2 - weave-trace: - name: weave-trace - commonEnvs: - - clickhouse - - kafka - - weaveTrace - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.78.0 - containers: - - name: weave-trace - args: - - "uvicorn" - - "src.trace_server:app" - - "--host" - - "0.0.0.0" - - "--port" - - "8080" - ports: - - containerPort: 8080 - name: weave-trace - protocol: TCP - livenessProbe: - httpGet: - path: /traces/health - readinessProbe: - httpGet: - path: /traces/health - service: - ports: - - port: 8080 - protocol: TCP - name: weave-trace - ingress: - paths: - - /traces - servicePort: "8080" - pathType: Prefix - jwtTokens: - - name: internal-jwt - mountPath: /tmp/weave-trace/internal-jwt - source: - kubernetesServiceAccount: - audience: internal-service - expirationSeconds: 600 - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 - weave-trace-worker: - name: weave-trace-worker - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.78.0 - env: - - name: DD_TRACE_ENABLED - value: "false" - commonEnvs: - - clickhouse - - kafka - - weaveTrace - containers: - - name: weave-trace-worker - args: - - "python" - - "-m" - - "src.workers.scoring_worker" - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 4 - minReplicas: 1 - weave-trace-evaluate-model-worker: - name: weave-trace-evaluate-model-worker - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.78.0 - env: - - name: DD_TRACE_ENABLED - value: "false" - commonEnvs: - - clickhouse - - kafka - - weaveTrace - containers: - - name: weave-trace-evaluate-model-worker - args: - - "python" - - "-m" - - "src.workers.evaluate_model_worker.evaluate_model_worker" - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - nginx-proxy: - name: nginx-proxy - features: - - proxy - image: - repository: nginxinc/nginx-unprivileged - tag: latest - containers: - - name: nginx-proxy - env: - - name: UPSTREAM_FRONTEND - sources: - - name: frontend - type: service - proto: "" - path: "" - - name: UPSTREAM_API - sources: - - name: api - type: service - proto: "" - path: "" - - name: UPSTREAM_WEAVE_TRACE - sources: - - name: weave-trace - type: service - proto: "" - path: "" - - name: UPSTREAM_WEAVE - sources: - - name: weave - type: service - proto: "" - path: "" - - name: BUCKET_HOST - sources: - - name: default - type: bucket - field: host - - name: BUCKET_PORT - sources: - - name: default - type: bucket - field: port - - name: UPSTREAM_BUCKET - value: "$(BUCKET_HOST):$(BUCKET_PORT)" - files: - - name: envvar.conf.template - mountPath: /etc/nginx/templates - fileName: envvar.conf.template - inline: | - upstream frontend { - server $UPSTREAM_FRONTEND; - } - - upstream api { - server $UPSTREAM_API; - } - - upstream weave_trace { - server $UPSTREAM_WEAVE_TRACE; - } - - upstream weave { - server $UPSTREAM_WEAVE; - } - - upstream bucket { - server $UPSTREAM_BUCKET; - } - - map $host $bucket{ - default $UPSTREAM_BUCKET; - } - - name: nginx.conf - mountPath: /etc/nginx - fileName: nginx.conf - inline: | - worker_processes auto; - - error_log /var/log/nginx/error.log notice; - pid /tmp/nginx.pid; - - events { - worker_connections 1024; - } - - http { - - include /etc/nginx/conf.d/envvar.conf; - - server { - listen 8080; - proxy_set_header Host $http_host; - client_max_body_size 0; - location / { - proxy_pass http://frontend; - } - - location /api { - proxy_pass http://api; - } - location /artifacts { - proxy_pass http://api; - } - location /artifactsV2 { - proxy_pass http://api; - } - - location /files { - proxy_pass http://api; - } - - location /graphql { - proxy_pass http://api; - } - - location /graphql2 { - proxy_pass http://api; - } - - location /oidc { - proxy_pass http://api; - } - - location /traces { - proxy_pass http://weave_trace; - } - - location /weave/ { - proxy_pass http://weave/; - } - - location /bucket { - proxy_ssl_verify off; - proxy_set_header Host $bucket; - proxy_pass https://bucket; - } - } - } - service: - type: NodePort - ports: - - name: http - port: 8080 - protocol: TCP - sizing: - default: - autoscaling: - enabled: true - minReplicas: 2 - maxReplicas: 4 - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 -migrations: - gorilla: - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - args: - - "migrate" - - "--db=$(GORILLA_METADATA_STORE)" - - "--runs-db=$(GORILLA_RUN_STORE)" - - "--usage-db=$(GORILLA_USAGE_STORE)" - - "--squash" - - "true" - commonEnvs: - - gorillaMysql - internal-signer: - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.0 - args: - - "secret-generation-job" - env: - - name: INTERNAL_SIGNER_K8S_SECRET_NAME - value: "wandb-internal-signer" - - name: INTERNAL_SIGNER_K8S_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - weave-trace: - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.78.0 - args: - - python - - migrator.py - commonEnvs: - - clickhouse diff --git a/hack/testing-manifests/server-manifest/0.78.0-pre/manifest.yaml b/hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.1/manifest.yaml similarity index 78% rename from hack/testing-manifests/server-manifest/0.78.0-pre/manifest.yaml rename to hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.1/manifest.yaml index 6b28115b..9bbdaf76 100644 --- a/hack/testing-manifests/server-manifest/0.78.0-pre/manifest.yaml +++ b/hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.1/manifest.yaml @@ -1,10 +1,8 @@ --- requiredOperatorVersion: ^2.0.0 -manifestVersion: v1alpha1 features: filestreamQueue: false - proxy: false bucket: default: @@ -13,9 +11,27 @@ bucket: - /bucket servicePort: "http-minio" pathType: Prefix + images: + seaweedfs: + registry: "docker.io" + repository: "chrislusf/seaweedfs" + tag: "4.35" clickhouse: - default: {} + default: + images: + server: + registry: "docker.io" + repository: "altinity/clickhouse-server" + tag: "25.8.16.10002.altinitystable" + +clickhouseKeeper: + default: + images: + keeper: + registry: "docker.io" + repository: "altinity/clickhouse-keeper" + tag: "25.8.16.10002.altinitystable" generatedSecrets: - name: session-key @@ -27,31 +43,68 @@ generatedSecrets: useExactName: true kafka: + images: + bufstream: + registry: "us-docker.pkg.dev" + repository: "buf-images-1/buf/images/bufstream" + tag: "0.4.15" + etcd: + registry: "quay.io" + repository: "coreos/etcd" + tag: "v3.5.31" + bucketEnsure: + repository: "amazon/aws-cli" + tag: "2.35.10" topics: - name: filestream features: - filestreamQueue topic: filestream - partitionCount: 48 + partitionCount: 96 - name: flat-run-fields-updater topic: flat-run-fields-updater - partitionCount: 48 + partitionCount: 96 - name: weave-worker topic: weave.call_ended - partitionCount: 48 + partitionCount: 16 - name: weave-evaluate-model-worker topic: weave.evaluate_model - partitionCount: 48 + partitionCount: 16 mysql: - default: {} + default: + images: + mysql: + registry: "ghcr.io" + repository: "cybozu-go/moco/mysql" + tag: "8.4.8" + exporter: + registry: "docker.io" + repository: "prom/mysqld-exporter" + tag: "v0.15.1" metadata: {} runs: {} usage: {} redis: default: - sentinelGroup: gorilla + images: + standalone: + registry: "quay.io" + repository: "opstree/redis" + tag: "v7.0.15" + replication: + registry: "quay.io" + repository: "opstree/redis" + tag: "v7.0.15" + sentinel: + registry: "quay.io" + repository: "opstree/redis-sentinel" + tag: "v7.0.12" + exporter: + registry: "quay.io" + repository: "opstree/redis-exporter" + tag: "v1.44.0" commonEnvvars: gorillaMysql: @@ -200,11 +253,20 @@ commonEnvvars: - name: session-key type: generatedSecret - name: BUCKET_PROXY - value: "true" + sources: + - name: bucket-proxy + type: custom-resource + field: spec.wandb.bucketProxy - name: GORILLA_GLUE_FILE_STORE_IS_PROXIED - value: "true" + sources: + - name: bucket-proxy + type: custom-resource + field: spec.wandb.bucketProxy - name: GORILLA_FILE_STORE_IS_PROXIED - value: "true" + sources: + - name: bucket-proxy + type: custom-resource + field: spec.wandb.bucketProxy - name: GORILLA_FILE_HOST sources: - name: hostname @@ -215,8 +277,6 @@ commonEnvvars: - name: hostname type: custom-resource field: status.wandb.hostname - - name: GORILLA_DISABLE_DYNAMIC_HOST - value: "true" - name: LICENSE sources: - name: license @@ -227,6 +287,32 @@ commonEnvvars: - name: license type: custom-resource field: spec.wandb.license + - name: GORILLA_OIDC_CLIENT_ID + sources: + - name: oidc + type: custom-resource + field: spec.wandb.oidc.clientId + - name: GORILLA_OIDC_SECRET + sources: + - name: oidc + type: custom-resource + field: spec.wandb.oidc.clientSecret + - name: GORILLA_OIDC_ISSUER + sources: + - name: oidc + type: custom-resource + field: spec.wandb.oidc.issuerUrl + - name: GORILLA_AUTH_METHOD + sources: + - name: oidc + type: custom-resource + field: spec.wandb.oidc.authMethod + - name: GORILLA_SESSION_LENGTH + sources: + - name: oidc + type: custom-resource + field: spec.wandb.oidc.sessionLength + defaultValue: "720h" gorillaCustomerSecrets: - name: GORILLA_CUSTOMER_SECRET_STORE_SOURCE value: "k8s-secretmanager://" @@ -248,7 +334,7 @@ commonEnvvars: sources: - name: default type: clickhouse - field: port + field: http-port - name: WF_CLICKHOUSE_USER sources: - name: default @@ -308,6 +394,8 @@ commonEnvvars: value: "/traces" - name: WEAVE_ENABLE_ONLINE_EVAL value: "true" + - name: WEAVE_ENABLE_AGENT_SCORING + value: "false" - name: WEAVE_ENABLE_EVALUATE_MODEL_WORKER value: "true" - name: WANDB_INTERNAL_SERVICE_TOKEN @@ -316,6 +404,22 @@ commonEnvvars: type: generatedSecret - name: WANDB_INTERNAL_SERVICE_TOKEN_SECRET_NAME value: "weave-worker-auth" + frontend: + - name: REACT_APP_HOST + source: + - name: hostname + type: custom-resource + field: status.wandb.hostname + - name: REACT_APP_ENVIRONMENT_NAME + value: "local" + - name: REACT_APP_ENVIRONMENT_IS_PRIVATE + value: "true" + - name: REACT_APP_ANALYTICS_DISABLED + value: "true" + - name: WEAVE_TRACES_ENABLED + value: "true" + - name: SERVER_FLAG_WEAVE_1_PERCENTAGE + value: "100" flatRunsV2Producer: - name: KAFKA_RUNS_V2_TOPIC_NAME value: "flat-run-fields-updater" @@ -375,7 +479,6 @@ commonEnvvars: sources: - type: telemetry field: gorillaTracer - commonVolumeMounts: internalSigner: - mountPath: /vol/env @@ -389,7 +492,7 @@ applications: name: anaconda2 image: repository: us-docker.pkg.dev/wandb-production/public/wandb/anaconda2 - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: anaconda2 ports: @@ -420,7 +523,6 @@ applications: - gorillaCustomerSecrets - kafka - flatRunsV2Producer - - telemetryOtel commonVolumeMounts: - internalSigner env: @@ -440,7 +542,7 @@ applications: type: mysql image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: api args: @@ -473,8 +575,6 @@ applications: pathType: Prefix executor: name: executor - args: - - executor commonEnvs: - gorillaMysql - gorillaBucket @@ -483,12 +583,13 @@ applications: - gorillaTaskQueueConsumer - gorillaService - gorillaOnprem - - telemetryOtel image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: executor + args: + - executor filemeta: name: filemeta args: @@ -499,10 +600,9 @@ applications: - gorillaRedis - gorillaService - gorillaOnprem - - telemetryOtel image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 filestream: name: filestream features: @@ -515,10 +615,9 @@ applications: - gorillaRedis - gorillaService - gorillaOnprem - - telemetryOtel image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: filestream flat-run-fields-updater: @@ -533,20 +632,21 @@ applications: - gorillaOnprem - kafka - flatRunsV2Consumer - - telemetryOtel env: - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE_SUBSCRIPTIONS_FLAT_RUN_FIELDS_UPDATER value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)?consumer_group_id=flat-run-fields-updater" image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: flat-run-fields-updater frontend: name: frontend image: repository: us-docker.pkg.dev/wandb-production/public/wandb/frontend-nginx - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 + commonEnvs: + - frontend env: - name: FRONTEND_APP_BACKEND sources: @@ -576,26 +676,6 @@ applications: value: "true" - name: OPERATOR_ENABLED value: "true" - - name: REACT_APP_HOST - sources: - - name: hostname - type: custom-resource - field: status.wandb.hostname - - name: REACT_APP_ENVIRONMENT_NAME - value: "local" - - name: REACT_APP_ENVIRONMENT_IS_PRIVATE - value: "true" - - name: REACT_APP_ANALYTICS_DISABLED - value: "true" - - name: WEAVE_TRACES_ENABLED - value: "true" - - name: SERVER_FLAG_WEAVE_1_PERCENTAGE - value: "100" - - name: PUBLIC_URL - sources: - - name: hostname - type: custom-resource - field: status.wandb.hostname containers: - name: frontend ports: @@ -618,6 +698,17 @@ applications: - / servicePort: "8080" pathType: Prefix + volumeMounts: + - name: tmp + mountPath: /tmp/ + source: + type: emptyDir + name: tmp + - name: nginx-cache + mountPath: /var/cache/nginx + source: + type: emptyDir + name: nginx-cache glue: name: glue args: @@ -629,7 +720,6 @@ applications: - gorillaHistoryStore - gorillaService - gorillaOnprem - - telemetryOtel commonVolumeMounts: - internalSigner env: @@ -645,7 +735,7 @@ applications: value: "memory://" image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: glue metric-observer: @@ -660,13 +750,12 @@ applications: - gorillaOnprem - kafka - flatRunsV2Consumer - - telemetryOtel env: - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE_SUBSCRIPTIONS_METRIC_OBSERVER value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)?consumer_group_id=metric-observer" image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: metric-observer parquet: @@ -681,10 +770,9 @@ applications: - gorillaTaskQueueProducer - gorillaService - gorillaOnprem - - telemetryOtel image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: parquet ports: @@ -706,9 +794,21 @@ applications: name: weave image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-python - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 commonEnvs: - - telemetryOtel + env: + - name: DATADOG_TRACE_ENABLED + value: "false" + service: + ports: + - port: 9239 + protocol: TCP + name: weave + ingress: + paths: + - /weave/ + servicePort: "9239" + pathType: Prefix containers: - name: weave ports: @@ -732,19 +832,6 @@ applications: requests: cpu: 100m memory: 128Mi - env: - - name: DATADOG_TRACE_ENABLED - value: "false" - service: - ports: - - port: 9239 - protocol: TCP - name: weave - ingress: - paths: - - /weave/ - servicePort: "9239" - pathType: Prefix volumeMounts: - name: temp-dir mountPath: /tmp/ @@ -758,14 +845,13 @@ applications: name: cache weave-trace: name: weave-trace + image: + repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace + tag: 0.83.0-clickhouse-keeper.2 commonEnvs: - clickhouse - kafka - weaveTrace - - telemetryOtel - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.78.2-rc.1773119635 containers: - name: weave-trace args: @@ -775,16 +861,16 @@ applications: - "0.0.0.0" - "--port" - "8080" - ports: - - containerPort: 8080 - name: weave-trace - protocol: TCP - livenessProbe: - httpGet: - path: /traces/health - readinessProbe: - httpGet: - path: /traces/health + ports: + - containerPort: 8080 + name: weave-trace + protocol: TCP + livenessProbe: + httpGet: + path: /traces/health + readinessProbe: + httpGet: + path: /traces/health service: ports: - port: 8080 @@ -806,7 +892,7 @@ applications: name: weave-trace-worker image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 env: - name: DD_TRACE_ENABLED value: "false" @@ -814,7 +900,6 @@ applications: - clickhouse - kafka - weaveTrace - - telemetryOtel containers: - name: weave-trace-worker args: @@ -823,9 +908,10 @@ applications: - "src.workers.scoring_worker" weave-trace-evaluate-model-worker: name: weave-trace-evaluate-model-worker + legacyKey: weave-evaluate-model-worker image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 env: - name: DD_TRACE_ENABLED value: "false" @@ -833,165 +919,18 @@ applications: - clickhouse - kafka - weaveTrace - - telemetryOtel containers: - name: weave-trace-evaluate-model-worker args: - "python" - "-m" - "src.workers.evaluate_model_worker.evaluate_model_worker" - nginx-proxy: - name: nginx-proxy - features: - - proxy - image: - repository: nginxinc/nginx-unprivileged - tag: latest - commonEnvs: - - telemetryOtel - env: - - name: UPSTREAM_FRONTEND - sources: - - name: frontend - type: service - proto: "" - path: "" - - name: UPSTREAM_API - sources: - - name: api - type: service - proto: "" - path: "" - - name: UPSTREAM_WEAVE_TRACE - sources: - - name: weave - type: service - proto: "" - path: "" - - name: UPSTREAM_WEAVE - sources: - - name: weave - type: service - proto: "" - path: "" - - name: BUCKET_HOST - sources: - - name: default - type: bucket - field: host - - name: BUCKET_PORT - sources: - - name: default - type: bucket - field: port - - name: UPSTREAM_BUCKET - value: "$(BUCKET_HOST):$(BUCKET_PORT)" - files: - - name: envvar.conf.template - mountPath: /etc/nginx/templates - fileName: envvar.conf.template - inline: | - upstream frontend { - server $UPSTREAM_FRONTEND; - } - - upstream api { - server $UPSTREAM_API; - } - - upstream weave_trace { - server $UPSTREAM_WEAVE_TRACE; - } - - upstream weave { - server $UPSTREAM_WEAVE; - } - - upstream bucket { - server $UPSTREAM_BUCKET; - } - - map $host $bucket{ - default $UPSTREAM_BUCKET; - } - - name: nginx.conf - mountPath: /etc/nginx - fileName: nginx.conf - inline: | - worker_processes auto; - - error_log /var/log/nginx/error.log notice; - pid /tmp/nginx.pid; - - events { - worker_connections 1024; - } - - http { - - include /etc/nginx/conf.d/envvar.conf; - - server { - listen 8080; - proxy_set_header Host $http_host; - client_max_body_size 0; - location / { - proxy_pass http://frontend; - } - - location /api { - proxy_pass http://api; - } - location /artifacts { - proxy_pass http://api; - } - location /artifactsV2 { - proxy_pass http://api; - } - - location /files { - proxy_pass http://api; - } - - location /graphql { - proxy_pass http://api; - } - - location /graphql2 { - proxy_pass http://api; - } - - location /oidc { - proxy_pass http://api; - } - - location /traces { - proxy_pass http://weave_trace; - } - - location /weave/ { - proxy_pass http://weave/; - } - - location /bucket { - proxy_ssl_verify off; - proxy_set_header Host $bucket; - proxy_pass https://bucket; - } - } - } - service: - type: NodePort - ports: - - name: http - port: 8080 - protocol: TCP migrations: gorilla: image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 args: - "migrate" - "--db=$(GORILLA_METADATA_STORE)" @@ -1005,7 +944,7 @@ migrations: internal-signer: image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 args: - "secret-generation-job" env: @@ -1018,7 +957,7 @@ migrations: weave-trace: image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.78.2-rc.1773119635 + tag: 0.83.0-clickhouse-keeper.2 args: - python - migrator.py diff --git a/hack/testing-manifests/server-manifest/0.78.0-pre/sizing.yaml b/hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.1/sizing.yaml similarity index 50% rename from hack/testing-manifests/server-manifest/0.78.0-pre/sizing.yaml rename to hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.1/sizing.yaml index 19c87fcf..29f7223b 100644 --- a/hack/testing-manifests/server-manifest/0.78.0-pre/sizing.yaml +++ b/hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.1/sizing.yaml @@ -3,18 +3,17 @@ bucket: sizing: default: replicas: 1 - pools: 1 # Not implemented yet, but will be needed if we commit to minio + pools: 1 volumeSize: 10Gi micro: replicas: 3 - pools: 1 - volumeSize: 10Gi + volumeSize: 50Gi resources: requests: - cpu: 500m + cpu: 1 memory: 4Gi limits: - cpu: 500m + cpu: 1 memory: 4Gi small: replicas: 3 @@ -31,42 +30,41 @@ bucket: volumeSize: 100Gi resources: requests: - cpu: 2 - memory: 8Gi + cpu: 4 + memory: 16Gi limits: - cpu: 2 - memory: 8Gi + cpu: 4 + memory: 16Gi large: replicas: 3 - volumeSize: 100Gi + volumeSize: 200Gi resources: requests: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 32Gi limits: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 32Gi xlarge: replicas: 3 - volumeSize: 100Gi + volumeSize: 200Gi resources: requests: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 32Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: + cpu: 8 + memory: 32Gi + 2xlarge: replicas: 3 - volumeSize: 100Gi + volumeSize: 200Gi resources: requests: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 32Gi limits: - cpu: 2 - memory: 8Gi - + cpu: 8 + memory: 32Gi clickhouse: default: sizing: @@ -75,19 +73,19 @@ clickhouse: replicas: 1 volumeSize: 10Gi micro: - replicas: 3 - volumeSize: 10Gi + replicas: 2 + volumeSize: 30Gi resources: requests: - cpu: 500m + cpu: 1 memory: 4Gi limits: - cpu: 500m + cpu: 1 memory: 4Gi small: shards: 1 replicas: 3 - volumeSize: 100Gi + volumeSize: 50Gi resources: requests: cpu: 2 @@ -96,79 +94,147 @@ clickhouse: cpu: 2 memory: 8Gi medium: + shards: 1 replicas: 3 volumeSize: 100Gi resources: requests: - cpu: 2 - memory: 8Gi + cpu: 4 + memory: 16Gi limits: - cpu: 2 - memory: 8Gi + cpu: 4 + memory: 16Gi large: + shards: 1 replicas: 3 - volumeSize: 100Gi + volumeSize: 200Gi + resources: + requests: + cpu: 8 + memory: 32Gi + limits: + cpu: 8 + memory: 32Gi + xlarge: + shards: 2 + replicas: 3 + volumeSize: 200Gi + resources: + requests: + cpu: 8 + memory: 32Gi + limits: + cpu: 8 + memory: 32Gi + 2xlarge: + shards: 4 + replicas: 3 + volumeSize: 200Gi + resources: + requests: + cpu: 8 + memory: 32Gi + limits: + cpu: 8 + memory: 32Gi +# Keeper coordinates ReplicatedMergeTree replication; it stores only the Raft +# log + metadata snapshots, so its volume is small and independent of CH data. +# replicas must be odd (Raft quorum): 1 for single-node, 3 for replicated CH. +clickhouseKeeper: + default: + sizing: + default: + replicas: 1 + volumeSize: 10Gi + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 500m + memory: 1Gi + micro: + replicas: 3 + volumeSize: 10Gi + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 500m + memory: 1Gi + small: + replicas: 3 + volumeSize: 20Gi + resources: + requests: + cpu: 1 + memory: 2Gi + limits: + cpu: 1 + memory: 2Gi + medium: + replicas: 3 + volumeSize: 20Gi + resources: + requests: + cpu: 1 + memory: 2Gi + limits: + cpu: 1 + memory: 2Gi + large: + replicas: 3 + volumeSize: 20Gi resources: requests: cpu: 2 - memory: 8Gi + memory: 4Gi limits: cpu: 2 - memory: 8Gi + memory: 4Gi xlarge: replicas: 3 - volumeSize: 100Gi + volumeSize: 20Gi resources: requests: cpu: 2 - memory: 8Gi + memory: 4Gi limits: cpu: 2 - memory: 8Gi - xxlarge: + memory: 4Gi + 2xlarge: replicas: 3 - volumeSize: 100Gi + volumeSize: 20Gi resources: requests: cpu: 2 - memory: 8Gi + memory: 4Gi limits: cpu: 2 - memory: 8Gi - + memory: 4Gi kafka: sizing: + # Bufstream brokers are stateless; replicas = broker count (floored at 2), volumeSize = etcd metadata PVC, replicationFactor = topic RF (1 for object-store backed). default: - replicas: 1 + replicas: 2 volumeSize: 10Gi replicationFactor: 1 - minInSyncReplicas: 1 - offsetsTopicRF: 1 - transactionStateRF: 1 - transactionStateISR: 1 micro: - replicas: 3 + replicas: 2 volumeSize: 10Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 + replicationFactor: 1 resources: requests: - cpu: 500m - memory: 1Gi + cpu: 1 + memory: 4Gi limits: - cpu: 500m - memory: 1Gi + cpu: 1 + memory: 4Gi small: - replicas: 3 - volumeSize: 100Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 + replicas: 2 + volumeSize: 10Gi + replicationFactor: 1 resources: requests: cpu: 2 @@ -178,65 +244,48 @@ kafka: memory: 8Gi medium: replicas: 3 - volumeSize: 100Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 + volumeSize: 10Gi + replicationFactor: 1 resources: requests: - cpu: 2 - memory: 8Gi + cpu: 4 + memory: 16Gi limits: - cpu: 2 - memory: 8Gi + cpu: 4 + memory: 16Gi large: replicas: 3 - volumeSize: 100Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 + volumeSize: 10Gi + replicationFactor: 1 resources: requests: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 30Gi limits: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 30Gi xlarge: - replicas: 3 - volumeSize: 100Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 + replicas: 4 + volumeSize: 10Gi + replicationFactor: 1 resources: requests: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 30Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 + cpu: 8 + memory: 30Gi + 2xlarge: + replicas: 6 + volumeSize: 10Gi + replicationFactor: 1 resources: requests: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 30Gi limits: - cpu: 2 - memory: 8Gi - + cpu: 8 + memory: 30Gi mysql: default: sizing: @@ -245,14 +294,14 @@ mysql: volumeSize: 10Gi micro: replicas: 3 - volumeSize: 10Gi + volumeSize: 50Gi resources: requests: - cpu: 500m - memory: 2Gi + cpu: 1 + memory: 4Gi limits: - cpu: 500m - memory: 2Gi + cpu: 1 + memory: 4Gi small: replicas: 3 volumeSize: 100Gi @@ -265,66 +314,65 @@ mysql: memory: 8Gi medium: replicas: 3 - volumeSize: 100Gi + volumeSize: 200Gi resources: requests: - cpu: 2 - memory: 8Gi + cpu: 4 + memory: 16Gi limits: - cpu: 2 - memory: 8Gi + cpu: 4 + memory: 16Gi large: replicas: 3 - volumeSize: 100Gi + volumeSize: 500Gi resources: requests: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 30Gi limits: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 30Gi xlarge: replicas: 3 - volumeSize: 100Gi + volumeSize: 1Ti resources: requests: - cpu: 2 - memory: 8Gi + cpu: 15 + memory: 60Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: + cpu: 15 + memory: 60Gi + 2xlarge: replicas: 3 - volumeSize: 100Gi + volumeSize: 2Ti resources: requests: - cpu: 2 - memory: 8Gi + cpu: 30 + memory: 120Gi limits: - cpu: 2 - memory: 8Gi - + cpu: 30 + memory: 120Gi redis: default: sizing: default: shards: 1 replicas: 1 - volumeSize: 10Gi + volumeSize: 5Gi micro: - replicas: 3 - volumeSize: 10Gi + replicas: 2 + volumeSize: 8Gi resources: requests: - cpu: 500m + cpu: 1 memory: 4Gi limits: - cpu: 500m + cpu: 1 memory: 4Gi small: shards: 1 replicas: 3 - volumeSize: 10Gi + volumeSize: 16Gi resources: requests: cpu: 2 @@ -333,57 +381,60 @@ redis: cpu: 2 memory: 8Gi medium: + shards: 1 replicas: 3 - volumeSize: 100Gi + volumeSize: 32Gi resources: requests: - cpu: 2 - memory: 8Gi + cpu: 4 + memory: 16Gi limits: - cpu: 2 - memory: 8Gi + cpu: 4 + memory: 16Gi large: + shards: 1 replicas: 3 - volumeSize: 100Gi + volumeSize: 64Gi resources: requests: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 30Gi limits: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 30Gi xlarge: + shards: 2 replicas: 3 - volumeSize: 100Gi + volumeSize: 128Gi resources: requests: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 30Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: + cpu: 8 + memory: 30Gi + 2xlarge: + shards: 4 replicas: 3 - volumeSize: 100Gi + volumeSize: 256Gi resources: requests: - cpu: 2 - memory: 8Gi + cpu: 8 + memory: 30Gi limits: - cpu: 2 - memory: 8Gi - + cpu: 8 + memory: 30Gi applications: anaconda2: sizing: micro: resources: limits: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 2Gi requests: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 2Gi autoscaling: horizontal: enabled: true @@ -403,61 +454,6 @@ applications: maxReplicas: 3 minReplicas: 2 medium: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - large: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - xlarge: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - api: - sizing: - micro: - resources: - limits: - cpu: 500m - memory: 2Gi - requests: - cpu: 500m - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: resources: limits: cpu: "4" @@ -470,46 +466,125 @@ applications: enabled: true maxReplicas: 3 minReplicas: 2 - medium: - replicas: 3 - volumeSize: 100Gi + large: resources: + limits: + cpu: "8" + memory: 16Gi requests: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + xlarge: + resources: limits: - cpu: 2 - memory: 8Gi - large: - replicas: 3 - volumeSize: 100Gi + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 6 + minReplicas: 4 + 2xlarge: resources: + limits: + cpu: "8" + memory: 16Gi requests: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 6 + minReplicas: 4 + api: + sizing: + micro: + resources: limits: - cpu: 2 - memory: 8Gi - xlarge: - replicas: 3 - volumeSize: 100Gi + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: resources: + limits: + cpu: "4" + memory: 8Gi requests: - cpu: 2 + cpu: "4" memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + medium: + resources: limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + large: resources: + limits: + cpu: "16" + memory: 32Gi requests: - cpu: 2 - memory: 8Gi + cpu: "16" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 6 + minReplicas: 4 + xlarge: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "16" + memory: 32Gi + requests: + cpu: "16" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 12 + minReplicas: 6 + 2xlarge: + resources: + limits: + cpu: "16" + memory: 32Gi + requests: + cpu: "16" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 16 + minReplicas: 8 executor: sizing: micro: @@ -539,55 +614,67 @@ applications: maxReplicas: 2 minReplicas: 1 medium: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 32Gi + requests: + cpu: "8" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 large: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 xlarge: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "16" + memory: 64Gi requests: - cpu: 2 - memory: 8Gi + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 16 + minReplicas: 8 + 2xlarge: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 32 + minReplicas: 16 filemeta: sizing: micro: resources: limits: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 2Gi requests: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 2Gi autoscaling: horizontal: enabled: true @@ -607,55 +694,67 @@ applications: maxReplicas: 2 minReplicas: 1 medium: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "4" + memory: 4Gi + requests: + cpu: "4" + memory: 4Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 large: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 xlarge: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "8" + memory: 16Gi requests: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + 2xlarge: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 filestream: sizing: micro: resources: limits: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 2Gi requests: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 2Gi autoscaling: horizontal: enabled: true @@ -675,55 +774,67 @@ applications: maxReplicas: 2 minReplicas: 1 medium: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 + cpu: "4" memory: 8Gi - large: - replicas: 3 - volumeSize: 100Gi - resources: requests: - cpu: 2 + cpu: "4" memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + large: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 xlarge: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "8" + memory: 16Gi requests: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 flat-run-fields-updater: sizing: micro: resources: limits: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 2Gi requests: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 2Gi autoscaling: horizontal: enabled: true @@ -740,58 +851,70 @@ applications: autoscaling: horizontal: enabled: true - maxReplicas: 5 + maxReplicas: 2 minReplicas: 1 medium: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 + cpu: "4" memory: 8Gi - large: - replicas: 3 - volumeSize: 100Gi - resources: requests: - cpu: 2 + cpu: "4" memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + large: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 xlarge: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "8" + memory: 16Gi requests: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 frontend: sizing: micro: resources: limits: - cpu: 500m - memory: 512Mi + cpu: "1" + memory: 2Gi requests: - cpu: 500m - memory: 512Mi + cpu: "1" + memory: 2Gi autoscaling: horizontal: enabled: true @@ -811,55 +934,67 @@ applications: maxReplicas: 3 minReplicas: 2 medium: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "4" + memory: 4Gi + requests: + cpu: "4" + memory: 4Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 large: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 + cpu: "8" memory: 8Gi - xlarge: - replicas: 3 - volumeSize: 100Gi - resources: requests: - cpu: 2 + cpu: "8" memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + xlarge: + resources: limits: - cpu: 2 + cpu: "8" memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: requests: - cpu: 2 + cpu: "8" memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + 2xlarge: + resources: limits: - cpu: 2 + cpu: "8" + memory: 8Gi + requests: + cpu: "8" memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 glue: sizing: micro: resources: limits: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 3Gi requests: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 3Gi small: resources: limits: @@ -869,55 +1004,47 @@ applications: cpu: "2" memory: 6Gi medium: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "4" + memory: 12Gi + requests: + cpu: "4" + memory: 12Gi large: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 24Gi + requests: + cpu: "8" + memory: 24Gi xlarge: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "16" + memory: 48Gi requests: - cpu: 2 - memory: 8Gi + cpu: "16" + memory: 48Gi + 2xlarge: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "30" + memory: 64Gi + requests: + cpu: "30" + memory: 64Gi metric-observer: sizing: micro: resources: limits: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 2Gi requests: - cpu: 500m - memory: 1Gi + cpu: "1" + memory: 2Gi autoscaling: horizontal: enabled: true @@ -934,58 +1061,70 @@ applications: autoscaling: horizontal: enabled: true - maxReplicas: 4 + maxReplicas: 2 minReplicas: 1 medium: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 + cpu: "4" memory: 8Gi - large: - replicas: 3 - volumeSize: 100Gi - resources: requests: - cpu: 2 + cpu: "4" memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + large: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 xlarge: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "8" + memory: 16Gi requests: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 parquet: sizing: micro: resources: limits: cpu: "1" - memory: 4Gi + memory: 8Gi requests: cpu: "1" - memory: 4Gi + memory: 8Gi autoscaling: horizontal: enabled: true @@ -1005,45 +1144,57 @@ applications: maxReplicas: 2 minReplicas: 2 medium: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 32Gi + requests: + cpu: "8" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 2 large: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 xlarge: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "16" + memory: 64Gi requests: - cpu: 2 - memory: 8Gi + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 6 + minReplicas: 4 + 2xlarge: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 weave: sizing: micro: @@ -1073,54 +1224,66 @@ applications: maxReplicas: 2 minReplicas: 2 medium: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 32Gi + requests: + cpu: "8" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 large: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - xlarge: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "8" + memory: 32Gi requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi + cpu: "8" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + xlarge: resources: + limits: + cpu: "8" + memory: 32Gi requests: - cpu: 2 - memory: 8Gi + cpu: "8" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 5 + minReplicas: 2 + 2xlarge: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 5 + minReplicas: 2 weave-trace: sizing: micro: resources: limits: - cpu: 500m + cpu: "1" memory: 2Gi requests: - cpu: 500m + cpu: "1" memory: 2Gi autoscaling: horizontal: @@ -1141,54 +1304,66 @@ applications: maxReplicas: 3 minReplicas: 2 medium: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "2" + memory: 16Gi + requests: + cpu: "2" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 large: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 xlarge: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "4" + memory: 32Gi requests: - cpu: 2 - memory: 8Gi + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 weave-trace-worker: sizing: micro: resources: limits: - cpu: 500m + cpu: "1" memory: 2Gi requests: - cpu: 500m + cpu: "1" memory: 2Gi autoscaling: horizontal: @@ -1209,54 +1384,66 @@ applications: maxReplicas: 4 minReplicas: 1 medium: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "2" + memory: 16Gi + requests: + cpu: "2" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 large: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 xlarge: - replicas: 3 - volumeSize: 100Gi resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "4" + memory: 32Gi requests: - cpu: 2 - memory: 8Gi + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: + resources: limits: - cpu: 2 - memory: 8Gi + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 weave-trace-evaluate-model-worker: sizing: micro: resources: limits: - cpu: 500m + cpu: "1" memory: 2Gi requests: - cpu: 500m + cpu: "1" memory: 2Gi autoscaling: horizontal: @@ -1277,109 +1464,54 @@ applications: maxReplicas: 2 minReplicas: 1 medium: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - large: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - xlarge: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - nginx-proxy: - sizing: - default: - autoscaling: - enabled: true - minReplicas: 2 - maxReplicas: 4 resources: limits: cpu: "2" - memory: 2Gi + memory: 16Gi requests: cpu: "2" - memory: 2Gi - micro: + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + large: resources: limits: - cpu: 500m - memory: 512Mi + cpu: "4" + memory: 32Gi requests: - cpu: 500m - memory: 512Mi + cpu: "4" + memory: 32Gi autoscaling: horizontal: enabled: true maxReplicas: 2 minReplicas: 1 - medium: - replicas: 3 - volumeSize: 100Gi + xlarge: resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - large: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "4" + memory: 32Gi requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi - xlarge: - replicas: 3 - volumeSize: 100Gi + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: resources: - requests: - cpu: 2 - memory: 8Gi limits: - cpu: 2 - memory: 8Gi - xxlarge: - replicas: 3 - volumeSize: 100Gi - resources: + cpu: "4" + memory: 32Gi requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 diff --git a/hack/testing-manifests/server-manifest/0.79.0/manifest.yaml b/hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.2/manifest.yaml similarity index 57% rename from hack/testing-manifests/server-manifest/0.79.0/manifest.yaml rename to hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.2/manifest.yaml index 4005af8a..9bbdaf76 100644 --- a/hack/testing-manifests/server-manifest/0.79.0/manifest.yaml +++ b/hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.2/manifest.yaml @@ -1,10 +1,8 @@ --- requiredOperatorVersion: ^2.0.0 -manifestVersion: v1alpha1 features: filestreamQueue: false - proxy: false bucket: default: @@ -13,61 +11,27 @@ bucket: - /bucket servicePort: "http-minio" pathType: Prefix - sizing: - default: - replicas: 1 - pools: 1 # Not implemented yet, but will be needed if we commit to minio - volumeSize: 10Gi - micro: - replicas: 3 - pools: 1 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi + images: + seaweedfs: + registry: "docker.io" + repository: "chrislusf/seaweedfs" + tag: "4.35" clickhouse: default: - sizing: - default: - shards: 1 - replicas: 1 - volumeSize: 10Gi - micro: - replicas: 3 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - shards: 1 - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi + images: + server: + registry: "docker.io" + repository: "altinity/clickhouse-server" + tag: "25.8.16.10002.altinitystable" + +clickhouseKeeper: + default: + images: + keeper: + registry: "docker.io" + repository: "altinity/clickhouse-keeper" + tag: "25.8.16.10002.altinitystable" generatedSecrets: - name: session-key @@ -79,119 +43,68 @@ generatedSecrets: useExactName: true kafka: - sizing: - default: - replicas: 1 - volumeSize: 10Gi - replicationFactor: 1 - minInSyncReplicas: 1 - offsetsTopicRF: 1 - transactionStateRF: 1 - transactionStateISR: 1 - micro: - replicas: 3 - volumeSize: 10Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - replicas: 3 - volumeSize: 100Gi - replicationFactor: 3 - minInSyncReplicas: 2 - offsetsTopicRF: 3 - transactionStateRF: 3 - transactionStateISR: 3 - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi + images: + bufstream: + registry: "us-docker.pkg.dev" + repository: "buf-images-1/buf/images/bufstream" + tag: "0.4.15" + etcd: + registry: "quay.io" + repository: "coreos/etcd" + tag: "v3.5.31" + bucketEnsure: + repository: "amazon/aws-cli" + tag: "2.35.10" topics: - name: filestream features: - filestreamQueue topic: filestream - partitionCount: 48 + partitionCount: 96 - name: flat-run-fields-updater topic: flat-run-fields-updater - partitionCount: 48 + partitionCount: 96 - name: weave-worker topic: weave.call_ended - partitionCount: 48 + partitionCount: 16 - name: weave-evaluate-model-worker topic: weave.evaluate_model - partitionCount: 48 + partitionCount: 16 mysql: default: - sizing: - default: - replicas: 1 - volumeSize: 10Gi - micro: - replicas: 3 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - replicas: 3 - volumeSize: 100Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi + images: + mysql: + registry: "ghcr.io" + repository: "cybozu-go/moco/mysql" + tag: "8.4.8" + exporter: + registry: "docker.io" + repository: "prom/mysqld-exporter" + tag: "v0.15.1" metadata: {} runs: {} usage: {} redis: default: - sizing: - default: - shards: 1 - replicas: 1 - volumeSize: 10Gi - micro: - replicas: 2 - volumeSize: 10Gi - resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 1 - memory: 4Gi - small: - shards: 1 - replicas: 3 - volumeSize: 10Gi - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 2 - memory: 8Gi + images: + standalone: + registry: "quay.io" + repository: "opstree/redis" + tag: "v7.0.15" + replication: + registry: "quay.io" + repository: "opstree/redis" + tag: "v7.0.15" + sentinel: + registry: "quay.io" + repository: "opstree/redis-sentinel" + tag: "v7.0.12" + exporter: + registry: "quay.io" + repository: "opstree/redis-exporter" + tag: "v1.44.0" commonEnvvars: gorillaMysql: @@ -340,11 +253,20 @@ commonEnvvars: - name: session-key type: generatedSecret - name: BUCKET_PROXY - value: "true" + sources: + - name: bucket-proxy + type: custom-resource + field: spec.wandb.bucketProxy - name: GORILLA_GLUE_FILE_STORE_IS_PROXIED - value: "true" + sources: + - name: bucket-proxy + type: custom-resource + field: spec.wandb.bucketProxy - name: GORILLA_FILE_STORE_IS_PROXIED - value: "true" + sources: + - name: bucket-proxy + type: custom-resource + field: spec.wandb.bucketProxy - name: GORILLA_FILE_HOST sources: - name: hostname @@ -412,7 +334,7 @@ commonEnvvars: sources: - name: default type: clickhouse - field: port + field: http-port - name: WF_CLICKHOUSE_USER sources: - name: default @@ -472,6 +394,8 @@ commonEnvvars: value: "/traces" - name: WEAVE_ENABLE_ONLINE_EVAL value: "true" + - name: WEAVE_ENABLE_AGENT_SCORING + value: "false" - name: WEAVE_ENABLE_EVALUATE_MODEL_WORKER value: "true" - name: WANDB_INTERNAL_SERVICE_TOKEN @@ -514,7 +438,47 @@ commonEnvvars: flatRunsV2Consumer: - name: KAFKA_RUNS_V2_TOPIC_NAME value: "flat-run-fields-updater" - + telemetryOtel: + - name: OTEL_EXPORTER_OTLP_PROTOCOL + sources: + - type: telemetry + field: protocol + - name: OTEL_TRACES_EXPORTER + sources: + - type: telemetry + field: tracesExporter + - name: OTEL_METRICS_EXPORTER + sources: + - type: telemetry + field: metricsExporter + - name: OTEL_LOGS_EXPORTER + sources: + - type: telemetry + field: logsExporter + - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT + sources: + - type: telemetry + field: metricsEndpoint + - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT + sources: + - type: telemetry + field: logsEndpoint + - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + sources: + - type: telemetry + field: tracesEndpoint + - name: OTEL_SERVICE_NAME + sources: + - type: telemetry + field: serviceName + - name: OTEL_RESOURCE_ATTRIBUTES + sources: + - type: telemetry + field: resourceAttributes + - name: GORILLA_TRACER + sources: + - type: telemetry + field: gorillaTracer commonVolumeMounts: internalSigner: - mountPath: /vol/env @@ -528,7 +492,7 @@ applications: name: anaconda2 image: repository: us-docker.pkg.dev/wandb-production/public/wandb/anaconda2 - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: anaconda2 ports: @@ -546,33 +510,6 @@ applications: - port: 8080 protocol: TCP name: anaconda2 - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 api: name: api commonEnvs: @@ -605,7 +542,7 @@ applications: type: mysql image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: api args: @@ -636,37 +573,8 @@ applications: - /oidc servicePort: "8080" pathType: Prefix - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "4" - memory: 8Gi - requests: - cpu: "4" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 executor: name: executor - args: - - executor commonEnvs: - gorillaMysql - gorillaBucket @@ -677,23 +585,11 @@ applications: - gorillaOnprem image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: executor - sizing: - small: - resources: - limits: - cpu: "4" - memory: 16Gi - requests: - cpu: "4" - memory: 16Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 + args: + - executor filemeta: name: filemeta args: @@ -706,34 +602,7 @@ applications: - gorillaOnprem image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.79.0 - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 + tag: 0.83.0-clickhouse-keeper.2 filestream: name: filestream features: @@ -748,36 +617,9 @@ applications: - gorillaOnprem image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: filestream - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 flat-run-fields-updater: name: flat-run-fields-updater args: @@ -795,41 +637,14 @@ applications: value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)?consumer_group_id=flat-run-fields-updater" image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: flat-run-fields-updater - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 5 - minReplicas: 1 frontend: name: frontend image: repository: us-docker.pkg.dev/wandb-production/public/wandb/frontend-nginx - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 commonEnvs: - frontend env: @@ -883,33 +698,17 @@ applications: - / servicePort: "8080" pathType: Prefix - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 + volumeMounts: + - name: tmp + mountPath: /tmp/ + source: + type: emptyDir + name: tmp + - name: nginx-cache + mountPath: /var/cache/nginx + source: + type: emptyDir + name: nginx-cache glue: name: glue args: @@ -936,26 +735,9 @@ applications: value: "memory://" image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: glue - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 3Gi - requests: - cpu: "1" - memory: 3Gi - small: - resources: - limits: - cpu: "2" - memory: 6Gi - requests: - cpu: "2" - memory: 6Gi metric-observer: name: metric-observer args: @@ -973,36 +755,9 @@ applications: value: "$(KAFKA_URL)/$(KAFKA_RUNS_V2_TOPIC_NAME)?consumer_group_id=metric-observer" image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: metric-observer - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: "2" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 4 - minReplicas: 1 parquet: name: parquet args: @@ -1017,7 +772,7 @@ applications: - gorillaOnprem image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 containers: - name: parquet ports: @@ -1035,38 +790,25 @@ applications: - port: 8080 protocol: TCP name: parquet - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "4" - memory: 16Gi - requests: - cpu: "4" - memory: 16Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 2 weave: name: weave image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-python - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 + commonEnvs: + env: + - name: DATADOG_TRACE_ENABLED + value: "false" + service: + ports: + - port: 9239 + protocol: TCP + name: weave + ingress: + paths: + - /weave/ + servicePort: "9239" + pathType: Prefix containers: - name: weave ports: @@ -1090,19 +832,6 @@ applications: requests: cpu: 100m memory: 128Mi - env: - - name: DATADOG_TRACE_ENABLED - value: "false" - service: - ports: - - port: 9239 - protocol: TCP - name: weave - ingress: - paths: - - /weave/ - servicePort: "9239" - pathType: Prefix volumeMounts: - name: temp-dir mountPath: /tmp/ @@ -1114,42 +843,15 @@ applications: source: type: emptyDir name: cache - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 4Gi - requests: - cpu: "1" - memory: 4Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "4" - memory: 16Gi - requests: - cpu: "4" - memory: 16Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 2 weave-trace: name: weave-trace + image: + repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace + tag: 0.83.0-clickhouse-keeper.2 commonEnvs: - clickhouse - kafka - weaveTrace - image: - repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.79.0 containers: - name: weave-trace args: @@ -1186,38 +888,11 @@ applications: kubernetesServiceAccount: audience: internal-service expirationSeconds: 600 - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 3 - minReplicas: 2 weave-trace-worker: name: weave-trace-worker image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 env: - name: DD_TRACE_ENABLED value: "false" @@ -1231,38 +906,12 @@ applications: - "python" - "-m" - "src.workers.scoring_worker" - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 4 - minReplicas: 1 weave-trace-evaluate-model-worker: name: weave-trace-evaluate-model-worker + legacyKey: weave-evaluate-model-worker image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 env: - name: DD_TRACE_ENABLED value: "false" @@ -1276,210 +925,12 @@ applications: - "python" - "-m" - "src.workers.evaluate_model_worker.evaluate_model_worker" - sizing: - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - small: - resources: - limits: - cpu: "1" - memory: 8Gi - requests: - cpu: "1" - memory: 8Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 - nginx-proxy: - name: nginx-proxy - features: - - proxy - image: - repository: nginxinc/nginx-unprivileged - tag: latest - containers: - - name: nginx-proxy - env: - - name: UPSTREAM_FRONTEND - sources: - - name: frontend - type: service - proto: "" - path: "" - - name: UPSTREAM_API - sources: - - name: api - type: service - proto: "" - path: "" - - name: UPSTREAM_WEAVE_TRACE - sources: - - name: weave-trace - type: service - proto: "" - path: "" - - name: UPSTREAM_WEAVE - sources: - - name: weave - type: service - proto: "" - path: "" - - name: BUCKET_HOST - sources: - - name: default - type: bucket - field: host - - name: BUCKET_PORT - sources: - - name: default - type: bucket - field: port - - name: UPSTREAM_BUCKET - value: "$(BUCKET_HOST):$(BUCKET_PORT)" - files: - - name: envvar.conf.template - mountPath: /etc/nginx/templates - fileName: envvar.conf.template - inline: | - upstream frontend { - server $UPSTREAM_FRONTEND; - } - - upstream api { - server $UPSTREAM_API; - } - - upstream weave_trace { - server $UPSTREAM_WEAVE_TRACE; - } - - upstream weave { - server $UPSTREAM_WEAVE; - } - - upstream bucket { - server $UPSTREAM_BUCKET; - } - - map $host $bucket{ - default $UPSTREAM_BUCKET; - } - - name: nginx.conf - mountPath: /etc/nginx - fileName: nginx.conf - inline: | - worker_processes auto; - - error_log /var/log/nginx/error.log notice; - pid /tmp/nginx.pid; - - events { - worker_connections 1024; - } - - http { - - include /etc/nginx/conf.d/envvar.conf; - - server { - listen 8080; - proxy_set_header Host $http_host; - client_max_body_size 0; - location / { - proxy_pass http://frontend; - } - - location /api { - proxy_pass http://api; - } - location /artifacts { - proxy_pass http://api; - } - location /artifactsV2 { - proxy_pass http://api; - } - - location /files { - proxy_pass http://api; - } - - location /graphql { - proxy_pass http://api; - } - - location /graphql2 { - proxy_pass http://api; - } - location /oidc { - proxy_pass http://api; - } - - location /traces { - proxy_pass http://weave_trace; - } - - location /weave/ { - proxy_pass http://weave/; - } - - location /bucket { - proxy_ssl_verify off; - proxy_set_header Host $bucket; - proxy_pass http://bucket; - } - } - } - service: - type: NodePort - ports: - - name: http - port: 8080 - protocol: TCP - sizing: - default: - autoscaling: - enabled: true - minReplicas: 2 - maxReplicas: 4 - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - micro: - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - autoscaling: - horizontal: - enabled: true - maxReplicas: 2 - minReplicas: 1 migrations: gorilla: image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 args: - "migrate" - "--db=$(GORILLA_METADATA_STORE)" @@ -1493,7 +944,7 @@ migrations: internal-signer: image: repository: us-docker.pkg.dev/wandb-production/public/wandb/megabinary - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 args: - "secret-generation-job" env: @@ -1506,7 +957,7 @@ migrations: weave-trace: image: repository: us-docker.pkg.dev/wandb-production/public/wandb/weave-trace - tag: 0.79.0 + tag: 0.83.0-clickhouse-keeper.2 args: - python - migrator.py diff --git a/hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.2/sizing.yaml b/hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.2/sizing.yaml new file mode 100644 index 00000000..5a97462b --- /dev/null +++ b/hack/testing-manifests/server-manifest/0.83.0-clickhouse-keeper.2/sizing.yaml @@ -0,0 +1,1518 @@ +bucket: + default: + sizing: + default: + replicas: 1 + pools: 1 + volumeSize: 10Gi + micro: + replicas: 3 + volumeSize: 50Gi + resources: + requests: + cpu: 1 + memory: 4Gi + limits: + cpu: 1 + memory: 4Gi + small: + replicas: 3 + volumeSize: 100Gi + resources: + requests: + cpu: 2 + memory: 8Gi + limits: + cpu: 2 + memory: 8Gi + medium: + replicas: 3 + volumeSize: 100Gi + resources: + requests: + cpu: 4 + memory: 16Gi + limits: + cpu: 4 + memory: 16Gi + large: + replicas: 3 + volumeSize: 200Gi + metadataVolumeSize: 40Gi + resources: + requests: + cpu: 8 + memory: 32Gi + limits: + cpu: 8 + memory: 32Gi + xlarge: + replicas: 3 + volumeSize: 200Gi + resources: + requests: + cpu: 8 + memory: 32Gi + limits: + cpu: 8 + memory: 32Gi + 2xlarge: + replicas: 3 + volumeSize: 200Gi + resources: + requests: + cpu: 8 + memory: 32Gi + limits: + cpu: 8 + memory: 32Gi +clickhouse: + default: + sizing: + default: + shards: 1 + replicas: 1 + volumeSize: 10Gi + micro: + replicas: 2 + volumeSize: 30Gi + resources: + requests: + cpu: 1 + memory: 4Gi + limits: + cpu: 1 + memory: 4Gi + small: + shards: 1 + replicas: 3 + volumeSize: 50Gi + resources: + requests: + cpu: 2 + memory: 8Gi + limits: + cpu: 2 + memory: 8Gi + medium: + shards: 1 + replicas: 3 + volumeSize: 100Gi + resources: + requests: + cpu: 4 + memory: 16Gi + limits: + cpu: 4 + memory: 16Gi + large: + shards: 1 + replicas: 3 + volumeSize: 200Gi + resources: + requests: + cpu: 8 + memory: 32Gi + limits: + cpu: 8 + memory: 32Gi + xlarge: + shards: 2 + replicas: 3 + volumeSize: 200Gi + resources: + requests: + cpu: 8 + memory: 32Gi + limits: + cpu: 8 + memory: 32Gi + 2xlarge: + shards: 4 + replicas: 3 + volumeSize: 200Gi + resources: + requests: + cpu: 8 + memory: 32Gi + limits: + cpu: 8 + memory: 32Gi +# Keeper coordinates ReplicatedMergeTree replication; it stores only the Raft +# log + metadata snapshots, so its volume is small and independent of CH data. +# replicas must be odd (Raft quorum): 1 for single-node, 3 for replicated CH. +clickhouseKeeper: + default: + sizing: + default: + replicas: 1 + volumeSize: 10Gi + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 500m + memory: 1Gi + micro: + replicas: 3 + volumeSize: 10Gi + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 500m + memory: 1Gi + small: + replicas: 3 + volumeSize: 20Gi + resources: + requests: + cpu: 1 + memory: 2Gi + limits: + cpu: 1 + memory: 2Gi + medium: + replicas: 3 + volumeSize: 20Gi + resources: + requests: + cpu: 1 + memory: 2Gi + limits: + cpu: 1 + memory: 2Gi + large: + replicas: 3 + volumeSize: 20Gi + resources: + requests: + cpu: 2 + memory: 4Gi + limits: + cpu: 2 + memory: 4Gi + xlarge: + replicas: 3 + volumeSize: 20Gi + resources: + requests: + cpu: 2 + memory: 4Gi + limits: + cpu: 2 + memory: 4Gi + 2xlarge: + replicas: 3 + volumeSize: 20Gi + resources: + requests: + cpu: 2 + memory: 4Gi + limits: + cpu: 2 + memory: 4Gi +kafka: + sizing: + # Bufstream brokers are stateless; replicas = broker count (floored at 2), volumeSize = etcd metadata PVC, replicationFactor = topic RF (1 for object-store backed). + default: + replicas: 2 + volumeSize: 10Gi + replicationFactor: 1 + micro: + replicas: 2 + volumeSize: 10Gi + replicationFactor: 1 + resources: + requests: + cpu: 1 + memory: 4Gi + limits: + cpu: 1 + memory: 4Gi + small: + replicas: 2 + volumeSize: 10Gi + replicationFactor: 1 + resources: + requests: + cpu: 2 + memory: 8Gi + limits: + cpu: 2 + memory: 8Gi + medium: + replicas: 3 + volumeSize: 10Gi + replicationFactor: 1 + resources: + requests: + cpu: 4 + memory: 16Gi + limits: + cpu: 4 + memory: 16Gi + large: + replicas: 3 + volumeSize: 10Gi + replicationFactor: 1 + resources: + requests: + cpu: 8 + memory: 30Gi + limits: + cpu: 8 + memory: 30Gi + xlarge: + replicas: 4 + volumeSize: 10Gi + replicationFactor: 1 + resources: + requests: + cpu: 8 + memory: 30Gi + limits: + cpu: 8 + memory: 30Gi + 2xlarge: + replicas: 6 + volumeSize: 10Gi + replicationFactor: 1 + resources: + requests: + cpu: 8 + memory: 30Gi + limits: + cpu: 8 + memory: 30Gi +mysql: + default: + sizing: + default: + replicas: 1 + volumeSize: 10Gi + micro: + replicas: 3 + volumeSize: 50Gi + resources: + requests: + cpu: 1 + memory: 4Gi + limits: + cpu: 1 + memory: 4Gi + small: + replicas: 3 + volumeSize: 100Gi + resources: + requests: + cpu: 2 + memory: 8Gi + limits: + cpu: 2 + memory: 8Gi + medium: + replicas: 3 + volumeSize: 200Gi + resources: + requests: + cpu: 4 + memory: 16Gi + limits: + cpu: 4 + memory: 16Gi + large: + replicas: 3 + volumeSize: 500Gi + resources: + requests: + cpu: 8 + memory: 30Gi + limits: + cpu: 8 + memory: 30Gi + xlarge: + replicas: 3 + volumeSize: 1Ti + resources: + requests: + cpu: 15 + memory: 60Gi + limits: + cpu: 15 + memory: 60Gi + 2xlarge: + replicas: 3 + volumeSize: 2Ti + resources: + requests: + cpu: 30 + memory: 120Gi + limits: + cpu: 30 + memory: 120Gi +redis: + default: + sizing: + default: + shards: 1 + replicas: 1 + volumeSize: 5Gi + micro: + replicas: 2 + volumeSize: 8Gi + resources: + requests: + cpu: 1 + memory: 4Gi + limits: + cpu: 1 + memory: 4Gi + small: + shards: 1 + replicas: 3 + volumeSize: 16Gi + resources: + requests: + cpu: 2 + memory: 8Gi + limits: + cpu: 2 + memory: 8Gi + medium: + shards: 1 + replicas: 3 + volumeSize: 32Gi + resources: + requests: + cpu: 4 + memory: 16Gi + limits: + cpu: 4 + memory: 16Gi + large: + shards: 1 + replicas: 3 + volumeSize: 64Gi + resources: + requests: + cpu: 8 + memory: 30Gi + limits: + cpu: 8 + memory: 30Gi + xlarge: + shards: 2 + replicas: 3 + volumeSize: 128Gi + resources: + requests: + cpu: 8 + memory: 30Gi + limits: + cpu: 8 + memory: 30Gi + 2xlarge: + shards: 4 + replicas: 3 + volumeSize: 256Gi + resources: + requests: + cpu: 8 + memory: 30Gi + limits: + cpu: 8 + memory: 30Gi +applications: + anaconda2: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "2" + memory: 4Gi + requests: + cpu: "2" + memory: 4Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + medium: + resources: + limits: + cpu: "4" + memory: 8Gi + requests: + cpu: "4" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + large: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + xlarge: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 6 + minReplicas: 4 + 2xlarge: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 6 + minReplicas: 4 + api: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "4" + memory: 8Gi + requests: + cpu: "4" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + medium: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + large: + resources: + limits: + cpu: "16" + memory: 32Gi + requests: + cpu: "16" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 6 + minReplicas: 4 + xlarge: + resources: + limits: + cpu: "16" + memory: 32Gi + requests: + cpu: "16" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 12 + minReplicas: 6 + 2xlarge: + resources: + limits: + cpu: "16" + memory: 32Gi + requests: + cpu: "16" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 16 + minReplicas: 8 + executor: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 4Gi + requests: + cpu: "1" + memory: 4Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "4" + memory: 16Gi + requests: + cpu: "4" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + medium: + resources: + limits: + cpu: "8" + memory: 32Gi + requests: + cpu: "8" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + large: + resources: + limits: + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 + xlarge: + resources: + limits: + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 16 + minReplicas: 8 + 2xlarge: + resources: + limits: + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 32 + minReplicas: 16 + filemeta: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + medium: + resources: + limits: + cpu: "4" + memory: 4Gi + requests: + cpu: "4" + memory: 4Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + large: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + xlarge: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + 2xlarge: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + filestream: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "2" + memory: 4Gi + requests: + cpu: "2" + memory: 4Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + medium: + resources: + limits: + cpu: "4" + memory: 8Gi + requests: + cpu: "4" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + large: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + xlarge: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 + flat-run-fields-updater: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "2" + memory: 4Gi + requests: + cpu: "2" + memory: 4Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + medium: + resources: + limits: + cpu: "4" + memory: 8Gi + requests: + cpu: "4" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + large: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + xlarge: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 + frontend: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + medium: + resources: + limits: + cpu: "4" + memory: 4Gi + requests: + cpu: "4" + memory: 4Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + large: + resources: + limits: + cpu: "8" + memory: 8Gi + requests: + cpu: "8" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + xlarge: + resources: + limits: + cpu: "8" + memory: 8Gi + requests: + cpu: "8" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + 2xlarge: + resources: + limits: + cpu: "8" + memory: 8Gi + requests: + cpu: "8" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + glue: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 3Gi + requests: + cpu: "1" + memory: 3Gi + small: + resources: + limits: + cpu: "2" + memory: 6Gi + requests: + cpu: "2" + memory: 6Gi + medium: + resources: + limits: + cpu: "4" + memory: 12Gi + requests: + cpu: "4" + memory: 12Gi + large: + resources: + limits: + cpu: "8" + memory: 24Gi + requests: + cpu: "8" + memory: 24Gi + xlarge: + resources: + limits: + cpu: "16" + memory: 48Gi + requests: + cpu: "16" + memory: 48Gi + 2xlarge: + resources: + limits: + cpu: "30" + memory: 64Gi + requests: + cpu: "30" + memory: 64Gi + metric-observer: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "2" + memory: 4Gi + requests: + cpu: "2" + memory: 4Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + medium: + resources: + limits: + cpu: "4" + memory: 8Gi + requests: + cpu: "4" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + large: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + xlarge: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: + resources: + limits: + cpu: "8" + memory: 16Gi + requests: + cpu: "8" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 + parquet: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 8Gi + requests: + cpu: "1" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "4" + memory: 16Gi + requests: + cpu: "4" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 2 + medium: + resources: + limits: + cpu: "8" + memory: 32Gi + requests: + cpu: "8" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 2 + large: + resources: + limits: + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + xlarge: + resources: + limits: + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 6 + minReplicas: 4 + 2xlarge: + resources: + limits: + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 + weave: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 4Gi + requests: + cpu: "1" + memory: 4Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "4" + memory: 16Gi + requests: + cpu: "4" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 2 + medium: + resources: + limits: + cpu: "8" + memory: 32Gi + requests: + cpu: "8" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + large: + resources: + limits: + cpu: "8" + memory: 32Gi + requests: + cpu: "8" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + xlarge: + resources: + limits: + cpu: "8" + memory: 32Gi + requests: + cpu: "8" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 5 + minReplicas: 2 + 2xlarge: + resources: + limits: + cpu: "16" + memory: 64Gi + requests: + cpu: "16" + memory: 64Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 5 + minReplicas: 2 + weave-trace: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "1" + memory: 8Gi + requests: + cpu: "1" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 3 + minReplicas: 2 + medium: + resources: + limits: + cpu: "2" + memory: 16Gi + requests: + cpu: "2" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + large: + resources: + limits: + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + xlarge: + resources: + limits: + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: + resources: + limits: + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 + weave-trace-worker: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "1" + memory: 8Gi + requests: + cpu: "1" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 1 + medium: + resources: + limits: + cpu: "2" + memory: 16Gi + requests: + cpu: "2" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + large: + resources: + limits: + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + xlarge: + resources: + limits: + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: + resources: + limits: + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 + weave-trace-evaluate-model-worker: + sizing: + micro: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + small: + resources: + limits: + cpu: "1" + memory: 8Gi + requests: + cpu: "1" + memory: 8Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + medium: + resources: + limits: + cpu: "2" + memory: 16Gi + requests: + cpu: "2" + memory: 16Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + large: + resources: + limits: + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 2 + minReplicas: 1 + xlarge: + resources: + limits: + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 4 + minReplicas: 2 + 2xlarge: + resources: + limits: + cpu: "4" + memory: 32Gi + requests: + cpu: "4" + memory: 32Gi + autoscaling: + horizontal: + enabled: true + maxReplicas: 8 + minReplicas: 4 diff --git a/hack/testing-manifests/test-infra/templates/_helpers.tpl b/hack/testing-manifests/test-infra/templates/_helpers.tpl index 330eeab7..2e800a8e 100644 --- a/hack/testing-manifests/test-infra/templates/_helpers.tpl +++ b/hack/testing-manifests/test-infra/templates/_helpers.tpl @@ -22,3 +22,27 @@ app.kubernetes.io/component: {{ .component }} {{- define "test-infra.seaweedfsHost" -}} {{ .Values.seaweedfs.service.name }}.{{ .Release.Namespace }}.svc.cluster.local {{- end -}} + +{{- define "test-infra.mysqlTLS" -}} +{{- if .Values.mysql.tls.enabled -}}true{{- end -}} +{{- end -}} + +{{- define "test-infra.redisTLS" -}} +{{- if .Values.redis.tls.enabled -}}true{{- end -}} +{{- end -}} + +{{- define "test-infra.anyTLS" -}} +{{- if or .Values.tls.enabled .Values.mysql.tls.enabled .Values.redis.tls.enabled -}}true{{- end -}} +{{- end -}} + +{{- define "test-infra.selfSignedIssuerName" -}} +{{ .Values.tls.issuer.selfSignedName }} +{{- end -}} + +{{- define "test-infra.caIssuerName" -}} +{{ .Values.tls.issuer.caName }} +{{- end -}} + +{{- define "test-infra.caSecretName" -}} +{{ .Values.tls.ca.secretName }} +{{- end -}} diff --git a/hack/testing-manifests/test-infra/templates/certificates.yaml b/hack/testing-manifests/test-infra/templates/certificates.yaml new file mode 100644 index 00000000..16b6d312 --- /dev/null +++ b/hack/testing-manifests/test-infra/templates/certificates.yaml @@ -0,0 +1,100 @@ +{{- if include "test-infra.anyTLS" . }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "test-infra.selfSignedIssuerName" . }} + labels: + {{- include "test-infra.componentLabels" (dict "root" . "component" "tls") | nindent 4 }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "test-infra.caSecretName" . }} + labels: + {{- include "test-infra.componentLabels" (dict "root" . "component" "tls") | nindent 4 }} +spec: + secretName: {{ include "test-infra.caSecretName" . }} + isCA: true + commonName: test-infra-ca + duration: {{ .Values.tls.duration }} + usages: + - cert sign + - crl sign + privateKey: + algorithm: RSA + size: 2048 + issuerRef: + name: {{ include "test-infra.selfSignedIssuerName" . }} + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "test-infra.caIssuerName" . }} + labels: + {{- include "test-infra.componentLabels" (dict "root" . "component" "tls") | nindent 4 }} +spec: + ca: + secretName: {{ include "test-infra.caSecretName" . }} +{{- if include "test-infra.mysqlTLS" . }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.mysql.tls.secretName }} + labels: + {{- include "test-infra.componentLabels" (dict "root" . "component" "mysql") | nindent 4 }} +spec: + secretName: {{ .Values.mysql.tls.secretName }} + commonName: {{ include "test-infra.mysqlHost" . }} + duration: {{ .Values.tls.duration }} + usages: + - digital signature + - key encipherment + - server auth + dnsNames: + - {{ .Values.mysql.service.name | quote }} + - {{ printf "%s.%s" .Values.mysql.service.name .Release.Namespace | quote }} + - {{ printf "%s.%s.svc" .Values.mysql.service.name .Release.Namespace | quote }} + - {{ include "test-infra.mysqlHost" . | quote }} + privateKey: + algorithm: RSA + size: 2048 + issuerRef: + name: {{ include "test-infra.caIssuerName" . }} + kind: Issuer + group: cert-manager.io +{{- end }} +{{- if include "test-infra.redisTLS" . }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.redis.tls.secretName }} + labels: + {{- include "test-infra.componentLabels" (dict "root" . "component" "redis") | nindent 4 }} +spec: + secretName: {{ .Values.redis.tls.secretName }} + commonName: {{ include "test-infra.redisHost" . }} + duration: {{ .Values.tls.duration }} + usages: + - digital signature + - key encipherment + - server auth + dnsNames: + - {{ .Values.redis.service.name | quote }} + - {{ printf "%s.%s" .Values.redis.service.name .Release.Namespace | quote }} + - {{ printf "%s.%s.svc" .Values.redis.service.name .Release.Namespace | quote }} + - {{ include "test-infra.redisHost" . | quote }} + privateKey: + algorithm: RSA + size: 2048 + issuerRef: + name: {{ include "test-infra.caIssuerName" . }} + kind: Issuer + group: cert-manager.io +{{- end }} +{{- end }} diff --git a/hack/testing-manifests/test-infra/templates/mysql-deployment.yaml b/hack/testing-manifests/test-infra/templates/mysql-deployment.yaml index 35d10d87..0eb425aa 100644 --- a/hack/testing-manifests/test-infra/templates/mysql-deployment.yaml +++ b/hack/testing-manifests/test-infra/templates/mysql-deployment.yaml @@ -22,6 +22,13 @@ spec: - name: mysql image: {{ .Values.mysql.image }} imagePullPolicy: IfNotPresent + {{- if include "test-infra.mysqlTLS" . }} + args: + - --require-secure-transport=ON + - --ssl-ca=/etc/mysql/tls/ca.crt + - --ssl-cert=/etc/mysql/tls/tls.crt + - --ssl-key=/etc/mysql/tls/tls.key + {{- end }} ports: - name: mysql containerPort: 3306 @@ -58,7 +65,17 @@ spec: volumeMounts: - name: data mountPath: /var/lib/mysql + {{- if include "test-infra.mysqlTLS" . }} + - name: tls + mountPath: /etc/mysql/tls + readOnly: true + {{- end }} volumes: - name: data emptyDir: {} + {{- if include "test-infra.mysqlTLS" . }} + - name: tls + secret: + secretName: {{ .Values.mysql.tls.secretName }} + {{- end }} {{- end }} diff --git a/hack/testing-manifests/test-infra/templates/redis-deployment.yaml b/hack/testing-manifests/test-infra/templates/redis-deployment.yaml index 63c96397..d27ef5fd 100644 --- a/hack/testing-manifests/test-infra/templates/redis-deployment.yaml +++ b/hack/testing-manifests/test-infra/templates/redis-deployment.yaml @@ -22,19 +22,34 @@ spec: - name: redis image: {{ .Values.redis.image }} imagePullPolicy: IfNotPresent - {{- if .Values.redis.password }} + {{- if include "test-infra.redisTLS" . }} + command: + - /bin/sh + - -c + args: + - > + exec redis-server + --port 0 + --tls-port {{ .Values.redis.service.port }} + --tls-cert-file /etc/redis/tls/tls.crt + --tls-key-file /etc/redis/tls/tls.key + --tls-ca-cert-file /etc/redis/tls/ca.crt + --tls-auth-clients no{{- if .Values.redis.password }} --requirepass "$REDIS_PASSWORD"{{- end }} + {{- else if .Values.redis.password }} command: - /bin/sh - -c - exec redis-server --requirepass "$REDIS_PASSWORD" + {{- else }} + args: ["redis-server"] + {{- end }} + {{- if .Values.redis.password }} env: - name: REDIS_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.redis.secret.name }} key: Password - {{- else }} - args: ["redis-server"] {{- end }} ports: - name: redis @@ -54,7 +69,17 @@ spec: volumeMounts: - name: data mountPath: /data + {{- if include "test-infra.redisTLS" . }} + - name: tls + mountPath: /etc/redis/tls + readOnly: true + {{- end }} volumes: - name: data emptyDir: {} + {{- if include "test-infra.redisTLS" . }} + - name: tls + secret: + secretName: {{ .Values.redis.tls.secretName }} + {{- end }} {{- end }} diff --git a/hack/testing-manifests/test-infra/templates/seaweedfs-deployment.yaml b/hack/testing-manifests/test-infra/templates/seaweedfs-deployment.yaml index 85783110..d84c47d6 100644 --- a/hack/testing-manifests/test-infra/templates/seaweedfs-deployment.yaml +++ b/hack/testing-manifests/test-infra/templates/seaweedfs-deployment.yaml @@ -55,7 +55,7 @@ spec: containerPort: 8333 readinessProbe: httpGet: - path: / + path: /status port: 8333 initialDelaySeconds: 5 periodSeconds: 5 diff --git a/hack/testing-manifests/test-infra/values.yaml b/hack/testing-manifests/test-infra/values.yaml index d21d01ef..9b59f9f8 100644 --- a/hack/testing-manifests/test-infra/values.yaml +++ b/hack/testing-manifests/test-infra/values.yaml @@ -1,3 +1,12 @@ +tls: + enabled: false + ca: + secretName: test-infra-ca + issuer: + selfSignedName: test-infra-selfsigned + caName: test-infra-ca + duration: 2160h + mysql: enabled: true image: mysql:8.0 @@ -10,6 +19,9 @@ mysql: port: 3306 secret: name: external-mysql-connection + tls: + enabled: false + secretName: external-mysql-tls resources: requests: cpu: 100m @@ -24,6 +36,9 @@ redis: port: 6379 secret: name: external-redis-connection + tls: + enabled: false + secretName: external-redis-tls resources: requests: cpu: 50m diff --git a/hack/testing-manifests/wandb/custom-ca-e2e/README.md b/hack/testing-manifests/wandb/custom-ca-e2e/README.md new file mode 100644 index 00000000..7e75740e --- /dev/null +++ b/hack/testing-manifests/wandb/custom-ca-e2e/README.md @@ -0,0 +1,52 @@ +# Custom CA Tilt E2E + +This runbook exercises operator v2 custom CA parity with composable Tilt +settings. Tilt generates test CA material at render time, writes it through the +normal W&B CR and user ConfigMap inputs, and installs TLS-enabled external +MySQL/Redis only when those external services are selected. + +## Full Parity Run + +Use all external infra plus custom CA: + +```python +SETTINGS = { + "includeCR": True, + "wandbNamespace": "wandb-ca-e2e", + "useExternalMysql": True, + "useExternalRedis": True, + "useExternalObjectStore": True, + "useCustomCA": True, +} +``` + +Then run: + +```bash +tilt up +``` + +Wait for `Test-Infra`, `wandb-operator`, `Wandb`, and `Wandb-Endpoint`, then +run the verifier from a terminal: + +```bash +./hack/scripts/verify-custom-ca-e2e.sh --namespace wandb-ca-e2e --name wandb +``` + +## Composable Variants + +The settings are independent: + +- `useExternalMysql=True` installs local MySQL from `test-infra` and points the + W&B CR at the generated connection Secret. +- `useExternalRedis=True` does the same for Redis. +- `useExternalObjectStore=True` does the same for SeaweedFS/S3. +- `useCustomCA=True` generates global custom CA material. If external MySQL or + Redis are also enabled, their test-infra services use TLS and the CR includes + `sslCa` selectors for the generated CA Secrets. + +## Clean Up + +```bash +./hack/scripts/tilt-down-dev-clean.sh --namespace wandb-ca-e2e --name wandb +``` diff --git a/hack/testing-manifests/wandb/oci.yaml b/hack/testing-manifests/wandb/oci.yaml new file mode 100644 index 00000000..49f968e1 --- /dev/null +++ b/hack/testing-manifests/wandb/oci.yaml @@ -0,0 +1,74 @@ +--- +apiVersion: apps.wandb.com/v1 +kind: WeightsAndBiases +metadata: + labels: + app.kubernetes.io/name: weightsandbiases + app.kubernetes.io/instance: weightsandbiases-sample + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: operator + name: wandb-default +spec: + chart: + url: oci://host.internal:5050/wandb/operator-wandb + version: "0.41.3" + plainHTTP: true + credentialSecret: + name: oci-registry-creds + values: + global: + bucket: + provider: "s3" + name: "minio.minio.svc.cluster.local:9000/bucket" + region: "us-east-1" + accessKey: "minio" + secretKey: "minio123" + + app: + resources: + requests: + cpu: "100m" + memory: "128Mi" + + parquet: + resources: + requests: + cpu: "100m" + memory: "128Mi" + + weave: + resources: + requests: + cpu: "100m" + memory: "128Mi" + + console: + resources: + requests: + cpu: "100m" + memory: "128Mi" + + ingress: + install: false + create: false + + mysql: + install: true + resources: + requests: + cpu: "100m" + memory: "128Mi" + + redis: + install: true + resources: + requests: + cpu: "100m" + memory: "128Mi" + + reloader: + install: true + + settingsMigrationJob: + install: false diff --git a/hack/testing-manifests/wandb/wandb-legacy-overrides-v1.yaml b/hack/testing-manifests/wandb/wandb-legacy-overrides-v1.yaml new file mode 100644 index 00000000..ef289118 --- /dev/null +++ b/hack/testing-manifests/wandb/wandb-legacy-overrides-v1.yaml @@ -0,0 +1,71 @@ +--- +# v1 CR exercising the legacyOverrides conversion path: +# - global env/extraEnv (env wins on BOTH_LAYERS) -> legacyOverrides.global +# - per-app env (api, incl. a valueFrom body) and resources (parquet sizing) +# - a renamed section (nginx -> nginx-proxy via the manifest's legacyKey) +# - sections with no v2 application (app, console) that the reconciler +# should log as unmapped and leave unapplied +# - a helm-templated env value the conversion should drop with a log +apiVersion: apps.wandb.com/v1 +kind: WeightsAndBiases +metadata: + labels: + app.kubernetes.io/name: weightsandbiases + app.kubernetes.io/instance: weightsandbiases-sample + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: operator + # Keep CR names <=16 chars: managed ClickHouse derives + # chk--clickhouse-keeper-deploy-confd-default-0-0 volume names that + # must fit the 63-char DNS label limit or the Altinity operator silently + # fails to create the StatefulSets. + name: wandb-legacy-v1 +spec: + chart: + url: https://charts.wandb.ai + name: "operator-wandb" + version: "0.37.1" + values: + global: + size: "dev" + env: + GLOBAL_FROM_ENV: "global-env" + BOTH_LAYERS: "from-env" + extraEnv: + BOTH_LAYERS: "from-extra-env" + HTTP_PROXY: "http://proxy.internal:3128" + NO_PROXY: "10.0.0.0/8,.svc.cluster.local" + TEMPLATED_DROPPED: "{{ .Release.Name }}-value" + + api: + env: + GORILLA_CUSTOM_FLAG: "true" + API_SECRET_VAR: + valueFrom: + secretKeyRef: + name: my-api-secret + key: token + + parquet: + sizing: + dev: + resources: + requests: + cpu: "250m" + memory: "256Mi" + limits: + memory: "1Gi" + + nginx: + env: + NGINX_EXTRA: "1" + + app: + image: + tag: 0.83.0-daily.17 + env: + MONOLITH_ONLY_VAR: "ignored-by-v2" + + console: + env: + CONSOLE_ONLY_VAR: "ignored-by-v2" diff --git a/hack/tilt/wandbcr/main.go b/hack/tilt/wandbcr/main.go index cf3388c8..d7841600 100644 --- a/hack/tilt/wandbcr/main.go +++ b/hack/tilt/wandbcr/main.go @@ -1,14 +1,22 @@ package main import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" "flag" "fmt" + "math/big" "os" "path/filepath" "strings" + "time" v2 "github.com/wandb/operator/api/v2" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" ) @@ -24,6 +32,14 @@ const ( defaultVersion = "0.80.0" defaultSize = v2.SizeDev defaultRetentionPolicy = v2.DetachOnDelete + customCAConfigMapName = "wandb-user-ca-certs" + customCAConfigMapKey = "user-ca.crt" + + externalMySQLSecret = "external-mysql-connection" + externalMySQLTLSSecret = "external-mysql-tls" + externalRedisSecret = "external-redis-connection" + externalRedisTLSSecret = "external-redis-tls" + externalObjectStoreSecret = "external-objectstore-connection" ) type Options struct { @@ -44,6 +60,13 @@ type Options struct { CreateCA bool CreateCASet bool IssuerName string + + ExternalMySQL bool + ExternalRedis bool + ExternalObjectStore bool + CustomCA bool + CustomCAConfigMapOut string + CustomCACertificatePEM string } func main() { @@ -64,6 +87,11 @@ func main() { flag.StringVar(&opts.IngressClass, "ingress-class", "nginx", "IngressClass name for ingress mode") flag.BoolVar(&opts.CreateCA, "create-ca", true, "Use the generated W&B CA issuer for HTTPS hostnames") flag.StringVar(&opts.IssuerName, "issuer-name", "", "Existing cert-manager issuer for HTTPS hostnames") + flag.BoolVar(&opts.ExternalMySQL, "external-mysql", false, "Use the test-infra external MySQL connection Secret") + flag.BoolVar(&opts.ExternalRedis, "external-redis", false, "Use the test-infra external Redis connection Secret") + flag.BoolVar(&opts.ExternalObjectStore, "external-objectstore", false, "Use the test-infra external object store connection Secret") + flag.BoolVar(&opts.CustomCA, "custom-ca", false, "Generate and configure custom CA material") + flag.StringVar(&opts.CustomCAConfigMapOut, "custom-ca-configmap-out", "", "Path to write the generated custom CA ConfigMap YAML") flag.Parse() flag.Visit(func(f *flag.Flag) { if f.Name == "create-ca" { @@ -78,7 +106,9 @@ func main() { } func Run(opts Options) error { - cr, err := BuildCR(opts) + applyDefaults(&opts) + + cr, configMap, err := BuildArtifacts(opts) if err != nil { return err } @@ -94,11 +124,40 @@ func Run(opts Options) error { if err := os.WriteFile(opts.OutPath, data, 0o644); err != nil { return fmt.Errorf("write generated CR: %w", err) } + if configMap != nil && opts.CustomCAConfigMapOut != "" { + data, err := marshalObjectYAML(configMap) + if err != nil { + return fmt.Errorf("marshal custom CA ConfigMap YAML: %w", err) + } + if err := os.MkdirAll(filepath.Dir(opts.CustomCAConfigMapOut), 0o755); err != nil { + return fmt.Errorf("create custom CA ConfigMap directory: %w", err) + } + if err := os.WriteFile(opts.CustomCAConfigMapOut, data, 0o644); err != nil { + return fmt.Errorf("write custom CA ConfigMap: %w", err) + } + } return nil } func marshalCRYAML(cr *v2.WeightsAndBiases) ([]byte, error) { - data, err := json.Marshal(cr) + obj, err := prunedObject(cr) + if err != nil { + return nil, err + } + delete(obj, "status") + return yaml.Marshal(obj) +} + +func marshalObjectYAML(value interface{}) ([]byte, error) { + obj, err := prunedObject(value) + if err != nil { + return nil, err + } + return yaml.Marshal(obj) +} + +func prunedObject(value interface{}) (map[string]interface{}, error) { + data, err := json.Marshal(value) if err != nil { return nil, err } @@ -107,10 +166,11 @@ func marshalCRYAML(cr *v2.WeightsAndBiases) ([]byte, error) { if err := json.Unmarshal(data, &obj); err != nil { return nil, err } - delete(obj, "status") - pruned, _ := pruneEmpty(obj) - return yaml.Marshal(pruned) + if _, keep := pruneEmpty(obj); !keep { + return map[string]interface{}{}, nil + } + return obj, nil } func pruneEmpty(value interface{}) (interface{}, bool) { @@ -142,30 +202,50 @@ func pruneEmpty(value interface{}) (interface{}, bool) { } func BuildCR(opts Options) (*v2.WeightsAndBiases, error) { + cr, _, err := BuildArtifacts(opts) + return cr, err +} + +func BuildArtifacts(opts Options) (*v2.WeightsAndBiases, *corev1.ConfigMap, error) { applyDefaults(&opts) + if opts.CustomCA && opts.CustomCACertificatePEM == "" { + certPEM, err := generateSelfSignedCACertificate() + if err != nil { + return nil, nil, err + } + opts.CustomCACertificatePEM = certPEM + } + cr, err := baseCR(opts.CRFile) if err != nil { - return nil, err + return nil, nil, err } ensureTypeMeta(cr) patchMetadata(cr, opts) patchScalarSpec(cr, opts) if err := patchManifestRepository(cr, opts.ManifestSource); err != nil { - return nil, err + return nil, nil, err } if err := patchLicense(cr, opts.LicenseFile); err != nil { - return nil, err + return nil, nil, err } if err := patchNetworking(cr, opts); err != nil { - return nil, err + return nil, nil, err } if err := patchTelemetry(cr, opts.ObservabilityMode); err != nil { - return nil, err + return nil, nil, err + } + patchExternalInfra(cr, opts) + + var configMap *corev1.ConfigMap + if opts.CustomCA { + patchCustomCA(cr, opts.CustomCACertificatePEM) + configMap = customCAConfigMap(cr.Namespace, opts.CustomCACertificatePEM) } - return cr, nil + return cr, configMap, nil } func applyDefaults(opts *Options) { @@ -234,11 +314,11 @@ func baseCR(crFile string) (*v2.WeightsAndBiases, error) { Features: map[string]bool{"proxy": true}, InternalServiceAuth: v2.InternalServiceAuth{Enabled: boolPtr(false)}, }, - MySQL: v2.MySQLSpec{ManagedMysql: &v2.ManagedMysqlSpec{}}, - Redis: v2.RedisSpec{ManagedRedis: &v2.ManagedRedisSpec{}}, + MySQL: map[string]v2.MySQLSpec{v2.DefaultInstanceName: {ManagedMysql: &v2.ManagedMysqlSpec{}}}, + Redis: map[string]v2.RedisSpec{v2.DefaultInstanceName: {ManagedRedis: &v2.ManagedRedisSpec{}}}, Kafka: v2.KafkaSpec{ManagedKafka: &v2.ManagedKafkaSpec{}}, - ObjectStore: v2.ObjectStoreSpec{ManagedObjectStore: &v2.ManagedObjectStoreSpec{}}, - ClickHouse: v2.ClickHouseSpec{ManagedClickHouse: &v2.ManagedClickHouseSpec{}}, + ObjectStore: map[string]v2.ObjectStoreSpec{v2.DefaultInstanceName: {ManagedObjectStore: &v2.ManagedObjectStoreSpec{}}}, + ClickHouse: map[string]v2.ClickHouseSpec{v2.DefaultInstanceName: {ManagedClickHouse: &v2.ManagedClickHouseSpec{}}}, }, }, nil } @@ -363,24 +443,129 @@ func patchTelemetry(cr *v2.WeightsAndBiases, observabilityMode string) error { return fmt.Errorf("observability-mode must be one of: off, full, forward") } - if cr.Spec.MySQL.ManagedMysql != nil { - cr.Spec.MySQL.ManagedMysql.Telemetry.Enabled = enabled + for _, spec := range cr.Spec.MySQL { + if spec.ManagedMysql != nil { + spec.ManagedMysql.Telemetry.Enabled = enabled + } } - if cr.Spec.Redis.ManagedRedis != nil { - cr.Spec.Redis.ManagedRedis.Telemetry.Enabled = enabled + for _, spec := range cr.Spec.Redis { + if spec.ManagedRedis != nil { + spec.ManagedRedis.Telemetry.Enabled = enabled + } } if cr.Spec.Kafka.ManagedKafka != nil { cr.Spec.Kafka.ManagedKafka.Telemetry.Enabled = enabled } - if cr.Spec.ObjectStore.ManagedObjectStore != nil { - cr.Spec.ObjectStore.ManagedObjectStore.Telemetry.Enabled = enabled + for _, spec := range cr.Spec.ObjectStore { + if spec.ManagedObjectStore != nil { + spec.ManagedObjectStore.Telemetry.Enabled = enabled + } } - if cr.Spec.ClickHouse.ManagedClickHouse != nil { - cr.Spec.ClickHouse.ManagedClickHouse.Telemetry.Enabled = enabled + for _, spec := range cr.Spec.ClickHouse { + if spec.ManagedClickHouse != nil { + spec.ManagedClickHouse.Telemetry.Enabled = enabled + } } return nil } +func patchExternalInfra(cr *v2.WeightsAndBiases, opts Options) { + if opts.ExternalMySQL { + conn := &v2.MysqlConnection{ + Host: secretKeySelector(externalMySQLSecret, "Host"), + Port: secretKeySelector(externalMySQLSecret, "Port"), + Database: secretKeySelector(externalMySQLSecret, "Database"), + Username: secretKeySelector(externalMySQLSecret, "Username"), + Password: secretKeySelector(externalMySQLSecret, "Password"), + } + if opts.CustomCA { + conn.SslCa = secretKeySelector(externalMySQLTLSSecret, "ca.crt") + } + cr.Spec.MySQL[v2.DefaultInstanceName] = v2.MySQLSpec{ExternalMysql: conn} + } + + if opts.ExternalRedis { + conn := &v2.RedisConnection{ + Host: secretKeySelector(externalRedisSecret, "Host"), + Port: secretKeySelector(externalRedisSecret, "Port"), + } + if opts.CustomCA { + conn.SslCa = secretKeySelector(externalRedisTLSSecret, "ca.crt") + } + cr.Spec.Redis[v2.DefaultInstanceName] = v2.RedisSpec{ExternalRedis: conn} + } + + if opts.ExternalObjectStore { + cr.Spec.ObjectStore[v2.DefaultInstanceName] = v2.ObjectStoreSpec{ExternalObjectStore: &v2.ObjectStoreConnection{ + Provider: secretKeySelector(externalObjectStoreSecret, "Provider"), + Endpoint: secretKeySelector(externalObjectStoreSecret, "Host"), + Port: secretKeySelector(externalObjectStoreSecret, "Port"), + Bucket: secretKeySelector(externalObjectStoreSecret, "Bucket"), + Region: secretKeySelector(externalObjectStoreSecret, "Region"), + AccessKey: secretKeySelector(externalObjectStoreSecret, "AccessKey"), + SecretKey: secretKeySelector(externalObjectStoreSecret, "SecretKey"), + }} + } +} + +func patchCustomCA(cr *v2.WeightsAndBiases, certPEM string) { + cr.Spec.Global.CustomCACerts = []string{certPEM} + cr.Spec.Global.CACertsConfigMap = customCAConfigMapName +} + +func customCAConfigMap(namespace, certPEM string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: customCAConfigMapName, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "tilt", + "app.kubernetes.io/part-of": "wandb", + }, + }, + Data: map[string]string{ + customCAConfigMapKey: certPEM, + }, + } +} + +func generateSelfSignedCACertificate() (string, error) { + serialLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialLimit) + if err != nil { + return "", fmt.Errorf("generate custom CA serial: %w", err) + } + + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return "", fmt.Errorf("generate custom CA key: %w", err) + } + + now := time.Now() + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: "wandb-tilt-custom-ca", + }, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + return "", fmt.Errorf("generate custom CA certificate: %w", err) + } + + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})), nil +} + func effectiveHostname(opts Options) string { if normalizeNetworkMode(opts.NetworkMode) == "ingress" && opts.Hostname == defaultHostname { return defaultIngressHostname @@ -420,3 +605,10 @@ func boolPtr(value bool) *bool { func stringPtr(value string) *string { return &value } + +func secretKeySelector(secretName, key string) corev1.SecretKeySelector { + return corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, + Key: key, + } +} diff --git a/hack/tilt/wandbcr/main_test.go b/hack/tilt/wandbcr/main_test.go index 0f5ee303..9787400f 100644 --- a/hack/tilt/wandbcr/main_test.go +++ b/hack/tilt/wandbcr/main_test.go @@ -1,15 +1,23 @@ package main import ( + "crypto/x509" + "encoding/pem" "os" "path/filepath" "strings" "testing" v2 "github.com/wandb/operator/api/v2" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/yaml" ) +const testCustomCAPEM = `-----BEGIN CERTIFICATE----- +MIIBtest +-----END CERTIFICATE----- +` + func TestBuildCRDefaultGateway(t *testing.T) { cr, err := BuildCR(Options{}) if err != nil { @@ -34,21 +42,33 @@ func TestBuildCRDefaultGateway(t *testing.T) { if cr.Spec.Wandb.InternalServiceAuth.Enabled == nil || *cr.Spec.Wandb.InternalServiceAuth.Enabled { t.Fatalf("internal service auth should be explicitly disabled") } - if cr.Spec.MySQL.ManagedMysql == nil || cr.Spec.MySQL.ManagedMysql.Telemetry.Enabled { + if cr.Spec.MySQL[v2.DefaultInstanceName].ManagedMysql == nil || cr.Spec.MySQL[v2.DefaultInstanceName].ManagedMysql.Telemetry.Enabled { t.Fatalf("mysql telemetry should be disabled by default") } - if cr.Spec.Redis.ManagedRedis == nil || cr.Spec.Redis.ManagedRedis.Telemetry.Enabled { + if cr.Spec.Redis[v2.DefaultInstanceName].ManagedRedis == nil || cr.Spec.Redis[v2.DefaultInstanceName].ManagedRedis.Telemetry.Enabled { t.Fatalf("redis telemetry should be disabled by default") } if cr.Spec.Kafka.ManagedKafka == nil || cr.Spec.Kafka.ManagedKafka.Telemetry.Enabled { t.Fatalf("kafka telemetry should be disabled by default") } - if cr.Spec.ObjectStore.ManagedObjectStore == nil || cr.Spec.ObjectStore.ManagedObjectStore.Telemetry.Enabled { + if cr.Spec.ObjectStore[v2.DefaultInstanceName].ManagedObjectStore == nil || cr.Spec.ObjectStore[v2.DefaultInstanceName].ManagedObjectStore.Telemetry.Enabled { t.Fatalf("object store telemetry should be disabled by default") } - if cr.Spec.ClickHouse.ManagedClickHouse == nil || cr.Spec.ClickHouse.ManagedClickHouse.Telemetry.Enabled { + if cr.Spec.ClickHouse[v2.DefaultInstanceName].ManagedClickHouse == nil || cr.Spec.ClickHouse[v2.DefaultInstanceName].ManagedClickHouse.Telemetry.Enabled { t.Fatalf("clickhouse telemetry should be disabled by default") } + if cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql != nil { + t.Fatalf("external mysql should be unset by default") + } + if cr.Spec.Redis[v2.DefaultInstanceName].ExternalRedis != nil { + t.Fatalf("external redis should be unset by default") + } + if cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore != nil { + t.Fatalf("external object store should be unset by default") + } + if len(cr.Spec.Global.CustomCACerts) != 0 || cr.Spec.Global.CACertsConfigMap != "" { + t.Fatalf("custom CA fields should be unset by default: %#v", cr.Spec.Global) + } if cr.Spec.Networking.Mode != v2.NetworkingModeGatewayAPI { t.Fatalf("networking mode = %q", cr.Spec.Networking.Mode) } @@ -57,6 +77,166 @@ func TestBuildCRDefaultGateway(t *testing.T) { } } +func TestBuildCRExternalMySQLOnly(t *testing.T) { + cr, err := BuildCR(Options{ExternalMySQL: true}) + if err != nil { + t.Fatal(err) + } + + if cr.Spec.MySQL[v2.DefaultInstanceName].ManagedMysql != nil { + t.Fatalf("managed mysql should be disabled") + } + if cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql == nil { + t.Fatalf("external mysql should be configured") + } + assertSelector(t, cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql.Host, externalMySQLSecret, "Host") + assertSelector(t, cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql.Port, externalMySQLSecret, "Port") + assertSelector(t, cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql.Database, externalMySQLSecret, "Database") + assertSelector(t, cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql.Username, externalMySQLSecret, "Username") + assertSelector(t, cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql.Password, externalMySQLSecret, "Password") + assertEmptySelector(t, cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql.SslCa, "mysql sslCa") + + if cr.Spec.Redis[v2.DefaultInstanceName].ManagedRedis == nil || cr.Spec.Redis[v2.DefaultInstanceName].ExternalRedis != nil { + t.Fatalf("redis should remain managed") + } + if cr.Spec.ObjectStore[v2.DefaultInstanceName].ManagedObjectStore == nil || cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore != nil { + t.Fatalf("object store should remain managed") + } +} + +func TestBuildCRExternalRedisOnly(t *testing.T) { + cr, err := BuildCR(Options{ExternalRedis: true}) + if err != nil { + t.Fatal(err) + } + + if cr.Spec.Redis[v2.DefaultInstanceName].ManagedRedis != nil { + t.Fatalf("managed redis should be disabled") + } + if cr.Spec.Redis[v2.DefaultInstanceName].ExternalRedis == nil { + t.Fatalf("external redis should be configured") + } + assertSelector(t, cr.Spec.Redis[v2.DefaultInstanceName].ExternalRedis.Host, externalRedisSecret, "Host") + assertSelector(t, cr.Spec.Redis[v2.DefaultInstanceName].ExternalRedis.Port, externalRedisSecret, "Port") + assertEmptySelector(t, cr.Spec.Redis[v2.DefaultInstanceName].ExternalRedis.Password, "redis password") + assertEmptySelector(t, cr.Spec.Redis[v2.DefaultInstanceName].ExternalRedis.SslCa, "redis sslCa") + + if cr.Spec.MySQL[v2.DefaultInstanceName].ManagedMysql == nil || cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql != nil { + t.Fatalf("mysql should remain managed") + } + if cr.Spec.ObjectStore[v2.DefaultInstanceName].ManagedObjectStore == nil || cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore != nil { + t.Fatalf("object store should remain managed") + } +} + +func TestBuildCRExternalObjectStoreOnly(t *testing.T) { + cr, err := BuildCR(Options{ExternalObjectStore: true}) + if err != nil { + t.Fatal(err) + } + + if cr.Spec.ObjectStore[v2.DefaultInstanceName].ManagedObjectStore != nil { + t.Fatalf("managed object store should be disabled") + } + if cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore == nil { + t.Fatalf("external object store should be configured") + } + assertSelector(t, cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore.Provider, externalObjectStoreSecret, "Provider") + assertSelector(t, cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore.Endpoint, externalObjectStoreSecret, "Host") + assertSelector(t, cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore.Port, externalObjectStoreSecret, "Port") + assertSelector(t, cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore.Bucket, externalObjectStoreSecret, "Bucket") + assertSelector(t, cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore.Region, externalObjectStoreSecret, "Region") + assertSelector(t, cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore.AccessKey, externalObjectStoreSecret, "AccessKey") + assertSelector(t, cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore.SecretKey, externalObjectStoreSecret, "SecretKey") + + if cr.Spec.MySQL[v2.DefaultInstanceName].ManagedMysql == nil || cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql != nil { + t.Fatalf("mysql should remain managed") + } + if cr.Spec.Redis[v2.DefaultInstanceName].ManagedRedis == nil || cr.Spec.Redis[v2.DefaultInstanceName].ExternalRedis != nil { + t.Fatalf("redis should remain managed") + } +} + +func TestBuildArtifactsCustomCAOnly(t *testing.T) { + cr, configMap, err := BuildArtifacts(Options{ + CustomCA: true, + CustomCACertificatePEM: testCustomCAPEM, + }) + if err != nil { + t.Fatal(err) + } + + if len(cr.Spec.Global.CustomCACerts) != 1 || cr.Spec.Global.CustomCACerts[0] != testCustomCAPEM { + t.Fatalf("custom CA certs not configured: %#v", cr.Spec.Global.CustomCACerts) + } + if cr.Spec.Global.CACertsConfigMap != customCAConfigMapName { + t.Fatalf("caCertsConfigMap = %q", cr.Spec.Global.CACertsConfigMap) + } + if configMap == nil { + t.Fatalf("custom CA ConfigMap should be generated") + } + if configMap.Name != customCAConfigMapName || configMap.Namespace != defaultNamespace { + t.Fatalf("unexpected ConfigMap metadata: %s/%s", configMap.Namespace, configMap.Name) + } + if configMap.Data[customCAConfigMapKey] != testCustomCAPEM { + t.Fatalf("ConfigMap cert data not populated") + } + if cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql != nil || cr.Spec.Redis[v2.DefaultInstanceName].ExternalRedis != nil || cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore != nil { + t.Fatalf("custom CA should not switch infra to external by itself") + } +} + +func TestBuildArtifactsGeneratesValidCustomCA(t *testing.T) { + cr, configMap, err := BuildArtifacts(Options{CustomCA: true}) + if err != nil { + t.Fatal(err) + } + + if len(cr.Spec.Global.CustomCACerts) != 1 { + t.Fatalf("expected one generated custom CA, got %d", len(cr.Spec.Global.CustomCACerts)) + } + if configMap == nil || configMap.Data[customCAConfigMapKey] != cr.Spec.Global.CustomCACerts[0] { + t.Fatalf("generated ConfigMap should contain the same CA cert as the CR") + } + + block, _ := pem.Decode([]byte(cr.Spec.Global.CustomCACerts[0])) + if block == nil { + t.Fatalf("generated custom CA is not PEM encoded") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatal(err) + } + if !cert.IsCA { + t.Fatalf("generated certificate is not a CA") + } +} + +func TestBuildArtifactsExternalInfraWithCustomCA(t *testing.T) { + cr, configMap, err := BuildArtifacts(Options{ + ExternalMySQL: true, + ExternalRedis: true, + ExternalObjectStore: true, + CustomCA: true, + CustomCACertificatePEM: testCustomCAPEM, + }) + if err != nil { + t.Fatal(err) + } + + if configMap == nil { + t.Fatalf("custom CA ConfigMap should be generated") + } + assertSelector(t, cr.Spec.MySQL[v2.DefaultInstanceName].ExternalMysql.SslCa, externalMySQLTLSSecret, "ca.crt") + assertSelector(t, cr.Spec.Redis[v2.DefaultInstanceName].ExternalRedis.SslCa, externalRedisTLSSecret, "ca.crt") + if cr.Spec.ObjectStore[v2.DefaultInstanceName].ExternalObjectStore == nil { + t.Fatalf("external object store should be configured") + } + if cr.Spec.ObjectStore[v2.DefaultInstanceName].ManagedObjectStore != nil || cr.Spec.MySQL[v2.DefaultInstanceName].ManagedMysql != nil || cr.Spec.Redis[v2.DefaultInstanceName].ManagedRedis != nil { + t.Fatalf("selected external infra should disable corresponding managed infra") + } +} + func TestBuildCRLocalManifestSource(t *testing.T) { cr, err := BuildCR(Options{ManifestSource: "local"}) if err != nil { @@ -114,11 +294,11 @@ func TestBuildCRFullObservabilityEnablesManagedTelemetry(t *testing.T) { t.Fatal(err) } - if !cr.Spec.MySQL.ManagedMysql.Telemetry.Enabled || - !cr.Spec.Redis.ManagedRedis.Telemetry.Enabled || + if !cr.Spec.MySQL[v2.DefaultInstanceName].ManagedMysql.Telemetry.Enabled || + !cr.Spec.Redis[v2.DefaultInstanceName].ManagedRedis.Telemetry.Enabled || !cr.Spec.Kafka.ManagedKafka.Telemetry.Enabled || - !cr.Spec.ObjectStore.ManagedObjectStore.Telemetry.Enabled || - !cr.Spec.ClickHouse.ManagedClickHouse.Telemetry.Enabled { + !cr.Spec.ObjectStore[v2.DefaultInstanceName].ManagedObjectStore.Telemetry.Enabled || + !cr.Spec.ClickHouse[v2.DefaultInstanceName].ManagedClickHouse.Telemetry.Enabled { t.Fatalf("managed telemetry was not enabled") } } @@ -271,3 +451,49 @@ func TestRunWritesStableYAML(t *testing.T) { t.Fatalf("generated CR contains empty runtime/defaulted fields:\n%s", rendered) } } + +func TestRunWritesCustomCAConfigMapYAML(t *testing.T) { + dir := t.TempDir() + out := filepath.Join(dir, "generated", "wandb.yaml") + configMapOut := filepath.Join(dir, "generated", "custom-ca-configmap.yaml") + + if err := Run(Options{ + OutPath: out, + Namespace: "custom-ns", + CustomCA: true, + CustomCACertificatePEM: testCustomCAPEM, + CustomCAConfigMapOut: configMapOut, + }); err != nil { + t.Fatal(err) + } + + data, err := os.ReadFile(configMapOut) + if err != nil { + t.Fatal(err) + } + + var configMap corev1.ConfigMap + if err := yaml.Unmarshal(data, &configMap); err != nil { + t.Fatal(err) + } + if configMap.Name != customCAConfigMapName || configMap.Namespace != "custom-ns" { + t.Fatalf("unexpected ConfigMap metadata: %s/%s", configMap.Namespace, configMap.Name) + } + if configMap.Data[customCAConfigMapKey] != testCustomCAPEM { + t.Fatalf("ConfigMap cert data not populated") + } +} + +func assertSelector(t *testing.T, selector corev1.SecretKeySelector, name, key string) { + t.Helper() + if selector.Name != name || selector.Key != key { + t.Fatalf("selector = %s/%s, want %s/%s", selector.Name, selector.Key, name, key) + } +} + +func assertEmptySelector(t *testing.T, selector corev1.SecretKeySelector, field string) { + t.Helper() + if selector.Name != "" || selector.Key != "" { + t.Fatalf("%s selector should be empty, got %s/%s", field, selector.Name, selector.Key) + } +} diff --git a/internal/controller/application_controller.go b/internal/controller/application_controller.go index 1d4dbbc0..77b092f5 100644 --- a/internal/controller/application_controller.go +++ b/internal/controller/application_controller.go @@ -23,6 +23,7 @@ import ( gkeGatewayApiNetworkingv1 "github.com/GoogleCloudPlatform/gke-gateway-api/apis/networking/v1" wandbv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/common" "github.com/wandb/operator/internal/logx" "github.com/wandb/operator/pkg/utils" v1alpha1 "github.com/wandb/operator/pkg/vendored/argo-rollouts/argoproj.io.rollouts/v1alpha1" @@ -30,6 +31,7 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -83,6 +85,7 @@ func (r *ApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) } logger.Info("Handling Application", "Application", app.Name) + statusBefore := app.DeepCopy().Status // Add finalizer if it doesn't exist if app.DeletionTimestamp == nil { @@ -232,6 +235,9 @@ func (r *ApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } + if apiequality.Semantic.DeepEqual(statusBefore, app.Status) { + return result, nil + } if err := r.Status().Update(ctx, &app); err != nil { logger.Error("Failed to update Application status", logx.ErrAttr(err)) return ctrl.Result{}, err @@ -247,6 +253,7 @@ func (r *ApplicationReconciler) reconcileDeployment(ctx context.Context, app *wa deployment := &appsv1.Deployment{} err := r.Get(ctx, client.ObjectKey{Namespace: app.Namespace, Name: app.Name}, deployment) + before := deployment.DeepCopy() if err != nil { if client.IgnoreNotFound(err) != nil { logger.Error("Failed to get Deployment", logx.ErrAttr(err)) @@ -257,10 +264,20 @@ func (r *ApplicationReconciler) reconcileDeployment(ctx context.Context, app *wa selectorLabels := getSelectorLabels(app) + // spec.selector is immutable. If it drifted (e.g. label-standard migration), + // delete the existing Deployment and recreate it on the next reconcile. + if !deployment.CreationTimestamp.IsZero() && selectorChanged(deployment.Spec.Selector, selectorLabels) { + logger.Info("Deployment selector changed; deleting for recreate", "Deployment", app.Name) + if err := r.Delete(ctx, deployment, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil && !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + deployment.Name = app.Name deployment.Namespace = app.Namespace - deployment.Spec.Template.Spec = *app.Spec.PodTemplate.Spec.DeepCopy() + deployment.Spec.Template.Spec = r.defaultPodSpec(app.Spec.PodTemplate.Spec) deployment.Spec.Template.SetLabels( utils.MergeMapsStringString( deployment.Spec.Template.GetLabels(), @@ -293,14 +310,19 @@ func (r *ApplicationReconciler) reconcileDeployment(ctx context.Context, app *wa return ctrl.Result{}, err } - logger.Debug("Deployment spec", "Deployment", deployment.Name, "Spec", deployment.Spec) + logger.Debug( + "Desired Deployment", + "Deployment", deployment.Name, + "containers", len(deployment.Spec.Template.Spec.Containers), + "initContainers", len(deployment.Spec.Template.Spec.InitContainers), + ) if deployment.CreationTimestamp.IsZero() { if err := r.Create(ctx, deployment); err != nil { logger.Error("Failed to create Deployment", logx.ErrAttr(err)) return ctrl.Result{}, err } - } else { + } else if !deploymentManagedFieldsEqual(before, deployment) { if err := r.Update(ctx, deployment); err != nil { logger.Error("Failed to update Deployment", logx.ErrAttr(err)) return ctrl.Result{}, err @@ -313,13 +335,66 @@ func (r *ApplicationReconciler) reconcileDeployment(ctx context.Context, app *wa return ctrl.Result{}, nil } +// getSelectorLabels returns the immutable label set used for a workload's +// spec.selector. It derives the selector from the operator/ownership family +// (weightsandbiases.apps.wandb.com/*) stamped onto the pod template by the +// WeightsAndBiases reconciler, which is stable and collision-free. Applications +// created before that family existed fall back to the legacy app.kubernetes.io +// selector so their live workloads keep matching. func getSelectorLabels(app *wandbv2.Application) map[string]string { + podLabels := app.Spec.PodTemplate.GetLabels() + if name, ok := podLabels[common.WandbNameLabel]; ok && name != "" { + selector := map[string]string{common.WandbNameLabel: name} + if component, ok := podLabels[common.WandbComponentLabel]; ok && component != "" { + selector[common.WandbComponentLabel] = component + } + return selector + } + // Legacy fallback (pre-operator-family Applications). return map[string]string{ "app.kubernetes.io/name": app.Name, "app.kubernetes.io/instance": app.Namespace, } } +// selectorChanged reports whether a workload's immutable spec.selector no longer +// matches the desired selector. A workload's selector cannot be mutated in place, +// so a change requires deleting and recreating the workload. +func selectorChanged(current *metav1.LabelSelector, desired map[string]string) bool { + if current == nil { + return false + } + return !reflect.DeepEqual(current.MatchLabels, desired) +} + +func managedObjectMetadataEqual(before, after client.Object) bool { + return apiequality.Semantic.DeepEqual(before.GetLabels(), after.GetLabels()) && + apiequality.Semantic.DeepEqual(before.GetAnnotations(), after.GetAnnotations()) && + apiequality.Semantic.DeepEqual(before.GetOwnerReferences(), after.GetOwnerReferences()) +} + +func (r *ApplicationReconciler) defaultPodSpec(spec corev1.PodSpec) corev1.PodSpec { + pod := &corev1.Pod{Spec: *spec.DeepCopy()} + if r.Scheme != nil { + r.Scheme.Default(pod) + } + return pod.Spec +} + +func deploymentManagedFieldsEqual(before, after *appsv1.Deployment) bool { + return managedObjectMetadataEqual(before, after) && + apiequality.Semantic.DeepEqual(before.Spec, after.Spec) +} + +func rolloutManagedFieldsEqual(before, after *v1alpha1.Rollout) bool { + return managedObjectMetadataEqual(before, after) && reflect.DeepEqual(before.Spec, after.Spec) +} + +func statefulSetManagedFieldsEqual(before, after *appsv1.StatefulSet) bool { + return managedObjectMetadataEqual(before, after) && + apiequality.Semantic.DeepEqual(before.Spec, after.Spec) +} + // deleteDeployment deletes the Deployment associated with the Application func (r *ApplicationReconciler) deleteDeployment(ctx context.Context, app *wandbv2.Application) error { logger := logx.GetSlog(ctx) @@ -353,6 +428,7 @@ func (r *ApplicationReconciler) reconcileRollout(ctx context.Context, app *wandb rollout := &v1alpha1.Rollout{} err := r.Get(ctx, client.ObjectKey{Namespace: app.Namespace, Name: app.Name}, rollout) + before := rollout.DeepCopy() if err != nil { if client.IgnoreNotFound(err) != nil { logger.Error("Failed to get Rollout", logx.ErrAttr(err)) @@ -363,10 +439,18 @@ func (r *ApplicationReconciler) reconcileRollout(ctx context.Context, app *wandb selectorLabels := getSelectorLabels(app) + if !rollout.CreationTimestamp.IsZero() && selectorChanged(rollout.Spec.Selector, selectorLabels) { + logger.Info("Rollout selector changed; deleting for recreate", "Rollout", app.Name) + if err := r.Delete(ctx, rollout, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil && !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + rollout.Name = app.Name rollout.Namespace = app.Namespace - rollout.Spec.Template.Spec = *app.Spec.PodTemplate.Spec.DeepCopy() + rollout.Spec.Template.Spec = r.defaultPodSpec(app.Spec.PodTemplate.Spec) rollout.Spec.Template.SetLabels( utils.MergeMapsStringString( rollout.Spec.Template.GetLabels(), @@ -399,14 +483,19 @@ func (r *ApplicationReconciler) reconcileRollout(ctx context.Context, app *wandb return ctrl.Result{}, err } - logger.Info("Rollout spec", "Rollout", rollout.Name, "Spec", rollout.Spec) + logger.Debug( + "Desired Rollout", + "Rollout", rollout.Name, + "containers", len(rollout.Spec.Template.Spec.Containers), + "initContainers", len(rollout.Spec.Template.Spec.InitContainers), + ) if rollout.CreationTimestamp.IsZero() { if err := r.Create(ctx, rollout); err != nil { logger.Error("Failed to create Rollout", logx.ErrAttr(err)) return ctrl.Result{}, err } - } else { + } else if !rolloutManagedFieldsEqual(before, rollout) { if err := r.Update(ctx, rollout); err != nil { logger.Error("Failed to update Rollout", logx.ErrAttr(err)) return ctrl.Result{}, err @@ -452,6 +541,7 @@ func (r *ApplicationReconciler) reconcileStatefulSet(ctx context.Context, app *w statefulSet := &appsv1.StatefulSet{} err := r.Get(ctx, client.ObjectKey{Namespace: app.Namespace, Name: app.Name}, statefulSet) + before := statefulSet.DeepCopy() if err != nil { if client.IgnoreNotFound(err) != nil { logger.Error("Failed to get StatefulSet", logx.ErrAttr(err)) @@ -462,10 +552,18 @@ func (r *ApplicationReconciler) reconcileStatefulSet(ctx context.Context, app *w selectorLabels := getSelectorLabels(app) + if !statefulSet.CreationTimestamp.IsZero() && selectorChanged(statefulSet.Spec.Selector, selectorLabels) { + logger.Info("StatefulSet selector changed; deleting for recreate", "StatefulSet", app.Name) + if err := r.Delete(ctx, statefulSet, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil && !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + statefulSet.Name = app.Name statefulSet.Namespace = app.Namespace - statefulSet.Spec.Template.Spec = *app.Spec.PodTemplate.Spec.DeepCopy() + statefulSet.Spec.Template.Spec = r.defaultPodSpec(app.Spec.PodTemplate.Spec) statefulSet.Spec.Template.SetLabels( utils.MergeMapsStringString( statefulSet.Spec.Template.GetLabels(), @@ -514,14 +612,19 @@ func (r *ApplicationReconciler) reconcileStatefulSet(ctx context.Context, app *w return ctrl.Result{}, err } - logger.Info("StatefulSet spec", "StatefulSet", statefulSet.Name, "Spec", statefulSet.Spec) + logger.Debug( + "Desired StatefulSet", + "StatefulSet", statefulSet.Name, + "containers", len(statefulSet.Spec.Template.Spec.Containers), + "initContainers", len(statefulSet.Spec.Template.Spec.InitContainers), + ) if statefulSet.CreationTimestamp.IsZero() { if err := r.Create(ctx, statefulSet); err != nil { logger.Error("Failed to create StatefulSet", logx.ErrAttr(err)) return ctrl.Result{}, err } - } else { + } else if !statefulSetManagedFieldsEqual(before, statefulSet) { if err := r.Update(ctx, statefulSet); err != nil { logger.Error("Failed to update StatefulSet", logx.ErrAttr(err)) return ctrl.Result{}, err @@ -720,6 +823,10 @@ func (r *ApplicationReconciler) reconcileCronJobs(ctx context.Context, app *wand } else { // Update existing cronjob cronJobToReconcile.ResourceVersion = currentCronJob.ResourceVersion + if managedObjectMetadataEqual(currentCronJob, cronJobToReconcile) && + apiequality.Semantic.DeepEqual(currentCronJob.Spec, cronJobToReconcile.Spec) { + continue + } if err := r.Update(ctx, cronJobToReconcile); err != nil { logger.Error("Failed to update CronJob", logx.ErrAttr(err), "CronJob", cronJobName) return err @@ -786,9 +893,16 @@ func (r *ApplicationReconciler) reconcileService(ctx context.Context, app *wandb desired.Annotations, app.Spec.MetaTemplate.Annotations, ) + // The API server stores empty annotations as nil; keep them nil so the + // steady-state metadata compare below settles. + if len(desired.Annotations) == 0 { + desired.Annotations = nil + } - // Copy spec from template + // Copy spec from template, filling the port defaults the API server would + // apply (templates written before normalization may lack them). desired.Spec = *app.Spec.ServiceTemplate.DeepCopy() + common.NormalizeServicePorts(desired.Spec.Ports) // Ensure selector targets the application's pods selectorLabels := getSelectorLabels(app) @@ -820,6 +934,7 @@ func (r *ApplicationReconciler) reconcileService(ctx context.Context, app *wandb logger.Info("Successfully created Service", "Service", desired.Name) return nil } + before := current.DeepCopy() // Update path: preserve immutable fields desired.ResourceVersion = current.ResourceVersion @@ -828,6 +943,7 @@ func (r *ApplicationReconciler) reconcileService(ctx context.Context, app *wandb desired.Spec.IPFamilies = current.Spec.IPFamilies desired.Spec.IPFamilyPolicy = current.Spec.IPFamilyPolicy desired.Spec.HealthCheckNodePort = current.Spec.HealthCheckNodePort + preserveServerDefaultedServiceFields(&desired.Spec, ¤t.Spec) // Only update if there are changes // Apply desired into current to minimize overwrite @@ -845,18 +961,57 @@ func (r *ApplicationReconciler) reconcileService(ctx context.Context, app *wandb current.Spec.LoadBalancerIP = desired.Spec.LoadBalancerIP current.Spec.LoadBalancerSourceRanges = desired.Spec.LoadBalancerSourceRanges - logger.Info("Updating Service", "Service", current.Name) - if err := r.Update(ctx, current); err != nil { - logger.Error("Failed to update Service", logx.ErrAttr(err)) - return err + if !managedObjectMetadataEqual(before, current) || !apiequality.Semantic.DeepEqual(before.Spec, current.Spec) { + logger.Info("Updating Service", "Service", current.Name) + if err := r.Update(ctx, current); err != nil { + logger.Error("Failed to update Service", logx.ErrAttr(err)) + return err + } + logger.Info("Successfully updated Service", "Service", current.Name) } app.Status.ServiceStatus = ¤t.Status - logger.Info("Successfully updated Service", "Service", current.Name) return nil } +// preserveServerDefaultedServiceFields keeps API-server-defaulted or -allocated +// values for fields the template leaves unset, so a settled Service compares +// equal and Update only fires on real changes. +func preserveServerDefaultedServiceFields(desired, current *corev1.ServiceSpec) { + if desired.Type == "" { + desired.Type = current.Type + } + if desired.SessionAffinity == "" { + desired.SessionAffinity = current.SessionAffinity + } + if desired.ExternalTrafficPolicy == "" { + desired.ExternalTrafficPolicy = current.ExternalTrafficPolicy + } + if desired.InternalTrafficPolicy == nil { + desired.InternalTrafficPolicy = current.InternalTrafficPolicy + } + if desired.AllocateLoadBalancerNodePorts == nil { + desired.AllocateLoadBalancerNodePorts = current.AllocateLoadBalancerNodePorts + } + if desired.Type != corev1.ServiceTypeNodePort && desired.Type != corev1.ServiceTypeLoadBalancer { + return + } + // NodePorts are allocated by the API server; keep them unless the template pins one. + for i := range desired.Ports { + p := &desired.Ports[i] + if p.NodePort != 0 { + continue + } + for j := range current.Ports { + if current.Ports[j].Name == p.Name { + p.NodePort = current.Ports[j].NodePort + break + } + } + } +} + // deleteService deletes the Service associated with the Application func (r *ApplicationReconciler) deleteService(ctx context.Context, app *wandbv2.Application) error { logger := logx.GetSlog(ctx) @@ -955,6 +1110,10 @@ func (r *ApplicationReconciler) reconcileHPA(ctx context.Context, app *wandbv2.A // Update path desired.ResourceVersion = current.ResourceVersion + if managedObjectMetadataEqual(current, desired) && apiequality.Semantic.DeepEqual(current.Spec, desired.Spec) { + app.Status.HPAStatus = ¤t.Status + return nil + } logger.Info("Updating HPA", "HPA", desired.Name) if err := r.Update(ctx, desired); err != nil { logger.Error("Failed to update HPA", logx.ErrAttr(err)) diff --git a/internal/controller/application_controller_test.go b/internal/controller/application_controller_test.go index 155dc41b..dee29a7f 100644 --- a/internal/controller/application_controller_test.go +++ b/internal/controller/application_controller_test.go @@ -22,11 +22,13 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/common" "github.com/wandb/operator/pkg/vendored/argo-rollouts/argoproj.io.rollouts/v1alpha1" appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,6 +37,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) +// serviceUpdateCounter counts Service Update calls passing through the client. +type serviceUpdateCounter struct { + client.Client + updates int +} + +func (c *serviceUpdateCounter) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if _, ok := obj.(*corev1.Service); ok { + c.updates++ + } + return c.Client.Update(ctx, obj, opts...) +} + var _ = Describe("Application Controller", func() { Context("When reconciling a resource", func() { ctx := context.Background() @@ -496,6 +511,99 @@ var _ = Describe("Application Controller", func() { Expect(found.Spec.Ports[0].Port).To(Equal(int32(80))) }) + It("round-trips a normalized ServiceTemplate without drift", func() { + // The Application CRD schema defaults serviceTemplate.ports[].protocol, + // so an un-normalized template reads back different from what was + // written — the drift that made the parent's update gate fire on every + // reconcile and kept Service-bearing Applications churning. + raw := &corev1.ServiceSpec{Ports: []corev1.ServicePort{{Name: "http", Port: 8080}}} + + rawApp := &apiv2.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "test-roundtrip-raw", Namespace: "default"}, + Spec: apiv2.ApplicationSpec{ + Kind: "Deployment", + PodTemplate: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "web", Image: "nginx"}}}, + }, + ServiceTemplate: raw.DeepCopy(), + }, + } + Expect(k8sClient.Create(ctx, rawApp)).To(Succeed()) + + fetched := &apiv2.Application{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "test-roundtrip-raw", Namespace: "default"}, fetched)).To(Succeed()) + Expect(fetched.Spec.ServiceTemplate.Ports[0].Protocol).To(Equal(corev1.ProtocolTCP), + "CRD schema defaulting fills ports[].protocol") + Expect(apiequality.Semantic.DeepEqual(fetched.Spec.ServiceTemplate, raw)).To(BeFalse(), + "un-normalized templates do not round-trip; reconcilers must not build them") + + // The normalized form (what reconcileApplications writes now) is stable. + normalized := raw.DeepCopy() + common.NormalizeServicePorts(normalized.Ports) + normApp := &apiv2.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "test-roundtrip-norm", Namespace: "default"}, + Spec: apiv2.ApplicationSpec{ + Kind: "Deployment", + PodTemplate: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "web", Image: "nginx"}}}, + }, + ServiceTemplate: normalized.DeepCopy(), + }, + } + Expect(k8sClient.Create(ctx, normApp)).To(Succeed()) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "test-roundtrip-norm", Namespace: "default"}, fetched)).To(Succeed()) + Expect(apiequality.Semantic.DeepEqual(fetched.Spec.ServiceTemplate, normalized)).To(BeTrue(), + "normalized templates round-trip unchanged, so the update gate settles") + }) + + It("does not update the Service on a steady-state reconcile", func() { + resourceName := "test-service-steady" + typeNamespacedName := types.NamespacedName{Name: resourceName, Namespace: "default"} + + resource := &apiv2.Application{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName, Namespace: "default"}, + Spec: apiv2.ApplicationSpec{ + Kind: "Deployment", + PodTemplate: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "web", Image: "nginx"}}}, + }, + // No protocol/targetPort: mirrors templates written before + // normalization existed. + ServiceTemplate: &corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Name: "http", Port: 80}}, + }, + }, + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + + // Count Service Update calls: the API server absorbs writes of + // defaulted-back fields as no-ops (resourceVersion holds), but each + // call still fires the Owns(Service) watch and re-queues the + // Application — the hot loop this test pins. + counter := &serviceUpdateCounter{Client: k8sClient} + controllerReconciler := &ApplicationReconciler{Client: counter, Scheme: k8sClient.Scheme()} + + // 1. Add finalizer; 2. create Deployment and Service. + _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + svc := &corev1.Service{} + Expect(k8sClient.Get(ctx, typeNamespacedName, svc)).To(Succeed()) + + // Steady state: the API server has defaulted fields the template leaves + // unset (protocol, targetPort, sessionAffinity, type, ...); reconciling + // again must not write the Service at all. + counter.updates = 0 + for i := 0; i < 3; i++ { + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + } + Expect(counter.updates).To(BeZero(), + "steady-state reconciles must not update the Service") + }) + It("should successfully reconcile Jobs and CronJobs", func() { resourceName := "test-jobs" typeNamespacedName := types.NamespacedName{ diff --git a/internal/controller/application_service_test.go b/internal/controller/application_service_test.go new file mode 100644 index 00000000..8e372c3a --- /dev/null +++ b/internal/controller/application_service_test.go @@ -0,0 +1,94 @@ +package controller + +import ( + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/wandb/operator/internal/controller/common" +) + +// defaultedClusterIPSpec is what the API server stores for a Service created +// from a template that only names a port. +func defaultedClusterIPSpec() *corev1.ServiceSpec { + itp := corev1.ServiceInternalTrafficPolicyCluster + return &corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "10.0.0.1", + SessionAffinity: corev1.ServiceAffinityNone, + InternalTrafficPolicy: &itp, + Ports: []corev1.ServicePort{ + {Name: "http", Port: 80, Protocol: corev1.ProtocolTCP, TargetPort: intstr.FromInt32(80)}, + }, + } +} + +func TestPreserveServerDefaultedServiceFields_SteadyStateSettles(t *testing.T) { + current := defaultedClusterIPSpec() + desired := &corev1.ServiceSpec{Ports: []corev1.ServicePort{{Name: "http", Port: 80}}} + + common.NormalizeServicePorts(desired.Ports) + preserveServerDefaultedServiceFields(desired, current) + + require.Equal(t, current.Type, desired.Type) + require.Equal(t, current.SessionAffinity, desired.SessionAffinity) + require.Equal(t, current.InternalTrafficPolicy, desired.InternalTrafficPolicy) + require.True(t, apiequality.Semantic.DeepEqual(current.Ports, desired.Ports), + "a template-derived spec must compare equal to the server-defaulted one at steady state") +} + +func TestPreserveServerDefaultedServiceFields_RealChangesStillDiffer(t *testing.T) { + current := defaultedClusterIPSpec() + desired := &corev1.ServiceSpec{Ports: []corev1.ServicePort{{Name: "http", Port: 9090}}} + + common.NormalizeServicePorts(desired.Ports) + preserveServerDefaultedServiceFields(desired, current) + + require.False(t, apiequality.Semantic.DeepEqual(current.Ports, desired.Ports), + "a genuine port change must still be detected") +} + +func TestPreserveServerDefaultedServiceFields_NodePorts(t *testing.T) { + current := &corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + {Name: "http", Port: 80, Protocol: corev1.ProtocolTCP, TargetPort: intstr.FromInt32(80), NodePort: 30080}, + }, + } + + desired := &corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{{Name: "http", Port: 80}}, + } + common.NormalizeServicePorts(desired.Ports) + preserveServerDefaultedServiceFields(desired, current) + require.Equal(t, int32(30080), desired.Ports[0].NodePort, "allocated NodePort is preserved") + + pinned := &corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{{Name: "http", Port: 80, NodePort: 31000}}, + } + common.NormalizeServicePorts(pinned.Ports) + preserveServerDefaultedServiceFields(pinned, current) + require.Equal(t, int32(31000), pinned.Ports[0].NodePort, "template-pinned NodePort wins") +} + +func TestPreserveServerDefaultedServiceFields_ClusterIPDropsNodePorts(t *testing.T) { + current := &corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + {Name: "http", Port: 80, Protocol: corev1.ProtocolTCP, TargetPort: intstr.FromInt32(80), NodePort: 30080}, + }, + } + desired := &corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{{Name: "http", Port: 80}}, + } + common.NormalizeServicePorts(desired.Ports) + preserveServerDefaultedServiceFields(desired, current) + require.Zero(t, desired.Ports[0].NodePort, + "switching to ClusterIP must not carry the stale NodePort") +} diff --git a/internal/controller/common/co_managed.go b/internal/controller/common/co_managed.go new file mode 100644 index 00000000..4adce0a2 --- /dev/null +++ b/internal/controller/common/co_managed.go @@ -0,0 +1,74 @@ +package common + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// WriteOwnedFields create-or-updates obj, writing ONLY the fields the operator +// owns and preserving everything a co-managing controller writes (finalizers, +// status, annotations). +// +// Use it for vendored CRs (e.g. the Altinity CHI/CHK) whose status holds +// unexported fields and uint64 values: controllerutil.CreateOrUpdate +// (apiequality.Semantic.DeepEqual) and CreateOrPatch (DeepCopyJSON of the +// status) both panic traversing that status. Here we fetch, mutate only owned +// fields, compare only those, and Update — which marshals via JSON and never +// deep-copies the status. +// +// applyOwned mutates the fetched object in place. ownedEqual must compare only +// owned fields (spec, labels, owner refs); excluding status keeps the +// panic-prone vendored status out of the comparison. +func WriteOwnedFields[T client.Object]( + ctx context.Context, + cl client.Client, + obj T, + applyOwned func(obj T), + ownedEqual func(a, b T) bool, +) (controllerutil.OperationResult, error) { + key := client.ObjectKeyFromObject(obj) + if err := cl.Get(ctx, key, obj); err != nil { + if !apierrors.IsNotFound(err) { + return controllerutil.OperationResultNone, err + } + obj.SetName(key.Name) + obj.SetNamespace(key.Namespace) + applyOwned(obj) + if err := cl.Create(ctx, obj); err != nil { + return controllerutil.OperationResultNone, err + } + return controllerutil.OperationResultCreated, nil + } + + before, ok := obj.DeepCopyObject().(T) + if !ok { + return controllerutil.OperationResultNone, fmt.Errorf("DeepCopyObject returned unexpected type for %T", obj) + } + applyOwned(obj) + if ownedEqual(before, obj) { + return controllerutil.OperationResultNone, nil + } + if err := cl.Update(ctx, obj); err != nil { + return controllerutil.OperationResultNone, err + } + return controllerutil.OperationResultUpdated, nil +} + +// JSONEqual reports whether a and b marshal to identical JSON. It is panic-safe +// for vendored types whose reflective deep-equal/deep-copy chokes on unexported +// or uint64 fields (json.Marshal skips unexported fields and accepts uint64). A +// marshal error is treated as not-equal so the caller writes rather than skips. +func JSONEqual(a, b any) bool { + ab, err1 := json.Marshal(a) + bb, err2 := json.Marshal(b) + if err1 != nil || err2 != nil { + return false + } + return bytes.Equal(ab, bb) +} diff --git a/internal/controller/common/condition.go b/internal/controller/common/condition.go index 37f910a0..091f8eae 100644 --- a/internal/controller/common/condition.go +++ b/internal/controller/common/condition.go @@ -18,6 +18,7 @@ const ( ResourceErrorReason = "ResourceError" UnknownReason = "Unknown" DetachedSpecMismatch = "DetachedSpecMismatch" + InvalidNameReason = "InvalidName" ) const ( diff --git a/internal/controller/common/condition_test.go b/internal/controller/common/condition_test.go index bff00202..69e7f49b 100644 --- a/internal/controller/common/condition_test.go +++ b/internal/controller/common/condition_test.go @@ -81,10 +81,18 @@ var _ = Describe("Condition", func() { result := ComputeConditionUpdates(oldConditions, currentConditions, currentGeneration, expiry) Expect(result).To(HaveLen(2)) - Expect(result[0].Reason).To(Equal("NewReason1")) - Expect(result[1].Reason).To(Equal("NewReason2")) - Expect(result[0].ObservedGeneration).To(Equal(currentGeneration)) - Expect(result[1].ObservedGeneration).To(Equal(currentGeneration)) + Expect(result).To(ConsistOf( + And( + HaveField("Type", "Type1"), + HaveField("Reason", "NewReason1"), + HaveField("ObservedGeneration", currentGeneration), + ), + And( + HaveField("Type", "Type2"), + HaveField("Reason", "NewReason2"), + HaveField("ObservedGeneration", currentGeneration), + ), + )) }) It("should keep old condition when current has no changes", func() { diff --git a/internal/controller/common/labels.go b/internal/controller/common/labels.go index 245a830b..218870c0 100644 --- a/internal/controller/common/labels.go +++ b/internal/controller/common/labels.go @@ -10,6 +10,78 @@ const ( WandbComponentLabel = "weightsandbiases.apps.wandb.com/component" ) +// Standard Kubernetes "recommended" label keys. These are descriptive labels for +// ecosystem tooling (kube-state-metrics, dashboards, kubectl) and NetworkPolicy +// selectors. They are intentionally distinct from the operator/ownership family +// above, which backs immutable spec.selectors and retention selectors. +const ( + StandardNameLabel = "app.kubernetes.io/name" + StandardInstanceLabel = "app.kubernetes.io/instance" + StandardComponentLabel = "app.kubernetes.io/component" + StandardPartOfLabel = "app.kubernetes.io/part-of" + StandardManagedByLabel = "app.kubernetes.io/managed-by" + StandardVersionLabel = "app.kubernetes.io/version" + + // PartOfValue is the value shared by every wandb-managed resource; it is the + // anchor for namespace-wide NetworkPolicies and release-wide queries. + PartOfValue = "wandb" + // ManagedByValue identifies resources reconciled by this operator. + ManagedByValue = "wandb-operator" +) + +// Architectural component roles. See docs/pod-labeling-standards.md. +const ( + RoleServer = "server" + RoleWorker = "worker" + RoleProxy = "proxy" + RoleDatabase = "database" + RoleCache = "cache" + RoleAnalyticsDB = "analytics-db" + RoleQueue = "queue" + RoleObjectStorage = "object-storage" + RoleMigration = "migration" +) + +// appComponentRoles maps known W&B application (manifest) names to their +// architectural role. Unknown apps default to RoleServer. +var appComponentRoles = map[string]string{ + "executor": RoleWorker, + "parquet": RoleWorker, + "weave-trace-worker": RoleWorker, + "weave-trace-evaluate-model-worker": RoleWorker, + "flat-run-fields-updater": RoleWorker, + "metric-observer": RoleWorker, + "nginx-proxy": RoleProxy, +} + +// AppComponentRole returns the architectural role for a W&B application name, +// defaulting to RoleServer for request-serving apps. +func AppComponentRole(appName string) string { + if role, ok := appComponentRoles[appName]; ok { + return role + } + return RoleServer +} + +// StandardLabels returns the descriptive app.kubernetes.io/* label set for a +// resource owned by the given CR. component and version are optional; empty +// values are omitted. instance is always the owning CR (release) name. +func StandardLabels(wandb *apiv2.WeightsAndBiases, name, component, version string) map[string]string { + l := map[string]string{ + StandardNameLabel: name, + StandardInstanceLabel: wandb.Name, + StandardPartOfLabel: PartOfValue, + StandardManagedByLabel: ManagedByValue, + } + if component != "" { + l[StandardComponentLabel] = component + } + if version != "" { + l[StandardVersionLabel] = version + } + return l +} + // HasAllLabelKeys reports whether existing contains every key present in desired, // regardless of value. func HasAllLabelKeys(existing, desired map[string]string) bool { diff --git a/internal/controller/common/labels_test.go b/internal/controller/common/labels_test.go new file mode 100644 index 00000000..2eadc614 --- /dev/null +++ b/internal/controller/common/labels_test.go @@ -0,0 +1,41 @@ +package common + +import ( + apiv2 "github.com/wandb/operator/api/v2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("StandardLabels", func() { + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "wandb-system"}, + } + + It("sets instance to the CR name, not the namespace", func() { + labels := StandardLabels(wandb, "api", RoleServer, "0.80.0") + Expect(labels[StandardInstanceLabel]).To(Equal("wandb")) + Expect(labels[StandardNameLabel]).To(Equal("api")) + Expect(labels[StandardComponentLabel]).To(Equal(RoleServer)) + Expect(labels[StandardPartOfLabel]).To(Equal(PartOfValue)) + Expect(labels[StandardManagedByLabel]).To(Equal(ManagedByValue)) + Expect(labels[StandardVersionLabel]).To(Equal("0.80.0")) + }) + + It("omits component and version when empty", func() { + labels := StandardLabels(wandb, "generated-secret", "", "") + Expect(labels).NotTo(HaveKey(StandardComponentLabel)) + Expect(labels).NotTo(HaveKey(StandardVersionLabel)) + Expect(labels).To(HaveKeyWithValue(StandardPartOfLabel, PartOfValue)) + }) +}) + +var _ = Describe("AppComponentRole", func() { + It("maps known workers and proxies, defaulting others to server", func() { + Expect(AppComponentRole("executor")).To(Equal(RoleWorker)) + Expect(AppComponentRole("parquet")).To(Equal(RoleWorker)) + Expect(AppComponentRole("nginx-proxy")).To(Equal(RoleProxy)) + Expect(AppComponentRole("api")).To(Equal(RoleServer)) + Expect(AppComponentRole("some-new-service")).To(Equal(RoleServer)) + }) +}) diff --git a/internal/controller/common/naming.go b/internal/controller/common/naming.go new file mode 100644 index 00000000..39c86321 --- /dev/null +++ b/internal/controller/common/naming.go @@ -0,0 +1,54 @@ +package common + +import ( + "crypto/sha256" + "fmt" + "strings" + + apiv2 "github.com/wandb/operator/api/v2" + "k8s.io/apimachinery/pkg/util/validation" +) + +// InstanceBaseName is what default infra names derive from: the CR name, plus +// the instance key for non-default instances. The key sits before the infra +// suffix so suffix-based derivations (Keeper pairing, connection-name trims) +// keep working. +func InstanceBaseName(crName, instanceKey string) string { + if instanceKey == "" || instanceKey == apiv2.DefaultInstanceName { + return crName + } + return crName + "-" + instanceKey +} + +// infraNameHashLen is enough to keep sibling CRs sharing a long prefix from colliding. +const infraNameHashLen = 5 + +// FitDefaultInfraName returns "", falling back to a +// deterministic "-" when that would not be a valid +// DNS-1123 label within budget — any CR name yields a usable default. +func FitDefaultInfraName(crName, suffix string, budget int) string { + plain := crName + suffix + if len(plain) <= budget && len(validation.IsDNS1123Label(plain)) == 0 { + return plain + } + + digest := fmt.Sprintf("%x", sha256.Sum256([]byte(crName)))[:infraNameHashLen] + prefix := sanitizeLabelPrefix(crName, budget-len(suffix)-infraNameHashLen-1) + if prefix == "" { + return digest + suffix + } + return fmt.Sprintf("%s-%s%s", prefix, digest, suffix) +} + +// sanitizeLabelPrefix truncates s to maxLen and strips what can't lead a +// DNS-1123 label (CR names may contain dots; truncation may leave hyphens). +func sanitizeLabelPrefix(s string, maxLen int) string { + if maxLen <= 0 { + return "" + } + s = strings.ReplaceAll(s, ".", "-") + if len(s) > maxLen { + s = s[:maxLen] + } + return strings.Trim(s, "-") +} diff --git a/internal/controller/common/naming_test.go b/internal/controller/common/naming_test.go new file mode 100644 index 00000000..f084fdb0 --- /dev/null +++ b/internal/controller/common/naming_test.go @@ -0,0 +1,49 @@ +package common + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/validation" +) + +var _ = Describe("FitDefaultInfraName", func() { + // deliberately tight so the cases exercise shortening + const budget = 27 + + It("keeps the plain name when it fits the budget", func() { + Expect(FitDefaultInfraName("wandb", "-clickhouse", budget)).To(Equal("wandb-clickhouse")) + }) + + It("shortens over-budget names deterministically and keeps the suffix", func() { + first := FitDefaultInfraName("wandb-legacy-overrides-v1", "-clickhouse", budget) + second := FitDefaultInfraName("wandb-legacy-overrides-v1", "-clickhouse", budget) + + Expect(first).To(Equal(second)) + Expect(len(first)).To(BeNumerically("<=", budget)) + Expect(first).To(HaveSuffix("-clickhouse")) + Expect(validation.IsDNS1123Label(first)).To(BeEmpty()) + }) + + It("derives distinct names for distinct CR names sharing a long prefix", func() { + one := FitDefaultInfraName("wandb-production-eu-west-1", "-clickhouse", budget) + two := FitDefaultInfraName("wandb-production-eu-west-2", "-clickhouse", budget) + + Expect(one).NotTo(Equal(two)) + }) + + It("sanitizes dots, which are legal in CR names but not in labels", func() { + name := FitDefaultInfraName("wandb.prod.eu", "-clickhouse", budget) + + Expect(name).NotTo(ContainSubstring(".")) + Expect(len(name)).To(BeNumerically("<=", budget)) + Expect(validation.IsDNS1123Label(name)).To(BeEmpty()) + }) + + It("survives truncation points that land on a hyphen", func() { + // prefix budget is 10 here; the 10th char of the CR name is '-' + name := FitDefaultInfraName("wandb-abcd-something-long-enough", "-clickhouse", budget) + + Expect(name).NotTo(ContainSubstring("--")) + Expect(validation.IsDNS1123Label(name)).To(BeEmpty()) + }) +}) diff --git a/internal/controller/common/pods.go b/internal/controller/common/pods.go new file mode 100644 index 00000000..3965750e --- /dev/null +++ b/internal/controller/common/pods.go @@ -0,0 +1,17 @@ +package common + +import corev1 "k8s.io/api/core/v1" + +// PodReady reports whether a pod is Running with its Ready condition true, so a +// starting or CrashLoopBackOff pod (Running but not Ready) is not counted. +func PodReady(pod *corev1.Pod) bool { + if pod == nil || pod.Status.Phase != corev1.PodRunning { + return false + } + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} diff --git a/internal/controller/common/resource.go b/internal/controller/common/resource.go index a9eca13a..73f4c069 100644 --- a/internal/controller/common/resource.go +++ b/internal/controller/common/resource.go @@ -2,10 +2,14 @@ package common import ( "context" + "encoding/base64" + "encoding/json" + "maps" "reflect" "github.com/wandb/operator/internal/logx" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -47,10 +51,11 @@ func GetResource[T client.Object]( type CrudAction string const ( - NoAction = "" - CreateAction = "Create" - UpdateAction = "Update" - DeleteAction = "Delete" + NoAction = "" + CreateAction = "Create" + UpdateAction = "Update" + UnchangedAction = "Unchanged" + DeleteAction = "Delete" ) // CrudResource is a generic function that gets a resource, and creates it if not found, or updates it if it exists. @@ -64,10 +69,13 @@ func CrudResource[T client.Object](ctx context.Context, c client.Client, desired actualExists := !IsNil(actual) && actual.GetName() != "" if actualExists && desiredExists { - action = UpdateAction - - desired.SetResourceVersion(actual.GetResourceVersion()) - err = c.Update(ctx, desired) + if resourceManagedFieldsEqual(desired, actual) { + action = UnchangedAction + } else { + action = UpdateAction + prepareResourceUpdate(desired, actual) + err = c.Update(ctx, desired) + } } if !actualExists && desiredExists { action = CreateAction @@ -77,7 +85,7 @@ func CrudResource[T client.Object](ctx context.Context, c client.Client, desired action = DeleteAction err = c.Delete(ctx, actual) } - if action != NoAction { + if action != NoAction && action != UnchangedAction { if desiredExists { log.Info(string(action), "namespace", desired.GetNamespace(), "name", desired.GetName()) } else if actualExists { @@ -90,6 +98,116 @@ func CrudResource[T client.Object](ctx context.Context, c client.Client, desired return action, err } +func resourceManagedFieldsEqual(desired, actual client.Object) bool { + if !mapContains(actual.GetLabels(), desired.GetLabels()) || + !mapContains(actual.GetAnnotations(), desired.GetAnnotations()) || + !ownerReferencesContain(actual.GetOwnerReferences(), desired.GetOwnerReferences()) { + return false + } + + return resourceContentEqual(desired, actual) +} + +func resourceContentEqual(desired, actual client.Object) bool { + toContent := func(obj client.Object) (map[string]any, bool) { + data, err := json.Marshal(obj) + if err != nil { + return nil, false + } + content := map[string]any{} + if err := json.Unmarshal(data, &content); err != nil { + return nil, false + } + delete(content, "apiVersion") + delete(content, "kind") + delete(content, "metadata") + delete(content, "status") + normalizeSecretStringData(content) + return content, true + } + + desiredContent, desiredOK := toContent(desired) + actualContent, actualOK := toContent(actual) + return desiredOK && actualOK && reflect.DeepEqual(desiredContent, actualContent) +} + +func normalizeSecretStringData(content map[string]any) { + stringData, ok := content["stringData"].(map[string]any) + if !ok { + return + } + data, _ := content["data"].(map[string]any) + if data == nil { + data = map[string]any{} + } + for key, value := range stringData { + text, ok := value.(string) + if !ok { + continue + } + data[key] = base64.StdEncoding.EncodeToString([]byte(text)) + } + content["data"] = data + delete(content, "stringData") +} + +func mapContains(actual, desired map[string]string) bool { + for key, value := range desired { + if actual[key] != value { + return false + } + } + return true +} + +func ownerReferencesContain(actual, desired []metav1.OwnerReference) bool { + for _, desiredRef := range desired { + found := false + for _, actualRef := range actual { + if reflect.DeepEqual(actualRef, desiredRef) { + found = true + break + } + } + if !found { + return false + } + } + return true +} + +func prepareResourceUpdate(desired, actual client.Object) { + desired.SetResourceVersion(actual.GetResourceVersion()) + desired.SetUID(actual.GetUID()) + desired.SetCreationTimestamp(actual.GetCreationTimestamp()) + desired.SetGeneration(actual.GetGeneration()) + desired.SetManagedFields(actual.GetManagedFields()) + desired.SetFinalizers(actual.GetFinalizers()) + desired.SetDeletionTimestamp(actual.GetDeletionTimestamp()) + + labels := maps.Clone(actual.GetLabels()) + if labels == nil { + labels = map[string]string{} + } + maps.Copy(labels, desired.GetLabels()) + desired.SetLabels(labels) + + annotations := maps.Clone(actual.GetAnnotations()) + if annotations == nil { + annotations = map[string]string{} + } + maps.Copy(annotations, desired.GetAnnotations()) + desired.SetAnnotations(annotations) + + ownerReferences := append([]metav1.OwnerReference(nil), actual.GetOwnerReferences()...) + for _, desiredRef := range desired.GetOwnerReferences() { + if !ownerReferencesContain(ownerReferences, []metav1.OwnerReference{desiredRef}) { + ownerReferences = append(ownerReferences, desiredRef) + } + } + desired.SetOwnerReferences(ownerReferences) +} + // IsNil checks if the generic value v is a pointer and if that pointer is nil. // It returns false if true is a non-pointer type, or if it's a non-nil pointer. func IsNil[T any](v T) bool { diff --git a/internal/controller/common/resource_test.go b/internal/controller/common/resource_test.go new file mode 100644 index 00000000..4d3e4c2f --- /dev/null +++ b/internal/controller/common/resource_test.go @@ -0,0 +1,123 @@ +package common + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type updateCountingClient struct { + client.Client + updates int +} + +func (c *updateCountingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + c.updates++ + return c.Client.Update(ctx, obj, opts...) +} + +func TestCrudResourceSkipsUnchangedManagedFields(t *testing.T) { + t.Parallel() + + actual := testConfigMap("value") + actual.Labels["co-manager"] = "preserve" + actual.Finalizers = []string{"co-manager/finalizer"} + c := newUpdateCountingClient(t, actual) + desired := testConfigMap("value") + + action, err := CrudResource(context.Background(), c, desired, actual.DeepCopy()) + if err != nil { + t.Fatalf("reconcile resource: %v", err) + } + if action != UnchangedAction { + t.Fatalf("action = %q, want %q", action, UnchangedAction) + } + if c.updates != 0 { + t.Fatalf("updates = %d, want 0", c.updates) + } +} + +func TestCrudResourcePreservesCoManagedMetadataOnUpdate(t *testing.T) { + t.Parallel() + + actual := testConfigMap("old") + actual.Labels["co-manager"] = "preserve" + actual.Finalizers = []string{"co-manager/finalizer"} + c := newUpdateCountingClient(t, actual) + desired := testConfigMap("new") + + action, err := CrudResource(context.Background(), c, desired, actual.DeepCopy()) + if err != nil { + t.Fatalf("reconcile resource: %v", err) + } + if action != UpdateAction { + t.Fatalf("action = %q, want %q", action, UpdateAction) + } + if c.updates != 1 { + t.Fatalf("updates = %d, want 1", c.updates) + } + + updated := &corev1.ConfigMap{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(actual), updated); err != nil { + t.Fatalf("get updated ConfigMap: %v", err) + } + if updated.Data["key"] != "new" { + t.Fatalf("data = %q, want new", updated.Data["key"]) + } + if updated.Labels["co-manager"] != "preserve" { + t.Fatal("co-managed label was removed") + } + if len(updated.Finalizers) != 1 || updated.Finalizers[0] != "co-manager/finalizer" { + t.Fatalf("finalizers = %v, want co-manager finalizer", updated.Finalizers) + } +} + +func TestCrudResourceTreatsSecretStringDataAsExistingData(t *testing.T) { + t.Parallel() + + actual := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "connection", Namespace: "default"}, + Data: map[string][]byte{"Password": []byte("secret")}, + } + c := newUpdateCountingClient(t, actual) + desired := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "connection", Namespace: "default"}, + StringData: map[string]string{"Password": "secret"}, + } + + action, err := CrudResource(context.Background(), c, desired, actual.DeepCopy()) + if err != nil { + t.Fatalf("reconcile resource: %v", err) + } + if action != UnchangedAction { + t.Fatalf("action = %q, want %q", action, UnchangedAction) + } + if c.updates != 0 { + t.Fatalf("updates = %d, want 0", c.updates) + } +} + +func testConfigMap(value string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "config", + Namespace: "default", + Labels: map[string]string{"managed-by": "wandb"}, + }, + Data: map[string]string{"key": value}, + } +} + +func newUpdateCountingClient(t *testing.T, objects ...client.Object) *updateCountingClient { + t.Helper() + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("add core API to scheme: %v", err) + } + return &updateCountingClient{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build()} +} diff --git a/internal/controller/common/service.go b/internal/controller/common/service.go new file mode 100644 index 00000000..d23498fb --- /dev/null +++ b/internal/controller/common/service.go @@ -0,0 +1,19 @@ +package common + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// NormalizeServicePorts fills the port fields the API server defaults (protocol, +// targetPort) so specs built from manifests round-trip equal to what is stored. +func NormalizeServicePorts(ports []corev1.ServicePort) { + for i := range ports { + if ports[i].Protocol == "" { + ports[i].Protocol = corev1.ProtocolTCP + } + if ports[i].TargetPort == (intstr.IntOrString{}) { + ports[i].TargetPort = intstr.FromInt32(ports[i].Port) + } + } +} diff --git a/internal/controller/common/service_test.go b/internal/controller/common/service_test.go new file mode 100644 index 00000000..360473a5 --- /dev/null +++ b/internal/controller/common/service_test.go @@ -0,0 +1,25 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func TestNormalizeServicePorts(t *testing.T) { + ports := []corev1.ServicePort{ + {Name: "http", Port: 8080}, + {Name: "grpc", Port: 9000, Protocol: corev1.ProtocolUDP, TargetPort: intstr.FromString("grpc")}, + } + NormalizeServicePorts(ports) + + require.Equal(t, corev1.ProtocolTCP, ports[0].Protocol) + require.Equal(t, intstr.FromInt32(8080), ports[0].TargetPort) + + require.Equal(t, corev1.ProtocolUDP, ports[1].Protocol, "explicit protocol is kept") + require.Equal(t, intstr.FromString("grpc"), ports[1].TargetPort, "explicit targetPort is kept") + + NormalizeServicePorts(nil) +} diff --git a/internal/controller/ctrlqueue/ctrl_state_test.go b/internal/controller/ctrlqueue/ctrl_state_test.go index 20395c7c..4b636c43 100644 --- a/internal/controller/ctrlqueue/ctrl_state_test.go +++ b/internal/controller/ctrlqueue/ctrl_state_test.go @@ -201,7 +201,6 @@ var _ = Describe("CtrlState", func() { result, err := state.ReconcilerResult() Expect(err).To(BeNil()) Expect(result).To(Equal(ctrl.Result{})) - Expect(result.Requeue).To(BeFalse()) // nolint:SA1019 Expect(result.RequeueAfter).To(Equal(time.Duration(0))) }) diff --git a/internal/controller/infra/external/clickhouse/clickhouse.go b/internal/controller/infra/external/clickhouse/clickhouse.go index 1700b934..79e5b2bd 100644 --- a/internal/controller/infra/external/clickhouse/clickhouse.go +++ b/internal/controller/infra/external/clickhouse/clickhouse.go @@ -2,6 +2,7 @@ package clickhouse import ( "context" + "fmt" apiv2 "github.com/wandb/operator/api/v2" "github.com/wandb/operator/internal/controller/infra/external" @@ -15,18 +16,27 @@ import ( const ConnectionSecretName = "wandb-clickhouse-connection" +func connectionSecretName(key string) string { + if key == "" || key == apiv2.DefaultInstanceName { + return ConnectionSecretName + } + return fmt.Sprintf("%s-%s", ConnectionSecretName, key) +} + func WriteState( ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, + key string, + spec *apiv2.ClickHouseConnection, ) []metav1.Condition { - spec := wandb.Spec.ClickHouse.ExternalClickHouse logger := ctrl.LoggerFrom(ctx) fields := map[string]corev1.SecretKeySelector{ "url": spec.URL, "Host": spec.Host, - "Port": spec.Port, + "HTTPPort": spec.HTTPPort, + "TCPPort": spec.TCPPort, "User": spec.Username, "Password": spec.Password, "Database": spec.Database, @@ -42,7 +52,7 @@ func WriteState( }} } - nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: ConnectionSecretName} + nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: connectionSecretName(key)} return external.WriteConnectionSecret(ctx, c, wandb, nsName, data) } @@ -50,9 +60,10 @@ func ReadState( ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, + key string, newConditions []metav1.Condition, ) ([]metav1.Condition, *apiv2.ClickHouseConnection) { - nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: ConnectionSecretName} + nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: connectionSecretName(key)} _, conditions, found := external.ReadConnectionSecret(ctx, c, nsName, newConditions) if !found { return conditions, nil @@ -62,16 +73,17 @@ func ReadState( return conditions, &apiv2.ClickHouseConnection{ URL: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "url", Optional: ptr.To(false)}, Host: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Host", Optional: ptr.To(false)}, - Port: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Port", Optional: ptr.To(false)}, + HTTPPort: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "HTTPPort", Optional: ptr.To(false)}, + TCPPort: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "TCPPort", Optional: ptr.To(false)}, Username: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "User", Optional: ptr.To(false)}, Password: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Password", Optional: ptr.To(false)}, Database: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Database", Optional: ptr.To(false)}, } } -func DeleteConnectionSecret(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases) error { +func DeleteConnectionSecret(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string) error { return external.DeleteConnectionSecret(ctx, c, types.NamespacedName{ Namespace: wandb.Namespace, - Name: ConnectionSecretName, + Name: connectionSecretName(key), }) } diff --git a/internal/controller/infra/external/common.go b/internal/controller/infra/external/common.go index 280d0ad7..1fc7cc21 100644 --- a/internal/controller/infra/external/common.go +++ b/internal/controller/infra/external/common.go @@ -135,11 +135,19 @@ func InferExternalStatus( generation int64, hasConnection bool, ) (string, bool, []metav1.Condition) { - state := common.HealthyState - ready := true - if !hasConnection { - state = common.ErrorState - ready = false + hasCurrentReconciledCondition := false + for _, condition := range newConditions { + if condition.Type == common.ReconciledType { + hasCurrentReconciledCondition = true + break + } + } + if hasConnection && !hasCurrentReconciledCondition { + newConditions = append(newConditions, metav1.Condition{ + Type: common.ReconciledType, + Status: metav1.ConditionTrue, + Reason: common.ResourceExistsReason, + }) } updatedConditions := common.ComputeConditionUpdates( @@ -148,6 +156,19 @@ func InferExternalStatus( generation, common.DefaultConditionExpiry, ) + + ready := hasConnection + for _, condition := range newConditions { + if condition.Type == common.ReconciledType && condition.Status != metav1.ConditionTrue { + ready = false + break + } + } + + state := common.HealthyState + if !ready { + state = common.ErrorState + } return state, ready, updatedConditions } diff --git a/internal/controller/infra/external/common_test.go b/internal/controller/infra/external/common_test.go new file mode 100644 index 00000000..a5ef26d7 --- /dev/null +++ b/internal/controller/infra/external/common_test.go @@ -0,0 +1,52 @@ +package external + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/wandb/operator/internal/controller/common" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestInferExternalStatusUsesCurrentReconcileCondition(t *testing.T) { + failed := metav1.Condition{ + Type: common.ReconciledType, + Status: metav1.ConditionFalse, + Reason: common.ResourceErrorReason, + Message: "invalid connection", + } + + state, ready, conditions := InferExternalStatus(nil, []metav1.Condition{failed}, 2, true) + + require.Equal(t, common.ErrorState, state) + require.False(t, ready) + require.Len(t, conditions, 1) + require.Equal(t, metav1.ConditionFalse, conditions[0].Status) + require.Equal(t, int64(2), conditions[0].ObservedGeneration) +} + +func TestInferExternalStatusClearsRecoveredFailure(t *testing.T) { + old := metav1.Condition{ + Type: common.ReconciledType, + Status: metav1.ConditionFalse, + Reason: common.ResourceErrorReason, + Message: "invalid connection", + ObservedGeneration: 1, + } + + state, ready, conditions := InferExternalStatus([]metav1.Condition{old}, nil, 2, true) + + require.Equal(t, common.HealthyState, state) + require.True(t, ready) + require.Len(t, conditions, 1) + require.Equal(t, metav1.ConditionTrue, conditions[0].Status) + require.Equal(t, common.ResourceExistsReason, conditions[0].Reason) + require.Equal(t, int64(2), conditions[0].ObservedGeneration) +} + +func TestInferExternalStatusRequiresConnection(t *testing.T) { + state, ready, _ := InferExternalStatus(nil, nil, 1, false) + + require.Equal(t, common.ErrorState, state) + require.False(t, ready) +} diff --git a/internal/controller/infra/external/mysql/mysql.go b/internal/controller/infra/external/mysql/mysql.go index 5eea5434..8ed76000 100644 --- a/internal/controller/infra/external/mysql/mysql.go +++ b/internal/controller/infra/external/mysql/mysql.go @@ -16,13 +16,27 @@ import ( ) const ConnectionSecretName = "wandb-mysql-connection" +const caCertPath = "/etc/ssl/certs/mysql_ca.pem" +const sslCertPath = "/etc/ssl/certs/mysql_ssl_cert.pem" +const sslKeyPath = "/etc/ssl/certs/mysql_ssl_key.pem" + +// connectionSecretName returns the connection secret name for an instance. The +// reserved default instance keeps the historical name for backward +// compatibility; other instances are suffixed with their key. +func connectionSecretName(key string) string { + if key == "" || key == apiv2.DefaultInstanceName { + return ConnectionSecretName + } + return fmt.Sprintf("%s-%s", ConnectionSecretName, key) +} func WriteState( ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, + key string, + spec *apiv2.MysqlConnection, ) []metav1.Condition { - spec := wandb.Spec.MySQL.ExternalMysql logger := ctrl.LoggerFrom(ctx) fields := map[string]corev1.SecretKeySelector{ @@ -53,10 +67,27 @@ func WriteState( User: url.UserPassword(data["Username"], data["Password"]), Path: data["Database"], } + values := dbUrl.Query() + if tls, ok := data["Tls"]; ok { + values.Set("tls", tls) + } + if _, ok := data["SslCa"]; ok { + if values.Get("tls") == "" { + values.Set("tls", "custom") + } + values.Set("ssl-ca", caCertPath) + } + if _, ok := data["SslCert"]; ok { + values.Set("ssl-cert", sslCertPath) + } + if _, ok := data["SslKey"]; ok { + values.Set("ssl-key", sslKeyPath) + } + dbUrl.RawQuery = values.Encode() data["url"] = dbUrl.String() - nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: ConnectionSecretName} + nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: connectionSecretName(key)} return external.WriteConnectionSecret(ctx, c, wandb, nsName, data) } @@ -64,9 +95,10 @@ func ReadState( ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, + key string, newConditions []metav1.Condition, ) ([]metav1.Condition, *apiv2.MysqlConnection) { - nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: ConnectionSecretName} + nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: connectionSecretName(key)} _, conditions, found := external.ReadConnectionSecret(ctx, c, nsName, newConditions) if !found { return conditions, nil @@ -87,9 +119,9 @@ func ReadState( } } -func DeleteConnectionSecret(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases) error { +func DeleteConnectionSecret(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string) error { return external.DeleteConnectionSecret(ctx, c, types.NamespacedName{ Namespace: wandb.Namespace, - Name: ConnectionSecretName, + Name: connectionSecretName(key), }) } diff --git a/internal/controller/infra/external/mysql/mysql_test.go b/internal/controller/infra/external/mysql/mysql_test.go new file mode 100644 index 00000000..77a82965 --- /dev/null +++ b/internal/controller/infra/external/mysql/mysql_test.go @@ -0,0 +1,90 @@ +package mysql + +import ( + "context" + "net/url" + "testing" + + "github.com/stretchr/testify/require" + apiv2 "github.com/wandb/operator/api/v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const mysqlSourceSecretName = "external-mysql" + +func mysqlSel(key string) corev1.SecretKeySelector { + return corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: mysqlSourceSecretName}, + Key: key, + } +} + +func TestWriteStateAddsCustomTLSParamsWhenCACertPresent(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, apiv2.AddToScheme(scheme)) + + source := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: mysqlSourceSecretName, Namespace: "default"}, + Data: map[string][]byte{ + "Host": []byte("mysql.example.com"), + "Port": []byte("3306"), + "Database": []byte("wandb"), + "Username": []byte("wandb"), + "Password": []byte("secret"), + "SslCa": []byte("---ca---"), + "SslCert": []byte("---cert---"), + "SslKey": []byte("---key---"), + }, + } + wandb := &apiv2.WeightsAndBiases{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps.wandb.com/v2", Kind: "WeightsAndBiases"}, + ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "default"}, + Spec: apiv2.WeightsAndBiasesSpec{ + MySQL: map[string]apiv2.MySQLSpec{apiv2.DefaultInstanceName: { + ExternalMysql: &apiv2.MysqlConnection{ + Host: mysqlSel("Host"), + Port: mysqlSel("Port"), + Database: mysqlSel("Database"), + Username: mysqlSel("Username"), + Password: mysqlSel("Password"), + SslCa: mysqlSel("SslCa"), + SslCert: mysqlSel("SslCert"), + SslKey: mysqlSel("SslKey"), + }, + }}, + }, + } + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(wandb, source).Build() + + conditions := WriteState(context.Background(), client, wandb, apiv2.DefaultInstanceName, wandb.Spec.MySQL[apiv2.DefaultInstanceName].ExternalMysql) + require.Nil(t, conditions) + + written := &corev1.Secret{} + require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: ConnectionSecretName, Namespace: "default"}, written)) + data := mysqlConnectionData(written) + parsed, err := url.Parse(data["url"]) + require.NoError(t, err) + require.Equal(t, "mysql", parsed.Scheme) + require.Equal(t, "mysql.example.com:3306", parsed.Host) + require.Equal(t, "/wandb", parsed.Path) + require.Equal(t, "custom", parsed.Query().Get("tls")) + require.Equal(t, caCertPath, parsed.Query().Get("ssl-ca")) + require.Equal(t, sslCertPath, parsed.Query().Get("ssl-cert")) + require.Equal(t, sslKeyPath, parsed.Query().Get("ssl-key")) +} + +func mysqlConnectionData(secret *corev1.Secret) map[string]string { + out := map[string]string{} + for k, v := range secret.Data { + out[k] = string(v) + } + for k, v := range secret.StringData { + out[k] = v + } + return out +} diff --git a/internal/controller/infra/external/objectstore/connection.go b/internal/controller/infra/external/objectstore/connection.go deleted file mode 100644 index fbd5facf..00000000 --- a/internal/controller/infra/external/objectstore/connection.go +++ /dev/null @@ -1,132 +0,0 @@ -package objectstore - -import ( - "fmt" - "net/url" - "strconv" - "strings" - - apiv2 "github.com/wandb/operator/api/v2" -) - -// ConnInfo is the resolved object-store connection: the read-side counterpart to WriteState, decoded back from the connection secret. -type ConnInfo struct { - Provider apiv2.ObjectStoreProvider - // URI is the provider-native location, e.g. "s3://bucket", "gs://bucket/prefix", or "https://acct.blob.core.windows.net/container". - URI string - // Bucket is the bare bucket/container name. - Bucket string - // Endpoint overrides the S3 API endpoint for S3-compatible providers (SeaweedFS, MinIO); empty for AWS S3, GCS, and Azure. - Endpoint string - Region string - AccessKey string - SecretKey string - // ForcePathStyle is required by most non-AWS S3-compatible providers. - ForcePathStyle bool -} - -// HasStaticCredentials reports whether explicit keys were provided; when false, credentials come from ambient identity (IAM role / workload identity). -func (c ConnInfo) HasStaticCredentials() bool { - return c.AccessKey != "" && c.SecretKey != "" -} - -// ParseConnection decodes an object-store connection secret's canonical `url` (scheme->provider, userinfo->creds, path->bucket, query->tls/region/forcePathStyle), falling back to discrete keys. -func ParseConnection(data map[string][]byte) (ConnInfo, error) { - get := func(k string) string { return string(data[k]) } - - raw := get("url") - if raw == "" { - return ConnInfo{}, fmt.Errorf("object store connection secret missing url") - } - u, err := url.Parse(raw) - if err != nil { - return ConnInfo{}, fmt.Errorf("parse object store url %q: %w", raw, err) - } - - info := ConnInfo{} - if u.User != nil { - info.AccessKey = u.User.Username() - if pw, ok := u.User.Password(); ok { - info.SecretKey = pw - } - } - if info.AccessKey == "" { - info.AccessKey = get("AccessKey") - } - if info.SecretKey == "" { - info.SecretKey = get("SecretKey") - } - - q := u.Query() - info.Region = q.Get("region") - if info.Region == "" { - info.Region = get("Region") - } - - switch strings.ToLower(u.Scheme) { - case "s3", "cw": - info.Provider = apiv2.ObjectStoreProviderS3 - bucket := strings.TrimPrefix(u.Path, "/") - host := u.Host - if bucket == "" { - // No path: bucket is the host (s3://my-bucket) or opaque part (s3:my-bucket), with no endpoint override. - if u.Opaque != "" { - bucket = u.Opaque - } else { - bucket = host - } - host = "" - } - info.Bucket = bucket - info.URI = "s3://" + bucket - // A host alongside a bucket path means an S3-compatible endpoint (SeaweedFS, MinIO); AWS S3 has no endpoint override. - if host != "" { - endpointScheme := "http" - if tls, _ := strconv.ParseBool(q.Get("tls")); tls { - endpointScheme = "https" - } - info.Endpoint = fmt.Sprintf("%s://%s", endpointScheme, host) - } - if fps := q.Get("forcePathStyle"); fps != "" { - info.ForcePathStyle, _ = strconv.ParseBool(fps) - } else { - // Non-AWS S3-compatible endpoints generally require path-style. - info.ForcePathStyle = info.Endpoint != "" - } - case "gs", "gcs": - info.Provider = apiv2.ObjectStoreProviderGCS - info.Bucket = u.Host - info.URI = "gs://" + u.Host + u.Path - case "azure", "az": - // az://// - info.Provider = apiv2.ObjectStoreProviderAzure - account := u.Host - container, prefix := splitBucketPath(u.Path) - info.Bucket = container - info.URI = azureBlobURI(account, container, prefix) - case "http", "https": - if !strings.Contains(u.Host, "blob.core.windows.net") { - return ConnInfo{}, fmt.Errorf("unsupported object store url scheme %q", u.Scheme) - } - info.Provider = apiv2.ObjectStoreProviderAzure - container, _ := splitBucketPath(u.Path) - info.Bucket = container - // Pass the container URI through verbatim (sans credentials/query). - info.URI = (&url.URL{Scheme: u.Scheme, Host: u.Host, Path: u.Path}).String() - default: - return ConnInfo{}, fmt.Errorf("unsupported object store url scheme %q", u.Scheme) - } - - if info.Bucket == "" && info.Provider != "" { - return ConnInfo{}, fmt.Errorf("object store url %q has no bucket/container", raw) - } - return info, nil -} - -func azureBlobURI(account, container, prefix string) string { - uri := fmt.Sprintf("https://%s.blob.core.windows.net/%s", account, container) - if prefix != "" { - uri += "/" + prefix - } - return uri -} diff --git a/internal/controller/infra/external/objectstore/connection_test.go b/internal/controller/infra/external/objectstore/connection_test.go deleted file mode 100644 index 4ef3b3bc..00000000 --- a/internal/controller/infra/external/objectstore/connection_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package objectstore - -import ( - "testing" - - "github.com/stretchr/testify/require" - - apiv2 "github.com/wandb/operator/api/v2" -) - -func TestParseConnectionS3Compatible(t *testing.T) { - // Managed SeaweedFS shape: s3://ak:sk@host:port/bucket?tls=false plus a - // discrete Region key. - data := map[string][]byte{ - "url": []byte("s3://ak:sk@seaweedfs.wandb.svc.cluster.local:8333/wandb-bucket?tls=false"), - "Region": []byte("us-east-1"), - } - - info, err := ParseConnection(data) - require.NoError(t, err) - require.Equal(t, apiv2.ObjectStoreProviderS3, info.Provider) - require.Equal(t, "wandb-bucket", info.Bucket) - require.Equal(t, "s3://wandb-bucket", info.URI) - require.Equal(t, "http://seaweedfs.wandb.svc.cluster.local:8333", info.Endpoint) - require.Equal(t, "us-east-1", info.Region) - require.Equal(t, "ak", info.AccessKey) - require.Equal(t, "sk", info.SecretKey) - require.True(t, info.ForcePathStyle) -} - -func TestParseConnectionS3TLS(t *testing.T) { - data := map[string][]byte{ - "url": []byte("s3://ak:sk@minio.example.com:9000/bucket?tls=true"), - } - info, err := ParseConnection(data) - require.NoError(t, err) - require.Equal(t, "https://minio.example.com:9000", info.Endpoint) -} - -func TestParseConnectionAWS(t *testing.T) { - // AWS S3 with IAM role: no host, no credentials. - data := map[string][]byte{ - "url": []byte("s3://my-bucket"), - "Region": []byte("us-west-2"), - } - info, err := ParseConnection(data) - require.NoError(t, err) - require.Equal(t, apiv2.ObjectStoreProviderS3, info.Provider) - require.Equal(t, "my-bucket", info.Bucket) - require.Empty(t, info.Endpoint) - require.False(t, info.ForcePathStyle) - require.False(t, info.HasStaticCredentials()) -} - -func TestParseConnectionDiscreteCredFallback(t *testing.T) { - data := map[string][]byte{ - "url": []byte("s3://host:9000/bucket"), - "AccessKey": []byte("ak"), - "SecretKey": []byte("sk"), - } - info, err := ParseConnection(data) - require.NoError(t, err) - require.Equal(t, "ak", info.AccessKey) - require.Equal(t, "sk", info.SecretKey) -} - -func TestParseConnectionGCS(t *testing.T) { - data := map[string][]byte{ - "url": []byte("gs://wandb-bucket/some/prefix"), - } - info, err := ParseConnection(data) - require.NoError(t, err) - require.Equal(t, apiv2.ObjectStoreProviderGCS, info.Provider) - require.Equal(t, "wandb-bucket", info.Bucket) - require.Equal(t, "gs://wandb-bucket/some/prefix", info.URI) -} - -func TestParseConnectionAzureHTTPS(t *testing.T) { - data := map[string][]byte{ - "url": []byte("https://acct.blob.core.windows.net/container/prefix"), - } - info, err := ParseConnection(data) - require.NoError(t, err) - require.Equal(t, apiv2.ObjectStoreProviderAzure, info.Provider) - require.Equal(t, "container", info.Bucket) - require.Equal(t, "https://acct.blob.core.windows.net/container/prefix", info.URI) -} - -func TestParseConnectionAzureScheme(t *testing.T) { - data := map[string][]byte{ - "url": []byte("az://acct:key@acct/container/prefix"), - } - info, err := ParseConnection(data) - require.NoError(t, err) - require.Equal(t, apiv2.ObjectStoreProviderAzure, info.Provider) - require.Equal(t, "container", info.Bucket) - require.Equal(t, "https://acct.blob.core.windows.net/container/prefix", info.URI) - require.Equal(t, "acct", info.AccessKey) - require.Equal(t, "key", info.SecretKey) -} - -func TestParseConnectionErrors(t *testing.T) { - _, err := ParseConnection(map[string][]byte{}) - require.Error(t, err) - - _, err = ParseConnection(map[string][]byte{"url": []byte("ftp://nope/bucket")}) - require.Error(t, err) -} diff --git a/internal/controller/infra/external/objectstore/objectstore.go b/internal/controller/infra/external/objectstore/objectstore.go index a4b68a7c..60928521 100644 --- a/internal/controller/infra/external/objectstore/objectstore.go +++ b/internal/controller/infra/external/objectstore/objectstore.go @@ -4,36 +4,53 @@ import ( "context" "fmt" "net/url" + "strconv" "strings" apiv2 "github.com/wandb/operator/api/v2" "github.com/wandb/operator/internal/controller/infra/external" + osconn "github.com/wandb/operator/internal/controller/infra/objectstore" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) const ConnectionSecretName = "wandb-objectstore-connection" +// connectionSecretName builds the connection secret name for an object-store +// instance key, using the shared default name for the primary instance. +func connectionSecretName(key string) string { + if key == "" || key == apiv2.DefaultInstanceName { + return ConnectionSecretName + } + return fmt.Sprintf("%s-%s", ConnectionSecretName, key) +} + +// WriteState resolves the external object-store fields into a connection secret +// and returns the reconcile conditions plus the resulting ObjectStoreConnection +// selectors (nil on error). func WriteState( ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, + key string, + spec *apiv2.ObjectStoreConnection, ) ([]metav1.Condition, *apiv2.ObjectStoreConnection) { - spec := wandb.Spec.ObjectStore.ExternalObjectStore logger := ctrl.LoggerFrom(ctx) fields := map[string]corev1.SecretKeySelector{ - "Host": spec.Endpoint, - "Port": spec.Port, - "AccessKey": spec.AccessKey, - "SecretKey": spec.SecretKey, - "Bucket": spec.Bucket, - "Region": spec.Region, - "Provider": spec.Provider, + "Host": spec.Endpoint, + "Port": spec.Port, + "AccessKey": spec.AccessKey, + "SecretKey": spec.SecretKey, + "Bucket": spec.Bucket, + "Path": spec.Path, + "Region": spec.Region, + "Provider": spec.Provider, + "TlsEnabled": spec.TlsEnabled, + "ForcePathStyle": spec.ForcePathStyle, } data, err := external.ResolveFields(ctx, c, wandb.Namespace, fields) @@ -46,49 +63,64 @@ func WriteState( }}, nil } + // Normalize the prefix once so every consumer joins it without stray slashes. + if trimmed := strings.Trim(data["Path"], "/"); trimmed != "" { + data["Path"] = trimmed + } else { + delete(data, "Path") + } + provider := apiv2.ObjectStoreProvider(data["Provider"]) if provider == "" { provider = apiv2.ObjectStoreProviderS3 } - data["Provider"] = string(provider) + + connInfo := osconn.ConnInfo{ + Provider: provider, + Endpoint: data["Host"], + Port: data["Port"], + AccessKey: data["AccessKey"], + SecretKey: data["SecretKey"], + Bucket: data["Bucket"], + Path: data["Path"], + Region: data["Region"], + } + if tls, err := strconv.ParseBool(data["TlsEnabled"]); err == nil { + connInfo.TlsEnabled = tls + } switch provider { case apiv2.ObjectStoreProviderGCS: - data["url"] = buildGCSURL(data) + connInfo.URL = buildGCSURL(data) case apiv2.ObjectStoreProviderAzure: - data["url"] = buildAzureURL(data) + connInfo.URL = buildAzureURL(data) default: - data["url"] = buildS3URL(data) + // Consumers (Bufstream) render this verbatim, so derive it when the CR doesn't say. + if fps, ok := data["ForcePathStyle"]; ok { + connInfo.ForcePathStyle, _ = strconv.ParseBool(fps) + } else { + connInfo.ForcePathStyle = osconn.RequiresPathStyle(data["Host"]) + } + connInfo.URL = buildS3URL(data) } - nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: ConnectionSecretName} - if conditions := external.WriteConnectionSecret(ctx, c, wandb, nsName, data); conditions != nil { + nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: connectionSecretName(key)} + if conditions := external.WriteConnectionSecret(ctx, c, wandb, nsName, connInfo.ToSecretData()); conditions != nil { return conditions, nil } - localRef := corev1.LocalObjectReference{Name: nsName.Name} - // ResolveFields only writes non-empty values, so any field that is - // legitimately absent for some deployment must be optional: Host (plain - // AWS S3 with no custom endpoint), AccessKey/SecretKey (IAM-role / - // workload-identity auth), Region (MinIO or region supplied out-of-band). - // url and Bucket are always written. - return nil, &apiv2.ObjectStoreConnection{ - Provider: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Provider", Optional: ptr.To(false)}, - URL: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "url", Optional: ptr.To(false)}, - Endpoint: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Host", Optional: ptr.To(true)}, - Port: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Port", Optional: ptr.To(true)}, - AccessKey: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "AccessKey", Optional: ptr.To(true)}, - SecretKey: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "SecretKey", Optional: ptr.To(true)}, - Bucket: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Bucket", Optional: ptr.To(false)}, - Region: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Region", Optional: ptr.To(true)}, - } + // ToSecretData only writes non-empty values, so any field that is + // legitimately absent for some deployment (Host for plain AWS S3, + // AccessKey/SecretKey for IAM-role / workload-identity auth, Region for + // MinIO) stays optional; url/Provider/Bucket are always required. + return nil, connInfo.ToObjectStoreConnection(nsName.Name, false) } -// buildS3URL assembles s3://[accessKey:secretKey@][host[:port]]/bucket; host and creds are omitted for native AWS S3 / IAM-role auth. +// buildS3URL assembles s3://[accessKey:secretKey@][host[:port]]/bucket[/path]; host and creds are omitted for native AWS S3 / IAM-role auth. func buildS3URL(data map[string]string) string { bucketURL := url.URL{ Scheme: "s3", - Path: data["Bucket"], + Path: joinBucketPrefix(data["Bucket"], data["Path"]), } if _, ok := data["Host"]; ok { if _, ok := data["Port"]; ok { @@ -105,7 +137,8 @@ func buildS3URL(data map[string]string) string { // buildGCSURL assembles gs://[/path]; creds default to workload identity, or accessKey (SA email) + secretKey (PEM key) as userinfo. func buildGCSURL(data map[string]string) string { - bucket, path := splitBucketPath(data["Bucket"]) + bucket, path := osconn.SplitBucketPath(data["Bucket"]) + path = joinBucketPrefix(path, data["Path"]) bucketURL := url.URL{Scheme: "gs", Host: bucket} if path != "" { bucketURL.Path = "/" + path @@ -119,7 +152,8 @@ func buildGCSURL(data map[string]string) string { // buildAzureURL assembles az:///[/path] from accessKey (account), bucket (container), and secretKey (account key, when set). func buildAzureURL(data map[string]string) string { account := data["AccessKey"] - container, path := splitBucketPath(data["Bucket"]) + container, path := osconn.SplitBucketPath(data["Bucket"]) + path = joinBucketPrefix(path, data["Path"]) bucketURL := url.URL{Scheme: "az", Host: account, Path: "/" + container} if path != "" { bucketURL.Path += "/" + path @@ -130,27 +164,36 @@ func buildAzureURL(data map[string]string) string { return bucketURL.String() } -// splitBucketPath splits "bucket/optional/prefix" into the leading bucket (or container) segment and the remaining object prefix. -func splitBucketPath(raw string) (bucket, path string) { - trimmed := strings.TrimPrefix(raw, "/") - if slash := strings.IndexByte(trimmed, '/'); slash >= 0 { - return trimmed[:slash], trimmed[slash+1:] +// joinBucketPrefix appends a normalized key prefix to base (a bucket or an existing prefix). +func joinBucketPrefix(base, prefix string) string { + prefix = strings.Trim(prefix, "/") + switch { + case prefix == "": + return base + case base == "": + return prefix + default: + return base + "/" + prefix } - return trimmed, "" } +// ReadState is a no-op for external object stores; it passes through the +// conditions produced by WriteState since there is no additional state to read. func ReadState( _ context.Context, _ client.Client, _ *apiv2.WeightsAndBiases, + _ string, newConditions []metav1.Condition, ) []metav1.Condition { return newConditions } -func DeleteConnectionSecret(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases) error { +// DeleteConnectionSecret removes the connection secret written for the given +// object-store instance key. +func DeleteConnectionSecret(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string) error { return external.DeleteConnectionSecret(ctx, c, types.NamespacedName{ Namespace: wandb.Namespace, - Name: ConnectionSecretName, + Name: connectionSecretName(key), }) } diff --git a/internal/controller/infra/external/objectstore/objectstore_test.go b/internal/controller/infra/external/objectstore/objectstore_test.go index 6cf4dd14..84630d75 100644 --- a/internal/controller/infra/external/objectstore/objectstore_test.go +++ b/internal/controller/infra/external/objectstore/objectstore_test.go @@ -66,20 +66,31 @@ func writeStateFixtureProvider(t *testing.T, provider apiv2.ObjectStoreProvider, if present["Bucket"] { ext.Bucket = sel("Bucket") } + if present["Path"] { + ext.Path = sel("Path") + } if present["Region"] { ext.Region = sel("Region") } + if present["TlsEnabled"] { + ext.TlsEnabled = sel("TlsEnabled") + } + if present["ForcePathStyle"] { + ext.ForcePathStyle = sel("ForcePathStyle") + } wandb := &apiv2.WeightsAndBiases{ TypeMeta: metav1.TypeMeta{APIVersion: "apps.wandb.com/v2", Kind: "WeightsAndBiases"}, ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "default"}, Spec: apiv2.WeightsAndBiasesSpec{ - ObjectStore: apiv2.ObjectStoreSpec{ExternalObjectStore: ext}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: {ExternalObjectStore: ext}, + }, }, } c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(wandb, source).Build() - conditions, conn := WriteState(context.Background(), c, wandb) + conditions, conn := WriteState(context.Background(), c, wandb, apiv2.DefaultInstanceName, ext) written := &corev1.Secret{} require.NoError(t, c.Get(context.Background(), types.NamespacedName{Name: ConnectionSecretName, Namespace: "default"}, written)) @@ -182,6 +193,90 @@ func TestWriteState_FullConfig(t *testing.T) { } } +func TestWriteState_PathPrefix(t *testing.T) { + // S3-compatible endpoint with a key prefix. + _, written, conditions, conn := writeStateFixture(t, + map[string]string{ + "Host": "minio.local", + "Port": "9000", + "AccessKey": "minio", + "SecretKey": "minio123", + "Bucket": "my-bucket", + "Path": "team/prefix", + }, + map[string]bool{"Host": true, "Port": true, "AccessKey": true, "SecretKey": true, "Bucket": true, "Path": true}, + ) + require.Nil(t, conditions) + data := connectionData(written) + require.Equal(t, "s3://minio:minio123@minio.local:9000/my-bucket/team/prefix", data["url"]) + require.Equal(t, "team/prefix", data["Path"]) + require.Equal(t, "Path", conn.Path.Key) + require.NotNil(t, conn.Path.Optional) + require.True(t, *conn.Path.Optional) + + // Native AWS with a prefix; slashes are normalized. + _, written, conditions, _ = writeStateFixture(t, + map[string]string{"Bucket": "my-bucket", "Path": "/prefix/"}, + map[string]bool{"Bucket": true, "Path": true}, + ) + require.Nil(t, conditions) + data = connectionData(written) + require.Equal(t, "s3://my-bucket/prefix", data["url"]) + require.Equal(t, "prefix", data["Path"], "the stored key is normalized, not raw") +} + +func TestWriteState_GCSPathPrefix(t *testing.T) { + _, written, conditions, _ := writeStateFixtureProvider(t, apiv2.ObjectStoreProviderGCS, + map[string]string{"Bucket": "my-gcs-bucket", "Path": "team/prefix"}, + map[string]bool{"Bucket": true, "Path": true}, + ) + require.Nil(t, conditions) + require.Equal(t, "gs://my-gcs-bucket/team/prefix", connectionData(written)["url"]) +} + +func TestWriteState_ForcePathStyleDerived(t *testing.T) { + // Custom endpoint → path-style. + _, written, conditions, _ := writeStateFixture(t, + map[string]string{"Host": "minio.local", "Bucket": "b"}, + map[string]bool{"Host": true, "Bucket": true}, + ) + require.Nil(t, conditions) + require.Equal(t, "true", connectionData(written)["ForcePathStyle"]) + + // No endpoint (native AWS) → virtual-hosted. + _, written, conditions, _ = writeStateFixture(t, + map[string]string{"Bucket": "b"}, + map[string]bool{"Bucket": true}, + ) + require.Nil(t, conditions) + require.Equal(t, "false", connectionData(written)["ForcePathStyle"]) + + // An explicit AWS endpoint override is still a custom endpoint → path-style. + _, written, conditions, _ = writeStateFixture(t, + map[string]string{"Host": "s3.us-east-1.amazonaws.com", "Bucket": "b"}, + map[string]bool{"Host": true, "Bucket": true}, + ) + require.Nil(t, conditions) + require.Equal(t, "true", connectionData(written)["ForcePathStyle"]) + + // CoreWeave object storage is virtual-hosted. + _, written, conditions, _ = writeStateFixture(t, + map[string]string{"Host": "cwobject.com", "Bucket": "b"}, + map[string]bool{"Host": true, "Bucket": true}, + ) + require.Nil(t, conditions) + require.Equal(t, "false", connectionData(written)["ForcePathStyle"]) +} + +func TestWriteState_ForcePathStyleExplicitWins(t *testing.T) { + _, written, conditions, _ := writeStateFixture(t, + map[string]string{"Host": "minio.local", "Bucket": "b", "ForcePathStyle": "false"}, + map[string]bool{"Host": true, "Bucket": true, "ForcePathStyle": true}, + ) + require.Nil(t, conditions) + require.Equal(t, "false", connectionData(written)["ForcePathStyle"], "explicit CR value must not be overridden by derivation") +} + func TestWriteState_GCSWorkloadIdentity(t *testing.T) { _, written, conditions, conn := writeStateFixtureProvider(t, apiv2.ObjectStoreProviderGCS, map[string]string{"Bucket": "my-gcs-bucket"}, @@ -195,6 +290,7 @@ func TestWriteState_GCSWorkloadIdentity(t *testing.T) { data := connectionData(written) require.Equal(t, "gs://my-gcs-bucket", data["url"], "workload identity carries no credentials") require.Equal(t, "gcs", data["Provider"]) + require.NotContains(t, data, "ForcePathStyle", "path-style is an S3-only concept") } func TestWriteState_GCSWithPrefixAndKey(t *testing.T) { diff --git a/internal/controller/infra/external/redis/redis.go b/internal/controller/infra/external/redis/redis.go index 9056916f..be9193e1 100644 --- a/internal/controller/infra/external/redis/redis.go +++ b/internal/controller/infra/external/redis/redis.go @@ -4,8 +4,11 @@ import ( "context" "fmt" "net/url" + "strconv" + "strings" apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/common" "github.com/wandb/operator/internal/controller/infra/external" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,13 +19,22 @@ import ( ) const ConnectionSecretName = "wandb-redis-connection" +const caCertPath = "/etc/ssl/certs/redis_ca.pem" + +func connectionSecretName(key string) string { + if key == "" || key == apiv2.DefaultInstanceName { + return ConnectionSecretName + } + return fmt.Sprintf("%s-%s", ConnectionSecretName, key) +} func WriteState( ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, + key string, + spec *apiv2.RedisConnection, ) []metav1.Condition { - spec := wandb.Spec.Redis.ExternalRedis logger := ctrl.LoggerFrom(ctx) fields := map[string]corev1.SecretKeySelector{ @@ -37,9 +49,20 @@ func WriteState( if err != nil { logger.Error(err, "failed to resolve external redis fields") return []metav1.Condition{{ - Type: "Reconciled", - Status: metav1.ConditionFalse, - Reason: "ApiError", + Type: common.ReconciledType, + Status: metav1.ConditionFalse, + Reason: common.ApiErrorReason, + Message: err.Error(), + }} + } + + if err := validateConnectionData(data); err != nil { + logger.Error(err, "invalid external redis connection") + return []metav1.Condition{{ + Type: common.ReconciledType, + Status: metav1.ConditionFalse, + Reason: common.ResourceErrorReason, + Message: err.Error(), }} } @@ -57,20 +80,46 @@ func WriteState( values.Add("tls", data["Tls"]) redisUrl.RawQuery = values.Encode() } + if _, ok := data["SslCa"]; ok { + values := redisUrl.Query() + if values.Get("tls") == "" { + values.Set("tls", "true") + } + values.Set("caCertPath", caCertPath) + redisUrl.RawQuery = values.Encode() + } data["url"] = redisUrl.String() - nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: ConnectionSecretName} + nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: connectionSecretName(key)} return external.WriteConnectionSecret(ctx, c, wandb, nsName, data) } +func validateConnectionData(data map[string]string) error { + host := strings.TrimSpace(data["Host"]) + if host == "" { + return fmt.Errorf("external Redis host is empty") + } + + portValue := strings.TrimSpace(data["Port"]) + port, err := strconv.Atoi(portValue) + if err != nil || port < 1 || port > 65535 { + return fmt.Errorf("external Redis port %q must be an integer between 1 and 65535", portValue) + } + + data["Host"] = host + data["Port"] = strconv.Itoa(port) + return nil +} + func ReadState( ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, + key string, newConditions []metav1.Condition, ) ([]metav1.Condition, *apiv2.RedisConnection) { - nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: ConnectionSecretName} + nsName := types.NamespacedName{Namespace: wandb.Namespace, Name: connectionSecretName(key)} _, conditions, found := external.ReadConnectionSecret(ctx, c, nsName, newConditions) if !found { return conditions, nil @@ -87,9 +136,9 @@ func ReadState( } } -func DeleteConnectionSecret(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases) error { +func DeleteConnectionSecret(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string) error { return external.DeleteConnectionSecret(ctx, c, types.NamespacedName{ Namespace: wandb.Namespace, - Name: ConnectionSecretName, + Name: connectionSecretName(key), }) } diff --git a/internal/controller/infra/external/redis/redis_test.go b/internal/controller/infra/external/redis/redis_test.go new file mode 100644 index 00000000..8a694bc8 --- /dev/null +++ b/internal/controller/infra/external/redis/redis_test.go @@ -0,0 +1,125 @@ +package redis + +import ( + "context" + "net/url" + "testing" + + "github.com/stretchr/testify/require" + apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/common" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const redisSourceSecretName = "external-redis" + +func redisSel(key string) corev1.SecretKeySelector { + return corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: redisSourceSecretName}, + Key: key, + } +} + +func redisWriteStateFixture(t *testing.T, sourceData map[string][]byte) (ctrlclient.Client, *apiv2.WeightsAndBiases) { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, apiv2.AddToScheme(scheme)) + + source := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: redisSourceSecretName, Namespace: "default"}, + Data: sourceData, + } + wandb := &apiv2.WeightsAndBiases{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps.wandb.com/v2", Kind: "WeightsAndBiases"}, + ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "default"}, + Spec: apiv2.WeightsAndBiasesSpec{ + Redis: map[string]apiv2.RedisSpec{apiv2.DefaultInstanceName: { + ExternalRedis: &apiv2.RedisConnection{ + Host: redisSel("Host"), + Port: redisSel("Port"), + }, + }}, + }, + } + if _, ok := sourceData["Password"]; ok { + connection := wandb.Spec.Redis[apiv2.DefaultInstanceName] + connection.ExternalRedis.Password = redisSel("Password") + wandb.Spec.Redis[apiv2.DefaultInstanceName] = connection + } + if _, ok := sourceData["SslCa"]; ok { + connection := wandb.Spec.Redis[apiv2.DefaultInstanceName] + connection.ExternalRedis.SslCa = redisSel("SslCa") + wandb.Spec.Redis[apiv2.DefaultInstanceName] = connection + } + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(wandb, source).Build(), wandb +} + +func TestWriteStateAddsCACertPathAndTLSWhenCACertPresent(t *testing.T) { + client, wandb := redisWriteStateFixture(t, map[string][]byte{ + "Host": []byte("redis.example.com"), + "Port": []byte("6379"), + "Password": []byte("secret"), + "SslCa": []byte("---ca---"), + }) + + conditions := WriteState(context.Background(), client, wandb, apiv2.DefaultInstanceName, wandb.Spec.Redis[apiv2.DefaultInstanceName].ExternalRedis) + require.Nil(t, conditions) + + written := &corev1.Secret{} + require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: ConnectionSecretName, Namespace: "default"}, written)) + data := redisConnectionData(written) + parsed, err := url.Parse(data["url"]) + require.NoError(t, err) + require.Equal(t, "redis", parsed.Scheme) + require.Equal(t, "redis.example.com:6379", parsed.Host) + require.Equal(t, "true", parsed.Query().Get("tls")) + require.Equal(t, caCertPath, parsed.Query().Get("caCertPath")) +} + +func TestWriteStateRejectsInvalidRequiredFields(t *testing.T) { + tests := []struct { + name string + data map[string][]byte + }{ + {name: "empty host", data: map[string][]byte{"Host": {}, "Port": []byte("6379")}}, + {name: "empty port", data: map[string][]byte{"Host": []byte("redis.example.com"), "Port": {}}}, + {name: "non-numeric port", data: map[string][]byte{"Host": []byte("redis.example.com"), "Port": []byte("redis")}}, + {name: "zero port", data: map[string][]byte{"Host": []byte("redis.example.com"), "Port": []byte("0")}}, + {name: "port above range", data: map[string][]byte{"Host": []byte("redis.example.com"), "Port": []byte("65536")}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client, wandb := redisWriteStateFixture(t, test.data) + + conditions := WriteState(context.Background(), client, wandb, apiv2.DefaultInstanceName, wandb.Spec.Redis[apiv2.DefaultInstanceName].ExternalRedis) + + require.Len(t, conditions, 1) + require.Equal(t, common.ReconciledType, conditions[0].Type) + require.Equal(t, metav1.ConditionFalse, conditions[0].Status) + require.Equal(t, common.ResourceErrorReason, conditions[0].Reason) + require.NotEmpty(t, conditions[0].Message) + + err := client.Get(context.Background(), types.NamespacedName{Name: ConnectionSecretName, Namespace: "default"}, &corev1.Secret{}) + require.True(t, apierrors.IsNotFound(err)) + }) + } +} + +func redisConnectionData(secret *corev1.Secret) map[string]string { + out := map[string]string{} + for k, v := range secret.Data { + out[k] = string(v) + } + for k, v := range secret.StringData { + out[k] = v + } + return out +} diff --git a/internal/controller/infra/managed/clickhouse/altinity/conn.go b/internal/controller/infra/managed/clickhouse/altinity/conn.go index 75d01806..1a64641f 100644 --- a/internal/controller/infra/managed/clickhouse/altinity/conn.go +++ b/internal/controller/infra/managed/clickhouse/altinity/conn.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "net/url" + "strconv" apiv2 "github.com/wandb/operator/api/v2" "github.com/wandb/operator/internal/controller/common" @@ -16,14 +18,26 @@ import ( type clickhouseConnInfo struct { Host string - Port string + TCPPort string + HTTPPort string User string Password string Database string + Tls bool } func (c *clickhouseConnInfo) toURL() string { - return fmt.Sprintf("clickhouse://%s:%s@%s:%s/%s", c.User, c.Password, c.Host, c.Port, c.Database) + values := url.Values{ + "tls": []string{strconv.FormatBool(c.Tls)}, + } + clickhouseUrl := url.URL{ + Scheme: "clickhouse", + Host: fmt.Sprintf("%s:%s", c.Host, c.TCPPort), + User: url.UserPassword(c.User, c.Password), + Path: c.Database, + RawQuery: values.Encode(), + } + return clickhouseUrl.String() } func writeClickHouseConnInfo( @@ -78,7 +92,8 @@ func writeClickHouseConnInfo( StringData: map[string]string{ urlKey: connInfo.toURL(), "Host": connInfo.Host, - "Port": connInfo.Port, + "TCPPort": connInfo.TCPPort, + "HTTPPort": connInfo.HTTPPort, "User": connInfo.User, "Password": connInfo.Password, "Database": connInfo.Database, @@ -93,7 +108,8 @@ func writeClickHouseConnInfo( return &apiv2.ClickHouseConnection{ URL: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: urlKey, Optional: ptr.To(false)}, Host: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Host", Optional: ptr.To(false)}, - Port: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Port", Optional: ptr.To(false)}, + HTTPPort: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "HTTPPort", Optional: ptr.To(false)}, + TCPPort: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "TCPPort", Optional: ptr.To(false)}, Username: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "User", Optional: ptr.To(false)}, Password: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Password", Optional: ptr.To(false)}, Database: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Database", Optional: ptr.To(false)}, diff --git a/internal/controller/infra/managed/clickhouse/altinity/keeper/keeper_suite_test.go b/internal/controller/infra/managed/clickhouse/altinity/keeper/keeper_suite_test.go new file mode 100644 index 00000000..de4aa549 --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/keeper/keeper_suite_test.go @@ -0,0 +1,13 @@ +package keeper + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestKeeper(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ClickHouse-Keeper Suite") +} diff --git a/internal/controller/infra/managed/clickhouse/altinity/keeper/naming.go b/internal/controller/infra/managed/clickhouse/altinity/keeper/naming.go new file mode 100644 index 00000000..db983209 --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/keeper/naming.go @@ -0,0 +1,35 @@ +package keeper + +import ( + "fmt" +) + +// chkNameSuffix is terse on purpose: it comes out of the DNS-1123 budget of +// every derived per-host name (see PerHostConfigVolumeName). +const chkNameSuffix = "-chk" + +// InstallationName derives the Keeper CR name from the base shared with the +// installation ("-chi" / "-chk"). Re-derived every reconcile: +// changing the scheme after managed ClickHouse ships needs a migration path. +func InstallationName(baseName string) string { + return baseName + chkNameSuffix +} + +// ClientServiceName is the Altinity-created client Service name ("keeper-"). +func ClientServiceName(baseName string) string { + return "keeper-" + InstallationName(baseName) +} + +// ClientServiceFQDN is the in-cluster DNS the CHI's config points at. +func ClientServiceFQDN(namespace, baseName string) string { + return fmt.Sprintf("%s.%s.svc.cluster.local", ClientServiceName(baseName), namespace) +} + +// PerHostConfigVolumeName mirrors the Altinity operator's per-host +// ConfigMap/StatefulSet-volume name (pkg/model/chk/namer/patterns.go). The +// longest derived name, and a DNS-1123 label: past 63 chars the apiserver +// rejects the StatefulSet and Altinity retries without surfacing the failure. +func PerHostConfigVolumeName(baseName string, shardOrdinal, replicaOrdinal int) string { + return fmt.Sprintf("chk-%s-deploy-confd-%s-%d-%d", + InstallationName(baseName), ClusterName, shardOrdinal, replicaOrdinal) +} diff --git a/internal/controller/infra/managed/clickhouse/altinity/keeper/read.go b/internal/controller/infra/managed/clickhouse/altinity/keeper/read.go new file mode 100644 index 00000000..ed24b015 --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/keeper/read.go @@ -0,0 +1,104 @@ +package keeper + +import ( + "context" + "fmt" + + "github.com/wandb/operator/internal/controller/common" + "github.com/wandb/operator/internal/logx" + chkv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ReadState reports Keeper pod readiness via KeeperReportedReadyType, which gates +// ClickHouse readiness. +func ReadState( + ctx context.Context, + cl client.Client, + keeperNsName types.NamespacedName, +) []metav1.Condition { + ctx, _ = logx.WithSlog(ctx, logx.ClickHouse) + var actual = &chkv1.ClickHouseKeeperInstallation{} + + found, err := common.GetResource(ctx, cl, keeperNsName, ResourceTypeName, actual) + if err != nil { + return []metav1.Condition{{ + Type: KeeperReportedReadyType, + Status: metav1.ConditionUnknown, + Reason: common.ApiErrorReason, + }} + } + if !found { + return []metav1.Condition{{ + Type: KeeperReportedReadyType, + Status: metav1.ConditionFalse, + Reason: common.NoResourceReason, + }} + } + + podsRunning, err := keeperPodsRunningStatus(ctx, cl, keeperNsName.Namespace, actual) + if err != nil { + return []metav1.Condition{{ + Type: KeeperReportedReadyType, + Status: metav1.ConditionUnknown, + Reason: common.ApiErrorReason, + }} + } + + return computeKeeperReadyCondition(ctx, podsRunning) +} + +func keeperPodsRunningStatus( + ctx context.Context, cl client.Client, namespace string, chk *chkv1.ClickHouseKeeperInstallation, +) (map[string]bool, error) { + result := make(map[string]bool) + if chk == nil || chk.Status == nil { + return result, nil + } + for _, podName := range chk.Status.Pods { + var pod = &corev1.Pod{} + nsName := types.NamespacedName{Namespace: namespace, Name: podName} + found, err := common.GetResource(ctx, cl, nsName, "KeeperPod", pod) + if err != nil { + return result, err + } + result[podName] = found && common.PodReady(pod) + } + return result, nil +} + +func computeKeeperReadyCondition(ctx context.Context, podsRunning map[string]bool) []metav1.Condition { + log := logx.GetSlog(ctx) + + var runningCount, podCount int + for _, isRunning := range podsRunning { + podCount++ + if isRunning { + runningCount++ + } + } + log.Info("Keeper pods status", "running", runningCount, "total", podCount) + + status := metav1.ConditionUnknown + reason := common.UnknownReason + message := "" + switch { + case podCount > 0 && podCount == runningCount: + status = metav1.ConditionTrue + reason = common.ResourceExistsReason + case podCount > 0: + status = metav1.ConditionFalse + reason = common.NoResourceReason + message = fmt.Sprintf("%d of %d keeper pods running", runningCount, podCount) + } + + return []metav1.Condition{{ + Type: KeeperReportedReadyType, + Status: status, + Reason: reason, + Message: message, + }} +} diff --git a/internal/controller/infra/managed/clickhouse/altinity/keeper/read_test.go b/internal/controller/infra/managed/clickhouse/altinity/keeper/read_test.go new file mode 100644 index 00000000..5db34fb3 --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/keeper/read_test.go @@ -0,0 +1,29 @@ +package keeper + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("Keeper readiness", func() { + It("is ready when all pods are running", func() { + conds := computeKeeperReadyCondition(context.Background(), map[string]bool{"a": true, "b": true, "c": true}) + Expect(conds).To(HaveLen(1)) + Expect(conds[0].Type).To(Equal(KeeperReportedReadyType)) + Expect(conds[0].Status).To(Equal(metav1.ConditionTrue)) + }) + + It("is not ready when some pods are not running", func() { + conds := computeKeeperReadyCondition(context.Background(), map[string]bool{"a": true, "b": false, "c": true}) + Expect(conds[0].Status).To(Equal(metav1.ConditionFalse)) + Expect(conds[0].Message).To(ContainSubstring("2 of 3")) + }) + + It("is unknown when no pods are reported yet", func() { + conds := computeKeeperReadyCondition(context.Background(), map[string]bool{}) + Expect(conds[0].Status).To(Equal(metav1.ConditionUnknown)) + }) +}) diff --git a/internal/controller/infra/managed/clickhouse/altinity/keeper/spec.go b/internal/controller/infra/managed/clickhouse/altinity/keeper/spec.go new file mode 100644 index 00000000..4eca4bfe --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/keeper/spec.go @@ -0,0 +1,167 @@ +package keeper + +import ( + "context" + "fmt" + + apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/common" + "github.com/wandb/operator/internal/logx" + "github.com/wandb/operator/pkg/utils" + chkv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1" + chiv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + "github.com/wandb/operator/pkg/wandb/manifest" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" +) + +func KeeperImage(img manifest.ImageRef, globalImageRegistry string) string { + if out := img.GetImage(globalImageRegistry); out != "" { + return out + } + // Fallback for older manifests that don't supply the image. + return defaultKeeperImage +} + +// ToKeeperVendorSpec builds the ClickHouseKeeperInstallation CR that coordinates +// ReplicatedMergeTree replication. nsName comes from altinity.KeeperNsName — +// this package never sees the "-chi"-suffixed spec name. +func ToKeeperVendorSpec( + ctx context.Context, + wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedClickHouseSpec, + scheme *runtime.Scheme, + nsName types.NamespacedName, + mfst manifest.Manifest, +) (*chkv1.ClickHouseKeeperInstallation, error) { + _, log := logx.WithSlog(ctx, logx.ClickHouse) + if spec == nil { + return nil, nil + } + + // Keeper sizing comes from the server manifest (clickhouseKeeper) or CR; there + // are no operator-side defaults, so fail loudly if the storage size is missing. + storageQuantity, err := resource.ParseQuantity(spec.Keeper.StorageSize) + if err != nil { + return nil, fmt.Errorf("invalid keeper storageSize %q (expected from the server manifest's clickhouseKeeper sizing): %w", spec.Keeper.StorageSize, err) + } + + labels := common.BuildWandbLabels(wandb, KeeperModuleName) + + podSpec := corev1.PodSpec{ + SecurityContext: keeperPodSecurityContext(), + Affinity: wandb.GetAffinity(spec.ManagedInfraSpec), + Tolerations: *wandb.GetTolerations(spec.ManagedInfraSpec), + Containers: []corev1.Container{ + { + Name: keeperContainerName, + Image: KeeperImage(mfst.ClickhouseKeeper["default"].Images["keeper"], wandb.Spec.Global.ImageRegistry), + SecurityContext: keeperContainerSecurityContext(), + }, + }, + } + if len(spec.Keeper.Config.Resources.Requests) > 0 || len(spec.Keeper.Config.Resources.Limits) > 0 { + podSpec.Containers[0].Resources = corev1.ResourceRequirements{ + Requests: spec.Keeper.Config.Resources.Requests, + Limits: spec.Keeper.Config.Resources.Limits, + } + } + + chk := &chkv1.ClickHouseKeeperInstallation{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName.Name, + Namespace: nsName.Namespace, + Labels: labels, + }, + Spec: chkv1.ChkSpec{ + Configuration: &chkv1.Configuration{ + Clusters: []*chkv1.Cluster{ + { + Name: ClusterName, + Layout: &chkv1.ChkClusterLayout{ + ReplicasCount: int(spec.Keeper.Replicas), + }, + }, + }, + }, + Defaults: &chiv1.Defaults{ + Templates: &chiv1.TemplatesList{ + PodTemplate: podTemplateName, + DataVolumeClaimTemplate: volumeTemplateName, + }, + }, + Templates: &chiv1.Templates{ + PodTemplates: []chiv1.PodTemplate{ + { + Name: podTemplateName, + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: podSpec, + }, + }, + VolumeClaimTemplates: []chiv1.VolumeClaimTemplate{ + { + Name: volumeTemplateName, + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: storageQuantity}, + }, + }, + }, + }, + }, + }, + } + + if err := ctrl.SetControllerReference(wandb, chk, scheme); err != nil { + log.Error("failed to set owner reference on CHK CR", logx.ErrAttr(err)) + return nil, fmt.Errorf("failed to set owner reference: %w", err) + } + + return chk, nil +} + +// BuildWandbKeeperLabels returns the standard W&B labels for Keeper resources. +func BuildWandbKeeperLabels(wandb *apiv2.WeightsAndBiases) map[string]string { + return common.BuildWandbLabels(wandb, KeeperModuleName) +} + +func keeperPodSecurityContext() *corev1.PodSecurityContext { + if utils.IsOpenShift() { + return &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + SeccompProfile: runtimeDefaultSeccompProfile(), + } + } + return &corev1.PodSecurityContext{ + RunAsUser: ptr.To(keeperRunAsUser), + RunAsGroup: ptr.To(keeperRunAsGroup), + RunAsNonRoot: ptr.To(true), + FSGroup: ptr.To(keeperFSGroup), + SeccompProfile: runtimeDefaultSeccompProfile(), + } +} + +func keeperContainerSecurityContext() *corev1.SecurityContext { + sc := &corev1.SecurityContext{ + RunAsNonRoot: ptr.To(true), + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + SeccompProfile: runtimeDefaultSeccompProfile(), + } + if !utils.IsOpenShift() { + sc.RunAsUser = ptr.To(keeperRunAsUser) + sc.RunAsGroup = ptr.To(keeperRunAsGroup) + } + return sc +} + +func runtimeDefaultSeccompProfile() *corev1.SeccompProfile { + return &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault} +} diff --git a/internal/controller/infra/managed/clickhouse/altinity/keeper/spec_test.go b/internal/controller/infra/managed/clickhouse/altinity/keeper/spec_test.go new file mode 100644 index 00000000..1376cdc7 --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/keeper/spec_test.go @@ -0,0 +1,139 @@ +package keeper + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/pkg/utils" + chkv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1" + chiv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + "github.com/wandb/operator/pkg/wandb/manifest" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("Keeper vendor spec", func() { + BeforeEach(func() { + utils.SetOpenShiftMode(false) + }) + + It("builds a CHK with explicit replicas, storage, and a hardened pod", func() { + wandb := keeperWandb() + wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.Keeper = apiv2.ClickHouseKeeperSpec{ + Replicas: 5, + StorageSize: "20Gi", + Config: apiv2.ClickHouseConfig{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("250m")}, + }, + }, + } + + chk, err := ToKeeperVendorSpec(context.Background(), wandb, wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse, keeperScheme(), keeperNsName(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(chk).NotTo(BeNil()) + Expect(chk.Name).To(Equal("clickhouse-chk")) + Expect(chk.Namespace).To(Equal("wandb")) + + Expect(chk.Spec.Configuration.Clusters).To(HaveLen(1)) + Expect(chk.Spec.Configuration.Clusters[0].Layout.ReplicasCount).To(Equal(5)) + + Expect(chk.Spec.Templates.VolumeClaimTemplates).To(HaveLen(1)) + storage := chk.Spec.Templates.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] + Expect(storage).To(Equal(resource.MustParse("20Gi"))) + + Expect(chk.Spec.Templates.PodTemplates).To(HaveLen(1)) + container := chk.Spec.Templates.PodTemplates[0].Spec.Containers[0] + Expect(container.Image).To(Equal(KeeperImage(manifest.ImageRef{}, ""))) + Expect(container.Name).To(Equal(keeperContainerName)) + Expect(container.Resources.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("250m"))) + + sc := chk.Spec.Templates.PodTemplates[0].Spec.SecurityContext + Expect(sc).NotTo(BeNil()) + Expect(sc.RunAsUser).NotTo(BeNil()) + Expect(*sc.RunAsUser).To(Equal(keeperRunAsUser)) + Expect(sc.RunAsNonRoot).NotTo(BeNil()) + Expect(*sc.RunAsNonRoot).To(BeTrue()) + }) + + It("uses the Keeper image from the server manifest", func() { + wandb := keeperWandb() + wandb.Spec.Global.ImageRegistry = "myregistry.io" + mfst := manifest.Manifest{ + ClickhouseKeeper: map[string]manifest.InfraConfig{ + "default": { + Images: map[string]manifest.ImageRef{ + "keeper": {Registry: "docker.io", Repository: "altinity/clickhouse-keeper", Tag: "25.8"}, + }, + }, + }, + } + + chk, err := ToKeeperVendorSpec(context.Background(), wandb, wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse, keeperScheme(), keeperNsName(), mfst) + Expect(err).NotTo(HaveOccurred()) + Expect(chk.Spec.Templates.PodTemplates[0].Spec.Containers[0].Image). + To(Equal("myregistry.io/docker.io/altinity/clickhouse-keeper:25.8")) + }) + + It("errors when keeper storage size is unset (no operator defaults)", func() { + wandb := keeperWandb() + wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.Keeper = apiv2.ClickHouseKeeperSpec{} + _, err := ToKeeperVendorSpec(context.Background(), wandb, wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse, keeperScheme(), keeperNsName(), manifest.Manifest{}) + Expect(err).To(HaveOccurred()) + }) + + It("omits fixed IDs in OpenShift mode", func() { + utils.SetOpenShiftMode(true) + wandb := keeperWandb() + chk, err := ToKeeperVendorSpec(context.Background(), wandb, wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse, keeperScheme(), keeperNsName(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + sc := chk.Spec.Templates.PodTemplates[0].Spec.SecurityContext + Expect(sc.RunAsUser).To(BeNil()) + Expect(sc.RunAsNonRoot).NotTo(BeNil()) + Expect(*sc.RunAsNonRoot).To(BeTrue()) + }) +}) + +// keeperNsName mirrors what altinity.KeeperNsName derives for keeperWandb(). +func keeperNsName() types.NamespacedName { + return types.NamespacedName{Namespace: "wandb", Name: InstallationName("clickhouse")} +} + +func keeperScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + Expect(apiv2.AddToScheme(scheme)).To(Succeed()) + Expect(chiv1.AddToScheme(scheme)).To(Succeed()) + Expect(chkv1.AddToScheme(scheme)).To(Succeed()) + return scheme +} + +func keeperWandb() *apiv2.WeightsAndBiases { + tolerations := []corev1.Toleration{} + return &apiv2.WeightsAndBiases{ + TypeMeta: metav1.TypeMeta{ + APIVersion: apiv2.GroupVersion.String(), + Kind: "WeightsAndBiases", + }, + ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "wandb"}, + Spec: apiv2.WeightsAndBiasesSpec{ + Tolerations: &tolerations, + ClickHouse: map[string]apiv2.ClickHouseSpec{ + apiv2.DefaultInstanceName: { + ManagedClickHouse: &apiv2.ManagedClickHouseSpec{ + Name: "clickhouse", + Namespace: "wandb", + Keeper: apiv2.ClickHouseKeeperSpec{ + Replicas: 3, + StorageSize: "10Gi", + }, + }, + }, + }, + }, + } +} diff --git a/internal/controller/infra/managed/clickhouse/altinity/keeper/values.go b/internal/controller/infra/managed/clickhouse/altinity/keeper/values.go new file mode 100644 index 00000000..df41cb37 --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/keeper/values.go @@ -0,0 +1,31 @@ +package keeper + +const ( + // KeeperModuleName is the W&B component label value for Keeper resources. + KeeperModuleName = "clickhouse-keeper" + + // TODO: remove this hardcoded default once all supported manifest versions + // supply clickhouseKeeper..images.keeper. Pinned to the managed + // ClickHouse server version. + defaultKeeperImage = "altinity/clickhouse-keeper:25.8.16.10002.altinitystable" + + // KeeperClientPort is the ZooKeeper-compatible client port. + KeeperClientPort = 2181 + + // ClusterName is the name of the single Keeper cluster. + ClusterName = "default" + + // KeeperCustomResourceType is the condition type reported for the CHK CR. + KeeperCustomResourceType = "KeeperCustomResource" + + // KeeperReportedReadyType reports Keeper pod readiness; it gates ClickHouse readiness. + KeeperReportedReadyType = "KeeperReportedReady" + + podTemplateName = "keeper-pod-template" + volumeTemplateName = "keeper-data-volume" + keeperContainerName = "clickhouse-keeper" + + keeperRunAsUser int64 = 101 + keeperRunAsGroup int64 = 101 + keeperFSGroup int64 = 101 +) diff --git a/internal/controller/infra/managed/clickhouse/altinity/keeper/write.go b/internal/controller/infra/managed/clickhouse/altinity/keeper/write.go new file mode 100644 index 00000000..2185b498 --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/keeper/write.go @@ -0,0 +1,71 @@ +package keeper + +import ( + "context" + + "github.com/wandb/operator/internal/controller/common" + "github.com/wandb/operator/internal/logx" + chkv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// ResourceTypeName is the kind used for logging/error reporting of the CHK CR. +const ResourceTypeName = "ClickHouseKeeperInstallation" + +// WriteState create-or-updates the CHK, setting only the fields we own (spec, +// labels, owner refs) and preserving the Altinity-managed finalizer/status. It +// compares owned fields via JSON, never the vendored status — whose uint64 and +// unexported fields panic controllerutil's reflective diff/copy. +func WriteState( + ctx context.Context, + cl client.Client, + keeperNsName types.NamespacedName, + desired *chkv1.ClickHouseKeeperInstallation, +) []metav1.Condition { + ctx, _ = logx.WithSlog(ctx, logx.ClickHouse) + + obj := &chkv1.ClickHouseKeeperInstallation{ + ObjectMeta: metav1.ObjectMeta{Name: keeperNsName.Name, Namespace: keeperNsName.Namespace}, + } + + op, err := common.WriteOwnedFields(ctx, cl, obj, + func(o *chkv1.ClickHouseKeeperInstallation) { applyOwnedKeeper(o, desired) }, + keeperOwnedEqual, + ) + if err != nil { + return []metav1.Condition{ + {Type: KeeperCustomResourceType, Status: metav1.ConditionUnknown, Reason: common.ApiErrorReason}, + } + } + + if op == controllerutil.OperationResultCreated { + return []metav1.Condition{ + {Type: KeeperCustomResourceType, Status: metav1.ConditionFalse, Reason: common.PendingCreateReason}, + } + } + return []metav1.Condition{ + {Type: KeeperCustomResourceType, Status: metav1.ConditionTrue, Reason: common.ResourceExistsReason}, + } +} + +func applyOwnedKeeper(obj, desired *chkv1.ClickHouseKeeperInstallation) { + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + } + for k, v := range desired.GetLabels() { + labels[k] = v + } + obj.SetLabels(labels) + obj.SetOwnerReferences(desired.GetOwnerReferences()) + obj.Spec = desired.Spec +} + +func keeperOwnedEqual(a, b *chkv1.ClickHouseKeeperInstallation) bool { + return common.JSONEqual(a.Spec, b.Spec) && + common.JSONEqual(a.Labels, b.Labels) && + common.JSONEqual(a.OwnerReferences, b.OwnerReferences) +} diff --git a/internal/controller/infra/managed/clickhouse/altinity/naming.go b/internal/controller/infra/managed/clickhouse/altinity/naming.go index 3158f915..9ffe32e5 100644 --- a/internal/controller/infra/managed/clickhouse/altinity/naming.go +++ b/internal/controller/infra/managed/clickhouse/altinity/naming.go @@ -2,8 +2,13 @@ package altinity import ( "fmt" + "strings" + apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/common" + "github.com/wandb/operator/internal/controller/infra/managed/clickhouse/altinity/keeper" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" ) type NsNameBuilder struct { @@ -70,3 +75,79 @@ func (n *NsNameBuilder) ConnectionNsName() types.NamespacedName { func createNsNameBuilder(baseNsName types.NamespacedName) *NsNameBuilder { return CreateNsNameBuilder(baseNsName) } + +const ( + // chiClusterName is the single cluster the CHI defines. + chiClusterName = "default" + + // defaultNameSuffix is appended to the CR name by the defaulting webhook; + // terse to leave the CR name as much of the derived-name budget as possible. + defaultNameSuffix = "-chi" + + // maxExpectedHostOrdinal reserves two digits each for shard and replica + // ordinals (largest expected cluster ~100 pods), so a persisted name can + // never overflow when the cluster is scaled up later. + maxExpectedHostOrdinal = 99 +) + +// baseName strips the "-chi" suffix; the Keeper builds its names on the base. +func baseName(specName string) string { + return strings.TrimSuffix(specName, defaultNameSuffix) +} + +// KeeperNsName is the Keeper CR's namespaced name for a managed CH spec. +func KeeperNsName(spec *apiv2.ManagedClickHouseSpec) types.NamespacedName { + return types.NamespacedName{ + Namespace: spec.Namespace, + Name: keeper.InstallationName(baseName(spec.Name)), + } +} + +// perHostConfigVolumeName mirrors the Altinity operator's per-host +// ConfigMap/StatefulSet-volume name for a CHI (pkg/model/chi/namer/patterns.go); +// the longest CHI-derived name and a DNS-1123 label. +func perHostConfigVolumeName(specName string, shardOrdinal, replicaOrdinal int) string { + return fmt.Sprintf("chi-%s-deploy-confd-%s-%d-%d", specName, chiClusterName, shardOrdinal, replicaOrdinal) +} + +// MaxSpecNameLength is the longest defaulted ("-chi") spec name whose +// derived names all fit DNS-1123 labels; Keeper names build on the base, so +// their room extends by the suffix the base gives back. +func MaxSpecNameLength() int { + chiRoom := validation.DNS1123LabelMaxLength - + len(perHostConfigVolumeName("", maxExpectedHostOrdinal, maxExpectedHostOrdinal)) + chkRoom := validation.DNS1123LabelMaxLength - + len(keeper.PerHostConfigVolumeName("", maxExpectedHostOrdinal, maxExpectedHostOrdinal)) + len(defaultNameSuffix) + return min(chiRoom, chkRoom) +} + +// DefaultSpecName derives the managed ClickHouse name for a CR instance, +// shortening it when the plain form would overflow the derived-name budget. +func DefaultSpecName(crName, instanceKey string) string { + return common.FitDefaultInfraName(common.InstanceBaseName(crName, instanceKey), defaultNameSuffix, MaxSpecNameLength()) +} + +// ValidateDerivedNames reports why a spec name cannot be deployed: derived +// per-host volume names must fit DNS-1123 labels, and the Altinity operator +// wedges silently when they don't. Nil when every derived name fits. +func ValidateDerivedNames(spec *apiv2.ManagedClickHouseSpec) error { + for _, derived := range []string{ + keeper.PerHostConfigVolumeName(baseName(spec.Name), maxExpectedHostOrdinal, maxExpectedHostOrdinal), + perHostConfigVolumeName(spec.Name, maxExpectedHostOrdinal, maxExpectedHostOrdinal), + } { + // derived length grows 1:1 with the name, so excess → max usable length + if over := len(derived) - validation.DNS1123LabelMaxLength; over > 0 { + return fmt.Errorf( + "managed ClickHouse name %q cannot be deployed: the Altinity operator derives object name %q from it, which exceeds the %d-character DNS-1123 label limit; use at most %d characters, e.g. by shortening the CR name or setting spec.clickhouse.managedClickhouse.name", + spec.Name, derived, validation.DNS1123LabelMaxLength, len(spec.Name)-over, + ) + } + if errs := validation.IsDNS1123Label(derived); len(errs) > 0 { + return fmt.Errorf( + "managed ClickHouse name %q cannot be deployed: the Altinity operator derives object name %q from it, which is not a valid DNS-1123 label (%s)", + spec.Name, derived, strings.Join(errs, "; "), + ) + } + } + return nil +} diff --git a/internal/controller/infra/managed/clickhouse/altinity/naming_test.go b/internal/controller/infra/managed/clickhouse/altinity/naming_test.go new file mode 100644 index 00000000..2f8a02f2 --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/naming_test.go @@ -0,0 +1,87 @@ +package altinity + +import ( + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apiv2 "github.com/wandb/operator/api/v2" + "k8s.io/apimachinery/pkg/util/validation" +) + +var _ = Describe("managed ClickHouse naming", func() { + Describe("KeeperNsName", func() { + It("pairs the Keeper with the installation via the shared base name", func() { + spec := &apiv2.ManagedClickHouseSpec{Name: "wandb-legacy-overrides-v1-chi", Namespace: "wandb"} + Expect(KeeperNsName(spec).Name).To(Equal("wandb-legacy-overrides-v1-chk")) + }) + + It("appends -chk to explicit names without the default suffix", func() { + spec := &apiv2.ManagedClickHouseSpec{Name: "myclickhouse", Namespace: "wandb"} + Expect(KeeperNsName(spec).Name).To(Equal("myclickhouse-chk")) + }) + }) + + Describe("MaxSpecNameLength", func() { + It("costs the Keeper chain no more than the CHI chain, thanks to the suffix swap", func() { + chiRoom := validation.DNS1123LabelMaxLength - + len(perHostConfigVolumeName("", maxExpectedHostOrdinal, maxExpectedHostOrdinal)) + Expect(MaxSpecNameLength()).To(Equal(chiRoom)) + }) + }) + + Describe("DefaultSpecName", func() { + It("keeps the plain '-chi' for CR names that fit", func() { + Expect(DefaultSpecName("wandb", apiv2.DefaultInstanceName)).To(Equal("wandb-chi")) + // 25 chars — wedged the old "-clickhouse"/"-keeper" naming + Expect(DefaultSpecName("wandb-legacy-overrides-v1", apiv2.DefaultInstanceName)).To(Equal("wandb-legacy-overrides-v1-chi")) + }) + + It("keys non-default instances before the suffix so derivations still work", func() { + name := DefaultSpecName("wandb", "analytics") + + Expect(name).To(Equal("wandb-analytics-chi")) + spec := &apiv2.ManagedClickHouseSpec{Name: name} + Expect(KeeperNsName(spec).Name).To(Equal("wandb-analytics-chk")) + Expect(ValidateDerivedNames(spec)).To(Succeed()) + }) + + It("derives a deployable name for CR names the plain default would wedge", func() { + // 32 chars: "-chi" would overflow the per-host volume names + name := DefaultSpecName("wandb-integration-environments-2", apiv2.DefaultInstanceName) + + Expect(name).To(HaveSuffix("-chi")) + Expect(ValidateDerivedNames(&apiv2.ManagedClickHouseSpec{Name: name})).To(Succeed()) + }) + }) + + Describe("ValidateDerivedNames", func() { + It("accepts a defaulted name", func() { + Expect(ValidateDerivedNames(&apiv2.ManagedClickHouseSpec{Name: "wandb-chi"})).To(Succeed()) + }) + + It("rejects a name whose Keeper volume name exceeds the DNS-1123 label limit", func() { + err := ValidateDerivedNames(&apiv2.ManagedClickHouseSpec{Name: "wandb-legacy-overrides-v1-clickhouse"}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("chk-wandb-legacy-overrides-v1-clickhouse-chk-deploy-confd")) + Expect(err.Error()).To(ContainSubstring("spec.clickhouse.managedClickhouse.name")) + }) + + It("reserves ordinal digits so a name at the budget survives scaling to 100 pods", func() { + atBudget := &apiv2.ManagedClickHouseSpec{ + Name: strings.Repeat("a", MaxSpecNameLength()-len(defaultNameSuffix)) + defaultNameSuffix, + Replicas: 99, + } + atBudget.Keeper.Replicas = 99 + Expect(ValidateDerivedNames(atBudget)).To(Succeed()) + + atBudget.Name = "a" + atBudget.Name + Expect(ValidateDerivedNames(atBudget)).To(HaveOccurred()) + }) + + It("rejects characters that are invalid in derived label names", func() { + Expect(ValidateDerivedNames(&apiv2.ManagedClickHouseSpec{Name: "wandb.prod"})).To(HaveOccurred()) + }) + }) +}) diff --git a/internal/controller/infra/managed/clickhouse/altinity/objectstorage.go b/internal/controller/infra/managed/clickhouse/altinity/objectstorage.go new file mode 100644 index 00000000..ba5dea7c --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/objectstorage.go @@ -0,0 +1,139 @@ +package altinity + +import ( + "context" + "fmt" + "strconv" + "strings" + + apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/infra/objectstore" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + chtypes "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/common/types" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // StoragePolicyName is the object-store-backed policy, set server-wide so all + // MergeTree tables land in the bucket without per-table DDL. + StoragePolicyName = "s3_main" + + // DefaultObjectStoragePrefix is the in-bucket prefix when unset (trailing slash matters). + DefaultObjectStoragePrefix = "clickhouse/" + + s3DiskName = "s3_disk" + // Must sort after s3DiskName ('/' < '_'): the renderer emits in sorted + // order and ClickHouse requires the wrapped disk before its cache disk. + s3CacheDiskName = "s3_disk_cache" + s3MetadataPath = "/var/lib/clickhouse/disks/s3_disk/" + s3CachePath = "/var/lib/clickhouse/disks/s3_disk_cache/" + + // storageConfigKey renders to the config section. + storageConfigKey = "storage_configuration" +) + +// ResolveObjectStorage resolves the connection and builds the S3 endpoint. +func ResolveObjectStorage( + ctx context.Context, + cl client.Client, + spec *apiv2.ManagedClickHouseSpec, + conn *apiv2.ObjectStoreConnection, +) (*objectstore.ConnInfo, string, error) { + if spec == nil { + return nil, "", nil + } + + ci, err := objectstore.Resolve(ctx, cl, spec.Namespace, conn) + if err != nil { + return nil, "", err + } + if ci.Bucket == "" { + return nil, "", fmt.Errorf("object store connection has no bucket reference") + } + + endpoint, err := buildEndpoint(ci, objectStoragePrefix(spec)) + if err != nil { + return nil, "", err + } + + return &ci, endpoint, nil +} + +// objectStoragePrefix returns the normalized in-bucket prefix for the spec. +func objectStoragePrefix(spec *apiv2.ManagedClickHouseSpec) string { + return normalizePrefix(spec.ObjectStorage.Prefix) +} + +// normalizePrefix strips leading slashes and ensures one trailing slash, defaulting when empty. +func normalizePrefix(prefix string) string { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + return DefaultObjectStoragePrefix + } + prefix = strings.Trim(prefix, "/") + return prefix + "/" +} + +// buildEndpoint builds the S3 disk endpoint: path-style for a custom endpoint, +// else the AWS virtual-hosted URL derived from the region. +func buildEndpoint(ci objectstore.ConnInfo, prefix string) (string, error) { + if base := ci.EndpointURL(); base != "" { + return fmt.Sprintf("%s/%s/%s", base, ci.Bucket, prefix), nil + } + + if ci.Region == "" { + return "", fmt.Errorf("object store has no Host and no Region; cannot derive an S3 endpoint") + } + return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", ci.Bucket, ci.Region, prefix), nil +} + +// applyStorageConfiguration sets the S3 disk, cache, and storage policy. +// TODO(dpanzella): only S3 supported; add Azure and GCS. +func applyStorageConfiguration(settings *v1.Settings, ci *objectstore.ConnInfo, endpoint string, cacheMaxSizeBytes int64) { + disk := diskKey(s3DiskName) + settings.Set(disk("type"), v1.NewSettingScalar("s3")) + settings.Set(disk("endpoint"), v1.NewSettingScalar(endpoint)) + settings.Set(disk("metadata_path"), v1.NewSettingScalar(s3MetadataPath)) + if ci.Region != "" { + settings.Set(disk("region"), v1.NewSettingScalar(ci.Region)) + } + if ci.AccessKey == "" { + settings.Set(disk("use_environment_credentials"), v1.NewSettingScalar("true")) + } else { + settings.Set(disk("access_key_id"), secretSetting(ci.AccessKeyRef)) + settings.Set(disk("secret_access_key"), secretSetting(ci.SecretKeyRef)) + } + + cache := diskKey(s3CacheDiskName) + settings.Set(cache("type"), v1.NewSettingScalar("cache")) + settings.Set(cache("disk"), v1.NewSettingScalar(s3DiskName)) + settings.Set(cache("path"), v1.NewSettingScalar(s3CachePath)) + settings.Set(cache("max_size"), v1.NewSettingScalar(strconv.FormatInt(cacheMaxSizeBytes, 10))) + + settings.Set( + storageConfigKey+"/policies/"+StoragePolicyName+"/volumes/main/disk", + v1.NewSettingScalar(s3CacheDiskName), + ) + + // Server-wide default so W&B tables use the bucket without per-table DDL. + // system_*_log tables ship a predefined (which can't take a separate + // storage_policy), so they inherit this and live in the bucket too. + settings.Set("merge_tree/storage_policy", v1.NewSettingScalar(StoragePolicyName)) +} + +// diskKey builds settings paths for a named disk, e.g. +// diskKey("s3_disk")("type") -> "storage_configuration/disks/s3_disk/type". +func diskKey(name string) func(string) string { + prefix := storageConfigKey + "/disks/" + name + "/" + return func(field string) string { return prefix + field } +} + +// secretSetting builds a setting sourced from a Kubernetes secret; the Altinity +// operator wires it as a pod env var + from_env. +func secretSetting(ref corev1.SecretKeySelector) *v1.Setting { + r := ref + return v1.NewSettingSource(&v1.SettingSource{ + ValueFrom: &chtypes.DataSource{SecretKeyRef: &r}, + }) +} diff --git a/internal/controller/infra/managed/clickhouse/altinity/objectstorage_test.go b/internal/controller/infra/managed/clickhouse/altinity/objectstorage_test.go new file mode 100644 index 00000000..4b1c468c --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/objectstorage_test.go @@ -0,0 +1,119 @@ +package altinity + +import ( + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/wandb/operator/internal/controller/infra/objectstore" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + corev1 "k8s.io/api/core/v1" +) + +var _ = Describe("Object storage endpoint", func() { + It("uses the scheme reported by the connection for a custom host", func() { + ep, err := buildEndpoint(objectstore.ConnInfo{ + Endpoint: "http://seaweedfs.wandb.svc.cluster.local", Port: "80", Bucket: "bucket", Region: "us-east-1", + }, "clickhouse/") + Expect(err).NotTo(HaveOccurred()) + Expect(ep).To(Equal("http://seaweedfs.wandb.svc.cluster.local:80/bucket/clickhouse/")) + }) + + It("defaults to https for an external host with tls enabled", func() { + ep, err := buildEndpoint(objectstore.ConnInfo{ + Endpoint: "minio.example.com", Port: "9000", Bucket: "data", TlsEnabled: true, + }, "clickhouse/") + Expect(err).NotTo(HaveOccurred()) + Expect(ep).To(Equal("https://minio.example.com:9000/data/clickhouse/")) + }) + + It("uses http for an external host when tls is disabled", func() { + ep, err := buildEndpoint(objectstore.ConnInfo{ + Endpoint: "minio.example.com", Port: "9000", Bucket: "data", + }, "clickhouse/") + Expect(err).NotTo(HaveOccurred()) + Expect(ep).To(Equal("http://minio.example.com:9000/data/clickhouse/")) + }) + + It("derives an AWS virtual-hosted endpoint when no host is set", func() { + ep, err := buildEndpoint(objectstore.ConnInfo{ + Bucket: "my-bucket", Region: "us-west-2", TlsEnabled: true, + }, "clickhouse/") + Expect(err).NotTo(HaveOccurred()) + Expect(ep).To(Equal("https://my-bucket.s3.us-west-2.amazonaws.com/clickhouse/")) + }) + + It("errors when neither host nor region is available", func() { + _, err := buildEndpoint(objectstore.ConnInfo{Bucket: "my-bucket"}, "clickhouse/") + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("Object storage prefix", func() { + It("defaults when empty", func() { + Expect(normalizePrefix("")).To(Equal(DefaultObjectStoragePrefix)) + }) + + It("normalizes leading and trailing slashes", func() { + Expect(normalizePrefix("/foo/bar/")).To(Equal("foo/bar/")) + Expect(normalizePrefix("foo")).To(Equal("foo/")) + }) +}) + +var _ = Describe("Storage configuration settings", func() { + It("defines an s3 disk, cache, policy, default routing, and local system logs", func() { + ref := corev1.LocalObjectReference{Name: "objstore-conn"} + ci := &objectstore.ConnInfo{ + Region: "us-east-1", + AccessKey: "AKIA", + SecretKey: "secret", + AccessKeyRef: corev1.SecretKeySelector{LocalObjectReference: ref, Key: "AccessKey"}, + SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: ref, Key: "SecretKey"}, + } + settings := v1.NewSettings() + applyStorageConfiguration(settings, ci, "http://host:80/bucket/clickhouse/", 8<<30) + + Expect(settings.Get("storage_configuration/disks/s3_disk/type").String()).To(Equal("s3")) + Expect(settings.Get("storage_configuration/disks/s3_disk/endpoint").String()).To(Equal("http://host:80/bucket/clickhouse/")) + Expect(settings.Get("storage_configuration/disks/s3_disk/region").String()).To(Equal("us-east-1")) + Expect(settings.Get("storage_configuration/disks/s3_disk_cache/disk").String()).To(Equal("s3_disk")) + Expect(settings.Get("storage_configuration/disks/s3_disk_cache/max_size").String()).To(Equal("8589934592")) + Expect(settings.Get("storage_configuration/policies/s3_main/volumes/main/disk").String()).To(Equal("s3_disk_cache")) + + // s3_main is the server-wide default for all MergeTree tables. + Expect(settings.Get("merge_tree/storage_policy").String()).To(Equal(StoragePolicyName)) + + // Credentials are secret references (operator renders from_env), not literals. + accessKey := settings.Get("storage_configuration/disks/s3_disk/access_key_id") + Expect(accessKey.IsSource()).To(BeTrue()) + Expect(accessKey.GetSecretKeyRef()).NotTo(BeNil()) + Expect(accessKey.GetSecretKeyRef().Name).To(Equal("objstore-conn")) + Expect(accessKey.GetSecretKeyRef().Key).To(Equal("AccessKey")) + Expect(settings.Has("storage_configuration/disks/s3_disk/use_environment_credentials")).To(BeFalse()) + }) + + It("uses ambient credentials when no access keys are present", func() { + ci := &objectstore.ConnInfo{} + settings := v1.NewSettings() + applyStorageConfiguration(settings, ci, "https://b.s3.us-east-1.amazonaws.com/clickhouse/", 1024) + + Expect(settings.Get("storage_configuration/disks/s3_disk/use_environment_credentials").String()).To(Equal("true")) + Expect(settings.Has("storage_configuration/disks/s3_disk/access_key_id")).To(BeFalse()) + Expect(settings.Has("storage_configuration/disks/s3_disk/region")).To(BeFalse()) + }) + + It("renders the s3 disk before the cache disk that wraps it", func() { + ci := &objectstore.ConnInfo{} + settings := v1.NewSettings() + applyStorageConfiguration(settings, ci, "http://host:80/bucket/clickhouse/", 1<<30) + + // ClickHouse initializes disks in document order and requires the wrapped + // disk to be defined before the cache disk; verify the rendered XML order. + rendered := settings.ClickHouseConfig() + diskIdx := strings.Index(rendered, "<"+s3DiskName+">") + cacheIdx := strings.Index(rendered, "<"+s3CacheDiskName+">") + Expect(diskIdx).To(BeNumerically(">=", 0)) + Expect(cacheIdx).To(BeNumerically(">=", 0)) + Expect(diskIdx).To(BeNumerically("<", cacheIdx), "s3 disk must be rendered before the cache disk that wraps it") + }) +}) diff --git a/internal/controller/infra/managed/clickhouse/altinity/read.go b/internal/controller/infra/managed/clickhouse/altinity/read.go index 91032c56..c5c3ac42 100644 --- a/internal/controller/infra/managed/clickhouse/altinity/read.go +++ b/internal/controller/infra/managed/clickhouse/altinity/read.go @@ -21,11 +21,13 @@ func readConnectionDetails(actual *chiv1.ClickHouseInstallation) *clickhouseConn } clickhouseHost := actual.Status.Endpoint - clickhousePort := strconv.Itoa(ClickHouseHTTPPort) + clickhouseHTTPPort := strconv.Itoa(ClickHouseHTTPPort) + clickhouseTCPPort := strconv.Itoa(ClickHouseNativePort) return &clickhouseConnInfo{ Host: clickhouseHost, - Port: clickhousePort, + HTTPPort: clickhouseHTTPPort, + TCPPort: clickhouseTCPPort, User: ClickHouseUser, Password: ClickHousePassword, Database: ClickHouseDatabase, @@ -160,7 +162,7 @@ func chPodsRunningStatus( return result, err } if found { - result[podName] = pod.Status.Phase == corev1.PodRunning + result[podName] = ctrlcommon.PodReady(pod) } else { result[podName] = false } diff --git a/internal/controller/infra/managed/clickhouse/altinity/spec.go b/internal/controller/infra/managed/clickhouse/altinity/spec.go index 6300763c..10763d51 100644 --- a/internal/controller/infra/managed/clickhouse/altinity/spec.go +++ b/internal/controller/infra/managed/clickhouse/altinity/spec.go @@ -7,9 +7,12 @@ import ( apiv2 "github.com/wandb/operator/api/v2" "github.com/wandb/operator/internal/controller/common" + "github.com/wandb/operator/internal/controller/infra/objectstore" + "github.com/wandb/operator/internal/controller/infra/managed/clickhouse/altinity/keeper" "github.com/wandb/operator/internal/logx" "github.com/wandb/operator/pkg/utils" "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + chtypes "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/common/types" "github.com/wandb/operator/pkg/wandb/manifest" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -26,8 +29,13 @@ const ( // TODO: remove this hardcoded default once all supported manifest versions // supply clickhouse..images.server. defaultClickHouseImage = "altinity/clickhouse-server:25.8.16.10002.altinitystable" + + objectStoreWaitMaxAttempts = 150 + objectStoreWaitDelaySeconds = 2 ) +// ClickHouseImage resolves the ClickHouse server image from the manifest, +// falling back to the hardcoded default for older manifests that omit it. func ClickHouseImage(img manifest.ImageRef, globalImageRegistry string) string { if out := img.GetImage(globalImageRegistry); out != "" { return out @@ -51,6 +59,8 @@ const ( clickHouseCapabilityAll corev1.Capability = "ALL" ) +// clickHousePodSecurityContext returns the pod security context, omitting the +// fixed UID/GID/FSGroup on OpenShift where the platform assigns them. func clickHousePodSecurityContext() *corev1.PodSecurityContext { if utils.IsOpenShift() { return &corev1.PodSecurityContext{ @@ -68,6 +78,8 @@ func clickHousePodSecurityContext() *corev1.PodSecurityContext { } } +// clickHouseContainerSecurityContext returns the container security context, +// pinning the fixed UID/GID off OpenShift and always dropping all capabilities. func clickHouseContainerSecurityContext() *corev1.SecurityContext { securityContext := &corev1.SecurityContext{ RunAsNonRoot: ptr.To(true), @@ -84,6 +96,8 @@ func clickHouseContainerSecurityContext() *corev1.SecurityContext { return securityContext } +// clickHouseWritableVolumes returns the emptyDir volumes that back the writable +// paths a read-only-root-filesystem ClickHouse container still needs. func clickHouseWritableVolumes() []corev1.Volume { return []corev1.Volume{ writableEmptyDirVolume(clickHouseTmpVolumeName), @@ -92,6 +106,8 @@ func clickHouseWritableVolumes() []corev1.Volume { } } +// clickHouseWritableVolumeMounts returns the mounts pairing the writable +// emptyDir volumes with their in-container paths. func clickHouseWritableVolumeMounts() []corev1.VolumeMount { return []corev1.VolumeMount{ {Name: clickHouseTmpVolumeName, MountPath: clickHouseTmpMountPath}, @@ -100,10 +116,12 @@ func clickHouseWritableVolumeMounts() []corev1.VolumeMount { } } +// clickHouseRuntimeDefaultSeccompProfile returns the RuntimeDefault seccomp profile. func clickHouseRuntimeDefaultSeccompProfile() *corev1.SeccompProfile { return &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault} } +// writableEmptyDirVolume returns a named emptyDir volume. func writableEmptyDirVolume(name string) corev1.Volume { return corev1.Volume{ Name: name, @@ -113,29 +131,82 @@ func writableEmptyDirVolume(name string) corev1.Volume { } } +// ToServiceAccount builds the ClickHouse ServiceAccount, automounting its token +// only when object-store credentials are ambient (IAM / workload identity). +// Returns nil when the spec opts out of ServiceAccount creation. +func ToServiceAccount( + wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedClickHouseSpec, + objStorage *objectstore.ConnInfo, + scheme *runtime.Scheme, +) (*corev1.ServiceAccount, error) { + if spec.ServiceAccount.Create != nil && !*spec.ServiceAccount.Create { + return nil, nil + } + + serviceAccount := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: clickHouseServiceAccountName(spec), + Namespace: spec.Namespace, + Labels: BuildWandbClickhouseLabels(wandb), + Annotations: spec.ServiceAccount.Annotations, + }, + // Ambient (IAM/workload-identity) credentials require the projected SA + // token; static access keys don't, so only automount when creds are ambient. + AutomountServiceAccountToken: ptr.To(!objStorage.HasStaticCredentials()), + } + if wandb.Namespace == spec.Namespace { + if err := ctrl.SetControllerReference(wandb, serviceAccount, scheme); err != nil { + return nil, fmt.Errorf("failed to set owner reference on ClickHouse ServiceAccount: %w", err) + } + } + return serviceAccount, nil +} + +// clickHouseServiceAccountName returns the configured ServiceAccount name, +// defaulting to the spec name when unset. +func clickHouseServiceAccountName(spec *apiv2.ManagedClickHouseSpec) string { + if spec.ServiceAccount.ServiceAccountName != "" { + return spec.ServiceAccount.ServiceAccountName + } + return spec.Name +} + // ToClickHouseVendorSpec converts a ClickHouseSpec to a ClickHouseInstallation CR. // This function translates the high-level ClickHouse spec into the vendor-specific // ClickHouseInstallation format used by the Altinity operator. func ToClickHouseVendorSpec( ctx context.Context, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedClickHouseSpec, scheme *runtime.Scheme, + objStorage *objectstore.ConnInfo, + objStorageEndpoint string, + waitForObjectStore bool, mfst manifest.Manifest, ) (*v1.ClickHouseInstallation, error) { _, log := logx.WithSlog(ctx, logx.ClickHouse) - spec := wandb.Spec.ClickHouse.ManagedClickHouse if spec == nil { return nil, nil } + // Managed ClickHouse stores table data in the object store; require it. + if objStorage == nil { + return nil, fmt.Errorf("managed ClickHouse requires object storage, but none was resolved") + } + nsnBuilder := CreateNsNameBuilder(types.NamespacedName{ Namespace: spec.Namespace, Name: spec.Name, }) - // Parse storage quantity + // This PV holds only metadata/system tables and the S3 read-through cache; + // table data lives in the bucket. storageQuantity := resource.MustParse(spec.StorageSize) + // Reserve ~20% of the local PV for metadata/system; the rest backs the cache. + cacheMaxSizeBytes := storageQuantity.Value() * 8 / 10 + // Create user settings with password passwordSha256 := fmt.Sprintf("%x", sha256.Sum256([]byte(ClickHousePassword))) userSettings := v1.NewSettings() @@ -155,6 +226,9 @@ func ToClickHouseVendorSpec( // Create server settings serverSettings := v1.NewSettings() + // Define the S3 disk + cache + storage policy and make it the server-wide default. + applyStorageConfiguration(serverSettings, objStorage, objStorageEndpoint, cacheMaxSizeBytes) + // Enable built-in Prometheus metrics endpoint if telemetry is enabled if spec.Telemetry.Enabled { serverSettings.Set("prometheus/endpoint", v1.NewSettingScalar("/metrics")) @@ -170,20 +244,26 @@ func ToClickHouseVendorSpec( reclaimPolicy = v1.PVCReclaimPolicyDelete } + clickHouseImage := ClickHouseImage(mfst.Clickhouse["default"].Images["server"], wandb.Spec.Global.ImageRegistry) podSpec := corev1.PodSpec{ - SecurityContext: clickHousePodSecurityContext(), - Affinity: wandb.GetAffinity(spec.ManagedInfraSpec), - Tolerations: *wandb.GetTolerations(spec.ManagedInfraSpec), - Volumes: clickHouseWritableVolumes(), + ServiceAccountName: clickHouseServiceAccountName(spec), + AutomountServiceAccountToken: ptr.To(!objStorage.HasStaticCredentials()), + SecurityContext: clickHousePodSecurityContext(), + Affinity: wandb.GetAffinity(spec.ManagedInfraSpec), + Tolerations: *wandb.GetTolerations(spec.ManagedInfraSpec), + Volumes: clickHouseWritableVolumes(), Containers: []corev1.Container{ { Name: "clickhouse", - Image: ClickHouseImage(mfst.Clickhouse["default"].Images["server"], wandb.Spec.Global.ImageRegistry), + Image: clickHouseImage, SecurityContext: clickHouseContainerSecurityContext(), VolumeMounts: clickHouseWritableVolumeMounts(), }, }, } + if waitForObjectStore { + podSpec.InitContainers = []corev1.Container{clickHouseObjectStoreWaitContainer(objStorageEndpoint, clickHouseImage)} + } if len(spec.Config.Resources.Requests) > 0 || len(spec.Config.Resources.Limits) > 0 { podSpec.Containers[0].Resources = corev1.ResourceRequirements{ @@ -204,7 +284,7 @@ func ToClickHouseVendorSpec( Configuration: &v1.Configuration{ Clusters: []*v1.Cluster{ { - Name: "default", + Name: chiClusterName, Layout: &v1.ChiClusterLayout{ ShardsCount: ShardsCount, ReplicasCount: int(spec.Replicas), @@ -213,6 +293,14 @@ func ToClickHouseVendorSpec( }, Users: userSettings, Settings: serverSettings, + Zookeeper: &v1.ZookeeperConfig{ + Nodes: v1.ZookeeperNodes{ + { + Host: keeper.ClientServiceFQDN(spec.Namespace, baseName(spec.Name)), + Port: chtypes.NewInt32(int32(keeper.KeeperClientPort)), + }, + }, + }, }, Defaults: &v1.Defaults{ Templates: &v1.TemplatesList{ @@ -225,7 +313,10 @@ func ToClickHouseVendorSpec( { Name: nsnBuilder.PodTemplateName(), ObjectMeta: metav1.ObjectMeta{ - Labels: BuildWandbClickhouseLabels(wandb), + Labels: utils.MergeMapsStringString( + BuildWandbClickhouseLabels(wandb), + common.StandardLabels(wandb, "clickhouse", common.RoleAnalyticsDB, ""), + ), }, Spec: podSpec, }, @@ -264,10 +355,45 @@ func ToClickHouseVendorSpec( return chi, nil } +// clickHouseObjectStoreWaitContainer returns an init container that blocks until +// the object-store endpoint is reachable, so ClickHouse does not start before +// its backing bucket is available. +func clickHouseObjectStoreWaitContainer(endpoint, image string) corev1.Container { + // The existing ClickHouse image includes wget. Any HTTP response below 500 + // proves DNS and the S3 API are reachable; authentication remains ClickHouse's + // responsibility when its main process starts. + script := fmt.Sprintf( + `attempt=1 +while [ "$attempt" -le %d ]; do + response="$(wget --no-check-certificate --server-response --spider "$1" 2>&1)" + result=$? + if [ "$result" -eq 0 ] || printf '%%s\n' "$response" | grep -Eq 'HTTP/[0-9.]+ [1-4][0-9][0-9]'; then + exit 0 + fi + attempt=$((attempt + 1)) + if [ "$attempt" -le %d ]; then sleep %d; fi +done +echo 'object-store endpoint did not become reachable before timeout' >&2 +exit 1`, + objectStoreWaitMaxAttempts, + objectStoreWaitMaxAttempts, + objectStoreWaitDelaySeconds, + ) + return corev1.Container{ + Name: "wait-object-store", + Image: image, + Command: []string{"/bin/sh", "-c"}, + Args: []string{script, "wait-object-store", endpoint}, + SecurityContext: clickHouseContainerSecurityContext(), + } +} + +// BuildWandbClickhouseLabels returns the standard W&B labels for the ClickHouse module. func BuildWandbClickhouseLabels(wandb *apiv2.WeightsAndBiases) map[string]string { return common.BuildWandbLabels(wandb, ClickhouseModuleName) } +// ToClickHouseOnDeleteRule builds the on-delete retention rule for the ClickHouse module. func ToClickHouseOnDeleteRule(wandb *apiv2.WeightsAndBiases, retentionPolicy apiv2.RetentionPolicy) common.OnDeleteRule { return common.ToOnDeleteRule(wandb, retentionPolicy, ClickhouseModuleName) } diff --git a/internal/controller/infra/managed/clickhouse/altinity/spec_test.go b/internal/controller/infra/managed/clickhouse/altinity/spec_test.go index 0496d207..b0d4bd46 100644 --- a/internal/controller/infra/managed/clickhouse/altinity/spec_test.go +++ b/internal/controller/infra/managed/clickhouse/altinity/spec_test.go @@ -6,9 +6,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apiv2 "github.com/wandb/operator/api/v2" - "github.com/wandb/operator/pkg/wandb/manifest" + "github.com/wandb/operator/internal/controller/infra/objectstore" + "github.com/wandb/operator/internal/controller/infra/managed/clickhouse/altinity/keeper" "github.com/wandb/operator/pkg/utils" chiv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + "github.com/wandb/operator/pkg/wandb/manifest" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,18 +25,35 @@ var _ = Describe("ClickHouse vendor specs", func() { It("renders hardened pod templates with writable runtime mounts", func() { wandb := clickHouseWandb() - chi, err := ToClickHouseVendorSpec(context.Background(), wandb, clickHouseScheme(), manifest.Manifest{}) + chi, err := ToClickHouseVendorSpec(context.Background(), wandb, wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse, clickHouseScheme(), testObjectStorageConn(), testObjectStorageEndpoint, true, manifest.Manifest{}) Expect(err).NotTo(HaveOccurred()) Expect(chi).NotTo(BeNil()) Expect(chi.Spec.Templates.PodTemplates).To(HaveLen(1)) podSpec := chi.Spec.Templates.PodTemplates[0].Spec + Expect(podSpec.ServiceAccountName).To(Equal("clickhouse")) + Expect(podSpec.AutomountServiceAccountToken).NotTo(BeNil()) + Expect(*podSpec.AutomountServiceAccountToken).To(BeFalse()) expectClickHouseDefaultPodSecurityContext(podSpec.SecurityContext) expectClickHouseWritableVolume(podSpec.Volumes, clickHouseTmpVolumeName) expectClickHouseWritableVolume(podSpec.Volumes, clickHouseLogVolumeName) expectClickHouseWritableVolume(podSpec.Volumes, clickHouseRunVolumeName) Expect(podSpec.Containers).To(HaveLen(1)) + Expect(podSpec.InitContainers).To(HaveLen(1)) + wait := podSpec.InitContainers[0] + Expect(wait.Name).To(Equal("wait-object-store")) + Expect(wait.Image).To(Equal(podSpec.Containers[0].Image)) + Expect(wait.Args).To(HaveLen(3)) + Expect(wait.Args[0]).To(ContainSubstring("wget")) + Expect(wait.Args[0]).To(ContainSubstring("HTTP/[0-9.]+ [1-4]")) + Expect(wait.Args[0]).To(ContainSubstring("sleep 2")) + Expect(wait.Args[0]).NotTo(ContainSubstring("aws")) + Expect(wait.Args[0]).NotTo(ContainSubstring("head-bucket")) + Expect(wait.Args[0]).NotTo(ContainSubstring("AccessKey")) + Expect(wait.Args[0]).NotTo(ContainSubstring("SecretKey")) + Expect(wait.Args[2]).To(Equal(testObjectStorageEndpoint)) + Expect(wait.Env).To(BeEmpty()) container := podSpec.Containers[0] Expect(container.Image).To(Equal(ClickHouseImage(manifest.ImageRef{}, ""))) Expect(container.Resources.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("500m"))) @@ -47,7 +66,8 @@ var _ = Describe("ClickHouse vendor specs", func() { It("omits fixed ClickHouse IDs in OpenShift mode", func() { utils.SetOpenShiftMode(true) - chi, err := ToClickHouseVendorSpec(context.Background(), clickHouseWandb(), clickHouseScheme(), manifest.Manifest{}) + wandb := clickHouseWandb() + chi, err := ToClickHouseVendorSpec(context.Background(), wandb, wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse, clickHouseScheme(), testObjectStorageConn(), testObjectStorageEndpoint, true, manifest.Manifest{}) Expect(err).NotTo(HaveOccurred()) Expect(chi).NotTo(BeNil()) @@ -55,8 +75,108 @@ var _ = Describe("ClickHouse vendor specs", func() { expectClickHouseOpenShiftPodSecurityContext(podSpec.SecurityContext) expectClickHouseOpenShiftContainerSecurityContext(podSpec.Containers[0].SecurityContext) }) + + It("backs storage with the object store, sets a default policy, and wires keeper", func() { + wandb := clickHouseWandb() + chi, err := ToClickHouseVendorSpec(context.Background(), wandb, wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse, clickHouseScheme(), testObjectStorageConn(), testObjectStorageEndpoint, true, manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(chi).NotTo(BeNil()) + + settings := chi.Spec.Configuration.Settings + + // Server-wide default storage policy routes all MergeTree tables to the bucket. + Expect(settings.Get("merge_tree/storage_policy").String()).To(Equal(StoragePolicyName)) + + // storage_configuration is expressed through the typed Settings API (no XML). + Expect(settings.Get("storage_configuration/disks/s3_disk/type").String()).To(Equal("s3")) + Expect(settings.Get("storage_configuration/disks/s3_disk/endpoint").String()). + To(Equal("http://seaweedfs.wandb.svc.cluster.local:80/bucket/clickhouse/")) + Expect(settings.Get("storage_configuration/policies/" + StoragePolicyName + "/volumes/main/disk").String()). + To(Equal("s3_disk_cache")) + + // Credentials are secret references; the operator renders from_env. No + // config file and no manually-injected env vars on our side. + accessKey := settings.Get("storage_configuration/disks/s3_disk/access_key_id") + Expect(accessKey.IsSource()).To(BeTrue()) + Expect(accessKey.GetSecretKeyRef()).NotTo(BeNil()) + Expect(accessKey.GetSecretKeyRef().Key).To(Equal("AccessKey")) + Expect(chi.Spec.Configuration.Files).To(BeNil()) + Expect(chi.Spec.Templates.PodTemplates[0].Spec.Containers[0].Env).To(BeEmpty()) + + // Keeper wired via the zookeeper config. + Expect(chi.Spec.Configuration.Zookeeper).NotTo(BeNil()) + Expect(chi.Spec.Configuration.Zookeeper.Nodes).To(HaveLen(1)) + Expect(chi.Spec.Configuration.Zookeeper.Nodes[0].Host).To(Equal(keeper.ClientServiceFQDN("wandb", "clickhouse"))) + }) + + It("does not gate ClickHouse for bring-your-own object storage", func() { + wandb := clickHouseWandb() + chi, err := ToClickHouseVendorSpec(context.Background(), wandb, wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse, clickHouseScheme(), testObjectStorageConn(), testObjectStorageEndpoint, false, manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(chi.Spec.Templates.PodTemplates[0].Spec.InitContainers).To(BeEmpty()) + }) + + It("wires ambient object storage credentials through the configured service account", func() { + wandb := clickHouseWandb() + spec := wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse + spec.ServiceAccount = apiv2.ManagedServiceAccountSpec{ + ServiceAccountName: "clickhouse-workload-identity", + Annotations: map[string]string{ + "eks.amazonaws.com/role-arn": "arn:aws:iam::123456789012:role/wandb-clickhouse", + }, + } + objStorage := testObjectStorageConn() + // Ambient credentials: no static access/secret keys are set. + objStorage.AccessKey = "" + objStorage.SecretKey = "" + + serviceAccount, err := ToServiceAccount(wandb, spec, objStorage, clickHouseScheme()) + Expect(err).NotTo(HaveOccurred()) + Expect(serviceAccount.Name).To(Equal("clickhouse-workload-identity")) + Expect(serviceAccount.Annotations).To(Equal(spec.ServiceAccount.Annotations)) + Expect(serviceAccount.AutomountServiceAccountToken).NotTo(BeNil()) + Expect(*serviceAccount.AutomountServiceAccountToken).To(BeTrue()) + + chi, err := ToClickHouseVendorSpec(context.Background(), wandb, spec, clickHouseScheme(), objStorage, testObjectStorageEndpoint, false, manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + podSpec := chi.Spec.Templates.PodTemplates[0].Spec + Expect(podSpec.ServiceAccountName).To(Equal(serviceAccount.Name)) + Expect(podSpec.AutomountServiceAccountToken).NotTo(BeNil()) + Expect(*podSpec.AutomountServiceAccountToken).To(BeTrue()) + }) + + It("can reference an existing ClickHouse service account", func() { + wandb := clickHouseWandb() + spec := wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse + create := false + spec.ServiceAccount = apiv2.ManagedServiceAccountSpec{ + Create: &create, + ServiceAccountName: "existing-clickhouse-identity", + } + + serviceAccount, err := ToServiceAccount(wandb, spec, testObjectStorageConn(), clickHouseScheme()) + Expect(err).NotTo(HaveOccurred()) + Expect(serviceAccount).To(BeNil()) + + chi, err := ToClickHouseVendorSpec(context.Background(), wandb, spec, clickHouseScheme(), testObjectStorageConn(), testObjectStorageEndpoint, false, manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(chi.Spec.Templates.PodTemplates[0].Spec.ServiceAccountName).To(Equal("existing-clickhouse-identity")) + }) }) +const testObjectStorageEndpoint = "http://seaweedfs.wandb.svc.cluster.local:80/bucket/clickhouse/" + +func testObjectStorageConn() *objectstore.ConnInfo { + ref := corev1.LocalObjectReference{Name: "objstore-conn"} + return &objectstore.ConnInfo{ + Region: "us-east-1", + AccessKey: "AKIA", + SecretKey: "secret", + AccessKeyRef: corev1.SecretKeySelector{LocalObjectReference: ref, Key: "AccessKey"}, + SecretKeyRef: corev1.SecretKeySelector{LocalObjectReference: ref, Key: "SecretKey"}, + } +} + func clickHouseScheme() *runtime.Scheme { scheme := runtime.NewScheme() Expect(apiv2.AddToScheme(scheme)).To(Succeed()) @@ -77,16 +197,18 @@ func clickHouseWandb() *apiv2.WeightsAndBiases { }, Spec: apiv2.WeightsAndBiasesSpec{ Tolerations: &tolerations, - ClickHouse: apiv2.ClickHouseSpec{ - ManagedClickHouse: &apiv2.ManagedClickHouseSpec{ - Name: "clickhouse", - Namespace: "wandb", - Replicas: 1, - StorageSize: "10Gi", - Config: apiv2.ClickHouseConfig{ - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), + ClickHouse: map[string]apiv2.ClickHouseSpec{ + apiv2.DefaultInstanceName: { + ManagedClickHouse: &apiv2.ManagedClickHouseSpec{ + Name: "clickhouse", + Namespace: "wandb", + Replicas: 1, + StorageSize: "10Gi", + Config: apiv2.ClickHouseConfig{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + }, }, }, }, diff --git a/internal/controller/infra/managed/clickhouse/altinity/status.go b/internal/controller/infra/managed/clickhouse/altinity/status.go index bd07c356..244521d1 100644 --- a/internal/controller/infra/managed/clickhouse/altinity/status.go +++ b/internal/controller/infra/managed/clickhouse/altinity/status.go @@ -7,6 +7,7 @@ import ( "github.com/samber/lo" apiv2 "github.com/wandb/operator/api/v2" "github.com/wandb/operator/internal/controller/common" + "github.com/wandb/operator/internal/controller/infra/managed/clickhouse/altinity/keeper" "github.com/wandb/operator/internal/logx" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -85,9 +86,21 @@ func inferInfraState( var events []corev1.Event impliedStates := make(map[string]string, len(conditions)) + // undeployable name = terminal config error; surface it in `kubectl describe` + if cond, found := lo.Find(conditions, func(c metav1.Condition) bool { + return c.Type == ClickHouseCustomResourceType && c.Reason == common.InvalidNameReason + }); found { + events = append(events, corev1.Event{ + Type: corev1.EventTypeWarning, + Reason: "ClickHouseInvalidName", + Message: cond.Message, + }) + } + impliedStates = inferStateFromCondition(ctx, ClickHouseCustomResourceType, impliedStates, conditions) impliedStates = inferStateFromCondition(ctx, ClickHouseConnectionInfoType, impliedStates, conditions) impliedStates = inferStateFromCondition(ctx, ClickHouseReportedReadyType, impliedStates, conditions) + impliedStates = inferStateFromCondition(ctx, keeper.KeeperReportedReadyType, impliedStates, conditions) hasImpliedState := func(target string) bool { return len(lo.FilterValues( @@ -145,6 +158,8 @@ func inferStateFromCondition(ctx context.Context, conditionType string, impliedS impliedStates[conditionType] = inferState_ClickHouseConnectionInfoType(ctx, cond) case ClickHouseReportedReadyType: impliedStates[conditionType] = inferState_ClickHouseReportedReadyType(ctx, cond) + case keeper.KeeperReportedReadyType: + impliedStates[conditionType] = inferState_KeeperReportedReadyType(ctx, cond) default: impliedStates[conditionType] = common.UnknownState } @@ -165,6 +180,9 @@ func inferState_ClickHouseCustomResourceType(ctx context.Context, condition meta if condition.Reason == common.PendingDeleteReason { result = common.UnavailableState } + if condition.Reason == common.InvalidNameReason { + result = common.ErrorState + } } log.Debug( "implied state", "state", result, "condition", condition.Type, @@ -189,6 +207,23 @@ func inferState_ClickHouseConnectionInfoType(ctx context.Context, condition meta return result } +func inferState_KeeperReportedReadyType(ctx context.Context, condition metav1.Condition) string { + log := logx.GetSlog(ctx) + result := common.UnknownState + if condition.Status == metav1.ConditionTrue { + result = common.HealthyState + } + if condition.Status == metav1.ConditionFalse { + // Keeper not ready yet: ClickHouse can't coordinate replication, so hold at pending. + result = common.PendingState + } + log.Debug( + "implied state", "state", result, "condition", condition.Type, + "reason", condition.Reason, "status", condition.Status, + ) + return result +} + func inferState_ClickHouseReportedReadyType(ctx context.Context, condition metav1.Condition) string { log := logx.GetSlog(ctx) result := common.UnknownState diff --git a/internal/controller/infra/managed/clickhouse/altinity/status_test.go b/internal/controller/infra/managed/clickhouse/altinity/status_test.go new file mode 100644 index 00000000..c3e8e11e --- /dev/null +++ b/internal/controller/infra/managed/clickhouse/altinity/status_test.go @@ -0,0 +1,58 @@ +package altinity + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/wandb/operator/internal/controller/common" + "github.com/wandb/operator/internal/controller/infra/managed/clickhouse/altinity/keeper" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("ClickHouse status keeper gating", func() { + healthyClickHouse := func() []metav1.Condition { + return []metav1.Condition{ + {Type: ClickHouseCustomResourceType, Status: metav1.ConditionTrue, Reason: common.ResourceExistsReason}, + {Type: ClickHouseConnectionInfoType, Status: metav1.ConditionTrue, Reason: common.ResourceExistsReason}, + {Type: ClickHouseReportedReadyType, Status: metav1.ConditionTrue, Reason: common.ResourceExistsReason}, + } + } + + It("holds at pending while keeper is not ready, even if ClickHouse is healthy", func() { + conditions := append(healthyClickHouse(), metav1.Condition{ + Type: keeper.KeeperReportedReadyType, + Status: metav1.ConditionFalse, + Reason: common.NoResourceReason, + }) + state, _ := inferInfraState(context.Background(), true, conditions) + Expect(state).To(Equal(common.PendingState)) + }) + + It("is healthy when both keeper and ClickHouse are ready", func() { + conditions := append(healthyClickHouse(), metav1.Condition{ + Type: keeper.KeeperReportedReadyType, + Status: metav1.ConditionTrue, + Reason: common.ResourceExistsReason, + }) + state, _ := inferInfraState(context.Background(), true, conditions) + Expect(state).To(Equal(common.HealthyState)) + }) + + It("reports an error state and event when the managed name cannot be deployed", func() { + conditions := []metav1.Condition{ + { + Type: ClickHouseCustomResourceType, + Status: metav1.ConditionFalse, + Reason: common.InvalidNameReason, + Message: "managed ClickHouse name is too long", + }, + } + state, events := inferInfraState(context.Background(), true, conditions) + + Expect(state).To(Equal(common.ErrorState)) + Expect(events).To(HaveLen(1)) + Expect(events[0].Reason).To(Equal("ClickHouseInvalidName")) + Expect(events[0].Message).To(ContainSubstring("too long")) + }) +}) diff --git a/internal/controller/infra/managed/clickhouse/altinity/write.go b/internal/controller/infra/managed/clickhouse/altinity/write.go index 25cdfe8d..e5456bba 100644 --- a/internal/controller/infra/managed/clickhouse/altinity/write.go +++ b/internal/controller/infra/managed/clickhouse/altinity/write.go @@ -4,11 +4,15 @@ import ( "context" "github.com/wandb/operator/internal/controller/common" + "github.com/wandb/operator/internal/controller/infra/managed/clickhouse/altinity/keeper" "github.com/wandb/operator/internal/logx" + chkv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1" chiv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) const ( @@ -16,75 +20,124 @@ const ( AppConnTypeName = "ClickHouseAppConn" ) +// WriteState reconciles the Keeper ensemble (first, since ReplicatedMergeTree +// depends on it) and the ClickHouse installation. func WriteState( ctx context.Context, client client.Client, specNamespacedName types.NamespacedName, + desiredServiceAccount *corev1.ServiceAccount, + desiredKeeper *chkv1.ClickHouseKeeperInstallation, desired *chiv1.ClickHouseInstallation, ) []metav1.Condition { ctx, _ = logx.WithSlog(ctx, logx.ClickHouse) - var actual = &chiv1.ClickHouseInstallation{} + results := make([]metav1.Condition, 0) - nsnBuilder := createNsNameBuilder(specNamespacedName) + if desiredServiceAccount != nil { + serviceAccountConditions := writeServiceAccount(ctx, client, desiredServiceAccount) + results = append(results, serviceAccountConditions...) + if len(serviceAccountConditions) > 0 { + return results + } + } + results = append(results, keeper.WriteState( + ctx, client, + types.NamespacedName{Namespace: desiredKeeper.Namespace, Name: desiredKeeper.Name}, + desiredKeeper, + )...) + results = append(results, writeClickHouseInstallation(ctx, client, specNamespacedName, desired)...) - found, err := common.GetResource( - ctx, client, nsnBuilder.InstallationNsName(), ResourceTypeName, actual, - ) + return results +} + +func writeServiceAccount( + ctx context.Context, + cl client.Client, + desired *corev1.ServiceAccount, +) []metav1.Condition { + actual := &corev1.ServiceAccount{} + found, err := common.GetResource(ctx, cl, client.ObjectKeyFromObject(desired), "ServiceAccount", actual) if err != nil { - return []metav1.Condition{ - { - Type: common.ReconciledType, - Status: metav1.ConditionFalse, - Reason: common.ApiErrorReason, - }, - { - Type: ClickHouseCustomResourceType, - Status: metav1.ConditionUnknown, - Reason: common.ApiErrorReason, - }, - } + return []metav1.Condition{{ + Type: common.ReconciledType, + Status: metav1.ConditionFalse, + Reason: common.ApiErrorReason, + }} } if !found { actual = nil } - - result := make([]metav1.Condition, 0) - - action, err := common.CrudResource(ctx, client, desired, actual) - if err != nil { - result = append(result, metav1.Condition{ + if _, err := common.CrudResource(ctx, cl, desired, actual); err != nil { + return []metav1.Condition{{ Type: common.ReconciledType, Status: metav1.ConditionFalse, Reason: common.ApiErrorReason, - }) + }} } + return nil +} - switch action { - case common.CreateAction: - result = append(result, metav1.Condition{ - Type: ClickHouseCustomResourceType, - Status: metav1.ConditionFalse, - Reason: common.PendingCreateReason, - }) - case common.DeleteAction: - result = append(result, metav1.Condition{ - Type: ClickHouseCustomResourceType, - Status: metav1.ConditionFalse, - Reason: common.PendingDeleteReason, - }) - case common.UpdateAction: - result = append(result, metav1.Condition{ - Type: ClickHouseCustomResourceType, - Status: metav1.ConditionTrue, - Reason: common.ResourceExistsReason, - }) - case common.NoAction: - result = append(result, metav1.Condition{ - Type: ClickHouseCustomResourceType, - Status: metav1.ConditionFalse, - Reason: common.NoResourceReason, - }) +// writeClickHouseInstallation create-or-updates the CHI, setting only the fields +// we own (spec, labels, owner refs) and preserving the Altinity-managed +// finalizer/status. It compares owned fields via JSON, never the vendored status +// — whose uint64 and unexported fields panic controllerutil's reflective +// diff/copy. +func writeClickHouseInstallation( + ctx context.Context, + cl client.Client, + specNamespacedName types.NamespacedName, + desired *chiv1.ClickHouseInstallation, +) []metav1.Condition { + nsnBuilder := createNsNameBuilder(specNamespacedName) + obj := &chiv1.ClickHouseInstallation{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsnBuilder.InstallationName(), + Namespace: nsnBuilder.Namespace(), + }, + } + + op, err := common.WriteOwnedFields(ctx, cl, obj, + func(o *chiv1.ClickHouseInstallation) { + applyOwnedMetadata(o, desired) + o.Spec = desired.Spec + }, + clickHouseOwnedEqual, + ) + if err != nil { + return []metav1.Condition{ + {Type: common.ReconciledType, Status: metav1.ConditionFalse, Reason: common.ApiErrorReason}, + {Type: ClickHouseCustomResourceType, Status: metav1.ConditionUnknown, Reason: common.ApiErrorReason}, + } + } + + return []metav1.Condition{customResourceConditionForOp(ClickHouseCustomResourceType, op)} +} + +func clickHouseOwnedEqual(a, b *chiv1.ClickHouseInstallation) bool { + return common.JSONEqual(a.Spec, b.Spec) && + common.JSONEqual(a.Labels, b.Labels) && + common.JSONEqual(a.OwnerReferences, b.OwnerReferences) +} + +// applyOwnedMetadata sets the metadata we own (merged labels, owner references), +// leaving finalizers/annotations owned by the resource's operator untouched. +func applyOwnedMetadata(obj, desired metav1.Object) { + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + } + for k, v := range desired.GetLabels() { + labels[k] = v } + obj.SetLabels(labels) + obj.SetOwnerReferences(desired.GetOwnerReferences()) +} - return result +// customResourceConditionForOp maps a create-or-update result to the resource's +// existence condition (created => pending, updated/unchanged => exists). +func customResourceConditionForOp(conditionType string, op controllerutil.OperationResult) metav1.Condition { + if op == controllerutil.OperationResultCreated { + return metav1.Condition{Type: conditionType, Status: metav1.ConditionFalse, Reason: common.PendingCreateReason} + } + return metav1.Condition{Type: conditionType, Status: metav1.ConditionTrue, Reason: common.ResourceExistsReason} } diff --git a/internal/controller/infra/managed/kafka/bufstream/config.go b/internal/controller/infra/managed/kafka/bufstream/config.go index 95ad97b3..7f232a46 100644 --- a/internal/controller/infra/managed/kafka/bufstream/config.go +++ b/internal/controller/infra/managed/kafka/bufstream/config.go @@ -2,12 +2,11 @@ package bufstream import ( "fmt" - "strings" "gopkg.in/yaml.v3" apiv2 "github.com/wandb/operator/api/v2" - "github.com/wandb/operator/internal/controller/infra/external/objectstore" + "github.com/wandb/operator/internal/controller/infra/objectstore" ) type dataSource struct { @@ -74,9 +73,15 @@ func renderBufstreamConfig(clusterName, advertiseHost string, etcdAddresses []st // Isolate Bufstream's objects under a dedicated key prefix (the cluster name) // so they never collide with W&B artifact data, which shares the same bucket. // storage is passed by value, so this only affects the rendered config. - storage.URI = strings.TrimSuffix(storage.URI, "/") + "/" + clusterName + uri := storage.ProviderURI() - data, err := renderData(storage) + if storage.Path != "" { + uri = fmt.Sprintf("%s/%s", uri, storage.Path) + } + + uri = fmt.Sprintf("%s/%s", uri, clusterName) + + data, err := renderData(storage, uri) if err != nil { return "", err } @@ -110,29 +115,33 @@ func renderBufstreamConfig(clusterName, advertiseHost string, etcdAddresses []st // renderData maps the resolved object-store connection onto Bufstream's // provider-specific data storage config. -func renderData(storage objectstore.ConnInfo) (bufstreamData, error) { +func renderData(storage objectstore.ConnInfo, uri string) (bufstreamData, error) { switch storage.Provider { case apiv2.ObjectStoreProviderS3: - return bufstreamData{S3: renderS3Storage(storage)}, nil + return bufstreamData{S3: renderS3Storage(storage, uri)}, nil case apiv2.ObjectStoreProviderGCS: // GCS authenticates via workload identity / ADC, so only the bucket URI is configured. - return bufstreamData{GCS: storage.URI}, nil + return bufstreamData{GCS: uri}, nil case apiv2.ObjectStoreProviderAzure: - return bufstreamData{Azure: renderAzureStorage(storage)}, nil + return bufstreamData{Azure: renderAzureStorage(storage, uri)}, nil default: return bufstreamData{}, fmt.Errorf("unsupported object-store provider %q", storage.Provider) } } -func renderS3Storage(storage objectstore.ConnInfo) *bufstreamS3 { +// renderS3Storage maps the resolved connection onto Bufstream's S3 data config, +// applying the region default, endpoint, and path-style, and wiring env-var +// credential sources only when static keys are present. +func renderS3Storage(storage objectstore.ConnInfo, uri string) *bufstreamS3 { region := storage.Region if region == "" { - region = "us-east-1" + region = objectstore.DefaultRegion } + s3 := &bufstreamS3{ - URI: storage.URI, + URI: uri, Region: region, - Endpoint: storage.Endpoint, + Endpoint: storage.EndpointURL(), ForcePathStyle: storage.ForcePathStyle, } if storage.HasStaticCredentials() { @@ -142,8 +151,10 @@ func renderS3Storage(storage objectstore.ConnInfo) *bufstreamS3 { return s3 } -func renderAzureStorage(storage objectstore.ConnInfo) *bufstreamAzure { - az := &bufstreamAzure{URI: storage.URI} +// renderAzureStorage maps the resolved connection onto Bufstream's Azure data +// config, wiring env-var credential sources only when static keys are present. +func renderAzureStorage(storage objectstore.ConnInfo, uri string) *bufstreamAzure { + az := &bufstreamAzure{URI: uri} if storage.HasStaticCredentials() { az.AccessKeyID = &dataSource{EnvVar: EnvStorageAccessKeyID} az.SecretAccessKey = &dataSource{EnvVar: EnvStorageSecretAccessKey} diff --git a/internal/controller/infra/managed/kafka/bufstream/config_test.go b/internal/controller/infra/managed/kafka/bufstream/config_test.go index 5422305e..a65554b4 100644 --- a/internal/controller/infra/managed/kafka/bufstream/config_test.go +++ b/internal/controller/infra/managed/kafka/bufstream/config_test.go @@ -8,7 +8,7 @@ import ( "gopkg.in/yaml.v3" apiv2 "github.com/wandb/operator/api/v2" - "github.com/wandb/operator/internal/controller/infra/external/objectstore" + "github.com/wandb/operator/internal/controller/infra/objectstore" ) func TestRenderBufstreamConfigS3(t *testing.T) { @@ -78,7 +78,7 @@ func TestRenderBufstreamConfigS3NoStaticCreds(t *testing.T) { func TestRenderBufstreamConfigGCS(t *testing.T) { storage := objectstore.ConnInfo{ Provider: apiv2.ObjectStoreProviderGCS, - URI: "gs://wandb-bucket/prefix", + Path: "prefix", Bucket: "wandb-bucket", } rendered, err := renderBufstreamConfig("k", "k.ns.svc", []string{"e:2379"}, storage) @@ -93,9 +93,8 @@ func TestRenderBufstreamConfigGCS(t *testing.T) { func TestRenderBufstreamConfigAzure(t *testing.T) { storage := objectstore.ConnInfo{ Provider: apiv2.ObjectStoreProviderAzure, - URI: "https://acct.blob.core.windows.net/container", Bucket: "container", - AccessKey: "acct", + AccessKey: "wandbstorageacct", SecretKey: "azsupersecret", } rendered, err := renderBufstreamConfig("k", "k.ns.svc", []string{"e:2379"}, storage) @@ -105,7 +104,8 @@ func TestRenderBufstreamConfigAzure(t *testing.T) { require.NoError(t, yaml.Unmarshal([]byte(rendered), &parsed)) require.Nil(t, parsed.Data.S3) require.NotNil(t, parsed.Data.Azure) - require.Equal(t, "https://acct.blob.core.windows.net/container/k", parsed.Data.Azure.URI) + require.Equal(t, "https://wandbstorageacct.blob.core.windows.net/container/k", parsed.Data.Azure.URI, + "the blob host must derive from the connection's storage account") require.NotNil(t, parsed.Data.Azure.AccessKeyID) require.Equal(t, EnvStorageAccessKeyID, parsed.Data.Azure.AccessKeyID.EnvVar) require.NotContains(t, rendered, "azsupersecret") diff --git a/internal/controller/infra/managed/kafka/bufstream/naming.go b/internal/controller/infra/managed/kafka/bufstream/naming.go index b86a1b36..cc93fa60 100644 --- a/internal/controller/infra/managed/kafka/bufstream/naming.go +++ b/internal/controller/infra/managed/kafka/bufstream/naming.go @@ -4,9 +4,29 @@ import ( "fmt" "strings" + "github.com/wandb/operator/internal/controller/common" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" ) +// assumedMaxEtcdOrdinal covers quorum-sized etcd ensembles (1, 3, 5). +const assumedMaxEtcdOrdinal = 9 + +const defaultNameSuffix = "-kafka" + +// MaxSpecNameLength is the room left by the longest label-constrained derived +// name, the etcd pod name. +func MaxSpecNameLength() int { + builder := CreateNsNameBuilder(types.NamespacedName{}) + return validation.DNS1123LabelMaxLength - len(builder.EtcdPodName(assumedMaxEtcdOrdinal)) +} + +// DefaultSpecName derives the managed Kafka name for a CR instance, shortened +// to the budget. +func DefaultSpecName(crName, instanceKey string) string { + return common.FitDefaultInfraName(common.InstanceBaseName(crName, instanceKey), defaultNameSuffix, MaxSpecNameLength()) +} + // NsNameBuilder derives the names of all resources that make up a managed // Bufstream deployment from the base Kafka spec name/namespace. type NsNameBuilder struct { @@ -39,6 +59,24 @@ func (n *NsNameBuilder) BufstreamHost() string { return fmt.Sprintf("%s.%s.svc.cluster.local", n.BufstreamName(), n.Namespace()) } +// ServiceAccountName is the shared etcd/Bufstream identity for the SCC grant. +func (n *NsNameBuilder) ServiceAccountName() string { + return n.SpecName() +} + +func (n *NsNameBuilder) ServiceAccountNsName() types.NamespacedName { + return types.NamespacedName{Namespace: n.Namespace(), Name: n.ServiceAccountName()} +} + +// SccRoleBindingName grants the Kafka SA use of nonroot-v2 (OpenShift only). +func (n *NsNameBuilder) SccRoleBindingName() string { + return fmt.Sprintf("%s-scc-nonroot-v2", n.SpecName()) +} + +func (n *NsNameBuilder) SccRoleBindingNsName() types.NamespacedName { + return types.NamespacedName{Namespace: n.Namespace(), Name: n.SccRoleBindingName()} +} + func (n *NsNameBuilder) ConfigMapName() string { return fmt.Sprintf("%s-config", n.SpecName()) } diff --git a/internal/controller/infra/managed/kafka/bufstream/spec.go b/internal/controller/infra/managed/kafka/bufstream/spec.go index 920af7cc..50c850e8 100644 --- a/internal/controller/infra/managed/kafka/bufstream/spec.go +++ b/internal/controller/infra/managed/kafka/bufstream/spec.go @@ -5,10 +5,11 @@ import ( apiv2 "github.com/wandb/operator/api/v2" "github.com/wandb/operator/internal/controller/common" - "github.com/wandb/operator/internal/controller/infra/external/objectstore" + "github.com/wandb/operator/internal/controller/infra/objectstore" "github.com/wandb/operator/pkg/utils" "github.com/wandb/operator/pkg/wandb/manifest" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -33,6 +34,9 @@ const ( imageKeyBufstream = "bufstream" imageKeyEtcd = "etcd" imageKeyBucketEnsure = "bucketEnsure" + + bucketEnsureMaxAttempts = 150 + bucketEnsureDelaySeconds = 2 ) // BufstreamImage resolves the Bufstream broker image from the manifest, falling @@ -54,6 +58,8 @@ func BucketEnsureImage(img manifest.ImageRef, globalImageRegistry string) string return resolveImage(img, globalImageRegistry, defaultBucketEnsureImage) } +// resolveImage returns the manifest-supplied image, falling back to the given +// default for older manifests that omit it. func resolveImage(img manifest.ImageRef, globalImageRegistry, fallback string) string { if out := img.GetImage(globalImageRegistry); out != "" { return out @@ -62,14 +68,17 @@ func resolveImage(img manifest.ImageRef, globalImageRegistry, fallback string) s return fallback } +// BuildWandbKafkaLabels returns the standard W&B labels for the Kafka module. func BuildWandbKafkaLabels(wandb *apiv2.WeightsAndBiases) map[string]string { return common.BuildWandbLabels(wandb, KafkaModuleName) } +// ToKafkaOnDeleteRule builds the on-delete retention rule for the Kafka module. func ToKafkaOnDeleteRule(wandb *apiv2.WeightsAndBiases, retentionPolicy apiv2.RetentionPolicy) common.OnDeleteRule { return common.ToOnDeleteRule(wandb, retentionPolicy, KafkaModuleName) } +// kafkaPodSecurityContext: etcd omits fixed IDs on OpenShift, pins them else. func kafkaPodSecurityContext() *corev1.PodSecurityContext { if utils.IsOpenShift() { return &corev1.PodSecurityContext{ @@ -77,7 +86,25 @@ func kafkaPodSecurityContext() *corev1.PodSecurityContext { SeccompProfile: kafkaRuntimeDefaultSeccompProfile(), } } + return bufstreamPodSecurityContext() +} + +// kafkaContainerSecurityContext returns the container security context, dropping +// the fixed UID/GID on OpenShift where the platform assigns them. +func kafkaContainerSecurityContext() *corev1.SecurityContext { + if utils.IsOpenShift() { + return &corev1.SecurityContext{ + RunAsNonRoot: ptr.To(true), + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{kafkaCapabilityAll}}, + SeccompProfile: kafkaRuntimeDefaultSeccompProfile(), + } + } + return bufstreamContainerSecurityContext() +} +// bufstreamPodSecurityContext pins UID/GID 65532 (its 0700 binary needs it). +func bufstreamPodSecurityContext() *corev1.PodSecurityContext { return &corev1.PodSecurityContext{ RunAsUser: ptr.To(kafkaRunAsUser), RunAsGroup: ptr.To(kafkaRunAsGroup), @@ -88,22 +115,20 @@ func kafkaPodSecurityContext() *corev1.PodSecurityContext { } } -func kafkaContainerSecurityContext() *corev1.SecurityContext { - securityContext := &corev1.SecurityContext{ +// bufstreamContainerSecurityContext pins UID/GID 65532, which the broker's 0700 +// binary requires, and drops all capabilities. +func bufstreamContainerSecurityContext() *corev1.SecurityContext { + return &corev1.SecurityContext{ + RunAsUser: ptr.To(kafkaRunAsUser), + RunAsGroup: ptr.To(kafkaRunAsGroup), RunAsNonRoot: ptr.To(true), AllowPrivilegeEscalation: ptr.To(false), - Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{kafkaCapabilityAll}, - }, - SeccompProfile: kafkaRuntimeDefaultSeccompProfile(), - } - if !utils.IsOpenShift() { - securityContext.RunAsUser = ptr.To(kafkaRunAsUser) - securityContext.RunAsGroup = ptr.To(kafkaRunAsGroup) + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{kafkaCapabilityAll}}, + SeccompProfile: kafkaRuntimeDefaultSeccompProfile(), } - return securityContext } +// kafkaRuntimeDefaultSeccompProfile returns the RuntimeDefault seccomp profile. func kafkaRuntimeDefaultSeccompProfile() *corev1.SeccompProfile { return &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault} } @@ -115,6 +140,8 @@ func sameNamespace(wandb *apiv2.WeightsAndBiases, nsnBuilder *NsNameBuilder) boo return wandb.Namespace == nsnBuilder.Namespace() } +// setOwner sets the WeightsAndBiases controller reference on obj, but only when +// it shares the CR's namespace, since owner references are namespace-scoped. func setOwner(wandb *apiv2.WeightsAndBiases, obj metav1.Object, nsnBuilder *NsNameBuilder, scheme *runtime.Scheme) error { if !sameNamespace(wandb, nsnBuilder) { return nil @@ -122,6 +149,7 @@ func setOwner(wandb *apiv2.WeightsAndBiases, obj metav1.Object, nsnBuilder *NsNa return ctrl.SetControllerReference(wandb, obj, scheme) } +// intstrFromInt converts a port number to an IntOrString for service/probe specs. func intstrFromInt(port int) intstr.IntOrString { return intstr.FromInt32(int32(port)) } @@ -192,6 +220,64 @@ func ToConfigMap( return cm, nil } +// ToServiceAccount builds the dedicated etcd/Bufstream SA for the SCC grant. +func ToServiceAccount( + wandb *apiv2.WeightsAndBiases, + nsnBuilder *NsNameBuilder, + storage objectstore.ConnInfo, + scheme *runtime.Scheme, +) (*corev1.ServiceAccount, error) { + spec := wandb.Spec.Kafka.ManagedKafka + if spec.ServiceAccount.Create != nil && !*spec.ServiceAccount.Create { + return nil, nil + } + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: kafkaServiceAccountName(spec), + Namespace: nsnBuilder.Namespace(), + Labels: BuildWandbKafkaLabels(wandb), + Annotations: spec.ServiceAccount.Annotations, + }, + AutomountServiceAccountToken: ptr.To(!storage.HasStaticCredentials()), + } + if err := setOwner(wandb, sa, nsnBuilder, scheme); err != nil { + return nil, err + } + return sa, nil +} + +// ToSccRoleBinding binds the Kafka SA to nonroot-v2 for UID 65532 (OpenShift). +func ToSccRoleBinding( + wandb *apiv2.WeightsAndBiases, + nsnBuilder *NsNameBuilder, + scheme *runtime.Scheme, +) (*rbacv1.RoleBinding, error) { + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsnBuilder.SccRoleBindingName(), + Namespace: nsnBuilder.Namespace(), + Labels: BuildWandbKafkaLabels(wandb), + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: nonRootV2SCCClusterRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: kafkaServiceAccountName(wandb.Spec.Kafka.ManagedKafka), + Namespace: nsnBuilder.Namespace(), + }, + }, + } + if err := setOwner(wandb, rb, nsnBuilder, scheme); err != nil { + return nil, err + } + return rb, nil +} + // ToEtcdApplication builds the Application CR that deploys etcd as a highly // available StatefulSet: an odd-sized cluster (EtcdReplicas) fronted by a // headless Service that gives each member a stable peer DNS identity. @@ -244,13 +330,17 @@ func ToEtcdApplication( Replicas: ptr.To(int32(EtcdReplicas)), ServiceName: nsnBuilder.EtcdName(), MetaTemplate: metav1.ObjectMeta{ - Labels: labels, + Labels: utils.MergeMapsStringString( + labels, common.StandardLabels(wandb, "etcd", common.RoleDatabase, ""), + ), }, PodTemplate: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - SecurityContext: kafkaPodSecurityContext(), - Affinity: spreadAffinity(wandb, infraSpec.ManagedInfraSpec, labels), - Tolerations: tolerations(wandb, infraSpec.ManagedInfraSpec), + ServiceAccountName: kafkaServiceAccountName(infraSpec), + AutomountServiceAccountToken: ptr.To(false), + SecurityContext: kafkaPodSecurityContext(), + Affinity: spreadAffinity(wandb, infraSpec.ManagedInfraSpec, labels), + Tolerations: tolerations(wandb, infraSpec.ManagedInfraSpec), Containers: []corev1.Container{ { Name: "etcd", @@ -349,21 +439,26 @@ func spreadAffinity(wandb *apiv2.WeightsAndBiases, spec apiv2.ManagedInfraSpec, func bucketEnsureContainer(nsnBuilder *NsNameBuilder, storage objectstore.ConnInfo, img manifest.ImageRef, globalImageRegistry string) corev1.Container { region := storage.Region if region == "" { - region = "us-east-1" + region = objectstore.DefaultRegion } credsName := nsnBuilder.CredentialsName() - // head-bucket succeeds when the bucket already exists; otherwise create it. - // Both paths tolerate concurrent creation by other brokers. - script := fmt.Sprintf( - "aws --endpoint-url %q s3api head-bucket --bucket %q || "+ - "aws --endpoint-url %q s3api create-bucket --bucket %q", - storage.Endpoint, storage.Bucket, storage.Endpoint, storage.Bucket, - ) + // Retry in this process so transient DNS and API startup failures do not + // become init-container restarts subject to kubelet exponential backoff. + script := fmt.Sprintf(`attempt=1 +while [ "$attempt" -le %d ]; do + if { aws --endpoint-url "$1" s3api head-bucket --bucket "$2" || aws --endpoint-url "$1" s3api create-bucket --bucket "$2"; } >/dev/null 2>&1; then + exit 0 + fi + attempt=$((attempt + 1)) + if [ "$attempt" -le %d ]; then sleep %d; fi +done +echo 'object-store bucket did not become ready before timeout' >&2 +exit 1`, bucketEnsureMaxAttempts, bucketEnsureMaxAttempts, bucketEnsureDelaySeconds) return corev1.Container{ Name: "ensure-bucket", Image: BucketEnsureImage(img, globalImageRegistry), Command: []string{"/bin/sh", "-c"}, - Args: []string{script}, + Args: []string{script, "ensure-bucket", storage.EndpointURL(), storage.Bucket}, SecurityContext: kafkaContainerSecurityContext(), Env: []corev1.EnvVar{ {Name: "AWS_REGION", Value: region}, @@ -419,10 +514,9 @@ func storageCredentialEnv(nsnBuilder *NsNameBuilder, storage objectstore.ConnInf } } -// needsBucketEnsure reports whether to run the S3 bucket-creation init -// container. It only applies to S3-compatible endpoints (SeaweedFS, MinIO), -// which is where the operator provisions the bucket; AWS S3, GCS, and Azure -// buckets are expected to already exist. +// needsBucketEnsure reports whether a connection supports the managed +// S3 bucket initializer. The caller separately verifies that the selected +// object store is operator-managed so BYOB endpoints are never mutated. func needsBucketEnsure(storage objectstore.ConnInfo) bool { return storage.Provider == apiv2.ObjectStoreProviderS3 && storage.Endpoint != "" } @@ -447,6 +541,7 @@ func ToBufstreamApplication( wandb *apiv2.WeightsAndBiases, nsnBuilder *NsNameBuilder, storage objectstore.ConnInfo, + ensureBucket bool, scheme *runtime.Scheme, mfst manifest.Manifest, ) (*apiv2.Application, error) { @@ -459,7 +554,7 @@ func ToBufstreamApplication( Name: "bufstream", Image: BufstreamImage(mfst.Kafka.Images[imageKeyBufstream], wandb.Spec.Global.ImageRegistry), Args: []string{"serve", "--config", fmt.Sprintf("%s/%s", ConfigMountPath, ConfigFileName)}, - SecurityContext: kafkaContainerSecurityContext(), + SecurityContext: bufstreamContainerSecurityContext(), Ports: []corev1.ContainerPort{ {Name: "kafka", ContainerPort: KafkaListenerPort}, {Name: "metrics", ContainerPort: DebugPort}, @@ -475,7 +570,7 @@ func ToBufstreamApplication( } var initContainers []corev1.Container - if needsBucketEnsure(storage) { + if ensureBucket && needsBucketEnsure(storage) { initContainers = append(initContainers, bucketEnsureContainer(nsnBuilder, storage, mfst.Kafka.Images[imageKeyBucketEnsure], wandb.Spec.Global.ImageRegistry)) } @@ -489,15 +584,19 @@ func ToBufstreamApplication( Kind: "Deployment", Replicas: ptr.To(replicas), MetaTemplate: metav1.ObjectMeta{ - Labels: labels, + Labels: utils.MergeMapsStringString( + labels, common.StandardLabels(wandb, "kafka", common.RoleQueue, ""), + ), }, PodTemplate: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - SecurityContext: kafkaPodSecurityContext(), - Affinity: spreadAffinity(wandb, infraSpec.ManagedInfraSpec, labels), - Tolerations: tolerations(wandb, infraSpec.ManagedInfraSpec), - InitContainers: initContainers, - Containers: []corev1.Container{container}, + ServiceAccountName: kafkaServiceAccountName(infraSpec), + AutomountServiceAccountToken: ptr.To(!storage.HasStaticCredentials()), + SecurityContext: bufstreamPodSecurityContext(), + Affinity: spreadAffinity(wandb, infraSpec.ManagedInfraSpec, labels), + Tolerations: tolerations(wandb, infraSpec.ManagedInfraSpec), + InitContainers: initContainers, + Containers: []corev1.Container{container}, Volumes: []corev1.Volume{ { Name: "config", @@ -526,3 +625,12 @@ func ToBufstreamApplication( } return app, nil } + +// kafkaServiceAccountName returns the configured ServiceAccount name, defaulting +// to the spec name when unset. +func kafkaServiceAccountName(spec *apiv2.ManagedKafkaSpec) string { + if spec.ServiceAccount.ServiceAccountName != "" { + return spec.ServiceAccount.ServiceAccountName + } + return spec.Name +} diff --git a/internal/controller/infra/managed/kafka/bufstream/spec_test.go b/internal/controller/infra/managed/kafka/bufstream/spec_test.go index 260c2b83..04551b09 100644 --- a/internal/controller/infra/managed/kafka/bufstream/spec_test.go +++ b/internal/controller/infra/managed/kafka/bufstream/spec_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" apiv2 "github.com/wandb/operator/api/v2" - "github.com/wandb/operator/internal/controller/infra/external/objectstore" + "github.com/wandb/operator/internal/controller/infra/objectstore" "github.com/wandb/operator/pkg/utils" "github.com/wandb/operator/pkg/wandb/manifest" corev1 "k8s.io/api/core/v1" @@ -104,11 +104,14 @@ func TestToEtcdApplicationHA(t *testing.T) { } func testStorage() objectstore.ConnInfo { + // Mirror the managed SeaweedFS shape: a bare host with the port/scheme carried + // separately, so EndpointURL() must reassemble "http://seaweedfs:8333". return objectstore.ConnInfo{ Provider: apiv2.ObjectStoreProviderS3, URI: "s3://bucket", Bucket: "bucket", - Endpoint: "http://seaweedfs:80", + Endpoint: "seaweedfs", + Port: "8333", Region: "us-east-1", AccessKey: "ak", SecretKey: "sk", @@ -121,15 +124,26 @@ func TestToBufstreamApplication(t *testing.T) { wandb := testWandb() nsn := CreateNsNameBuilder(types.NamespacedName{Namespace: "default", Name: "wandb-kafka"}) - app, err := ToBufstreamApplication(wandb, nsn, testStorage(), testScheme(t), manifest.Manifest{}) + app, err := ToBufstreamApplication(wandb, nsn, testStorage(), true, testScheme(t), manifest.Manifest{}) require.NoError(t, err) require.Equal(t, "wandb-kafka", app.Name) require.Equal(t, "Deployment", app.Spec.Kind) require.NotNil(t, app.Spec.Replicas) require.Len(t, app.Spec.PodTemplate.Spec.InitContainers, 1) - require.Equal(t, "ensure-bucket", app.Spec.PodTemplate.Spec.InitContainers[0].Name) - requireKafkaContainerSecurityContext(t, app.Spec.PodTemplate.Spec.InitContainers[0].SecurityContext) + ensureBucket := app.Spec.PodTemplate.Spec.InitContainers[0] + require.Equal(t, "ensure-bucket", ensureBucket.Name) + requireKafkaContainerSecurityContext(t, ensureBucket.SecurityContext) + require.Len(t, ensureBucket.Args, 4) + require.Contains(t, ensureBucket.Args[0], "head-bucket") + require.Contains(t, ensureBucket.Args[0], "create-bucket") + require.Contains(t, ensureBucket.Args[0], "sleep 2") + require.Contains(t, ensureBucket.Args[0], "-le 150") + require.NotContains(t, ensureBucket.Args[0], testStorage().AccessKey) + require.NotContains(t, ensureBucket.Args[0], testStorage().SecretKey) + require.Equal(t, "http://seaweedfs:8333", ensureBucket.Args[2]) + require.Equal(t, testStorage().EndpointURL(), ensureBucket.Args[2]) + require.Equal(t, testStorage().Bucket, ensureBucket.Args[3]) require.Equal(t, int32(2), *app.Spec.Replicas) require.Len(t, app.Spec.PodTemplate.Spec.Containers, 1) @@ -147,34 +161,157 @@ func TestToBufstreamApplication(t *testing.T) { require.True(t, envNames[EnvStorageSecretAccessKey]) } +func TestBringYourOwnObjectStoresDoNotGetBucketInitializer(t *testing.T) { + setOpenShiftMode(t, false) + wandb := testWandb() + nsn := CreateNsNameBuilder(types.NamespacedName{Namespace: "default", Name: "wandb-kafka"}) + + tests := map[string]objectstore.ConnInfo{ + "custom S3": testStorage(), + "native AWS": {Provider: apiv2.ObjectStoreProviderS3, URI: "s3://bucket", Bucket: "bucket", Region: "us-east-1"}, + "GCS": {Provider: apiv2.ObjectStoreProviderGCS, URI: "gs://bucket", Bucket: "bucket"}, + "Azure": {Provider: apiv2.ObjectStoreProviderAzure, URI: "https://account.blob.core.windows.net/container", Bucket: "container"}, + } + for name, storage := range tests { + t.Run(name, func(t *testing.T) { + app, err := ToBufstreamApplication(wandb, nsn, storage, false, testScheme(t), manifest.Manifest{}) + require.NoError(t, err) + require.Empty(t, app.Spec.PodTemplate.Spec.InitContainers) + }) + } +} + func TestToBufstreamApplicationDefaultsReplicas(t *testing.T) { setOpenShiftMode(t, false) wandb := testWandb() wandb.Spec.Kafka.ManagedKafka.Replicas = 0 nsn := CreateNsNameBuilder(types.NamespacedName{Namespace: "default", Name: "wandb-kafka"}) - app, err := ToBufstreamApplication(wandb, nsn, testStorage(), testScheme(t), manifest.Manifest{}) + app, err := ToBufstreamApplication(wandb, nsn, testStorage(), true, testScheme(t), manifest.Manifest{}) require.NoError(t, err) require.Equal(t, int32(BufstreamReplicas), *app.Spec.Replicas) } -func TestApplicationsOmitFixedIDsInOpenShiftMode(t *testing.T) { +func TestApplicationsSecurityContextInOpenShiftMode(t *testing.T) { setOpenShiftMode(t, true) wandb := testWandb() nsn := CreateNsNameBuilder(types.NamespacedName{Namespace: "default", Name: "wandb-kafka"}) + // etcd tolerates an arbitrary UID, so it omits fixed IDs for restricted-v2. etcd, err := ToEtcdApplication(wandb, nsn, testScheme(t), manifest.Manifest{}) require.NoError(t, err) requireOpenShiftKafkaPodSecurityContext(t, etcd.Spec.PodTemplate.Spec.SecurityContext) requireOpenShiftKafkaContainerSecurityContext(t, etcd.Spec.PodTemplate.Spec.Containers[0].SecurityContext) - bufstream, err := ToBufstreamApplication(wandb, nsn, testStorage(), testScheme(t), manifest.Manifest{}) + // Bufstream keeps its fixed UID even on OpenShift (nonroot-v2 admits it). + bufstream, err := ToBufstreamApplication(wandb, nsn, testStorage(), true, testScheme(t), manifest.Manifest{}) require.NoError(t, err) - requireOpenShiftKafkaPodSecurityContext(t, bufstream.Spec.PodTemplate.Spec.SecurityContext) - requireOpenShiftKafkaContainerSecurityContext(t, bufstream.Spec.PodTemplate.Spec.Containers[0].SecurityContext) + requireKafkaPodSecurityContext(t, bufstream.Spec.PodTemplate.Spec.SecurityContext) + requireKafkaContainerSecurityContext(t, bufstream.Spec.PodTemplate.Spec.Containers[0].SecurityContext) + // The bucket-ensure init container omits a fixed UID and inherits the pod's. requireOpenShiftKafkaContainerSecurityContext(t, bufstream.Spec.PodTemplate.Spec.InitContainers[0].SecurityContext) } +func TestApplicationsUseDedicatedServiceAccount(t *testing.T) { + setOpenShiftMode(t, false) + wandb := testWandb() + nsn := CreateNsNameBuilder(types.NamespacedName{Namespace: "default", Name: "wandb-kafka"}) + + etcd, err := ToEtcdApplication(wandb, nsn, testScheme(t), manifest.Manifest{}) + require.NoError(t, err) + require.Equal(t, nsn.ServiceAccountName(), etcd.Spec.PodTemplate.Spec.ServiceAccountName) + require.NotNil(t, etcd.Spec.PodTemplate.Spec.AutomountServiceAccountToken) + require.False(t, *etcd.Spec.PodTemplate.Spec.AutomountServiceAccountToken) + + bufstream, err := ToBufstreamApplication(wandb, nsn, testStorage(), true, testScheme(t), manifest.Manifest{}) + require.NoError(t, err) + require.Equal(t, nsn.ServiceAccountName(), bufstream.Spec.PodTemplate.Spec.ServiceAccountName) + require.NotNil(t, bufstream.Spec.PodTemplate.Spec.AutomountServiceAccountToken) + require.False(t, *bufstream.Spec.PodTemplate.Spec.AutomountServiceAccountToken) +} + +func TestApplicationsUseWorkloadIdentityForAmbientCredentials(t *testing.T) { + setOpenShiftMode(t, false) + wandb := testWandb() + wandb.Spec.Kafka.ManagedKafka.ServiceAccount = apiv2.ManagedServiceAccountSpec{ + ServiceAccountName: "kafka-workload-identity", + Annotations: map[string]string{ + "eks.amazonaws.com/role-arn": "arn:aws:iam::123456789012:role/wandb-kafka", + }, + } + nsn := CreateNsNameBuilder(types.NamespacedName{Namespace: "default", Name: "wandb-kafka"}) + storage := testStorage() + storage.AccessKey = "" + storage.SecretKey = "" + + sa, err := ToServiceAccount(wandb, nsn, storage, testScheme(t)) + require.NoError(t, err) + require.Equal(t, "kafka-workload-identity", sa.Name) + require.Equal(t, wandb.Spec.Kafka.ManagedKafka.ServiceAccount.Annotations, sa.Annotations) + require.NotNil(t, sa.AutomountServiceAccountToken) + require.True(t, *sa.AutomountServiceAccountToken) + + bufstream, err := ToBufstreamApplication(wandb, nsn, storage, false, testScheme(t), manifest.Manifest{}) + require.NoError(t, err) + require.Equal(t, sa.Name, bufstream.Spec.PodTemplate.Spec.ServiceAccountName) + require.NotNil(t, bufstream.Spec.PodTemplate.Spec.AutomountServiceAccountToken) + require.True(t, *bufstream.Spec.PodTemplate.Spec.AutomountServiceAccountToken) + + etcd, err := ToEtcdApplication(wandb, nsn, testScheme(t), manifest.Manifest{}) + require.NoError(t, err) + require.Equal(t, sa.Name, etcd.Spec.PodTemplate.Spec.ServiceAccountName) + require.NotNil(t, etcd.Spec.PodTemplate.Spec.AutomountServiceAccountToken) + require.False(t, *etcd.Spec.PodTemplate.Spec.AutomountServiceAccountToken) +} + +func TestToServiceAccount(t *testing.T) { + wandb := testWandb() + nsn := CreateNsNameBuilder(types.NamespacedName{Namespace: "default", Name: "wandb-kafka"}) + + sa, err := ToServiceAccount(wandb, nsn, testStorage(), testScheme(t)) + require.NoError(t, err) + require.Equal(t, nsn.ServiceAccountName(), sa.Name) + require.Equal(t, "default", sa.Namespace) + require.NotNil(t, sa.AutomountServiceAccountToken) + require.False(t, *sa.AutomountServiceAccountToken) + // Same-namespace resources are owned by the CR for GC. + require.Len(t, sa.OwnerReferences, 1) +} + +func TestToServiceAccountCanReferenceExistingIdentity(t *testing.T) { + wandb := testWandb() + create := false + wandb.Spec.Kafka.ManagedKafka.ServiceAccount = apiv2.ManagedServiceAccountSpec{ + Create: &create, + ServiceAccountName: "existing-kafka-identity", + } + nsn := CreateNsNameBuilder(types.NamespacedName{Namespace: "default", Name: "wandb-kafka"}) + + sa, err := ToServiceAccount(wandb, nsn, testStorage(), testScheme(t)) + require.NoError(t, err) + require.Nil(t, sa) + + app, err := ToBufstreamApplication(wandb, nsn, testStorage(), false, testScheme(t), manifest.Manifest{}) + require.NoError(t, err) + require.Equal(t, "existing-kafka-identity", app.Spec.PodTemplate.Spec.ServiceAccountName) +} + +func TestToSccRoleBinding(t *testing.T) { + wandb := testWandb() + nsn := CreateNsNameBuilder(types.NamespacedName{Namespace: "default", Name: "wandb-kafka"}) + + rb, err := ToSccRoleBinding(wandb, nsn, testScheme(t)) + require.NoError(t, err) + require.Equal(t, nsn.SccRoleBindingName(), rb.Name) + require.Equal(t, "default", rb.Namespace) + require.Equal(t, "ClusterRole", rb.RoleRef.Kind) + require.Equal(t, nonRootV2SCCClusterRole, rb.RoleRef.Name) + require.Len(t, rb.Subjects, 1) + require.Equal(t, "ServiceAccount", rb.Subjects[0].Kind) + require.Equal(t, nsn.ServiceAccountName(), rb.Subjects[0].Name) + require.Equal(t, "default", rb.Subjects[0].Namespace) +} + func TestToCredentialsSecret(t *testing.T) { wandb := testWandb() nsn := CreateNsNameBuilder(types.NamespacedName{Namespace: "default", Name: "wandb-kafka"}) diff --git a/internal/controller/infra/managed/kafka/bufstream/values.go b/internal/controller/infra/managed/kafka/bufstream/values.go index 78adb59b..0d12e4d8 100644 --- a/internal/controller/infra/managed/kafka/bufstream/values.go +++ b/internal/controller/infra/managed/kafka/bufstream/values.go @@ -15,9 +15,6 @@ const ( // etcd image, which is configured via native ETCD_* environment variables. defaultEtcdImage = "quay.io/coreos/etcd:v3.5.31" - // defaultBucketEnsureImage runs a one-shot init container that creates the - // object-store bucket Bufstream expects, since Bufstream does not create it - // itself and reads from it on startup. defaultBucketEnsureImage = "amazon/aws-cli:2.35.10" // Kafka-compatible listener exposed by Bufstream. @@ -73,8 +70,13 @@ const ( ) const ( - ApplicationResourceType = "Application" - ConfigMapResourceType = "ConfigMap" - SecretResourceType = "Secret" - AppConnTypeName = "KafkaAppConn" + ApplicationResourceType = "Application" + ConfigMapResourceType = "ConfigMap" + SecretResourceType = "Secret" + ServiceAccountResourceType = "ServiceAccount" + RoleBindingResourceType = "RoleBinding" + AppConnTypeName = "KafkaAppConn" ) + +// nonRootV2SCCClusterRole is the auto-generated ClusterRole for nonroot-v2. +const nonRootV2SCCClusterRole = "system:openshift:scc:nonroot-v2" diff --git a/internal/controller/infra/managed/kafka/bufstream/write.go b/internal/controller/infra/managed/kafka/bufstream/write.go index beba27eb..27a916a3 100644 --- a/internal/controller/infra/managed/kafka/bufstream/write.go +++ b/internal/controller/infra/managed/kafka/bufstream/write.go @@ -5,10 +5,12 @@ import ( apiv2 "github.com/wandb/operator/api/v2" "github.com/wandb/operator/internal/controller/common" - "github.com/wandb/operator/internal/controller/infra/external/objectstore" + "github.com/wandb/operator/internal/controller/infra/objectstore" "github.com/wandb/operator/internal/logx" + "github.com/wandb/operator/pkg/utils" "github.com/wandb/operator/pkg/wandb/manifest" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,7 +30,7 @@ func WriteState( spec := wandb.Spec.Kafka.ManagedKafka nsnBuilder := CreateNsNameBuilder(types.NamespacedName{Namespace: spec.Namespace, Name: spec.Name}) - storage, ready, err := resolveStorage(ctx, cl, wandb) + storage, ready, err := resolveStorage(ctx, cl, wandb, spec) if err != nil { log.Error("failed to resolve object store connection for bufstream", logx.ErrAttr(err)) return []metav1.Condition{ @@ -42,6 +44,8 @@ func WriteState( {Type: ObjectStoreReadyType, Status: metav1.ConditionFalse, Reason: common.PendingCreateReason}, } } + //objectStoreSpec, _ := apiv2.ResolveInstance(wandb.Spec.ObjectStore, bufstreamObjectStoreInstance) + ensureBucket := false credsSecret, err := ToCredentialsSecret(wandb, nsnBuilder, storage, cl.Scheme()) if err != nil { @@ -51,11 +55,23 @@ func WriteState( if err != nil { return translateError(err) } + serviceAccount, err := ToServiceAccount(wandb, nsnBuilder, storage, cl.Scheme()) + if err != nil { + return translateError(err) + } + // On OpenShift, bind the SA to nonroot-v2 so broker runs as its fixed UID. + var sccRoleBinding *rbacv1.RoleBinding + if utils.IsOpenShift() { + sccRoleBinding, err = ToSccRoleBinding(wandb, nsnBuilder, cl.Scheme()) + if err != nil { + return translateError(err) + } + } etcdApp, err := ToEtcdApplication(wandb, nsnBuilder, cl.Scheme(), mfst) if err != nil { return translateError(err) } - bufstreamApp, err := ToBufstreamApplication(wandb, nsnBuilder, storage, cl.Scheme(), mfst) + bufstreamApp, err := ToBufstreamApplication(wandb, nsnBuilder, storage, ensureBucket, cl.Scheme(), mfst) if err != nil { return translateError(err) } @@ -65,12 +81,23 @@ func WriteState( } results = append(results, writeResource(ctx, cl, common.ReconciledType, SecretResourceType, credsSecret, &corev1.Secret{})...) results = append(results, writeResource(ctx, cl, common.ReconciledType, ConfigMapResourceType, configMap, &corev1.ConfigMap{})...) + if serviceAccount != nil { + serviceAccountConditions := writeResource(ctx, cl, common.ReconciledType, ServiceAccountResourceType, serviceAccount, &corev1.ServiceAccount{}) + results = append(results, serviceAccountConditions...) + if len(serviceAccountConditions) > 0 { + return results + } + } + if sccRoleBinding != nil { + results = append(results, writeResource(ctx, cl, common.ReconciledType, RoleBindingResourceType, sccRoleBinding, &rbacv1.RoleBinding{})...) + } results = append(results, writeResource(ctx, cl, EtcdApplicationType, ApplicationResourceType, etcdApp, &apiv2.Application{})...) results = append(results, writeResource(ctx, cl, BufstreamApplicationType, ApplicationResourceType, bufstreamApp, &apiv2.Application{})...) return results } +// translateError wraps an error as a failed Reconciled condition. func translateError(err error) []metav1.Condition { return []metav1.Condition{ {Type: common.ReconciledType, Status: metav1.ConditionFalse, Reason: common.ControllerErrorReason, Message: err.Error()}, @@ -114,19 +141,25 @@ func writeResource[T client.Object]( return []metav1.Condition{actionToCondition(conditionType, action)} } +// actionToCondition maps a CRUD action onto the status condition it implies. func actionToCondition(conditionType string, action common.CrudAction) metav1.Condition { switch action { case common.CreateAction: return metav1.Condition{Type: conditionType, Status: metav1.ConditionFalse, Reason: common.PendingCreateReason} case common.DeleteAction: return metav1.Condition{Type: conditionType, Status: metav1.ConditionFalse, Reason: common.PendingDeleteReason} - case common.UpdateAction: + case common.UpdateAction, common.UnchangedAction: return metav1.Condition{Type: conditionType, Status: metav1.ConditionTrue, Reason: common.ResourceExistsReason} default: return metav1.Condition{Type: conditionType, Status: metav1.ConditionFalse, Reason: common.NoResourceReason} } } +// bufstreamObjectStoreInstance is the object-store instance name Bufstream +// prefers for message storage; ResolveInstance falls back to the default +// instance when it is not provisioned. +const bufstreamObjectStoreInstance = "bufstream" + // resolveStorage reads the object store connection secret and parses its // connection string into the provider-specific values needed to configure // Bufstream. Returns ready=false when the object store is not yet available. @@ -134,32 +167,16 @@ func resolveStorage( ctx context.Context, cl client.Client, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedKafkaSpec, ) (objectstore.ConnInfo, bool, error) { - if !wandb.Status.ObjectStoreStatus.Ready { - return objectstore.ConnInfo{}, false, nil - } - - secretName := wandb.Status.ObjectStoreStatus.Connection.URL.Name - if secretName == "" { - return objectstore.ConnInfo{}, false, nil - } - - secret := &corev1.Secret{} - found, err := common.GetResource( - ctx, cl, - types.NamespacedName{Namespace: wandb.Namespace, Name: secretName}, - "Secret", secret, - ) - if err != nil { - return objectstore.ConnInfo{}, false, err - } - if !found { + status, ok := apiv2.ResolveInstance(wandb.Status.ObjectStoreStatus, bufstreamObjectStoreInstance) + if !ok || !status.Ready { return objectstore.ConnInfo{}, false, nil } - info, err := objectstore.ParseConnection(secret.Data) + connInfo, err := objectstore.Resolve(ctx, cl, spec.Namespace, &status.Connection) if err != nil { - return objectstore.ConnInfo{}, false, err + return objectstore.ConnInfo{}, true, err } - return info, true, nil + return connInfo, true, nil } diff --git a/internal/controller/infra/managed/kafka/bufstream/write_test.go b/internal/controller/infra/managed/kafka/bufstream/write_test.go new file mode 100644 index 00000000..c1b78b03 --- /dev/null +++ b/internal/controller/infra/managed/kafka/bufstream/write_test.go @@ -0,0 +1,102 @@ +package bufstream + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + apiv2 "github.com/wandb/operator/api/v2" +) + +// resolveStorageFixture seeds a connection secret with the given keys and a +// wandb whose default object-store status selects every key from it. +func resolveStorageFixture(t *testing.T, data map[string]string) (ctrlclient.Client, *apiv2.WeightsAndBiases, *apiv2.ManagedKafkaSpec) { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, apiv2.AddToScheme(scheme)) + + secretData := map[string][]byte{} + for k, v := range data { + secretData[k] = []byte(v) + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb-objectstore-connection", Namespace: "wandb"}, + Data: secretData, + } + + sel := func(key string) corev1.SecretKeySelector { + return corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: secret.Name}, + Key: key, + } + } + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "wandb"}, + Status: apiv2.WeightsAndBiasesStatus{ + ObjectStoreStatus: map[string]apiv2.ObjectStoreInfraStatus{ + apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + Connection: apiv2.ObjectStoreConnection{ + Provider: sel("Provider"), + Bucket: sel("Bucket"), + Endpoint: sel("Host"), + Port: sel("Port"), + Region: sel("Region"), + AccessKey: sel("AccessKey"), + SecretKey: sel("SecretKey"), + ForcePathStyle: sel("ForcePathStyle"), + TlsEnabled: sel("TlsEnabled"), + }, + }, + }, + }, + } + + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(secret).Build() + return cl, wandb, &apiv2.ManagedKafkaSpec{Namespace: "wandb"} +} + +func TestResolveStorage_ForcePathStyleFallbackOnMissingKey(t *testing.T) { + cl, wandb, spec := resolveStorageFixture(t, map[string]string{ + "Provider": "s3", + "Bucket": "wandb", + "Host": "minio.wandb.localhost", + "Port": "8080", + }) + + info, ready, err := resolveStorage(context.Background(), cl, wandb, spec) + require.NoError(t, err) + require.True(t, ready) + require.True(t, info.ForcePathStyle, "secrets predating the derived key must fall back to the endpoint rule") +} + +func TestResolveStorage_ForcePathStyleExplicitRespected(t *testing.T) { + cl, wandb, spec := resolveStorageFixture(t, map[string]string{ + "Provider": "s3", + "Bucket": "wandb", + "Host": "minio.wandb.localhost", + "ForcePathStyle": "false", + }) + + info, _, err := resolveStorage(context.Background(), cl, wandb, spec) + require.NoError(t, err) + require.False(t, info.ForcePathStyle) +} + +func TestResolveStorage_NoEndpointStaysVirtualHosted(t *testing.T) { + cl, wandb, spec := resolveStorageFixture(t, map[string]string{ + "Provider": "s3", + "Bucket": "wandb", + }) + + info, _, err := resolveStorage(context.Background(), cl, wandb, spec) + require.NoError(t, err) + require.False(t, info.ForcePathStyle, "native AWS S3 must not force path-style") +} diff --git a/internal/controller/infra/managed/mysql/moco/naming.go b/internal/controller/infra/managed/mysql/moco/naming.go index 82fc9787..86409661 100644 --- a/internal/controller/infra/managed/mysql/moco/naming.go +++ b/internal/controller/infra/managed/mysql/moco/naming.go @@ -3,9 +3,21 @@ package moco import ( "fmt" + "github.com/wandb/operator/internal/controller/common" "k8s.io/apimachinery/pkg/types" ) +// MaxClusterNameLength mirrors Moco's admission cap on MySQLCluster names. +const MaxClusterNameLength = 40 + +const defaultNameSuffix = "-mysql" + +// DefaultSpecName derives the managed MySQL name for a CR instance, shortened +// to Moco's cap. +func DefaultSpecName(crName, instanceKey string) string { + return common.FitDefaultInfraName(common.InstanceBaseName(crName, instanceKey), defaultNameSuffix, MaxClusterNameLength) +} + type NsNameBuilder struct { baseNsName types.NamespacedName } diff --git a/internal/controller/infra/managed/mysql/moco/spec.go b/internal/controller/infra/managed/mysql/moco/spec.go index 89c88c48..79612a93 100644 --- a/internal/controller/infra/managed/mysql/moco/spec.go +++ b/internal/controller/infra/managed/mysql/moco/spec.go @@ -80,6 +80,12 @@ func ToMocoMySQLClusterSpec( Replicas: replicas, MySQLConfigMapName: ptr.To(MyCnfConfigMapName(spec.Name)), PodTemplate: mocov1beta2.PodTemplateSpec{ + ObjectMeta: mocov1beta2.ObjectMeta{ + Labels: utils.MergeMapsStringString( + BuildWandbMysqlLabels(wandb), + common.StandardLabels(wandb, "mysql", common.RoleDatabase, ""), + ), + }, Spec: buildMocoPodSpec(spec.Config.Resources, mfst.Mysql["default"].Images["mysql"], wandb), OverwriteContainers: mocoOverwriteContainers(), }, diff --git a/internal/controller/infra/managed/mysql/moco/write.go b/internal/controller/infra/managed/mysql/moco/write.go index 1376fd04..15325fb2 100644 --- a/internal/controller/infra/managed/mysql/moco/write.go +++ b/internal/controller/infra/managed/mysql/moco/write.go @@ -155,7 +155,7 @@ func WriteState( Status: metav1.ConditionFalse, Reason: common.PendingDeleteReason, }) - case common.UpdateAction: + case common.UpdateAction, common.UnchangedAction: result = append(result, metav1.Condition{ Type: MySQLCustomResourceType, Status: metav1.ConditionTrue, diff --git a/internal/controller/infra/managed/objectstore/seaweedfs/conn.go b/internal/controller/infra/managed/objectstore/seaweedfs/conn.go index 2a29f883..cf5b577a 100644 --- a/internal/controller/infra/managed/objectstore/seaweedfs/conn.go +++ b/internal/controller/infra/managed/objectstore/seaweedfs/conn.go @@ -7,6 +7,7 @@ import ( apiv2 "github.com/wandb/operator/api/v2" "github.com/wandb/operator/internal/controller/common" + "github.com/wandb/operator/internal/controller/infra/objectstore" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -19,52 +20,61 @@ const ( S3Port = "8333" ) -type s3ConnInfo struct { - AccessKey string - SecretKey string - Host string - Port string - Bucket string - TLS bool -} - +// buildS3ConnInfo assembles the ConnInfo for the managed SeaweedFS S3 gateway: +// the in-cluster service host/port, path-style addressing, and default region. func buildS3ConnInfo( accessKey, secretKey string, nsnBuilder *NsNameBuilder, tls bool, -) *s3ConnInfo { - namespace := nsnBuilder.Namespace() - serviceName := fmt.Sprintf("%s-s3", SeaweedName(nsnBuilder.SpecName())) - return &s3ConnInfo{ - AccessKey: accessKey, - TLS: tls, - SecretKey: secretKey, - Host: fmt.Sprintf("%s.%s.svc.cluster.local", serviceName, namespace), - Port: "80", - Bucket: "bucket", +) *objectstore.ConnInfo { + connInfo := &objectstore.ConnInfo{ + Provider: apiv2.ObjectStoreProviderS3, + AccessKey: accessKey, + SecretKey: secretKey, + Endpoint: s3ServiceHost(nsnBuilder.SpecName(), nsnBuilder.Namespace()), + Port: S3Port, + Region: objectstore.DefaultRegion, + Bucket: "bucket", + Scheme: objectstore.SchemeForTLS(tls), + TlsEnabled: tls, + ForcePathStyle: true, } + connInfo.URL = managedS3URL(connInfo) + return connInfo } -func (s *s3ConnInfo) toUrl() *url.URL { - return &url.URL{ +// managedS3URL builds the canonical connection URL the W&B server signs against: +// s3://:@:/?tls=. +func managedS3URL(connInfo *objectstore.ConnInfo) string { + s3URL := &url.URL{ Scheme: S3UrlScheme, - Host: fmt.Sprintf("%s:%s", s.Host, s.Port), - User: url.UserPassword(s.AccessKey, s.SecretKey), - Path: s.Bucket, + Host: fmt.Sprintf("%s:%s", connInfo.Endpoint, connInfo.Port), + User: url.UserPassword(connInfo.AccessKey, connInfo.SecretKey), + Path: connInfo.Bucket, } + return fmt.Sprintf("%s?tls=%t", s3URL.String(), connInfo.TlsEnabled) } -func (s *s3ConnInfo) scheme() string { - if s.TLS { - return "https" - } - return "http" +// s3ServiceHost returns the in-cluster FQDN of the SeaweedFS S3 service. +func s3ServiceHost(specName, namespace string) string { + return fmt.Sprintf("%s-s3.%s.svc.cluster.local", SeaweedName(specName), namespace) } +// s3ExternalURL is the endpoint the W&B server signs S3 requests against +// (it presigns with this host and rewrites the URL for external clients +// without re-signing). The s3 gateway must verify signatures against this +// host rather than the Host/X-Forwarded-Host of proxied requests. +func s3ExternalURL(specName, namespace string, tls bool) string { + return fmt.Sprintf("%s://%s:%s", objectstore.SchemeForTLS(tls), s3ServiceHost(specName, namespace), S3Port) +} + +// writeWandbConnInfo writes the connection secret consumed by W&B and returns +// the ObjectStoreConnection with every selector required, since managed +// SeaweedFS always persists the full key set. func writeWandbConnInfo( ctx context.Context, cl client.Client, owner client.Object, nsnBuilder *NsNameBuilder, - connInfo *s3ConnInfo, + connInfo *objectstore.ConnInfo, ) ( *apiv2.ObjectStoreConnection, error, ) { @@ -74,7 +84,6 @@ func writeWandbConnInfo( var actual = &corev1.Secret{} nsName := nsnBuilder.ConnectionNsName() - urlKey := "url" if found, err = common.GetResource( ctx, cl, nsName, AppConnTypeName, actual, @@ -103,31 +112,14 @@ func writeWandbConnInfo( Namespace: nsName.Namespace, OwnerReferences: []metav1.OwnerReference{ref}, }, - Type: corev1.SecretTypeOpaque, - StringData: map[string]string{ - urlKey: fmt.Sprintf("%s?tls=%t", connInfo.toUrl().String(), connInfo.TLS), - "Host": connInfo.Host, - "Port": connInfo.Port, - "AccessKey": connInfo.AccessKey, - "SecretKey": connInfo.SecretKey, - "Region": "us-east-1", - "Bucket": connInfo.Bucket, - "Scheme": connInfo.scheme(), - }, + Type: corev1.SecretTypeOpaque, + StringData: connInfo.ToSecretData(), } if _, err = common.CrudResource(ctx, cl, desired, actual); err != nil { return nil, err } - localRef := corev1.LocalObjectReference{Name: nsName.Name} - return &apiv2.ObjectStoreConnection{ - URL: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: urlKey, Optional: ptr.To(false)}, - Endpoint: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Host", Optional: ptr.To(false)}, - Port: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Port", Optional: ptr.To(false)}, - AccessKey: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "AccessKey", Optional: ptr.To(false)}, - SecretKey: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "SecretKey", Optional: ptr.To(false)}, - Region: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Region", Optional: ptr.To(false)}, - Bucket: corev1.SecretKeySelector{LocalObjectReference: localRef, Key: "Bucket", Optional: ptr.To(false)}, - }, nil + // Managed SeaweedFS always writes the full key set, so every selector is required. + return connInfo.ToObjectStoreConnection(nsName.Name, true), nil } diff --git a/internal/controller/infra/managed/objectstore/seaweedfs/naming.go b/internal/controller/infra/managed/objectstore/seaweedfs/naming.go index f43ef902..84bed20e 100644 --- a/internal/controller/infra/managed/objectstore/seaweedfs/naming.go +++ b/internal/controller/infra/managed/objectstore/seaweedfs/naming.go @@ -4,9 +4,23 @@ import ( "fmt" "strings" + "github.com/wandb/operator/internal/controller/common" "k8s.io/apimachinery/pkg/types" ) +// MaxSpecNameLength is a conservative budget: the seaweedfs operator suffixes +// the Seaweed name ("-master", "-volume", "-filer", peer Services, ordinals); +// 40 leaves 23 chars of DNS-1123 label headroom. +const MaxSpecNameLength = 40 + +const defaultNameSuffix = "-seaweedfs" + +// DefaultSpecName derives the object-store name for a CR instance, shortened +// to the budget; the suffix is preserved so ConnectionName can still strip it. +func DefaultSpecName(crName, instanceKey string) string { + return common.FitDefaultInfraName(common.InstanceBaseName(crName, instanceKey), defaultNameSuffix, MaxSpecNameLength) +} + type NsNameBuilder struct { baseNsName types.NamespacedName } diff --git a/internal/controller/infra/managed/objectstore/seaweedfs/read.go b/internal/controller/infra/managed/objectstore/seaweedfs/read.go index 76f405a9..0c1803fc 100644 --- a/internal/controller/infra/managed/objectstore/seaweedfs/read.go +++ b/internal/controller/infra/managed/objectstore/seaweedfs/read.go @@ -2,6 +2,13 @@ package seaweedfs import ( "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" ctrlcommon "github.com/wandb/operator/internal/controller/common" "github.com/wandb/operator/internal/logx" @@ -11,6 +18,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const seaweedProbeTimeout = 5 * time.Second + func ReadState( ctx context.Context, k8sClient client.Client, @@ -65,12 +74,145 @@ func ReadState( } if actualResource != nil { - conditions = append(conditions, computeSeaweedReportedReadyCondition(ctx, actualResource)...) + readyConditions := computeSeaweedReportedReadyCondition(ctx, actualResource) + conditions = append(conditions, readyConditions...) + if readyConditions[0].Status == metav1.ConditionTrue { + conditions = append(conditions, computeSeaweedWritableCondition(ctx, actualResource)) + conditions = append(conditions, computeSeaweedS3ReachableCondition(ctx, actualResource)) + } } - log.Debug("read", "actualResource", actualResource, "rule", onDeleteRule.Policy) + log.Debug("read", "resourceExists", actualResource != nil, "rule", onDeleteRule.Policy) return conditions } +type seaweedAssignResponse struct { + FID string `json:"fid"` + Error string `json:"error"` +} + +func computeSeaweedWritableCondition(ctx context.Context, cr *seaweedv1.Seaweed) metav1.Condition { + scheme := "http" + transport := http.DefaultTransport + if cr.Spec.TLS != nil && cr.Spec.TLS.Enabled { + scheme = "https" + tlsTransport := http.DefaultTransport.(*http.Transport).Clone() + // The Seaweed operator generates an internal certificate whose CA is not + // mounted into this controller. The request never leaves the cluster DNS + // name and only verifies that the master can allocate writable storage. + tlsTransport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: true} // #nosec G402 + transport = tlsTransport + } + + endpoint := url.URL{ + Scheme: scheme, + Host: fmt.Sprintf( + "%s-master.%s.svc.cluster.local:%d", + cr.Name, + cr.Namespace, + seaweedv1.MasterHTTPPort, + ), + Path: "/dir/assign", + } + query := endpoint.Query() + query.Set("count", "1") + if cr.Spec.Master != nil && cr.Spec.Master.DefaultReplication != nil { + query.Set("replication", *cr.Spec.Master.DefaultReplication) + } + endpoint.RawQuery = query.Encode() + + return probeSeaweedAllocation(ctx, &http.Client{Transport: transport, Timeout: seaweedProbeTimeout}, endpoint.String()) +} + +func computeSeaweedS3ReachableCondition(ctx context.Context, cr *seaweedv1.Seaweed) metav1.Condition { + scheme := "http" + transport := http.DefaultTransport + if cr.Spec.TLS != nil && cr.Spec.TLS.Enabled { + scheme = "https" + tlsTransport := http.DefaultTransport.(*http.Transport).Clone() + // The generated internal CA is not mounted into this controller. This + // unauthenticated probe stays on the cluster-local service address. + tlsTransport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: true} // #nosec G402 + transport = tlsTransport + } + endpoint := url.URL{ + Scheme: scheme, + Host: fmt.Sprintf("%s-s3.%s.svc.cluster.local:%s", cr.Name, cr.Namespace, S3Port), + Path: "/", + } + return probeSeaweedS3(ctx, &http.Client{Transport: transport, Timeout: seaweedProbeTimeout}, endpoint.String()) +} + +func probeSeaweedS3(ctx context.Context, client *http.Client, endpoint string) metav1.Condition { + condition := metav1.Condition{Type: SeaweedS3ReachableType, Status: metav1.ConditionFalse, Reason: "EndpointUnavailable"} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + condition.Message = err.Error() + return condition + } + response, err := client.Do(req) + if err != nil { + condition.Message = err.Error() + return condition + } + defer response.Body.Close() + // An unauthenticated S3 service commonly returns 403. Any non-5xx response + // proves cluster DNS resolved and the S3 API accepted the connection. + if response.StatusCode >= http.StatusInternalServerError { + condition.Message = fmt.Sprintf("S3 endpoint returned HTTP %d", response.StatusCode) + return condition + } + condition.Status = metav1.ConditionTrue + condition.Reason = "EndpointReachable" + return condition +} + +func probeSeaweedAllocation(ctx context.Context, client *http.Client, endpoint string) metav1.Condition { + condition := metav1.Condition{ + Type: SeaweedWritableType, + Status: metav1.ConditionFalse, + Reason: "AllocationFailed", + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + condition.Message = err.Error() + return condition + } + response, err := client.Do(req) + if err != nil { + condition.Message = err.Error() + return condition + } + defer response.Body.Close() + + body, err := io.ReadAll(io.LimitReader(response.Body, 64*1024)) + if err != nil { + condition.Message = err.Error() + return condition + } + var assign seaweedAssignResponse + if err := json.Unmarshal(body, &assign); err != nil { + condition.Message = fmt.Sprintf("invalid allocation response: %v", err) + return condition + } + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + condition.Message = fmt.Sprintf("allocation returned HTTP %d: %s", response.StatusCode, assign.Error) + return condition + } + if assign.Error != "" { + condition.Message = assign.Error + return condition + } + if assign.FID == "" { + condition.Message = "allocation response did not include a file ID" + return condition + } + + condition.Status = metav1.ConditionTrue + condition.Reason = "AllocationSucceeded" + return condition +} + func computeSeaweedReportedReadyCondition(_ context.Context, cr *seaweedv1.Seaweed) []metav1.Condition { if cr == nil { return []metav1.Condition{} diff --git a/internal/controller/infra/managed/objectstore/seaweedfs/read_test.go b/internal/controller/infra/managed/objectstore/seaweedfs/read_test.go new file mode 100644 index 00000000..b92239fb --- /dev/null +++ b/internal/controller/infra/managed/objectstore/seaweedfs/read_test.go @@ -0,0 +1,81 @@ +package seaweedfs + +import ( + "context" + "net/http" + "net/http/httptest" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("SeaweedFS writable storage probe", func() { + It("reports writable storage after an allocation succeeds", func() { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + Expect(request.URL.Path).To(Equal("/dir/assign")) + _, err := response.Write([]byte(`{"fid":"3,01637037d6","url":"volume:8444","count":1}`)) + Expect(err).NotTo(HaveOccurred()) + })) + DeferCleanup(server.Close) + + condition := probeSeaweedAllocation(context.Background(), server.Client(), server.URL+"/dir/assign") + + Expect(condition.Type).To(Equal(SeaweedWritableType)) + Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + Expect(condition.Reason).To(Equal("AllocationSucceeded")) + }) + + It("reports allocation errors as not writable", func() { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.WriteHeader(http.StatusInternalServerError) + _, err := response.Write([]byte(`{"error":"No writable volumes and no free volumes left"}`)) + Expect(err).NotTo(HaveOccurred()) + })) + DeferCleanup(server.Close) + + condition := probeSeaweedAllocation(context.Background(), server.Client(), server.URL) + + Expect(condition.Status).To(Equal(metav1.ConditionFalse)) + Expect(condition.Reason).To(Equal("AllocationFailed")) + Expect(condition.Message).To(ContainSubstring("No writable volumes")) + }) + + It("rejects successful responses without an allocation", func() { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + _, err := response.Write([]byte(`{"count":0}`)) + Expect(err).NotTo(HaveOccurred()) + })) + DeferCleanup(server.Close) + + condition := probeSeaweedAllocation(context.Background(), server.Client(), server.URL) + + Expect(condition.Status).To(Equal(metav1.ConditionFalse)) + Expect(condition.Message).To(ContainSubstring("file ID")) + }) + + It("reports an unauthenticated S3 API response as reachable", func() { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.WriteHeader(http.StatusForbidden) + })) + DeferCleanup(server.Close) + + condition := probeSeaweedS3(context.Background(), server.Client(), server.URL) + + Expect(condition.Type).To(Equal(SeaweedS3ReachableType)) + Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + Expect(condition.Reason).To(Equal("EndpointReachable")) + }) + + It("rejects an unavailable S3 API", func() { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.WriteHeader(http.StatusServiceUnavailable) + })) + DeferCleanup(server.Close) + + condition := probeSeaweedS3(context.Background(), server.Client(), server.URL) + + Expect(condition.Status).To(Equal(metav1.ConditionFalse)) + Expect(condition.Reason).To(Equal("EndpointUnavailable")) + }) +}) diff --git a/internal/controller/infra/managed/objectstore/seaweedfs/spec.go b/internal/controller/infra/managed/objectstore/seaweedfs/spec.go index 9852bec0..d2bb9145 100644 --- a/internal/controller/infra/managed/objectstore/seaweedfs/spec.go +++ b/internal/controller/infra/managed/objectstore/seaweedfs/spec.go @@ -37,14 +37,37 @@ const ( seaweedWritableTmpVolumeName = "seaweedfs-tmp" seaweedWritableTmpMountPath = "/tmp" seaweedFilerDataMountPath = "/data/filerldb2" + + // Filer holds only the leveldb2 path index; its size scales with object count, + // not total data. 20Gi is a safe default for large counts; override with + // FilerStorageSize when a deployment needs more. + seaweedFilerStorageSize = "20Gi" ) const ( seaweedMasterMetricsPort int32 = 9091 seaweedVolumeMetricsPort int32 = 9092 seaweedFilerMetricsPort int32 = 9093 + seaweedVolumeSizeLimitMB int64 = 1024 + + seaweedVolumeReadinessPeriodSeconds int32 = 15 ) +func volumeLayout(storageQuantity resource.Quantity) (int32, int32) { + storageMB := storageQuantity.Value() / (1024 * 1024) + volumeSizeMB := min(seaweedVolumeSizeLimitMB, storageMB/2) + if volumeSizeMB < 1 { + volumeSizeMB = 1 + } + + maxVolumeCount := storageMB/volumeSizeMB - 1 + if maxVolumeCount < 1 { + maxVolumeCount = 1 + } + + return int32(volumeSizeMB), int32(maxVolumeCount) +} + func seaweedWritableVolumes() []corev1.Volume { return []corev1.Volume{ { @@ -65,11 +88,11 @@ func seaweedWritableVolumeMounts() []corev1.VolumeMount { func ToObjectStoreVendorSpec( ctx context.Context, wandb *apiv2.WeightsAndBiases, + infraSpec *apiv2.ManagedObjectStoreSpec, scheme *runtime.Scheme, mfst manifest.Manifest, ) (*seaweedv1.Seaweed, error) { _, log := logx.WithSlog(ctx, logx.ObjectStore) - infraSpec := wandb.Spec.ObjectStore.ManagedObjectStore if infraSpec == nil { return nil, nil } @@ -81,12 +104,24 @@ func ToObjectStoreVendorSpec( return nil, fmt.Errorf("invalid storage size %q: %w", infraSpec.StorageSize, err) } - replication := "000" - if infraSpec.Replicas > 1 { - replication = "001" + replication := seaweedReplication(infraSpec.Copies, infraSpec.Replicas) + + volumeSizeLimitMB, maxVolumeCount := volumeLayout(storageQuantity) + + // Merge the storage request (PVC size) with any configured cpu/memory so neither drops the other. + volumeRequests := corev1.ResourceList{corev1.ResourceStorage: storageQuantity} + for name, qty := range infraSpec.Config.Resources.Requests { + volumeRequests[name] = qty } - volumeSizeLimitMB := int32(storageQuantity.Value() / (1024 * 1024)) + filerStorageSize := seaweedFilerStorageSize + if infraSpec.SeaweedObjectStoreSpec.FilerStorageSize != "" { + filerStorageSize = infraSpec.SeaweedObjectStoreSpec.FilerStorageSize + } + filerStorageQuantity, err := resource.ParseQuantity(filerStorageSize) + if err != nil { + return nil, fmt.Errorf("invalid filer storage size %q: %w", filerStorageSize, err) + } labels := BuildWandbObjectStoreLabels(wandb) labels["app"] = SeaweedName(specName) @@ -110,20 +145,26 @@ func ToObjectStoreVendorSpec( ComponentSpec: seaweedv1.ComponentSpec{ Volumes: seaweedWritableVolumes(), VolumeMounts: seaweedWritableVolumeMounts(), + ExtraArgs: []string{"-ip.bind=0.0.0.0"}, }, }, Volume: &seaweedv1.VolumeSpec{ Replicas: infraSpec.Replicas, VolumeServerConfig: seaweedv1.VolumeServerConfig{ - MetricsPort: ptr.To(seaweedVolumeMetricsPort), + MetricsPort: ptr.To(seaweedVolumeMetricsPort), + MaxVolumeCounts: ptr.To(maxVolumeCount), ComponentSpec: seaweedv1.ComponentSpec{ Volumes: seaweedWritableVolumes(), VolumeMounts: seaweedWritableVolumeMounts(), + ExtraArgs: []string{"-ip.bind=0.0.0.0"}, + ReadinessProbe: &seaweedv1.ProbeOverride{ + PeriodSeconds: ptr.To(seaweedVolumeReadinessPeriodSeconds), + }, }, + // Operator sizes the data PVC from Requests[storage] — a persistent disk, not ephemeral. ResourceRequirements: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: storageQuantity, - }, + Requests: volumeRequests, + Limits: infraSpec.Config.Resources.Limits, }, }, }, @@ -131,6 +172,15 @@ func ToObjectStoreVendorSpec( ComponentSpec: seaweedv1.ComponentSpec{ Affinity: wandb.GetAffinity(infraSpec.ManagedInfraSpec), Tolerations: *wandb.GetTolerations(infraSpec.ManagedInfraSpec), + Env: []corev1.EnvVar{{ + // W&B presigns S3 URLs against the in-cluster endpoint and + // rewrites the host for external clients without re-signing; + // pin signature verification to that endpoint so presigned + // requests arriving through an ingress proxy (whose + // Host/X-Forwarded-Host is the external hostname) validate. + Name: "S3_EXTERNAL_URL", + Value: s3ExternalURL(specName, infraSpec.Namespace, infraSpec.SeaweedObjectStoreSpec.TlsEnabled), + }}, }, ResourceRequirements: corev1.ResourceRequirements{}, Replicas: 1, @@ -140,7 +190,6 @@ func ToObjectStoreVendorSpec( }, Key: "config.json", }, - Port: new(int32(80)), DomainName: nil, }, Filer: &seaweedv1.FilerSpec{ @@ -150,13 +199,14 @@ func ToObjectStoreVendorSpec( ComponentSpec: seaweedv1.ComponentSpec{ Volumes: seaweedWritableVolumes(), VolumeMounts: seaweedWritableVolumeMounts(), + ExtraArgs: []string{"-ip.bind=0.0.0.0"}, }, Persistence: &seaweedv1.PersistenceSpec{ Enabled: true, MountPath: ptr.To(seaweedFilerDataMountPath), Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceStorage: storageQuantity, + corev1.ResourceStorage: filerStorageQuantity, }, }, }, @@ -166,13 +216,6 @@ func ToObjectStoreVendorSpec( }, } - if len(infraSpec.Config.Resources.Requests) > 0 || len(infraSpec.Config.Resources.Limits) > 0 { - seaweedCR.Spec.Volume.ResourceRequirements = corev1.ResourceRequirements{ - Requests: infraSpec.Config.Resources.Requests, - Limits: infraSpec.Config.Resources.Limits, - } - } - if err := ctrl.SetControllerReference(wandb, seaweedCR, scheme); err != nil { log.Error("failed to set owner reference on Seaweed CR", logx.ErrAttr(err)) return nil, fmt.Errorf("failed to set owner reference: %w", err) @@ -181,6 +224,27 @@ func ToObjectStoreVendorSpec( return seaweedCR, nil } +// seaweedReplication builds the SeaweedFS replication code from the neutral copy +// count, clamped to the data-node count so we never request more copies than servers. +func seaweedReplication(copies, replicas int32) string { + // Unset copies keeps the legacy behavior: one extra copy once there is more than one server. + if copies <= 0 { + if replicas > 1 { + return "001" + } + return "000" + } + // Never request more copies than there are other servers to hold them. + maxCopies := replicas - 1 + if maxCopies < 0 { + maxCopies = 0 + } + if copies > maxCopies { + copies = maxCopies + } + return fmt.Sprintf("00%d", copies) +} + func ToObjectStoreEnvConfig( ctx context.Context, spec apiv2.ManagedObjectStoreSpec, diff --git a/internal/controller/infra/managed/objectstore/seaweedfs/spec_test.go b/internal/controller/infra/managed/objectstore/seaweedfs/spec_test.go index a1f0adbb..770cb5cf 100644 --- a/internal/controller/infra/managed/objectstore/seaweedfs/spec_test.go +++ b/internal/controller/infra/managed/objectstore/seaweedfs/spec_test.go @@ -2,12 +2,13 @@ package seaweedfs import ( "context" + "encoding/json" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apiv2 "github.com/wandb/operator/api/v2" - "github.com/wandb/operator/pkg/wandb/manifest" seaweedv1 "github.com/wandb/operator/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1" + "github.com/wandb/operator/pkg/wandb/manifest" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,7 +19,7 @@ var _ = Describe("SeaweedFS vendor specs", func() { It("renders writable runtime mounts for SeaweedFS components", func() { wandb := seaweedWandb() - seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, seaweedScheme(), manifest.Manifest{}) + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) Expect(err).NotTo(HaveOccurred()) Expect(seaweed).NotTo(BeNil()) @@ -33,6 +34,9 @@ var _ = Describe("SeaweedFS vendor specs", func() { expectSeaweedWritableMount(seaweed.Spec.Volume.VolumeMounts) expectSeaweedWritableVolume(seaweed.Spec.Filer.Volumes) expectSeaweedWritableMount(seaweed.Spec.Filer.VolumeMounts) + Expect(seaweed.Spec.Master.ExtraArgs).To(ContainElement("-ip.bind=0.0.0.0")) + Expect(seaweed.Spec.Volume.ExtraArgs).To(ContainElement("-ip.bind=0.0.0.0")) + Expect(seaweed.Spec.Filer.ExtraArgs).To(ContainElement("-ip.bind=0.0.0.0")) }) It("retargets the image to spec.global.imageRegistry when set", func() { @@ -49,14 +53,15 @@ var _ = Describe("SeaweedFS vendor specs", func() { }, } - seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, seaweedScheme(), mfst) + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), mfst) Expect(err).NotTo(HaveOccurred()) Expect(seaweed).NotTo(BeNil()) Expect(seaweed.Spec.Image).To(Equal("reg.corp:5000/chrislusf/seaweedfs:latest")) }) It("keeps the filer writable data path explicit", func() { - seaweed, err := ToObjectStoreVendorSpec(context.Background(), seaweedWandb(), seaweedScheme(), manifest.Manifest{}) + wandb := seaweedWandb() + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) Expect(err).NotTo(HaveOccurred()) Expect(seaweed).NotTo(BeNil()) Expect(seaweed.Spec.Filer.Config).NotTo(BeNil()) @@ -67,14 +72,63 @@ var _ = Describe("SeaweedFS vendor specs", func() { }) It("preserves managed resource overrides", func() { - seaweed, err := ToObjectStoreVendorSpec(context.Background(), seaweedWandb(), seaweedScheme(), manifest.Manifest{}) + wandb := seaweedWandb() + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) Expect(err).NotTo(HaveOccurred()) Expect(seaweed).NotTo(BeNil()) Expect(seaweed.Spec.Volume.ResourceRequirements.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("500m"))) }) + It("reserves storage headroom for writable volumes", func() { + wandb := seaweedWandb() + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(seaweed.Spec.Master.VolumeSizeLimitMB).NotTo(BeNil()) + Expect(*seaweed.Spec.Master.VolumeSizeLimitMB).To(Equal(int32(1024))) + Expect(seaweed.Spec.Volume.MaxVolumeCounts).NotTo(BeNil()) + Expect(*seaweed.Spec.Volume.MaxVolumeCounts).To(Equal(int32(9))) + }) + + DescribeTable("computes a writable volume layout", + func(storage string, expectedSizeMB, expectedMaxVolumes int32) { + size, count := volumeLayout(resource.MustParse(storage)) + Expect(size).To(Equal(expectedSizeMB)) + Expect(count).To(Equal(expectedMaxVolumes)) + }, + Entry("a development volume", "10Gi", int32(1024), int32(9)), + Entry("the upstream minimum example", "2Gi", int32(1024), int32(1)), + Entry("a sub-gibibyte volume", "512Mi", int32(256), int32(1)), + Entry("a large volume", "1Ti", int32(1024), int32(1023)), + ) + + It("pins s3 gateway signature verification to the in-cluster endpoint", func() { + wandb := seaweedWandb() + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed).NotTo(BeNil()) + Expect(seaweed.Spec.S3.Env).To(ContainElement(corev1.EnvVar{ + Name: "S3_EXTERNAL_URL", + Value: "http://" + SeaweedName("object-store") + "-s3.wandb.svc.cluster.local:" + S3Port, + })) + }) + + It("uses https for the s3 external URL when TLS is enabled", func() { + wandb := seaweedWandb() + wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.SeaweedObjectStoreSpec.TlsEnabled = true + + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed).NotTo(BeNil()) + Expect(seaweed.Spec.S3.Env).To(ContainElement(corev1.EnvVar{ + Name: "S3_EXTERNAL_URL", + Value: "https://" + SeaweedName("object-store") + "-s3.wandb.svc.cluster.local:" + S3Port, + })) + }) + It("sets metrics ports on master, volume, and filer", func() { - seaweed, err := ToObjectStoreVendorSpec(context.Background(), seaweedWandb(), seaweedScheme(), manifest.Manifest{}) + wandb := seaweedWandb() + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) Expect(err).NotTo(HaveOccurred()) Expect(seaweed).NotTo(BeNil()) @@ -87,6 +141,168 @@ var _ = Describe("SeaweedFS vendor specs", func() { Expect(seaweed.Spec.Filer.MetricsPort).NotTo(BeNil()) Expect(*seaweed.Spec.Filer.MetricsPort).To(Equal(seaweedFilerMetricsPort)) }) + + It("uses a fast readiness cadence for volume servers", func() { + wandb := seaweedWandb() + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed).NotTo(BeNil()) + + Expect(seaweed.Spec.Volume.ReadinessProbe).NotTo(BeNil()) + Expect(seaweed.Spec.Volume.ReadinessProbe.PeriodSeconds).NotTo(BeNil()) + Expect(*seaweed.Spec.Volume.ReadinessProbe.PeriodSeconds).To(Equal(seaweedVolumeReadinessPeriodSeconds)) + Expect(seaweed.Spec.Volume.LivenessProbe).To(BeNil()) + + encoded, err := json.Marshal(seaweed) + Expect(err).NotTo(HaveOccurred()) + Expect(string(encoded)).To(ContainSubstring(`"readinessProbe":{"periodSeconds":15}`)) + Expect(string(encoded)).NotTo(ContainSubstring(`"livenessProbe"`)) + }) + + It("keeps the volume storage request when cpu/memory overrides are set", func() { + wandb := seaweedWandb() + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed).NotTo(BeNil()) + + requests := seaweed.Spec.Volume.Requests + Expect(requests[corev1.ResourceStorage]).To(Equal(resource.MustParse("10Gi"))) + Expect(requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("500m"))) + }) + + It("sizes the filer disk independently of the data volumes", func() { + wandb := seaweedWandb() + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed).NotTo(BeNil()) + + filerStorage := seaweed.Spec.Filer.Persistence.Resources.Requests[corev1.ResourceStorage] + Expect(filerStorage).To(Equal(resource.MustParse(seaweedFilerStorageSize))) + Expect(filerStorage).NotTo(Equal(resource.MustParse("10Gi"))) + }) + + It("honors a configured filer storage size over the default", func() { + wandb := seaweedWandb() + wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.SeaweedObjectStoreSpec.FilerStorageSize = "50Gi" + seaweed, err := ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed).NotTo(BeNil()) + Expect(seaweed.Spec.Filer.Persistence.Resources.Requests[corev1.ResourceStorage]).To(Equal(resource.MustParse("50Gi"))) + }) +}) + +var _ = Describe("SeaweedFS translation edge cases", func() { + DescribeTable("maps replica count to a replication code", + func(replicas int32, wantReplication string) { + w := seaweedWandb() + w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Replicas = replicas + seaweed, err := ToObjectStoreVendorSpec(context.Background(), w, w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed.Spec.Volume.Replicas).To(Equal(replicas)) + Expect(*seaweed.Spec.Master.DefaultReplication).To(Equal(wantReplication)) + }, + Entry("zero replicas", int32(0), "000"), + Entry("single replica", int32(1), "000"), + Entry("two replicas", int32(2), "001"), + Entry("three replicas", int32(3), "001"), + Entry("five replicas", int32(5), "001"), + ) + + DescribeTable("derives the replication code from an explicit copies count", + func(copies, replicas int32, wantReplication string) { + w := seaweedWandb() + w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Copies = copies + w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Replicas = replicas + seaweed, err := ToObjectStoreVendorSpec(context.Background(), w, w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*seaweed.Spec.Master.DefaultReplication).To(Equal(wantReplication)) + }, + Entry("unset copies keeps a single copy on one node", int32(0), int32(1), "000"), + Entry("unset copies falls back to legacy one copy", int32(0), int32(3), "001"), + Entry("one extra copy", int32(1), int32(3), "001"), + Entry("two extra copies", int32(2), int32(3), "002"), + Entry("three extra copies", int32(3), int32(4), "003"), + Entry("copies clamped to data-node count", int32(5), int32(3), "002"), + Entry("copies clamped to zero on a single node", int32(2), int32(1), "000"), + Entry("one copy on two nodes", int32(1), int32(2), "001"), + Entry("negative copies treated as unset (never yields a bad code)", int32(-1), int32(3), "001"), + Entry("negative copies on a single node", int32(-3), int32(1), "000"), + ) + + It("layers cpu/memory requests and limits onto the volume without dropping storage", func() { + w := seaweedWandb() + w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.StorageSize = "100Gi" + w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Config.Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("8Gi"), + }, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, + } + seaweed, err := ToObjectStoreVendorSpec(context.Background(), w, w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + req := seaweed.Spec.Volume.Requests + Expect(req[corev1.ResourceStorage]).To(Equal(resource.MustParse("100Gi"))) + Expect(req[corev1.ResourceCPU]).To(Equal(resource.MustParse("2"))) + Expect(req[corev1.ResourceMemory]).To(Equal(resource.MustParse("8Gi"))) + Expect(seaweed.Spec.Volume.Limits[corev1.ResourceCPU]).To(Equal(resource.MustParse("4"))) + }) + + It("sets no cpu request and no limits when the CR configures none", func() { + w := seaweedWandb() + w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Config.Resources = corev1.ResourceRequirements{} + seaweed, err := ToObjectStoreVendorSpec(context.Background(), w, w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed.Spec.Volume.Requests).To(HaveKey(corev1.ResourceStorage)) + Expect(seaweed.Spec.Volume.Requests).NotTo(HaveKey(corev1.ResourceCPU)) + Expect(seaweed.Spec.Volume.Limits).To(BeNil()) + }) + + DescribeTable("keeps the filer disk fixed regardless of data volume size", + func(storage string) { + w := seaweedWandb() + w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.StorageSize = storage + seaweed, err := ToObjectStoreVendorSpec(context.Background(), w, w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed.Spec.Volume.Requests[corev1.ResourceStorage]).To(Equal(resource.MustParse(storage))) + Expect(seaweed.Spec.Filer.Persistence.Resources.Requests[corev1.ResourceStorage]).To(Equal(resource.MustParse(seaweedFilerStorageSize))) + }, + Entry("small data disk", "10Gi"), + Entry("large data disk", "1Ti"), + ) + + DescribeTable("rejects an unparseable storage size", + func(storage string) { + w := seaweedWandb() + w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.StorageSize = storage + _, err := ToObjectStoreVendorSpec(context.Background(), w, w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).To(HaveOccurred()) + }, + Entry("empty", ""), + Entry("wrong unit", "10GB"), + Entry("garbage", "abc"), + ) + + It("returns nil when no managed object store is configured", func() { + w := seaweedWandb() + w.Spec.ObjectStore[apiv2.DefaultInstanceName] = apiv2.ObjectStoreSpec{} + seaweed, err := ToObjectStoreVendorSpec(context.Background(), w, w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed).To(BeNil()) + }) + + DescribeTable("propagates the TLS toggle", + func(tls bool) { + w := seaweedWandb() + w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.SeaweedObjectStoreSpec.TlsEnabled = tls + seaweed, err := ToObjectStoreVendorSpec(context.Background(), w, w.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, seaweedScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed.Spec.TLS).NotTo(BeNil()) + Expect(seaweed.Spec.TLS.Enabled).To(Equal(tls)) + }, + Entry("enabled", true), + Entry("disabled", false), + ) }) func seaweedScheme() *runtime.Scheme { @@ -109,17 +325,19 @@ func seaweedWandb() *apiv2.WeightsAndBiases { }, Spec: apiv2.WeightsAndBiasesSpec{ Tolerations: &tolerations, - ObjectStore: apiv2.ObjectStoreSpec{ - ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{ - Name: "object-store", - Namespace: "wandb", - Replicas: 1, - StorageSize: "10Gi", - Config: apiv2.ObjectStoreConfig{ - AccessKey: "admin", - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: { + ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{ + Name: "object-store", + Namespace: "wandb", + Replicas: 1, + StorageSize: "10Gi", + Config: apiv2.ObjectStoreConfig{ + AccessKey: "admin", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + }, }, }, }, diff --git a/internal/controller/infra/managed/objectstore/seaweedfs/status.go b/internal/controller/infra/managed/objectstore/seaweedfs/status.go index b8eebdf6..7053b7b3 100644 --- a/internal/controller/infra/managed/objectstore/seaweedfs/status.go +++ b/internal/controller/infra/managed/objectstore/seaweedfs/status.go @@ -17,6 +17,8 @@ const ( SeaweedCustomResourceType = "SeaweedCustomResource" SeaweedConnectionInfoType = "SeaweedConnectionInfo" SeaweedReportedReadyType = "SeaweedReportedReady" + SeaweedWritableType = "SeaweedWritable" + SeaweedS3ReachableType = "SeaweedS3Reachable" ) func ComputeStatus( @@ -70,6 +72,20 @@ func applyDefaultConditions(conditions []metav1.Condition) []metav1.Condition { Reason: common.NoResourceReason, }) } + if !common.ContainsType(conditions, SeaweedWritableType) { + conditions = append(conditions, metav1.Condition{ + Type: SeaweedWritableType, + Status: metav1.ConditionUnknown, + Reason: common.NoResourceReason, + }) + } + if !common.ContainsType(conditions, SeaweedS3ReachableType) { + conditions = append(conditions, metav1.Condition{ + Type: SeaweedS3ReachableType, + Status: metav1.ConditionUnknown, + Reason: common.NoResourceReason, + }) + } return conditions } @@ -88,6 +104,8 @@ func inferInfraState( impliedStates = inferStateFromCondition(ctx, SeaweedCustomResourceType, impliedStates, conditions) impliedStates = inferStateFromCondition(ctx, SeaweedConnectionInfoType, impliedStates, conditions) impliedStates = inferStateFromCondition(ctx, SeaweedReportedReadyType, impliedStates, conditions) + impliedStates = inferStateFromCondition(ctx, SeaweedWritableType, impliedStates, conditions) + impliedStates = inferStateFromCondition(ctx, SeaweedS3ReachableType, impliedStates, conditions) hasImpliedState := func(target string) bool { return len(lo.FilterValues( @@ -141,6 +159,8 @@ func inferStateFromCondition(ctx context.Context, conditionType string, impliedS impliedStates[conditionType] = inferState_SeaweedConnectionInfoType(ctx, cond) case SeaweedReportedReadyType: impliedStates[conditionType] = inferState_SeaweedReportedReadyType(ctx, cond) + case SeaweedWritableType, SeaweedS3ReachableType: + impliedStates[conditionType] = inferState_SeaweedWritableType(ctx, cond) default: impliedStates[conditionType] = common.UnknownState } @@ -148,6 +168,22 @@ func inferStateFromCondition(ctx context.Context, conditionType string, impliedS return impliedStates } +func inferState_SeaweedWritableType(ctx context.Context, condition metav1.Condition) string { + log := logx.GetSlog(ctx) + result := common.PendingState + if condition.Status == metav1.ConditionTrue { + result = common.HealthyState + } + if condition.Status == metav1.ConditionFalse { + result = common.ErrorState + } + log.Debug( + "implied state", "state", result, "condition", condition.Type, + "reason", condition.Reason, "status", condition.Status, + ) + return result +} + func inferState_SeaweedCustomResourceType(ctx context.Context, condition metav1.Condition) string { log := logx.GetSlog(ctx) result := common.UnknownState diff --git a/internal/controller/infra/managed/objectstore/seaweedfs/status_test.go b/internal/controller/infra/managed/objectstore/seaweedfs/status_test.go new file mode 100644 index 00000000..cc6aeca4 --- /dev/null +++ b/internal/controller/infra/managed/objectstore/seaweedfs/status_test.go @@ -0,0 +1,69 @@ +package seaweedfs + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/wandb/operator/internal/controller/common" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("SeaweedFS status", func() { + It("does not report healthy from component readiness alone", func() { + status, _, _ := ComputeStatus( + context.Background(), + true, + nil, + []metav1.Condition{ + {Type: SeaweedCustomResourceType, Status: metav1.ConditionTrue}, + {Type: SeaweedConnectionInfoType, Status: metav1.ConditionTrue}, + {Type: SeaweedReportedReadyType, Status: metav1.ConditionTrue}, + }, + nil, + 1, + ) + + Expect(status.Ready).To(BeFalse()) + Expect(status.State).To(Equal(common.PendingState)) + }) + + It("reports an allocation failure as an error", func() { + status, _, _ := ComputeStatus( + context.Background(), + true, + nil, + []metav1.Condition{ + {Type: SeaweedCustomResourceType, Status: metav1.ConditionTrue}, + {Type: SeaweedConnectionInfoType, Status: metav1.ConditionTrue}, + {Type: SeaweedReportedReadyType, Status: metav1.ConditionTrue}, + {Type: SeaweedWritableType, Status: metav1.ConditionFalse, Reason: "AllocationFailed"}, + }, + nil, + 1, + ) + + Expect(status.Ready).To(BeFalse()) + Expect(status.State).To(Equal(common.ErrorState)) + }) + + It("reports healthy only after allocation succeeds", func() { + status, _, _ := ComputeStatus( + context.Background(), + true, + nil, + []metav1.Condition{ + {Type: SeaweedCustomResourceType, Status: metav1.ConditionTrue}, + {Type: SeaweedConnectionInfoType, Status: metav1.ConditionTrue}, + {Type: SeaweedReportedReadyType, Status: metav1.ConditionTrue}, + {Type: SeaweedWritableType, Status: metav1.ConditionTrue, Reason: "AllocationSucceeded"}, + {Type: SeaweedS3ReachableType, Status: metav1.ConditionTrue, Reason: "EndpointReachable"}, + }, + nil, + 1, + ) + + Expect(status.Ready).To(BeTrue()) + Expect(status.State).To(Equal(common.HealthyState)) + }) +}) diff --git a/internal/controller/infra/managed/objectstore/seaweedfs/write.go b/internal/controller/infra/managed/objectstore/seaweedfs/write.go index 49500ea6..7daac715 100644 --- a/internal/controller/infra/managed/objectstore/seaweedfs/write.go +++ b/internal/controller/infra/managed/objectstore/seaweedfs/write.go @@ -7,6 +7,7 @@ import ( "github.com/Masterminds/goutils" apiv2 "github.com/wandb/operator/api/v2" "github.com/wandb/operator/internal/controller/common" + "github.com/wandb/operator/internal/controller/infra/objectstore" "github.com/wandb/operator/internal/logx" seaweedv1 "github.com/wandb/operator/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1" corev1 "k8s.io/api/core/v1" @@ -23,6 +24,9 @@ const ( AppConnTypeName = "SeaweedAppConn" ) +// WriteState reconciles the managed SeaweedFS CR, its S3 identity config, and +// the W&B connection secret, returning the reconcile conditions plus the +// resulting ObjectStoreConnection (nil until the connection is available). func WriteState( ctx context.Context, kubeClient client.Client, @@ -66,6 +70,7 @@ func WriteState( Status: metav1.ConditionFalse, Reason: common.ApiErrorReason, }) + return result, nil } switch action { @@ -81,7 +86,7 @@ func WriteState( Status: metav1.ConditionFalse, Reason: common.PendingDeleteReason, }) - case common.UpdateAction: + case common.UpdateAction, common.UnchangedAction: result = append(result, metav1.Condition{ Type: SeaweedCustomResourceType, Status: metav1.ConditionTrue, @@ -137,13 +142,16 @@ func WriteState( return result, nil } +// writeSeaweedS3Config persists the SeaweedFS S3 identity config secret, +// preserving the existing secret key when one is already present so credentials +// stay stable across reconciles, and returns the resolved ConnInfo. func writeSeaweedS3Config( ctx context.Context, client client.Client, owner *seaweedv1.Seaweed, nsnBuilder *NsNameBuilder, envConfig SeaweedS3Config, -) (*s3ConnInfo, error) { +) (*objectstore.ConnInfo, error) { var err error var found bool var gvk schema.GroupVersionKind diff --git a/internal/controller/infra/managed/objectstore/seaweedfs/write_test.go b/internal/controller/infra/managed/objectstore/seaweedfs/write_test.go new file mode 100644 index 00000000..8d2f78bb --- /dev/null +++ b/internal/controller/infra/managed/objectstore/seaweedfs/write_test.go @@ -0,0 +1,87 @@ +package seaweedfs + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/common" + seaweedv1 "github.com/wandb/operator/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1" + "github.com/wandb/operator/pkg/wandb/manifest" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func writeScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + Expect(clientgoscheme.AddToScheme(scheme)).To(Succeed()) + Expect(apiv2.AddToScheme(scheme)).To(Succeed()) + Expect(seaweedv1.AddToScheme(scheme)).To(Succeed()) + return scheme +} + +func hasCondition(conds []metav1.Condition, condType string, status metav1.ConditionStatus) bool { + for _, c := range conds { + if c.Type == condType && c.Status == status { + return true + } + } + return false +} + +var _ = Describe("SeaweedFS WriteState", func() { + var ( + ctx context.Context + wandb *apiv2.WeightsAndBiases + desired *seaweedv1.Seaweed + envCfg SeaweedS3Config + specNsn types.NamespacedName + errWrite = errors.New("boom: apiserver write failed") + ) + + BeforeEach(func() { + ctx = context.Background() + wandb = seaweedWandb() + var err error + desired, err = ToObjectStoreVendorSpec(ctx, wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, writeScheme(), manifest.Manifest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(desired).NotTo(BeNil()) + envCfg = SeaweedS3Config{AccessKey: "admin"} + specNsn = types.NamespacedName{Namespace: "wandb", Name: "object-store"} + }) + + It("returns early and never reports a healthy connection when the CR write fails", func() { + cl := fake.NewClientBuilder(). + WithScheme(writeScheme()). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { + return errWrite + }, + }). + Build() + + conds, conn := WriteState(ctx, cl, specNsn, desired, envCfg, wandb) + + Expect(conn).To(BeNil()) + Expect(hasCondition(conds, common.ReconciledType, metav1.ConditionFalse)).To(BeTrue()) + // A failed CR write must not fall through to a "connection ready" report. + Expect(hasCondition(conds, SeaweedConnectionInfoType, metav1.ConditionTrue)).To(BeFalse()) + }) + + It("reports pending-create and writes a connection when the CR write succeeds", func() { + cl := fake.NewClientBuilder().WithScheme(writeScheme()).Build() + + conds, conn := WriteState(ctx, cl, specNsn, desired, envCfg, wandb) + + Expect(hasCondition(conds, SeaweedCustomResourceType, metav1.ConditionFalse)).To(BeTrue()) + Expect(conn).NotTo(BeNil()) + Expect(hasCondition(conds, SeaweedConnectionInfoType, metav1.ConditionTrue)).To(BeTrue()) + }) +}) diff --git a/internal/controller/infra/managed/redis/opstree/naming.go b/internal/controller/infra/managed/redis/opstree/naming.go index 3daed78c..5075de0e 100644 --- a/internal/controller/infra/managed/redis/opstree/naming.go +++ b/internal/controller/infra/managed/redis/opstree/naming.go @@ -3,9 +3,23 @@ package opstree import ( "fmt" + "github.com/wandb/operator/internal/controller/common" "k8s.io/apimachinery/pkg/types" ) +// MaxSpecNameLength is a conservative budget: this package and the opstree +// operator suffix the spec name ("-replica", "-sentinel", "-headless", ...); +// 40 leaves 23 chars of DNS-1123 label headroom. +const MaxSpecNameLength = 40 + +const defaultNameSuffix = "-redis" + +// DefaultSpecName derives the managed Redis name for a CR instance, shortened +// to the budget. +func DefaultSpecName(crName, instanceKey string) string { + return common.FitDefaultInfraName(common.InstanceBaseName(crName, instanceKey), defaultNameSuffix, MaxSpecNameLength) +} + type NsNameBuilder struct { baseNsName types.NamespacedName } diff --git a/internal/controller/infra/managed/redis/opstree/spec.go b/internal/controller/infra/managed/redis/opstree/spec.go index 570b8a32..606ac2cd 100644 --- a/internal/controller/infra/managed/redis/opstree/spec.go +++ b/internal/controller/infra/managed/redis/opstree/spec.go @@ -156,11 +156,11 @@ func createRedisExporterConfig(telemetry apiv2.Telemetry, img manifest.ImageRef, func ToRedisStandaloneVendorSpec( ctx context.Context, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedRedisSpec, scheme *runtime.Scheme, mfst manifest.Manifest, ) (*redisv1beta2.Redis, error) { _, log := logx.WithSlog(ctx, logx.Redis) - spec := wandb.Spec.Redis.ManagedRedis if spec == nil { return nil, nil } @@ -237,11 +237,11 @@ func ToRedisStandaloneVendorSpec( func ToRedisSentinelVendorSpec( ctx context.Context, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedRedisSpec, scheme *runtime.Scheme, mfst manifest.Manifest, ) (*redissentinelv1beta2.RedisSentinel, error) { _, log := logx.WithSlog(ctx, logx.Redis) - spec := wandb.Spec.Redis.ManagedRedis if spec == nil { return nil, nil } @@ -315,11 +315,11 @@ func ToRedisSentinelVendorSpec( func ToRedisReplicationVendorSpec( ctx context.Context, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedRedisSpec, scheme *runtime.Scheme, mfst manifest.Manifest, ) (*redisreplicationv1beta2.RedisReplication, error) { _, log := logx.WithSlog(ctx, logx.Redis) - spec := wandb.Spec.Redis.ManagedRedis if spec == nil { return nil, nil } diff --git a/internal/controller/infra/managed/redis/opstree/spec_test.go b/internal/controller/infra/managed/redis/opstree/spec_test.go index c34c832e..766b8b8f 100644 --- a/internal/controller/infra/managed/redis/opstree/spec_test.go +++ b/internal/controller/infra/managed/redis/opstree/spec_test.go @@ -24,7 +24,7 @@ var _ = Describe("Redis vendor specs", func() { It("renders hardened standalone Redis settings", func() { wandb := redisWandb(false) - redis, err := ToRedisStandaloneVendorSpec(context.Background(), wandb, redisScheme(), manifest.Manifest{}) + redis, err := ToRedisStandaloneVendorSpec(context.Background(), wandb, wandb.Spec.Redis[apiv2.DefaultInstanceName].ManagedRedis, redisScheme(), manifest.Manifest{}) Expect(err).NotTo(HaveOccurred()) Expect(redis).NotTo(BeNil()) @@ -38,7 +38,7 @@ var _ = Describe("Redis vendor specs", func() { It("renders hardened sentinel and replication Redis settings", func() { wandb := redisWandb(true) - sentinel, err := ToRedisSentinelVendorSpec(context.Background(), wandb, redisScheme(), manifest.Manifest{}) + sentinel, err := ToRedisSentinelVendorSpec(context.Background(), wandb, wandb.Spec.Redis[apiv2.DefaultInstanceName].ManagedRedis, redisScheme(), manifest.Manifest{}) Expect(err).NotTo(HaveOccurred()) Expect(sentinel).NotTo(BeNil()) expectRedisDefaultPodSecurityContext(sentinel.Spec.PodSecurityContext) @@ -46,7 +46,7 @@ var _ = Describe("Redis vendor specs", func() { Expect(sentinel.Spec.VolumeMount).NotTo(BeNil()) expectRedisWritableTmpMount(sentinel.Spec.VolumeMount.MountPath) - replication, err := ToRedisReplicationVendorSpec(context.Background(), wandb, redisScheme(), manifest.Manifest{}) + replication, err := ToRedisReplicationVendorSpec(context.Background(), wandb, wandb.Spec.Redis[apiv2.DefaultInstanceName].ManagedRedis, redisScheme(), manifest.Manifest{}) Expect(err).NotTo(HaveOccurred()) Expect(replication).NotTo(BeNil()) expectRedisDefaultPodSecurityContext(replication.Spec.PodSecurityContext) @@ -57,7 +57,8 @@ var _ = Describe("Redis vendor specs", func() { It("omits fixed Redis IDs in OpenShift mode", func() { utils.SetOpenShiftMode(true) - redis, err := ToRedisStandaloneVendorSpec(context.Background(), redisWandb(false), redisScheme(), manifest.Manifest{}) + wandb := redisWandb(false) + redis, err := ToRedisStandaloneVendorSpec(context.Background(), wandb, wandb.Spec.Redis[apiv2.DefaultInstanceName].ManagedRedis, redisScheme(), manifest.Manifest{}) Expect(err).NotTo(HaveOccurred()) Expect(redis).NotTo(BeNil()) @@ -87,13 +88,15 @@ func redisWandb(sentinel bool) *apiv2.WeightsAndBiases { Namespace: "wandb", }, Spec: apiv2.WeightsAndBiasesSpec{ - Redis: apiv2.RedisSpec{ - ManagedRedis: &apiv2.ManagedRedisSpec{ - Name: "redis", - Namespace: "wandb", - StorageSize: "1Gi", - Telemetry: apiv2.Telemetry{Enabled: true}, - Sentinel: apiv2.RedisSentinelSpec{Enabled: sentinel}, + Redis: map[string]apiv2.RedisSpec{ + apiv2.DefaultInstanceName: { + ManagedRedis: &apiv2.ManagedRedisSpec{ + Name: "redis", + Namespace: "wandb", + StorageSize: "1Gi", + Telemetry: apiv2.Telemetry{Enabled: true}, + Sentinel: apiv2.RedisSentinelSpec{Enabled: sentinel}, + }, }, }, }, diff --git a/internal/controller/infra/managed/redis/opstree/write.go b/internal/controller/infra/managed/redis/opstree/write.go index e13f6cc7..a08c58e5 100644 --- a/internal/controller/infra/managed/redis/opstree/write.go +++ b/internal/controller/infra/managed/redis/opstree/write.go @@ -213,7 +213,7 @@ func writeStandaloneState( Status: metav1.ConditionFalse, Reason: common.PendingDeleteReason, }) - case common.UpdateAction: + case common.UpdateAction, common.UnchangedAction: result = append(result, metav1.Condition{ Type: RedisStandaloneCustomResourceType, Status: metav1.ConditionTrue, @@ -283,7 +283,7 @@ func writeSentinelState( Status: metav1.ConditionFalse, Reason: common.PendingDeleteReason, }) - case common.UpdateAction: + case common.UpdateAction, common.UnchangedAction: result = append(result, metav1.Condition{ Type: RedisSentinelCustomResourceType, Status: metav1.ConditionTrue, @@ -353,7 +353,7 @@ func writeReplicationState( Status: metav1.ConditionFalse, Reason: common.PendingDeleteReason, }) - case common.UpdateAction: + case common.UpdateAction, common.UnchangedAction: result = append(result, metav1.Condition{ Type: RedisReplicationCustomResourceType, Status: metav1.ConditionTrue, diff --git a/internal/controller/infra/objectstore/conn.go b/internal/controller/infra/objectstore/conn.go new file mode 100644 index 00000000..54048b65 --- /dev/null +++ b/internal/controller/infra/objectstore/conn.go @@ -0,0 +1,65 @@ +// Package objectstore holds the unified object-store connection model shared by +// the managed (SeaweedFS) and external reconcile paths: a single ConnInfo struct +// plus mapper functions that convert between it, the connection Secret's data, +// and the apiv2.ObjectStoreConnection selectors. +package objectstore + +import ( + apiv2 "github.com/wandb/operator/api/v2" + corev1 "k8s.io/api/core/v1" +) + +// DefaultRegion is the region assumed when a connection carries none; S3 SDKs and +// S3-compatible backends (SeaweedFS, MinIO) require some region to be set. +const DefaultRegion = "us-east-1" + +// ConnInfo is the resolved object-store connection: the read-side counterpart to +// the connection secret, and the value both write paths populate before encoding +// it back out via ToSecretData / ToObjectStoreConnection. +type ConnInfo struct { + Provider apiv2.ObjectStoreProvider + // URI is the provider-native location, e.g. "s3://bucket", "gs://bucket/prefix", or "https://acct.blob.core.windows.net/container". + URI string + // URL is the canonical connection URL persisted under the secret's "url" key. + // It is built per-flow (managed appends "?tls="; external is provider-specific), so ToSecretData serializes it verbatim. + URL string + // Bucket is the bare bucket/container name. + Bucket string + // Endpoint is the S3 API endpoint for S3-compatible providers (SeaweedFS, MinIO); empty for AWS S3, GCS, and Azure. It maps to the secret's "Host" key. + Endpoint string + Region string + AccessKey string + SecretKey string + // Scheme is the object-store URL scheme ("http"/"https"); only set by the managed path, persisted under the secret's "Scheme" key. + Scheme string + // ForcePathStyle is required by most non-AWS S3-compatible providers. + ForcePathStyle bool + TlsEnabled bool + Port string + Path string + // Credential selectors, kept for consumers that inject creds by reference. + AccessKeyRef corev1.SecretKeySelector + SecretKeyRef corev1.SecretKeySelector +} + +// HasStaticCredentials reports whether explicit keys were provided; when false, credentials come from ambient identity (IAM role / workload identity). +func (c ConnInfo) HasStaticCredentials() bool { + return c.AccessKey != "" && c.SecretKey != "" +} + +// ProviderURI builds the provider-native base URI for the connection's bucket: +// "s3://", "gs://", or the Azure blob container URL. It returns "" +// for an unknown provider. Any object prefix is the caller's to append. +func (c ConnInfo) ProviderURI() string { + switch c.Provider { + case apiv2.ObjectStoreProviderS3: + return "s3://" + c.Bucket + case apiv2.ObjectStoreProviderGCS: + return "gs://" + c.Bucket + case apiv2.ObjectStoreProviderAzure: + // Azure carries the storage account in AccessKey. + return AzureBlobURI(c.AccessKey, c.Bucket, "") + default: + return "" + } +} diff --git a/internal/controller/infra/objectstore/endpoint.go b/internal/controller/infra/objectstore/endpoint.go new file mode 100644 index 00000000..b4aa71ff --- /dev/null +++ b/internal/controller/infra/objectstore/endpoint.go @@ -0,0 +1,92 @@ +package objectstore + +import ( + "fmt" + "net" + "strconv" + "strings" +) + +// SchemeForTLS returns the URL scheme implied by whether TLS is enabled. +func SchemeForTLS(tls bool) string { + if tls { + return "https" + } + return "http" +} + +// SplitScheme separates a "scheme://rest" endpoint into its scheme and remainder. +// When no scheme is present it returns an empty scheme and the input unchanged. +func SplitScheme(endpoint string) (scheme, rest string) { + if i := strings.Index(endpoint, "://"); i >= 0 { + return endpoint[:i], endpoint[i+len("://"):] + } + return "", endpoint +} + +// EndpointURL renders the S3-compatible API endpoint as "scheme://host[:port]", +// or "" when no endpoint override is set (i.e. plain AWS S3). A scheme already +// present in Endpoint is preserved; otherwise it is derived from TlsEnabled. Port +// is appended only when set and the host does not already carry one. +func (c ConnInfo) EndpointURL() string { + if c.Endpoint == "" { + return "" + } + scheme, host := SplitScheme(c.Endpoint) + if scheme == "" { + scheme = SchemeForTLS(c.TlsEnabled) + } + if c.Port != "" && !strings.Contains(host, ":") { + host += ":" + c.Port + } + return fmt.Sprintf("%s://%s", scheme, host) +} + +// AzureBlobURI builds the Azure Blob container URL for a storage account, e.g. +// "https://.blob.core.windows.net/[/]". +func AzureBlobURI(account, container, prefix string) string { + uri := fmt.Sprintf("https://%s.blob.core.windows.net/%s", account, container) + if prefix != "" { + uri += "/" + prefix + } + return uri +} + +// ParseLegacyBucket splits a v1 bucket.name/path into endpoint, port, bucket and prefix. +func ParseLegacyBucket(provider, name, path string) (endpoint, port, bucket, prefix string) { + prefix = strings.Trim(path, "/") + + // host[:port]/bucket + if slash := strings.IndexByte(name, '/'); slash >= 0 { + host, bkt := name[:slash], name[slash+1:] + if colon := strings.IndexByte(host, ':'); colon >= 0 { + return host[:colon], host[colon+1:], bkt, prefix + } + return host, "", bkt, prefix + } + + // host:port name, bucket in path + if host, hostPort := splitHostPort(name); host != "" && prefix != "" && S3Compatible(provider) { + bkt, rest, _ := strings.Cut(prefix, "/") + return host, hostPort, bkt, rest + } + + return "", "", name, prefix +} + +// splitHostPort parses "host:port", returning "","" if the port is missing or invalid. +func splitHostPort(name string) (host, port string) { + host, port, err := net.SplitHostPort(name) + if err != nil || host == "" { + return "", "" + } + if n, err := strconv.ParseUint(port, 10, 16); err != nil || n == 0 { + return "", "" + } + return host, port +} + +// S3Compatible reports whether the provider uses an S3-style endpoint. +func S3Compatible(provider string) bool { + return provider == "" || provider == "s3" || provider == "cw" +} diff --git a/internal/controller/infra/objectstore/endpoint_test.go b/internal/controller/infra/objectstore/endpoint_test.go new file mode 100644 index 00000000..5a0eee2e --- /dev/null +++ b/internal/controller/infra/objectstore/endpoint_test.go @@ -0,0 +1,102 @@ +package objectstore + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSchemeForTLS(t *testing.T) { + require.Equal(t, "https", SchemeForTLS(true)) + require.Equal(t, "http", SchemeForTLS(false)) +} + +func TestSplitScheme(t *testing.T) { + cases := []struct { + in string + wantScheme string + wantRest string + }{ + {"https://minio.example.com:9000", "https", "minio.example.com:9000"}, + {"http://host", "http", "host"}, + {"host:9000", "", "host:9000"}, + {"host", "", "host"}, + {"", "", ""}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + scheme, rest := SplitScheme(tc.in) + require.Equal(t, tc.wantScheme, scheme) + require.Equal(t, tc.wantRest, rest) + }) + } +} + +func TestEndpointURL(t *testing.T) { + cases := []struct { + name string + ci ConnInfo + want string + }{ + {"no endpoint (AWS S3)", ConnInfo{}, ""}, + {"host with port, tls off", ConnInfo{Endpoint: "minio.example.com", Port: "9000"}, "http://minio.example.com:9000"}, + {"host with port, tls on", ConnInfo{Endpoint: "minio.example.com", Port: "9000", TlsEnabled: true}, "https://minio.example.com:9000"}, + {"host without port", ConnInfo{Endpoint: "minio.example.com", TlsEnabled: true}, "https://minio.example.com"}, + {"scheme preserved over tls", ConnInfo{Endpoint: "http://seaweedfs.svc", Port: "80", TlsEnabled: true}, "http://seaweedfs.svc:80"}, + {"scheme and port already in endpoint", ConnInfo{Endpoint: "http://objstore.svc:8333"}, "http://objstore.svc:8333"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, tc.ci.EndpointURL()) + }) + } +} + +func TestAzureBlobURI(t *testing.T) { + require.Equal(t, + "https://acct.blob.core.windows.net/container", + AzureBlobURI("acct", "container", "")) + require.Equal(t, + "https://acct.blob.core.windows.net/container/some/prefix", + AzureBlobURI("acct", "container", "some/prefix")) +} + +func TestS3Compatible(t *testing.T) { + for _, p := range []string{"", "s3", "cw"} { + require.True(t, S3Compatible(p), p) + } + for _, p := range []string{"gcs", "az", "azure"} { + require.False(t, S3Compatible(p), p) + } +} + +func TestParseLegacyBucket(t *testing.T) { + cases := []struct { + desc string + provider, name, path string + endpoint, port, bkt, prefix string + }{ + {"empty", "", "", "", "", "", "", ""}, + {"bare bucket", "s3", "my-bucket", "", "", "", "my-bucket", ""}, + {"bare bucket with prefix", "s3", "my-bucket", "prefix", "", "", "my-bucket", "prefix"}, + {"embedded host", "", "minio.example.com/wandb", "", "minio.example.com", "", "wandb", ""}, + {"embedded host:port", "", "minio.example.com:9000/wandb", "", "minio.example.com", "9000", "wandb", ""}, + {"embedded short host:port", "", "minio:9000/wandb", "", "minio", "9000", "wandb", ""}, + {"embedded fqdn host:port", "", "minio.minio.svc.cluster.local:9000/bucket", "", "minio.minio.svc.cluster.local", "9000", "bucket", ""}, + {"host:port name, bucket in path", "s3", "minio.minio.svc.cluster.local:9000", "lsahu-minio-bucket", "minio.minio.svc.cluster.local", "9000", "lsahu-minio-bucket", ""}, + {"host:port name, bucket + prefix in path", "s3", "minio:9000", "/bucket/team/project/", "minio", "9000", "bucket", "team/project"}, + {"host:port name, no provider", "", "minio:9000", "bucket", "minio", "9000", "bucket", ""}, + {"aws bucket with path is not endpoint", "s3", "my-aws-bucket", "prefix", "", "", "my-aws-bucket", "prefix"}, + {"host:port name but non-s3 provider", "gcs", "foo:9000", "bucket", "", "", "foo:9000", "bucket"}, + {"ipv6 host:port name", "s3", "[fd00::1]:9000", "bucket", "fd00::1", "9000", "bucket", ""}, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + e, p, b, pre := ParseLegacyBucket(tc.provider, tc.name, tc.path) + require.Equal(t, tc.endpoint, e) + require.Equal(t, tc.port, p) + require.Equal(t, tc.bkt, b) + require.Equal(t, tc.prefix, pre) + }) + } +} diff --git a/internal/controller/infra/objectstore/pathstyle.go b/internal/controller/infra/objectstore/pathstyle.go new file mode 100644 index 00000000..e69b1eb9 --- /dev/null +++ b/internal/controller/infra/objectstore/pathstyle.go @@ -0,0 +1,31 @@ +package objectstore + +import ( + "net" + "strings" +) + +// coreweaveDomains identify CoreWeave AI Object Storage endpoints, which are +// virtual-hosted (matching the server's cw:// handling). +var coreweaveDomains = []string{"cwobject.com", "cwlota.com", "coreweave.com"} + +// RequiresPathStyle reports whether an S3 endpoint needs path-style addressing: +// true for any custom endpoint except CoreWeave's virtual-hosted object storage. +func RequiresPathStyle(endpoint string) bool { + host := strings.ToLower(endpoint) + if i := strings.Index(host, "://"); i >= 0 { + host = host[i+3:] + } + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + if host == "" { + return false + } + for _, domain := range coreweaveDomains { + if host == domain || strings.HasSuffix(host, "."+domain) { + return false + } + } + return true +} diff --git a/internal/controller/infra/objectstore/pathstyle_test.go b/internal/controller/infra/objectstore/pathstyle_test.go new file mode 100644 index 00000000..1e5bb516 --- /dev/null +++ b/internal/controller/infra/objectstore/pathstyle_test.go @@ -0,0 +1,40 @@ +package objectstore + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRequiresPathStyle(t *testing.T) { + cases := []struct { + endpoint string + want bool + }{ + {"", false}, + // Any explicit endpoint is path-style, AWS's own included: path-style + // works against AWS, and VPC endpoints require it. + {"s3.us-east-1.amazonaws.com", true}, + {"bucket.vpce-0abc.s3.us-west-2.vpce.amazonaws.com", true}, + {"minio.wandb.localhost", true}, + {"minio.wandb.localhost:8080", true}, + {"minio", true}, + {"minio:9000", true}, + {"seaweedfs.seaweedfs.svc.cluster.local:8333", true}, + {"http://minio.local:9000", true}, + {"https://s3.example.com", true}, + // CoreWeave object storage is virtual-hosted. + {"cwobject.com", false}, + {"accel-object.ord1.coreweave.com", false}, + {"foo.cwlota.com", false}, + {"COREWEAVE.COM", false}, + // Suffix match must anchor on a label boundary. + {"cwobject.com.evil.example", true}, + {"evil-cwobject.com", true}, + } + for _, tc := range cases { + t.Run(tc.endpoint, func(t *testing.T) { + require.Equal(t, tc.want, RequiresPathStyle(tc.endpoint)) + }) + } +} diff --git a/internal/controller/infra/objectstore/resolve_test.go b/internal/controller/infra/objectstore/resolve_test.go new file mode 100644 index 00000000..b8103f74 --- /dev/null +++ b/internal/controller/infra/objectstore/resolve_test.go @@ -0,0 +1,163 @@ +package objectstore + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + apiv2 "github.com/wandb/operator/api/v2" +) + +const connSecretName = "wandb-objectstore-connection" + +// resolveFixture builds a fake client holding a single connection secret with +// the given keys and an ObjectStoreConnection whose selectors point at them. +func resolveFixture(t *testing.T, data map[string]string) (*apiv2.ObjectStoreConnection, ConnInfo, error) { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + + raw := map[string][]byte{} + for k, v := range data { + raw[k] = []byte(v) + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: connSecretName, Namespace: "default"}, + Data: raw, + } + + connSel := func(key string) corev1.SecretKeySelector { + return corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: connSecretName}, + Key: key, + } + } + conn := &apiv2.ObjectStoreConnection{ + Provider: connSel("Provider"), + Endpoint: connSel("Host"), + Port: connSel("Port"), + AccessKey: connSel("AccessKey"), + SecretKey: connSel("SecretKey"), + Bucket: connSel("Bucket"), + Path: connSel("Path"), + Region: connSel("Region"), + TlsEnabled: connSel("TlsEnabled"), + ForcePathStyle: connSel("ForcePathStyle"), + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(secret).Build() + info, err := Resolve(context.Background(), c, "default", conn) + return conn, info, err +} + +func TestResolve_NilConnection(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + + _, err := Resolve(context.Background(), c, "default", nil) + require.Error(t, err) +} + +func TestResolve_ExternalS3WithStaticCredentials(t *testing.T) { + conn, info, err := resolveFixture(t, map[string]string{ + "Provider": "s3", + "Host": "minio.local", + "Port": "9000", + "AccessKey": "minio", + "SecretKey": "minio123", + "Bucket": "my-bucket", + "Path": "team/prefix", + "Region": "us-east-1", + "TlsEnabled": "true", + "ForcePathStyle": "true", + }) + require.NoError(t, err) + + require.Equal(t, apiv2.ObjectStoreProviderS3, info.Provider) + require.Equal(t, "minio.local", info.Endpoint) + require.Equal(t, "9000", info.Port) + require.Equal(t, "minio", info.AccessKey) + require.Equal(t, "minio123", info.SecretKey) + require.Equal(t, "my-bucket", info.Bucket) + require.Equal(t, "team/prefix", info.Path) + require.Equal(t, "us-east-1", info.Region) + require.True(t, info.TlsEnabled) + require.True(t, info.ForcePathStyle) + require.True(t, info.HasStaticCredentials()) + + // The credential selectors are preserved for consumers that inject by ref. + require.Equal(t, conn.AccessKey, info.AccessKeyRef) + require.Equal(t, conn.SecretKey, info.SecretKeyRef) +} + +func TestResolve_AmbientCredentials(t *testing.T) { + _, info, err := resolveFixture(t, map[string]string{ + "Provider": "s3", + "Bucket": "my-bucket", + "Region": "us-west-2", + }) + require.NoError(t, err) + + require.Empty(t, info.AccessKey) + require.Empty(t, info.SecretKey) + require.False(t, info.HasStaticCredentials()) + require.Equal(t, "us-west-2", info.Region) +} + +func TestResolve_ForcePathStyleFallbackFromEndpoint(t *testing.T) { + // A custom endpoint with no ForcePathStyle key falls back to RequiresPathStyle. + _, info, err := resolveFixture(t, map[string]string{ + "Host": "minio.local", + "Bucket": "my-bucket", + }) + require.NoError(t, err) + require.True(t, info.ForcePathStyle) + + // No endpoint (native AWS) means virtual-hosted addressing. + _, info, err = resolveFixture(t, map[string]string{ + "Bucket": "my-bucket", + }) + require.NoError(t, err) + require.False(t, info.ForcePathStyle) +} + +func TestResolve_DefaultsEmptyProviderToS3(t *testing.T) { + // Legacy-migrated connections omit Provider; Resolve must default it to S3 so + // downstream provider switches (ProviderURI, ToSecretData) don't fall through. + _, info, err := resolveFixture(t, map[string]string{ + "Bucket": "my-bucket", + }) + require.NoError(t, err) + require.Equal(t, apiv2.ObjectStoreProviderS3, info.Provider) +} + +func TestResolve_RejectsPartialCredentialPair(t *testing.T) { + // Access key without a secret key. + _, _, err := resolveFixture(t, map[string]string{ + "Bucket": "my-bucket", + "AccessKey": "only-access", + }) + require.Error(t, err) + + // Secret key without an access key. + _, _, err = resolveFixture(t, map[string]string{ + "Bucket": "my-bucket", + "SecretKey": "only-secret", + }) + require.Error(t, err) +} + +func TestResolve_MissingTlsDefaultsFalse(t *testing.T) { + _, info, err := resolveFixture(t, map[string]string{ + "Host": "minio.local", + "Bucket": "my-bucket", + }) + require.NoError(t, err) + require.False(t, info.TlsEnabled) +} diff --git a/internal/controller/infra/objectstore/secret.go b/internal/controller/infra/objectstore/secret.go new file mode 100644 index 00000000..2186439b --- /dev/null +++ b/internal/controller/infra/objectstore/secret.go @@ -0,0 +1,185 @@ +package objectstore + +import ( + "context" + "fmt" + "strconv" + "strings" + + apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/pkg/utils" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// connectionRequiredKeys are the secret keys every consumer must be able to read; +// their selectors are never marked optional regardless of the optionality policy. +var connectionRequiredKeys = map[string]bool{"url": true, "Provider": true, "Bucket": true} + +// ToSecretData encodes the connection into the Secret's StringData: the canonical +// "url", the discrete keys consumers read, and the bool flags. Empty discrete +// values are omitted so absent fields stay absent (matching the resolver's +// treatment of missing keys). ForcePathStyle is S3-only. +func (c ConnInfo) ToSecretData() map[string]string { + data := map[string]string{"url": c.URL} + + put := func(key, value string) { + if value != "" { + data[key] = value + } + } + put("Provider", string(c.Provider)) + put("Bucket", c.Bucket) + put("Host", c.Endpoint) + put("Port", c.Port) + put("AccessKey", c.AccessKey) + put("SecretKey", c.SecretKey) + put("Region", c.Region) + put("Path", c.Path) + put("Scheme", c.Scheme) + + data["TlsEnabled"] = strconv.FormatBool(c.TlsEnabled) + if c.Provider == apiv2.ObjectStoreProviderS3 { + data["ForcePathStyle"] = strconv.FormatBool(c.ForcePathStyle) + } + return data +} + +// ToObjectStoreConnection builds the selector view of the connection secret named +// secretName. It emits a selector only for keys ToSecretData actually writes. +// When requireAll is true every selector is required (managed SeaweedFS always +// writes the full key set); otherwise only url/Provider/Bucket are required and +// the rest are optional (external configs omit provider-dependent keys). +func (c ConnInfo) ToObjectStoreConnection(secretName string, requireAll bool) *apiv2.ObjectStoreConnection { + data := c.ToSecretData() + localRef := corev1.LocalObjectReference{Name: secretName} + + sel := func(key string) corev1.SecretKeySelector { + optional := !requireAll && !connectionRequiredKeys[key] + return corev1.SecretKeySelector{LocalObjectReference: localRef, Key: key, Optional: ptr.To(optional)} + } + has := func(key string) bool { _, ok := data[key]; return ok } + + conn := &apiv2.ObjectStoreConnection{} + if has("url") { + conn.URL = sel("url") + } + if has("Provider") { + conn.Provider = sel("Provider") + } + if has("Host") { + conn.Endpoint = sel("Host") + } + if has("Port") { + conn.Port = sel("Port") + } + if has("AccessKey") { + conn.AccessKey = sel("AccessKey") + } + if has("SecretKey") { + conn.SecretKey = sel("SecretKey") + } + if has("Bucket") { + conn.Bucket = sel("Bucket") + } + if has("Path") { + conn.Path = sel("Path") + } + if has("Region") { + conn.Region = sel("Region") + } + if has("TlsEnabled") { + conn.TlsEnabled = sel("TlsEnabled") + } + if has("ForcePathStyle") { + conn.ForcePathStyle = sel("ForcePathStyle") + } + return conn +} + +// Resolve reads the connection's secret selectors into a ConnInfo. +func Resolve( + ctx context.Context, + cl client.Client, + namespace string, + conn *apiv2.ObjectStoreConnection, +) (ConnInfo, error) { + if conn == nil { + return ConnInfo{}, fmt.Errorf("object store connection is not available yet") + } + + resolver := &utils.ConnSecretResolver{Client: cl, Namespace: namespace, Cache: map[string]*corev1.Secret{}} + + info := ConnInfo{ + AccessKeyRef: conn.AccessKey, + SecretKeyRef: conn.SecretKey, + } + + provider, err := resolver.Value(ctx, conn.Provider) + if err != nil { + return ConnInfo{}, err + } + // Legacy-migrated connections never set Provider; match WriteState's default so + // downstream provider switches (ProviderURI, ToSecretData) stay correct. + if provider == "" { + provider = string(apiv2.ObjectStoreProviderS3) + } + info.Provider = apiv2.ObjectStoreProvider(provider) + + if info.Bucket, err = resolver.Value(ctx, conn.Bucket); err != nil { + return ConnInfo{}, err + } + if info.Endpoint, err = resolver.Value(ctx, conn.Endpoint); err != nil { + return ConnInfo{}, err + } + if info.Port, err = resolver.Value(ctx, conn.Port); err != nil { + return ConnInfo{}, err + } + if info.Region, err = resolver.Value(ctx, conn.Region); err != nil { + return ConnInfo{}, err + } + if info.AccessKey, err = resolver.Value(ctx, conn.AccessKey); err != nil { + return ConnInfo{}, err + } + if info.SecretKey, err = resolver.Value(ctx, conn.SecretKey); err != nil { + return ConnInfo{}, err + } + // A half-configured pair silently picks the wrong credential mode downstream. + if (info.AccessKey == "") != (info.SecretKey == "") { + return ConnInfo{}, fmt.Errorf("object store access key and secret key must be configured together") + } + if info.Path, err = resolver.Value(ctx, conn.Path); err != nil { + return ConnInfo{}, err + } + + forcePathStyleString, err := resolver.Value(ctx, conn.ForcePathStyle) + if err != nil { + return ConnInfo{}, err + } + if fps, parseErr := strconv.ParseBool(forcePathStyleString); parseErr == nil { + info.ForcePathStyle = fps + } else { + // Connection secrets written before the operator derived this key lack it. + info.ForcePathStyle = RequiresPathStyle(info.Endpoint) + } + + tlsEnabledString, err := resolver.Value(ctx, conn.TlsEnabled) + if err != nil { + return ConnInfo{}, err + } + if tls, parseErr := strconv.ParseBool(tlsEnabledString); parseErr == nil { + info.TlsEnabled = tls + } + + return info, nil +} + +// SplitBucketPath splits "bucket/optional/prefix" into the leading bucket (or container) segment and the remaining object prefix. +func SplitBucketPath(raw string) (bucket, path string) { + trimmed := strings.TrimPrefix(raw, "/") + if slash := strings.IndexByte(trimmed, '/'); slash >= 0 { + return trimmed[:slash], trimmed[slash+1:] + } + return trimmed, "" +} diff --git a/internal/controller/infra/objectstore/secret_test.go b/internal/controller/infra/objectstore/secret_test.go new file mode 100644 index 00000000..9e0883ad --- /dev/null +++ b/internal/controller/infra/objectstore/secret_test.go @@ -0,0 +1,123 @@ +package objectstore + +import ( + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + + apiv2 "github.com/wandb/operator/api/v2" +) + +func TestToSecretData_ManagedSeaweedShape(t *testing.T) { + // Mirrors the values the managed SeaweedFS path populates; the resulting + // secret must contain exactly the historical key set (no Path). + ci := ConnInfo{ + Provider: apiv2.ObjectStoreProviderS3, + URL: "s3://ak:sk@object-store-s3.wandb.svc.cluster.local:8333/bucket?tls=false", + Endpoint: "object-store-s3.wandb.svc.cluster.local", + Port: "8333", + AccessKey: "ak", + SecretKey: "sk", + Region: "us-east-1", + Bucket: "bucket", + Scheme: "http", + TlsEnabled: false, + ForcePathStyle: true, + } + + require.Equal(t, map[string]string{ + "url": "s3://ak:sk@object-store-s3.wandb.svc.cluster.local:8333/bucket?tls=false", + "Host": "object-store-s3.wandb.svc.cluster.local", + "Port": "8333", + "AccessKey": "ak", + "SecretKey": "sk", + "Region": "us-east-1", + "Bucket": "bucket", + "Scheme": "http", + "TlsEnabled": "false", + "Provider": "s3", + "ForcePathStyle": "true", + }, ci.ToSecretData()) +} + +func TestToSecretData_OmitsEmptyDiscreteKeys(t *testing.T) { + ci := ConnInfo{ + Provider: apiv2.ObjectStoreProviderS3, + URL: "s3://my-bucket", + Bucket: "my-bucket", + } + data := ci.ToSecretData() + require.Equal(t, "s3://my-bucket", data["url"]) + require.NotContains(t, data, "Host") + require.NotContains(t, data, "AccessKey") + require.NotContains(t, data, "SecretKey") + require.NotContains(t, data, "Region") + require.NotContains(t, data, "Path") + require.NotContains(t, data, "Scheme") + // TlsEnabled is always written; ForcePathStyle is written for S3. + require.Equal(t, "false", data["TlsEnabled"]) + require.Equal(t, "false", data["ForcePathStyle"]) +} + +func TestToSecretData_ForcePathStyleIsS3Only(t *testing.T) { + for _, provider := range []apiv2.ObjectStoreProvider{apiv2.ObjectStoreProviderGCS, apiv2.ObjectStoreProviderAzure} { + ci := ConnInfo{Provider: provider, URL: "gs://b", Bucket: "b"} + require.NotContains(t, ci.ToSecretData(), "ForcePathStyle", "path-style is an S3-only concept") + } +} + +func TestToObjectStoreConnection_RequireAll(t *testing.T) { + ci := ConnInfo{ + Provider: apiv2.ObjectStoreProviderS3, + URL: "s3://ak:sk@host:8333/bucket?tls=false", + Endpoint: "host", + Port: "8333", + AccessKey: "ak", + SecretKey: "sk", + Region: "us-east-1", + Bucket: "bucket", + Scheme: "http", + ForcePathStyle: true, + } + conn := ci.ToObjectStoreConnection("conn-secret", true) + + // Every emitted selector points at conn-secret and is required. + for _, s := range []corev1.SecretKeySelector{ + conn.URL, conn.Provider, conn.Endpoint, conn.Port, conn.AccessKey, + conn.SecretKey, conn.Region, conn.Bucket, conn.TlsEnabled, conn.ForcePathStyle, + } { + require.Equal(t, "conn-secret", s.Name) + require.NotNil(t, s.Optional) + require.False(t, *s.Optional) + } + require.Equal(t, "Host", conn.Endpoint.Key) + require.Equal(t, "url", conn.URL.Key) + // Path is not written for the managed shape, so its selector stays empty. + require.Empty(t, conn.Path.Name) +} + +func TestToObjectStoreConnection_ExternalOptionality(t *testing.T) { + ci := ConnInfo{ + Provider: apiv2.ObjectStoreProviderS3, + URL: "s3://ak:sk@host:9000/bucket", + Endpoint: "host", + Port: "9000", + AccessKey: "ak", + SecretKey: "sk", + Region: "us-west-2", + Bucket: "bucket", + } + conn := ci.ToObjectStoreConnection("conn-secret", false) + + // url/Provider/Bucket are required... + for _, s := range []corev1.SecretKeySelector{conn.URL, conn.Provider, conn.Bucket} { + require.NotNil(t, s.Optional) + require.False(t, *s.Optional) + } + // ...everything else is optional. + for _, s := range []corev1.SecretKeySelector{conn.Endpoint, conn.Port, conn.AccessKey, conn.SecretKey, conn.Region} { + require.NotNil(t, s.Optional) + require.True(t, *s.Optional) + } +} diff --git a/internal/controller/reconciler/cleanup_legacy_v1.go b/internal/controller/reconciler/cleanup_legacy_v1.go index 8069539c..fa1b05ea 100644 --- a/internal/controller/reconciler/cleanup_legacy_v1.go +++ b/internal/controller/reconciler/cleanup_legacy_v1.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "sort" apiv2 "github.com/wandb/operator/api/v2" "github.com/wandb/operator/internal/logx" @@ -56,20 +57,34 @@ func buildDesiredAppNames(manifest serverManifest.Manifest) map[string]bool { return out } -func appsHealthy( - appStatuses map[string]apiv2.ApplicationStatus, +// deploymentsHealthy reports whether every manifest-desired application's +// Deployment is fully rolled out, plus the sorted names still blocking. It +// reads live Deployments rather than status.wandb.applications so the legacy +// cleanup gate cannot act on a stale status snapshot. +func deploymentsHealthy( + ctx context.Context, + c ctrlClient.Client, + namespace string, desiredAppNames map[string]bool, -) bool { +) (bool, []string) { if len(desiredAppNames) == 0 { - return false + return false, nil } + var notReady []string for name := range desiredAppNames { - s, ok := appStatuses[name] - if !ok || !s.Ready { - return false + dep := &appsv1.Deployment{} + if err := c.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, dep); err != nil { + notReady = append(notReady, name) + continue + } + if dep.Status.ObservedGeneration != dep.Generation || + dep.Status.ReadyReplicas != dep.Status.Replicas || + dep.Status.Replicas == 0 { + notReady = append(notReady, name) } } - return true + sort.Strings(notReady) + return len(notReady) == 0, notReady } func cleanupLegacyV1Deployments( diff --git a/internal/controller/reconciler/cleanup_legacy_v1_test.go b/internal/controller/reconciler/cleanup_legacy_v1_test.go index 00b80a2c..6e64a8b0 100644 --- a/internal/controller/reconciler/cleanup_legacy_v1_test.go +++ b/internal/controller/reconciler/cleanup_legacy_v1_test.go @@ -54,48 +54,92 @@ func legacyDeployment(wandbName, suffix, namespace string) *appsv1.Deployment { } } -func TestLegacyV1AppsHealthy(t *testing.T) { +// readyDeployment builds a fully rolled-out Deployment for gate tests. +func readyDeployment(name, namespace string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Generation: 2}, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 2, + Replicas: 1, + ReadyReplicas: 1, + }, + } +} + +func TestDeploymentsHealthy(t *testing.T) { + const namespace = "default" + desired := map[string]bool{"api": true, "console": true} + tests := []struct { name string - statuses map[string]apiv2.ApplicationStatus desired map[string]bool + deployments []*appsv1.Deployment wantHealthy bool + wantBlocked []string }{ { name: "empty desired set is never healthy", - statuses: map[string]apiv2.ApplicationStatus{"api": {Ready: true}}, desired: map[string]bool{}, wantHealthy: false, }, { - name: "missing status entry blocks gate", - statuses: map[string]apiv2.ApplicationStatus{"api": {Ready: true}}, - desired: map[string]bool{"api": true, "console": true}, + name: "missing deployment blocks gate", + desired: desired, + deployments: []*appsv1.Deployment{readyDeployment("api", namespace)}, wantHealthy: false, + wantBlocked: []string{"console"}, }, { - name: "one not-ready app blocks gate", - statuses: map[string]apiv2.ApplicationStatus{"api": {Ready: true}, "console": {Ready: false}}, - desired: map[string]bool{"api": true, "console": true}, + name: "mid-rollout deployment blocks gate", + desired: desired, + deployments: func() []*appsv1.Deployment { + rolling := readyDeployment("console", namespace) + rolling.Status.ReadyReplicas = 0 + return []*appsv1.Deployment{readyDeployment("api", namespace), rolling} + }(), wantHealthy: false, + wantBlocked: []string{"console"}, }, { - name: "all desired apps ready opens gate", - statuses: map[string]apiv2.ApplicationStatus{"api": {Ready: true}, "console": {Ready: true}}, - desired: map[string]bool{"api": true, "console": true}, - wantHealthy: true, + name: "zero replicas blocks gate", + desired: desired, + deployments: func() []*appsv1.Deployment { + scaled := readyDeployment("console", namespace) + scaled.Status.Replicas = 0 + scaled.Status.ReadyReplicas = 0 + return []*appsv1.Deployment{readyDeployment("api", namespace), scaled} + }(), + wantHealthy: false, + wantBlocked: []string{"console"}, + }, + { + name: "stale observedGeneration blocks gate", + desired: desired, + deployments: func() []*appsv1.Deployment { + stale := readyDeployment("console", namespace) + stale.Status.ObservedGeneration = 1 + return []*appsv1.Deployment{readyDeployment("api", namespace), stale} + }(), + wantHealthy: false, + wantBlocked: []string{"console"}, }, { - name: "extra status entries do not block gate", - statuses: map[string]apiv2.ApplicationStatus{"api": {Ready: true}, "console": {Ready: true}, "stale": {Ready: false}}, - desired: map[string]bool{"api": true, "console": true}, + name: "all deployments rolled out opens gate", + desired: desired, + deployments: []*appsv1.Deployment{readyDeployment("api", namespace), readyDeployment("console", namespace)}, wantHealthy: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.wantHealthy, appsHealthy(tc.statuses, tc.desired)) + builder := fake.NewClientBuilder().WithScheme(newCleanupFixtureScheme(t)) + for _, dep := range tc.deployments { + builder = builder.WithObjects(dep) + } + healthy, blocked := deploymentsHealthy(context.Background(), builder.Build(), namespace, tc.desired) + require.Equal(t, tc.wantHealthy, healthy) + require.Equal(t, tc.wantBlocked, blocked) }) } } @@ -233,3 +277,11 @@ func TestCleanupLegacyV1Deployments(t *testing.T) { } }) } + +func TestDeploymentsHealthy_BlockedListSorted(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(newCleanupFixtureScheme(t)).Build() + healthy, blocked := deploymentsHealthy(context.Background(), cl, "default", + map[string]bool{"weave": true, "api": true, "glue": true}) + require.False(t, healthy) + require.Equal(t, []string{"api", "glue", "weave"}, blocked) +} diff --git a/internal/controller/reconciler/clickhouse.go b/internal/controller/reconciler/clickhouse.go index 76b545f4..ed98ba24 100644 --- a/internal/controller/reconciler/clickhouse.go +++ b/internal/controller/reconciler/clickhouse.go @@ -8,6 +8,7 @@ import ( "github.com/wandb/operator/internal/controller/infra/external" externalch "github.com/wandb/operator/internal/controller/infra/external/clickhouse" "github.com/wandb/operator/internal/controller/infra/managed/clickhouse/altinity" + "github.com/wandb/operator/internal/controller/infra/managed/clickhouse/altinity/keeper" "github.com/wandb/operator/pkg/utils" "github.com/wandb/operator/pkg/wandb/manifest" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -17,34 +18,48 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +// clickHouseObjectStoreInstance is the object-store instance name managed +// ClickHouse prefers for its S3 disk; ResolveInstance falls back to the +// default instance when it is not provisioned. +const clickHouseObjectStoreInstance = "clickhouse" + func clickHouseWriteState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, mfst manifest.Manifest, -) []metav1.Condition { - if wandb.Spec.ClickHouse.ManagedClickHouse != nil { - return managedClickHouseWriteState(ctx, client, wandb, mfst) - } - if wandb.Spec.ClickHouse.ExternalClickHouse != nil { - return externalClickHouseWriteState(ctx, client, wandb) +) map[string][]metav1.Condition { + out := map[string][]metav1.Condition{} + for key, spec := range wandb.Spec.ClickHouse { + switch { + case spec.ManagedClickHouse != nil: + out[key] = managedClickHouseWriteState(ctx, client, wandb, spec.ManagedClickHouse, mfst) + case spec.ExternalClickHouse != nil: + out[key] = externalch.WriteState(ctx, client, wandb, key, spec.ExternalClickHouse) + } } - return nil + return out } func clickHouseReadState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, - newConditions []metav1.Condition, -) ([]metav1.Condition, *apiv2.ClickHouseConnection) { - if wandb.Spec.ClickHouse.ManagedClickHouse != nil { - return managedClickHouseReadState(ctx, client, wandb, newConditions) - } - if wandb.Spec.ClickHouse.ExternalClickHouse != nil { - return externalClickHouseReadState(ctx, client, wandb, newConditions) + conditions map[string][]metav1.Condition, +) (map[string][]metav1.Condition, map[string]*apiv2.ClickHouseConnection) { + outConds := map[string][]metav1.Condition{} + outConns := map[string]*apiv2.ClickHouseConnection{} + for key, spec := range wandb.Spec.ClickHouse { + switch { + case spec.ManagedClickHouse != nil: + outConds[key], outConns[key] = managedClickHouseReadState(ctx, client, wandb, spec.ManagedClickHouse, conditions[key]) + case spec.ExternalClickHouse != nil: + outConds[key], outConns[key] = externalch.ReadState(ctx, client, wandb, key, conditions[key]) + default: + outConds[key] = conditions[key] + } } - return newConditions, nil + return outConds, outConns } func clickHouseInferStatus( @@ -52,30 +67,62 @@ func clickHouseInferStatus( client client.Client, recorder record.EventRecorder, wandb *apiv2.WeightsAndBiases, - newConditions []metav1.Condition, - newInfraConn *apiv2.ClickHouseConnection, + conditions map[string][]metav1.Condition, + infraConns map[string]*apiv2.ClickHouseConnection, ) (ctrl.Result, error) { - if wandb.Spec.ClickHouse.ManagedClickHouse != nil { - return managedClickHouseInferStatus(ctx, client, recorder, wandb, newConditions, newInfraConn) + if wandb.Status.ClickHouseStatus == nil { + wandb.Status.ClickHouseStatus = map[string]apiv2.ClickHouseInfraStatus{} } - if wandb.Spec.ClickHouse.ExternalClickHouse != nil { - return externalClickHouseInferStatus(ctx, client, wandb, newConditions, newInfraConn) + var results []ctrl.Result + var firstErr error + for key, spec := range wandb.Spec.ClickHouse { + var res ctrl.Result + var err error + switch { + case spec.ManagedClickHouse != nil: + res, err = managedClickHouseInferStatus(ctx, client, recorder, wandb, key, conditions[key], infraConns[key]) + case spec.ExternalClickHouse != nil: + res, err = externalClickHouseInferStatus(ctx, client, wandb, key, conditions[key], infraConns[key]) + } + results = append(results, res) + if err != nil && firstErr == nil { + firstErr = err + } } - return ctrl.Result{}, nil + return consolidateResults(results), firstErr +} + +func runClickHouseRetentionFinalizer(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string, spec apiv2.ClickHouseSpec) error { + switch wandb.GetRetentionPolicy(clickHouseInstanceInfraSpec(spec)).OnDelete { + case apiv2.PurgeOnDelete: + return clickHousePurgeFinalizer(ctx, c, wandb, key, spec) + case apiv2.DetachOnDelete: + return clickHouseDetachFinalizer(ctx, c, wandb, key, spec) + } + return nil +} + +func clickHouseInstanceInfraSpec(spec apiv2.ClickHouseSpec) apiv2.ManagedInfraSpec { + if spec.ManagedClickHouse != nil { + return spec.ManagedClickHouse.ManagedInfraSpec + } + return apiv2.ManagedInfraSpec{} } func clickHousePurgeFinalizer( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + key string, + spec apiv2.ClickHouseSpec, ) error { - if spec := wandb.Spec.ClickHouse.ManagedClickHouse; spec != nil { - specNamespacedName := managedClickHouseSpecNamespacedName(spec) - onDeleteRule := altinity.ToClickHouseOnDeleteRule(wandb, wandb.GetRetentionPolicy(spec.ManagedInfraSpec)) + if managed := spec.ManagedClickHouse; managed != nil { + specNamespacedName := managedClickHouseSpecNamespacedName(managed) + onDeleteRule := altinity.ToClickHouseOnDeleteRule(wandb, wandb.GetRetentionPolicy(managed.ManagedInfraSpec)) return altinity.PurgeFinalizer(ctx, client, specNamespacedName, onDeleteRule) } - if wandb.Spec.ClickHouse.ExternalClickHouse != nil { - return externalch.DeleteConnectionSecret(ctx, client, wandb) + if spec.ExternalClickHouse != nil { + return externalch.DeleteConnectionSecret(ctx, client, wandb, key) } return nil } @@ -84,12 +131,14 @@ func clickHouseDetachFinalizer( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + _ string, + spec apiv2.ClickHouseSpec, ) error { - spec := wandb.Spec.ClickHouse.ManagedClickHouse - if spec == nil { + managed := spec.ManagedClickHouse + if managed == nil { return nil } - specNamespacedName := managedClickHouseSpecNamespacedName(spec) + specNamespacedName := managedClickHouseSpecNamespacedName(managed) return altinity.DetachFinalizer(ctx, client, specNamespacedName, wandb) } @@ -99,13 +148,81 @@ func managedClickHouseWriteState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedClickHouseSpec, mfst manifest.Manifest, ) []metav1.Condition { - spec := wandb.Spec.ClickHouse.ManagedClickHouse - - var specNamespacedName = managedClickHouseSpecNamespacedName(spec) log := ctrl.LoggerFrom(ctx) - desired, err := altinity.ToClickHouseVendorSpec(ctx, wandb, client.Scheme(), mfst) + + // Altinity swallows the apiserver's rejection of over-long derived names, so + // fail the status loudly here; also covers CRs that predate admission checks. + if err := altinity.ValidateDerivedNames(spec); err != nil { + log.Error(err, "managed ClickHouse name cannot be deployed") + return []metav1.Condition{ + { + Type: common.ReconciledType, + Status: metav1.ConditionFalse, + Reason: common.InvalidNameReason, + Message: err.Error(), + }, + { + Type: altinity.ClickHouseCustomResourceType, + Status: metav1.ConditionFalse, + Reason: common.InvalidNameReason, + Message: err.Error(), + }, + } + } + + // ClickHouse table data lives in the object store: use the "clickhouse" + // instance when provisioned, otherwise the default instance. + objStoreStatus, _ := apiv2.ResolveInstance(wandb.Status.ObjectStoreStatus, clickHouseObjectStoreInstance) + objStoreSpec, _ := apiv2.ResolveInstance(wandb.Spec.ObjectStore, clickHouseObjectStoreInstance) + waitForObjectStore := objStoreSpec.ManagedObjectStore != nil + + // Resolve the bucket connection; wait and requeue if it isn't ready yet. + objStorage, objStorageEndpoint, err := altinity.ResolveObjectStorage(ctx, client, spec, &objStoreStatus.Connection) + if err != nil { + log.Error(err, "object storage not ready for ClickHouse") + return []metav1.Condition{ + { + Type: common.ReconciledType, + Status: metav1.ConditionFalse, + Reason: common.PendingCreateReason, + }, + { + Type: altinity.ClickHouseCustomResourceType, + Status: metav1.ConditionFalse, + Reason: common.PendingCreateReason, + }, + } + } + + // Translate the Keeper and ClickHouse CRs; WriteState writes Keeper first. + desiredKeeper, err := keeper.ToKeeperVendorSpec(ctx, wandb, spec, client.Scheme(), altinity.KeeperNsName(spec), mfst) + if err != nil { + log.Error(err, "failed to translate Keeper spec to vendor spec") + return []metav1.Condition{ + { + Type: common.ReconciledType, + Status: metav1.ConditionFalse, + Reason: common.ControllerErrorReason, + }, + } + } + + desiredServiceAccount, err := altinity.ToServiceAccount(wandb, spec, objStorage, client.Scheme()) + if err != nil { + log.Error(err, "failed to translate ClickHouse ServiceAccount") + return []metav1.Condition{ + { + Type: common.ReconciledType, + Status: metav1.ConditionFalse, + Reason: common.ControllerErrorReason, + }, + } + } + + desired, err := altinity.ToClickHouseVendorSpec(ctx, wandb, spec, client.Scheme(), objStorage, objStorageEndpoint, waitForObjectStore, mfst) if err != nil { log.Error(err, "failed to translate ClickHouse spec to vendor spec") return []metav1.Condition{ @@ -117,11 +234,15 @@ func managedClickHouseWriteState( } } + specNamespacedName := managedClickHouseSpecNamespacedName(spec) + if conditions := altinity.CheckDetached(ctx, client, specNamespacedName, wandb.GetUID()); conditions != nil { return conditions } - results := altinity.WriteState(ctx, client, specNamespacedName, desired) + results := make([]metav1.Condition, 0) + results = append(results, altinity.WriteState(ctx, client, specNamespacedName, desiredServiceAccount, desiredKeeper, desired)...) + return results } @@ -129,14 +250,17 @@ func managedClickHouseReadState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedClickHouseSpec, newConditions []metav1.Condition, ) ([]metav1.Condition, *apiv2.ClickHouseConnection) { - spec := wandb.Spec.ClickHouse.ManagedClickHouse - specNamespacedName := managedClickHouseSpecNamespacedName(spec) onDeleteRule := altinity.ToClickHouseOnDeleteRule(wandb, wandb.GetRetentionPolicy(spec.ManagedInfraSpec)) readConditions, newInfraConn := altinity.ReadState(ctx, client, specNamespacedName, wandb, onDeleteRule) newConditions = append(newConditions, readConditions...) + + // Keeper readiness gates ClickHouse readiness (see inferInfraState). + newConditions = append(newConditions, keeper.ReadState(ctx, client, altinity.KeeperNsName(spec))...) + return newConditions, newInfraConn } @@ -145,12 +269,15 @@ func managedClickHouseInferStatus( client client.Client, recorder record.EventRecorder, wandb *apiv2.WeightsAndBiases, + key string, newConditions []metav1.Condition, newInfraConn *apiv2.ClickHouseConnection, ) (ctrl.Result, error) { + statusBefore := wandb.DeepCopy().Status enabled := true - oldConditions := wandb.Status.ClickHouseStatus.Conditions - oldInfraConn := wandb.Status.ClickHouseStatus.Connection + oldStatus := wandb.Status.ClickHouseStatus[key] + oldConditions := oldStatus.Conditions + oldInfraConn := oldStatus.Connection updatedStatus, events, ctrlResult := altinity.ComputeStatus( ctx, @@ -163,32 +290,26 @@ func managedClickHouseInferStatus( for _, e := range events { recorder.Event(wandb, e.Type, e.Reason, e.Message) } - wandb.Status.ClickHouseStatus = updatedStatus - err := client.Status().Update(ctx, wandb) + wandb.Status.ClickHouseStatus[key] = updatedStatus + err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore) return ctrlResult, err } // external -func externalClickHouseWriteState(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases) []metav1.Condition { - return externalch.WriteState(ctx, c, wandb) -} - -func externalClickHouseReadState(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, newConditions []metav1.Condition) ([]metav1.Condition, *apiv2.ClickHouseConnection) { - return externalch.ReadState(ctx, c, wandb, newConditions) -} - -func externalClickHouseInferStatus(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, newConditions []metav1.Condition, newInfraConn *apiv2.ClickHouseConnection) (ctrl.Result, error) { - oldInfraConn := wandb.Status.ClickHouseStatus.Connection - state, ready, updatedConditions := external.InferExternalStatus(wandb.Status.ClickHouseStatus.Conditions, newConditions, wandb.Generation, newInfraConn != nil) +func externalClickHouseInferStatus(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string, newConditions []metav1.Condition, newInfraConn *apiv2.ClickHouseConnection) (ctrl.Result, error) { + statusBefore := wandb.DeepCopy().Status + oldStatus := wandb.Status.ClickHouseStatus[key] + oldInfraConn := oldStatus.Connection + state, ready, updatedConditions := external.InferExternalStatus(oldStatus.Conditions, newConditions, wandb.Generation, newInfraConn != nil) conn := utils.Coalesce(newInfraConn, &oldInfraConn) - wandb.Status.ClickHouseStatus = apiv2.ClickHouseInfraStatus{ + wandb.Status.ClickHouseStatus[key] = apiv2.ClickHouseInfraStatus{ WBInfraStatus: apiv2.WBInfraStatus{Ready: ready, State: state, Conditions: updatedConditions}, Connection: *conn, } - return ctrl.Result{}, c.Status().Update(ctx, wandb) + return ctrl.Result{}, updateWandbStatusIfChanged(ctx, c, wandb, statusBefore) } // helpers diff --git a/internal/controller/reconciler/custom_ca.go b/internal/controller/reconciler/custom_ca.go new file mode 100644 index 00000000..8dfe6c31 --- /dev/null +++ b/internal/controller/reconciler/custom_ca.go @@ -0,0 +1,477 @@ +package reconciler + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + + apiv2 "github.com/wandb/operator/api/v2" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const ( + customCACertsChecksumAnnotation = "weightsandbiases.apps.wandb.com/ca-certs-checksum" + + customCACertsRootVolumeName = "wandb-ca-certs-root" + customCACertsInlineVolumeName = "wandb-ca-certs" + customCACertsConfigMapVolumeName = "wandb-ca-certs-user" + + customCACertsRootMountPath = "/usr/local/share/ca-certificates/" + customCACertsInlineMountPath = "/usr/local/share/ca-certificates/inline" + customCACertsConfigMapMountPath = "/usr/local/share/ca-certificates/configmap" + + mysqlCACertVolumeName = "mysql-ca" + mysqlCACertPath = "/etc/ssl/certs/mysql_ca.pem" + mysqlCACertFileName = "mysql_ca.pem" + + mysqlSSLCertVolumeName = "mysql-ssl-cert" + mysqlSSLCertPath = "/etc/ssl/certs/mysql_ssl_cert.pem" + mysqlSSLCertFileName = "mysql_ssl_cert.pem" + + mysqlSSLKeyVolumeName = "mysql-ssl-key" + mysqlSSLKeyPath = "/etc/ssl/certs/mysql_ssl_key.pem" + mysqlSSLKeyFileName = "mysql_ssl_key.pem" + + redisCACertVolumeName = "redis-ca" + redisCACertPath = "/etc/ssl/certs/redis_ca.pem" + redisCACertFileName = "redis_ca.pem" +) + +var customCACertsEnvVars = []corev1.EnvVar{ + {Name: "SSL_CERT_FILE", Value: "/etc/ssl/certs/ca-certificates.crt"}, + {Name: "SSL_CERT_DIR", Value: "/etc/ssl/certs"}, + {Name: "REQUESTS_CA_BUNDLE", Value: "/etc/ssl/certs/ca-certificates.crt"}, +} + +func customCACertsConfigMapName(wandb *apiv2.WeightsAndBiases) string { + return fmt.Sprintf("%s-ca-certs", wandb.Name) +} + +func hasGlobalCustomCACertConfig(wandb *apiv2.WeightsAndBiases) bool { + return len(wandb.Spec.Global.CustomCACerts) > 0 || wandb.Spec.Global.CACertsConfigMap != "" +} + +// defaultMySQLConnection returns the default MySQL instance's connection. The +// app's TLS env vars (MYSQL_CA_CERT_PATH etc.) are singular, so only the +// default instance's certificate material is mounted. +func defaultMySQLConnection(wandb *apiv2.WeightsAndBiases) apiv2.MysqlConnection { + status, _ := apiv2.ResolveInstance(wandb.Status.MySQLStatus, "") + return status.Connection +} + +// defaultRedisConnection returns the default Redis instance's connection; see +// defaultMySQLConnection. +func defaultRedisConnection(wandb *apiv2.WeightsAndBiases) apiv2.RedisConnection { + status, _ := apiv2.ResolveInstance(wandb.Status.RedisStatus, "") + return status.Connection +} + +func secretSelectorConfigured(sel corev1.SecretKeySelector) bool { + return sel.Name != "" && sel.Key != "" +} + +func hasOwnerReference(obj ctrlClient.Object, owner ctrlClient.Object) bool { + ownerUID := owner.GetUID() + for _, ref := range obj.GetOwnerReferences() { + if ownerUID != "" && ref.UID == ownerUID { + return true + } + } + return false +} + +func reconcileCustomCACerts(ctx context.Context, c ctrlClient.Client, wandb *apiv2.WeightsAndBiases) error { + nsName := types.NamespacedName{Name: customCACertsConfigMapName(wandb), Namespace: wandb.Namespace} + actual := &corev1.ConfigMap{} + err := c.Get(ctx, nsName, actual) + if err != nil && !apiErrors.IsNotFound(err) { + return err + } + + if len(wandb.Spec.Global.CustomCACerts) == 0 { + if apiErrors.IsNotFound(err) { + return nil + } + if !hasOwnerReference(actual, wandb) { + return nil + } + return c.Delete(ctx, actual) + } + + data := make(map[string]string, len(wandb.Spec.Global.CustomCACerts)) + for i, pem := range wandb.Spec.Global.CustomCACerts { + data[fmt.Sprintf("customCA%d.crt", i)] = pem + } + + desired := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName.Name, + Namespace: nsName.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "wandb-operator", + "app.kubernetes.io/instance": wandb.Name, + "app.kubernetes.io/part-of": "wandb", + }, + }, + Data: data, + } + if err := controllerutil.SetOwnerReference(wandb, desired, c.Scheme()); err != nil { + return err + } + + if apiErrors.IsNotFound(err) { + return c.Create(ctx, desired) + } + desired.ResourceVersion = actual.ResourceVersion + return c.Update(ctx, desired) +} + +func applyCustomCACertsToWorkload( + ctx context.Context, + c ctrlClient.Client, + wandb *apiv2.WeightsAndBiases, + envVars []corev1.EnvVar, + volumes []corev1.Volume, + volumeMounts []corev1.VolumeMount, +) ([]corev1.EnvVar, []corev1.Volume, []corev1.VolumeMount, string, error) { + mysqlConn := defaultMySQLConnection(wandb) + redisConn := defaultRedisConnection(wandb) + + if hasGlobalCustomCACertConfig(wandb) { + envVars = appendMissingEnvVars(envVars, customCACertsEnvVars) + volumes = upsertVolume(volumes, corev1.Volume{ + Name: customCACertsRootVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + volumeMounts = upsertVolumeMount(volumeMounts, corev1.VolumeMount{ + Name: customCACertsRootVolumeName, + MountPath: customCACertsRootMountPath, + ReadOnly: false, + }) + + if len(wandb.Spec.Global.CustomCACerts) > 0 { + volumes = upsertVolume(volumes, corev1.Volume{ + Name: customCACertsInlineVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: customCACertsConfigMapName(wandb)}, + }, + }, + }) + volumeMounts = upsertVolumeMount(volumeMounts, corev1.VolumeMount{ + Name: customCACertsInlineVolumeName, + MountPath: customCACertsInlineMountPath, + ReadOnly: true, + }) + } + + if wandb.Spec.Global.CACertsConfigMap != "" { + volumes = upsertVolume(volumes, corev1.Volume{ + Name: customCACertsConfigMapVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: wandb.Spec.Global.CACertsConfigMap}, + Optional: boolPtr(true), + }, + }, + }) + volumeMounts = upsertVolumeMount(volumeMounts, corev1.VolumeMount{ + Name: customCACertsConfigMapVolumeName, + MountPath: customCACertsConfigMapMountPath, + ReadOnly: true, + }) + } + } + + if hasValue, err := secretSelectorHasValue(ctx, c, wandb.Namespace, mysqlConn.SslCa); err != nil { + return nil, nil, nil, "", err + } else if hasValue { + envVars = appendMissingEnvVars(envVars, []corev1.EnvVar{{Name: "MYSQL_CA_CERT_PATH", Value: mysqlCACertPath}}) + volumes = upsertVolume(volumes, corev1.Volume{ + Name: mysqlCACertVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: secretCACertVolumeSource(mysqlConn.SslCa, mysqlCACertFileName), + }, + }) + volumeMounts = upsertVolumeMount(volumeMounts, corev1.VolumeMount{ + Name: mysqlCACertVolumeName, + MountPath: mysqlCACertPath, + SubPath: mysqlCACertFileName, + ReadOnly: true, + }) + } + if hasValue, err := secretSelectorHasValue(ctx, c, wandb.Namespace, mysqlConn.SslCert); err != nil { + return nil, nil, nil, "", err + } else if hasValue { + volumes = upsertVolume(volumes, corev1.Volume{ + Name: mysqlSSLCertVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: secretCACertVolumeSource(mysqlConn.SslCert, mysqlSSLCertFileName), + }, + }) + volumeMounts = upsertVolumeMount(volumeMounts, corev1.VolumeMount{ + Name: mysqlSSLCertVolumeName, + MountPath: mysqlSSLCertPath, + SubPath: mysqlSSLCertFileName, + ReadOnly: true, + }) + } + if hasValue, err := secretSelectorHasValue(ctx, c, wandb.Namespace, mysqlConn.SslKey); err != nil { + return nil, nil, nil, "", err + } else if hasValue { + volumes = upsertVolume(volumes, corev1.Volume{ + Name: mysqlSSLKeyVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: secretCACertVolumeSource(mysqlConn.SslKey, mysqlSSLKeyFileName), + }, + }) + volumeMounts = upsertVolumeMount(volumeMounts, corev1.VolumeMount{ + Name: mysqlSSLKeyVolumeName, + MountPath: mysqlSSLKeyPath, + SubPath: mysqlSSLKeyFileName, + ReadOnly: true, + }) + } + + if hasValue, err := secretSelectorHasValue(ctx, c, wandb.Namespace, redisConn.SslCa); err != nil { + return nil, nil, nil, "", err + } else if hasValue { + volumes = upsertVolume(volumes, corev1.Volume{ + Name: redisCACertVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: secretCACertVolumeSource(redisConn.SslCa, redisCACertFileName), + }, + }) + volumeMounts = upsertVolumeMount(volumeMounts, corev1.VolumeMount{ + Name: redisCACertVolumeName, + MountPath: redisCACertPath, + SubPath: redisCACertFileName, + ReadOnly: true, + }) + } + + checksum, err := customCACertsChecksum(ctx, c, wandb) + if err != nil { + return nil, nil, nil, "", err + } + return envVars, volumes, volumeMounts, checksum, nil +} + +func setCustomCACertsChecksumAnnotation(podTemplate *corev1.PodTemplateSpec, checksum string) { + annotations := podTemplate.GetAnnotations() + if checksum == "" { + if annotations == nil { + return + } + delete(annotations, customCACertsChecksumAnnotation) + if len(annotations) == 0 { + annotations = nil + } + podTemplate.SetAnnotations(annotations) + return + } + if annotations == nil { + annotations = map[string]string{} + } + annotations[customCACertsChecksumAnnotation] = checksum + podTemplate.SetAnnotations(annotations) +} + +func secretCACertVolumeSource(sel corev1.SecretKeySelector, fileName string) *corev1.SecretVolumeSource { + return &corev1.SecretVolumeSource{ + SecretName: sel.Name, + Items: []corev1.KeyToPath{{ + Key: sel.Key, + Path: fileName, + }}, + Optional: sel.Optional, + } +} + +func upsertVolume(volumes []corev1.Volume, volume corev1.Volume) []corev1.Volume { + for i := range volumes { + if volumes[i].Name == volume.Name { + volumes[i] = volume + return volumes + } + } + return append(volumes, volume) +} + +func upsertVolumeMount(volumeMounts []corev1.VolumeMount, mount corev1.VolumeMount) []corev1.VolumeMount { + for i := range volumeMounts { + if volumeMounts[i].Name == mount.Name { + volumeMounts[i] = mount + return volumeMounts + } + } + return append(volumeMounts, mount) +} + +func secretSelectorHasValue(ctx context.Context, c ctrlClient.Client, namespace string, sel corev1.SecretKeySelector) (bool, error) { + if !secretSelectorConfigured(sel) { + return false, nil + } + + secret := &corev1.Secret{} + err := c.Get(ctx, types.NamespacedName{Name: sel.Name, Namespace: namespace}, secret) + if apiErrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + + if _, ok := secret.Data[sel.Key]; ok { + return true, nil + } + if _, ok := secret.StringData[sel.Key]; ok { + return true, nil + } + return false, nil +} + +func customCACertsChecksum(ctx context.Context, c ctrlClient.Client, wandb *apiv2.WeightsAndBiases) (string, error) { + mysqlConn := defaultMySQLConnection(wandb) + redisConn := defaultRedisConnection(wandb) + + hasMySQLCA, err := secretSelectorHasValue(ctx, c, wandb.Namespace, mysqlConn.SslCa) + if err != nil { + return "", err + } + hasMySQLCert, err := secretSelectorHasValue(ctx, c, wandb.Namespace, mysqlConn.SslCert) + if err != nil { + return "", err + } + hasMySQLKey, err := secretSelectorHasValue(ctx, c, wandb.Namespace, mysqlConn.SslKey) + if err != nil { + return "", err + } + hasRedisCA, err := secretSelectorHasValue(ctx, c, wandb.Namespace, redisConn.SslCa) + if err != nil { + return "", err + } + + if !hasGlobalCustomCACertConfig(wandb) && !hasMySQLCA && !hasMySQLCert && !hasMySQLKey && !hasRedisCA { + return "", nil + } + + hash := sha256.New() + for i, pem := range wandb.Spec.Global.CustomCACerts { + _, _ = fmt.Fprintf(hash, "inline:%d:%s\n", i, pem) + } + + if wandb.Spec.Global.CACertsConfigMap != "" { + _, _ = fmt.Fprintf(hash, "configmap:%s\n", wandb.Spec.Global.CACertsConfigMap) + if err := hashConfigMapData(ctx, c, wandb.Namespace, wandb.Spec.Global.CACertsConfigMap, hashWriteString(hash)); err != nil { + return "", err + } + } + + if sel := mysqlConn.SslCa; hasMySQLCA { + _, _ = fmt.Fprintf(hash, "mysql:%s/%s\n", sel.Name, sel.Key) + if err := hashSecretKeyData(ctx, c, wandb.Namespace, sel, hashWriteString(hash)); err != nil { + return "", err + } + } + if sel := mysqlConn.SslCert; hasMySQLCert { + _, _ = fmt.Fprintf(hash, "mysql-cert:%s/%s\n", sel.Name, sel.Key) + if err := hashSecretKeyData(ctx, c, wandb.Namespace, sel, hashWriteString(hash)); err != nil { + return "", err + } + } + if sel := mysqlConn.SslKey; hasMySQLKey { + _, _ = fmt.Fprintf(hash, "mysql-key:%s/%s\n", sel.Name, sel.Key) + if err := hashSecretKeyData(ctx, c, wandb.Namespace, sel, hashWriteString(hash)); err != nil { + return "", err + } + } + + if sel := redisConn.SslCa; hasRedisCA { + _, _ = fmt.Fprintf(hash, "redis:%s/%s\n", sel.Name, sel.Key) + if err := hashSecretKeyData(ctx, c, wandb.Namespace, sel, hashWriteString(hash)); err != nil { + return "", err + } + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} + +type writeStringFunc func(string) + +func hashWriteString(hash interface{ Write([]byte) (int, error) }) writeStringFunc { + return func(s string) { + _, _ = hash.Write([]byte(s)) + } +} + +func hashConfigMapData(ctx context.Context, c ctrlClient.Client, namespace, name string, write writeStringFunc) error { + configMap := &corev1.ConfigMap{} + err := c.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, configMap) + if apiErrors.IsNotFound(err) { + write("missing-configmap\n") + return nil + } + if err != nil { + return err + } + + keys := make([]string, 0, len(configMap.Data)+len(configMap.BinaryData)) + for k := range configMap.Data { + keys = append(keys, "data:"+k) + } + for k := range configMap.BinaryData { + keys = append(keys, "binary:"+k) + } + sort.Strings(keys) + for _, typedKey := range keys { + write(typedKey) + write("=") + switch { + case len(typedKey) > len("data:") && typedKey[:len("data:")] == "data:": + write(configMap.Data[typedKey[len("data:"):]]) + case len(typedKey) > len("binary:") && typedKey[:len("binary:")] == "binary:": + write(string(configMap.BinaryData[typedKey[len("binary:"):]])) + } + write("\n") + } + return nil +} + +func hashSecretKeyData(ctx context.Context, c ctrlClient.Client, namespace string, sel corev1.SecretKeySelector, write writeStringFunc) error { + secret := &corev1.Secret{} + err := c.Get(ctx, types.NamespacedName{Name: sel.Name, Namespace: namespace}, secret) + if apiErrors.IsNotFound(err) { + write("missing-secret\n") + return nil + } + if err != nil { + return err + } + + if data, ok := secret.Data[sel.Key]; ok { + write(string(data)) + write("\n") + return nil + } + if stringData, ok := secret.StringData[sel.Key]; ok { + write(stringData) + write("\n") + return nil + } + write("missing-key\n") + return nil +} + +func boolPtr(v bool) *bool { + return &v +} diff --git a/internal/controller/reconciler/custom_ca_test.go b/internal/controller/reconciler/custom_ca_test.go new file mode 100644 index 00000000..ea43e3df --- /dev/null +++ b/internal/controller/reconciler/custom_ca_test.go @@ -0,0 +1,270 @@ +package reconciler + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + apiv2 "github.com/wandb/operator/api/v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func customCATestClient(t *testing.T, objects ...ctrlClient.Object) *fake.ClientBuilder { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, apiv2.AddToScheme(scheme)) + + builder := fake.NewClientBuilder().WithScheme(scheme) + if len(objects) > 0 { + builder.WithObjects(objects...) + } + return builder +} + +func TestReconcileCustomCACertsCreatesInlineConfigMap(t *testing.T) { + wandb := &apiv2.WeightsAndBiases{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps.wandb.com/v2", Kind: "WeightsAndBiases"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "wandb", + Namespace: "default", + UID: "wandb-uid", + }, + Spec: apiv2.WeightsAndBiasesSpec{ + Global: apiv2.GlobalSpec{ + CustomCACerts: []string{"---cert-one---", "---cert-two---"}, + }, + }, + } + builder := customCATestClient(t, wandb) + client := builder.Build() + + require.NoError(t, reconcileCustomCACerts(context.Background(), client, wandb)) + + var cm corev1.ConfigMap + require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: "wandb-ca-certs", Namespace: "default"}, &cm)) + require.Equal(t, "---cert-one---", cm.Data["customCA0.crt"]) + require.Equal(t, "---cert-two---", cm.Data["customCA1.crt"]) + require.Len(t, cm.OwnerReferences, 1) + require.Equal(t, "wandb", cm.OwnerReferences[0].Name) +} + +func TestReconcileCustomCACertsDoesNotDeleteUnownedGeneratedConfigMap(t *testing.T) { + wandb := &apiv2.WeightsAndBiases{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps.wandb.com/v2", Kind: "WeightsAndBiases"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "wandb", + Namespace: "default", + UID: "wandb-uid", + }, + } + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb-ca-certs", Namespace: "default"}, + Data: map[string]string{"user.crt": "---user---"}, + } + builder := customCATestClient(t, wandb, configMap) + client := builder.Build() + + require.NoError(t, reconcileCustomCACerts(context.Background(), client, wandb)) + + var cm corev1.ConfigMap + require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: "wandb-ca-certs", Namespace: "default"}, &cm)) + require.Equal(t, "---user---", cm.Data["user.crt"]) +} + +func TestApplyCustomCACertsToWorkloadAddsGlobalAndInfraMounts(t *testing.T) { + wandb := &apiv2.WeightsAndBiases{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps.wandb.com/v2", Kind: "WeightsAndBiases"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "wandb", + Namespace: "default", + }, + Spec: apiv2.WeightsAndBiasesSpec{ + Global: apiv2.GlobalSpec{ + CustomCACerts: []string{"---inline---"}, + CACertsConfigMap: "user-ca-certs", + }, + }, + Status: apiv2.WeightsAndBiasesStatus{ + MySQLStatus: map[string]apiv2.MysqlInfraStatus{ + apiv2.DefaultInstanceName: apiv2.MysqlInfraStatus{ + Connection: apiv2.MysqlConnection{ + SslCa: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "wandb-mysql-connection"}, + Key: "SslCa", + }, + SslCert: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "wandb-mysql-connection"}, + Key: "SslCert", + }, + SslKey: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "wandb-mysql-connection"}, + Key: "SslKey", + }, + }, + }, + }, + RedisStatus: map[string]apiv2.RedisInfraStatus{ + apiv2.DefaultInstanceName: apiv2.RedisInfraStatus{ + Connection: apiv2.RedisConnection{ + SslCa: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "wandb-redis-connection"}, + Key: "SslCa", + Optional: ptr.To(true), + }, + }, + }, + }, + }, + } + mysqlSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb-mysql-connection", Namespace: "default"}, + Data: map[string][]byte{ + "SslCa": []byte("---mysql-ca---"), + "SslCert": []byte("---mysql-cert---"), + "SslKey": []byte("---mysql-key---"), + }, + } + redisSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb-redis-connection", Namespace: "default"}, + Data: map[string][]byte{"SslCa": []byte("---redis-ca---")}, + } + userCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "user-ca-certs", Namespace: "default"}, + Data: map[string]string{"corp.crt": "---corp---"}, + } + builder := customCATestClient(t, wandb, mysqlSecret, redisSecret, userCM) + client := builder.Build() + + envs, volumes, mounts, checksum, err := applyCustomCACertsToWorkload(context.Background(), client, wandb, nil, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, checksum) + + requireContainsEnv(t, envs, "SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt") + requireContainsEnv(t, envs, "SSL_CERT_DIR", "/etc/ssl/certs") + requireContainsEnv(t, envs, "REQUESTS_CA_BUNDLE", "/etc/ssl/certs/ca-certificates.crt") + requireContainsEnv(t, envs, "MYSQL_CA_CERT_PATH", mysqlCACertPath) + + requireVolume(t, volumes, customCACertsRootVolumeName) + requireVolume(t, volumes, customCACertsInlineVolumeName) + requireVolume(t, volumes, customCACertsConfigMapVolumeName) + requireVolume(t, volumes, mysqlCACertVolumeName) + requireVolume(t, volumes, mysqlSSLCertVolumeName) + requireVolume(t, volumes, mysqlSSLKeyVolumeName) + requireVolume(t, volumes, redisCACertVolumeName) + + requireMount(t, mounts, customCACertsRootVolumeName, customCACertsRootMountPath) + requireMount(t, mounts, customCACertsInlineVolumeName, customCACertsInlineMountPath) + requireMount(t, mounts, customCACertsConfigMapVolumeName, customCACertsConfigMapMountPath) + requireMount(t, mounts, mysqlCACertVolumeName, mysqlCACertPath) + requireMount(t, mounts, mysqlSSLCertVolumeName, mysqlSSLCertPath) + requireMount(t, mounts, mysqlSSLKeyVolumeName, mysqlSSLKeyPath) + requireMount(t, mounts, redisCACertVolumeName, redisCACertPath) + + podTemplate := &corev1.PodTemplateSpec{} + setCustomCACertsChecksumAnnotation(podTemplate, checksum) + require.Equal(t, checksum, podTemplate.Annotations[customCACertsChecksumAnnotation]) +} + +func TestApplyCustomCACertsToWorkloadSkipsMissingOptionalInfraKeys(t *testing.T) { + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{ + Name: "wandb", + Namespace: "default", + }, + Status: apiv2.WeightsAndBiasesStatus{ + MySQLStatus: map[string]apiv2.MysqlInfraStatus{ + apiv2.DefaultInstanceName: apiv2.MysqlInfraStatus{ + Connection: apiv2.MysqlConnection{ + SslCa: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "wandb-mysql-connection"}, + Key: "SslCa", + Optional: ptr.To(true), + }, + }, + }, + }, + RedisStatus: map[string]apiv2.RedisInfraStatus{ + apiv2.DefaultInstanceName: apiv2.RedisInfraStatus{ + Connection: apiv2.RedisConnection{ + SslCa: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "wandb-redis-connection"}, + Key: "SslCa", + Optional: ptr.To(true), + }, + }, + }, + }, + }, + } + mysqlSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb-mysql-connection", Namespace: "default"}, + Data: map[string][]byte{"url": []byte("mysql://user:pass@db:3306/wandb")}, + } + redisSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb-redis-connection", Namespace: "default"}, + Data: map[string][]byte{"url": []byte("redis://redis:6379")}, + } + builder := customCATestClient(t, wandb, mysqlSecret, redisSecret) + client := builder.Build() + + envs, volumes, mounts, checksum, err := applyCustomCACertsToWorkload(context.Background(), client, wandb, nil, nil, nil) + require.NoError(t, err) + require.Empty(t, checksum) + requireNoEnv(t, envs, "MYSQL_CA_CERT_PATH") + requireNoVolume(t, volumes, mysqlCACertVolumeName) + requireNoVolume(t, volumes, redisCACertVolumeName) + require.Empty(t, mounts) +} + +func requireContainsEnv(t *testing.T, envs []corev1.EnvVar, name, value string) { + t.Helper() + for _, env := range envs { + if env.Name == name { + require.Equal(t, value, env.Value) + return + } + } + t.Fatalf("env var %q not found in %+v", name, envs) +} + +func requireNoEnv(t *testing.T, envs []corev1.EnvVar, name string) { + t.Helper() + for _, env := range envs { + require.NotEqual(t, name, env.Name) + } +} + +func requireVolume(t *testing.T, volumes []corev1.Volume, name string) { + t.Helper() + for _, volume := range volumes { + if volume.Name == name { + return + } + } + t.Fatalf("volume %q not found in %+v", name, volumes) +} + +func requireNoVolume(t *testing.T, volumes []corev1.Volume, name string) { + t.Helper() + for _, volume := range volumes { + require.NotEqual(t, name, volume.Name) + } +} + +func requireMount(t *testing.T, mounts []corev1.VolumeMount, name, mountPath string) { + t.Helper() + for _, mount := range mounts { + if mount.Name == name { + require.Equal(t, mountPath, mount.MountPath) + return + } + } + t.Fatalf("volume mount %q not found in %+v", name, mounts) +} diff --git a/internal/controller/reconciler/gateway.go b/internal/controller/reconciler/gateway.go index 4e50e588..bb371f37 100644 --- a/internal/controller/reconciler/gateway.go +++ b/internal/controller/reconciler/gateway.go @@ -301,17 +301,25 @@ func buildAllowedRoutes(wandb *apiv2.WeightsAndBiases) *gatewayv1.AllowedRoutes func requiresCrossNamespaceInfraRoutes(wandb *apiv2.WeightsAndBiases) bool { namespaces := []string{} - if spec := wandb.Spec.ObjectStore.ManagedObjectStore; spec != nil { - namespaces = append(namespaces, spec.Namespace) + for _, instance := range wandb.Spec.ObjectStore { + if spec := instance.ManagedObjectStore; spec != nil { + namespaces = append(namespaces, spec.Namespace) + } } - if spec := wandb.Spec.ClickHouse.ManagedClickHouse; spec != nil { - namespaces = append(namespaces, spec.Namespace) + for _, instance := range wandb.Spec.ClickHouse { + if spec := instance.ManagedClickHouse; spec != nil { + namespaces = append(namespaces, spec.Namespace) + } } - if spec := wandb.Spec.MySQL.ManagedMysql; spec != nil { - namespaces = append(namespaces, spec.Namespace) + for _, instance := range wandb.Spec.MySQL { + if spec := instance.ManagedMysql; spec != nil { + namespaces = append(namespaces, spec.Namespace) + } } - if spec := wandb.Spec.Redis.ManagedRedis; spec != nil { - namespaces = append(namespaces, spec.Namespace) + for _, instance := range wandb.Spec.Redis { + if spec := instance.ManagedRedis; spec != nil { + namespaces = append(namespaces, spec.Namespace) + } } for _, ns := range namespaces { diff --git a/internal/controller/reconciler/infra_routes.go b/internal/controller/reconciler/infra_routes.go index 8592ea0a..13c4faad 100644 --- a/internal/controller/reconciler/infra_routes.go +++ b/internal/controller/reconciler/infra_routes.go @@ -44,19 +44,23 @@ const infraHTTPRouteComponent = "infra-route" func resolveInfraRoutes(ctx context.Context, c ctrlClient.Client, wandb *apiv2.WeightsAndBiases, manifest serverManifest.Manifest) ([]infraRouteEntry, error) { var entries []infraRouteEntry - if objectStoreSpec := wandb.Spec.ObjectStore.ManagedObjectStore; objectStoreSpec != nil { + for crKey, instance := range wandb.Spec.ObjectStore { + objectStoreSpec := instance.ManagedObjectStore + if objectStoreSpec == nil { + continue + } for _, instanceName := range sortedInfraConfigNames(manifest.Bucket) { cfg := manifest.Bucket[instanceName] if cfg.Ingress == nil { continue } svcName := fmt.Sprintf("%s-s3", objectStoreSpec.Name) - port, err := resolveInfraServicePort(ctx, c, types.NamespacedName{Name: svcName, Namespace: wandb.Spec.ObjectStore.ManagedObjectStore.Namespace}, cfg.Ingress, 8333) + port, err := resolveInfraServicePort(ctx, c, types.NamespacedName{Name: svcName, Namespace: objectStoreSpec.Namespace}, cfg.Ingress, 8333) if err != nil { return nil, fmt.Errorf("bucket instance %q: %w", instanceName, err) } entries = append(entries, infraRouteEntry{ - name: fmt.Sprintf("%s-bucket-%s", wandb.Name, instanceName), + name: fmt.Sprintf("%s-bucket-%s", wandb.Name, infraRouteInstanceName(crKey, instanceName)), namespace: objectStoreSpec.Namespace, serviceName: svcName, servicePort: port, @@ -67,7 +71,11 @@ func resolveInfraRoutes(ctx context.Context, c ctrlClient.Client, wandb *apiv2.W } } - if chSpec := wandb.Spec.ClickHouse.ManagedClickHouse; chSpec != nil { + for crKey, instance := range wandb.Spec.ClickHouse { + chSpec := instance.ManagedClickHouse + if chSpec == nil { + continue + } for _, instanceName := range sortedInfraConfigNames(manifest.Clickhouse) { cfg := manifest.Clickhouse[instanceName] if cfg.Ingress == nil { @@ -78,7 +86,7 @@ func resolveInfraRoutes(ctx context.Context, c ctrlClient.Client, wandb *apiv2.W port, err := resolveInfraServicePort( ctx, c, - types.NamespacedName{Name: svcName, Namespace: wandb.Spec.ClickHouse.ManagedClickHouse.Namespace}, + types.NamespacedName{Name: svcName, Namespace: chSpec.Namespace}, cfg.Ingress, 8123, ) @@ -86,7 +94,7 @@ func resolveInfraRoutes(ctx context.Context, c ctrlClient.Client, wandb *apiv2.W return nil, fmt.Errorf("clickhouse instance %q: %w", instanceName, err) } entries = append(entries, infraRouteEntry{ - name: fmt.Sprintf("%s-clickhouse-%s", wandb.Name, instanceName), + name: fmt.Sprintf("%s-clickhouse-%s", wandb.Name, infraRouteInstanceName(crKey, instanceName)), namespace: chSpec.Namespace, serviceName: svcName, servicePort: port, @@ -100,6 +108,16 @@ func resolveInfraRoutes(ctx context.Context, c ctrlClient.Client, wandb *apiv2.W return entries, nil } +// infraRouteInstanceName composes a unique route suffix from the CR instance key +// and the manifest infra-config name. The default CR instance keeps the +// historical suffix (manifest name only) to preserve existing route names. +func infraRouteInstanceName(crKey, manifestInstanceName string) string { + if crKey == apiv2.DefaultInstanceName { + return manifestInstanceName + } + return fmt.Sprintf("%s-%s", crKey, manifestInstanceName) +} + func resolveInfraServicePort(ctx context.Context, c ctrlClient.Client, serviceRef types.NamespacedName, ingress *serverManifest.AppIngressSpec, defaultPort int32) (gatewayv1.PortNumber, error) { if ingress != nil && ingress.ServicePort != "" { parsed := intstr.Parse(ingress.ServicePort) @@ -281,8 +299,6 @@ func buildInfraHTTPRoute( }, } - hostnameOverride := fmt.Sprintf("%s.%s.svc.cluster.local", entry.serviceName, entry.namespace) - return &gatewayv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: entry.name, @@ -297,19 +313,6 @@ func buildInfraHTTPRoute( Rules: []gatewayv1.HTTPRouteRule{{ Matches: matches, BackendRefs: []gatewayv1.HTTPBackendRef{backendRef}, - Filters: []gatewayv1.HTTPRouteFilter{ - { - Type: gatewayv1.HTTPRouteFilterRequestHeaderModifier, - RequestHeaderModifier: &gatewayv1.HTTPHeaderFilter{ - Remove: []string{"X-Forwarded-Host", "X-Forwarded-Port"}, - }, - }, { - Type: gatewayv1.HTTPRouteFilterURLRewrite, - URLRewrite: &gatewayv1.HTTPURLRewriteFilter{ - Hostname: (*gatewayv1.PreciseHostname)(&hostnameOverride), - }, - }, - }, }}, }, } diff --git a/internal/controller/reconciler/ingress.go b/internal/controller/reconciler/ingress.go index 0d3845b9..75410b57 100644 --- a/internal/controller/reconciler/ingress.go +++ b/internal/controller/reconciler/ingress.go @@ -69,6 +69,29 @@ func reconcileConsolidatedIngress(ctx context.Context, c ctrlClient.Client, wand } } + infraRoutes, err := resolveInfraRoutes(ctx, c, wandb, manifest) + if err != nil { + return err + } + for _, route := range infraRoutes { + if len(route.ingress.Paths) == 0 { + continue + } + pathType := networkingv1.PathType(route.ingress.PathType) + paths = append(paths, networkingv1.HTTPIngressPath{ + Path: route.ingress.Paths[0], + PathType: &pathType, + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: route.serviceName, + Port: networkingv1.ServiceBackendPort{ + Number: route.servicePort, + }, + }, + }, + }) + } + if len(paths) == 0 { return nil } @@ -93,6 +116,7 @@ func reconcileConsolidatedIngress(ctx context.Context, c ctrlClient.Client, wand for k, v := range wandb.Spec.Networking.Annotations { annotations[k] = v } + if wandb.Spec.Networking.TLS != nil && wandb.Spec.Networking.TLS.CertManager != nil { cm := wandb.Spec.Networking.TLS.CertManager if cm.ClusterIssuer != "" { @@ -120,6 +144,9 @@ func reconcileConsolidatedIngress(ctx context.Context, c ctrlClient.Client, wand if wandb.Spec.Networking.Ingress != nil { desired.Spec.IngressClassName = wandb.Spec.Networking.Ingress.IngressClassName + if *desired.Spec.IngressClassName == "nginx" { + desired.Annotations["nginx.ingress.kubernetes.io/proxy-body-size"] = "0" + } } if wandb.Spec.Networking.TLS != nil && wandb.Spec.Networking.TLS.SecretName != "" { @@ -136,7 +163,7 @@ func reconcileConsolidatedIngress(ctx context.Context, c ctrlClient.Client, wand } current := &networkingv1.Ingress{} - err := c.Get(ctx, types.NamespacedName{Name: ingressName, Namespace: wandb.Namespace}, current) + err = c.Get(ctx, types.NamespacedName{Name: ingressName, Namespace: wandb.Namespace}, current) if err != nil { if apiErrors.IsNotFound(err) { if err := c.Create(ctx, desired); err != nil { diff --git a/internal/controller/reconciler/kafka.go b/internal/controller/reconciler/kafka.go index 797206db..e99b9d6e 100644 --- a/internal/controller/reconciler/kafka.go +++ b/internal/controller/reconciler/kafka.go @@ -127,6 +127,7 @@ func managedKafkaInferStatus( newConditions []metav1.Condition, newInfraConn *apiv2.KafkaConnection, ) (ctrl.Result, error) { + statusBefore := wandb.DeepCopy().Status oldConditions := wandb.Status.KafkaStatus.Conditions oldInfraConn := wandb.Status.KafkaStatus.Connection @@ -143,7 +144,7 @@ func managedKafkaInferStatus( recorder.Event(wandb, e.Type, e.Reason, e.Message) } wandb.Status.KafkaStatus = updatedStatus - err := client.Status().Update(ctx, wandb) + err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore) return ctrlResult, err } diff --git a/internal/controller/reconciler/legacy_overrides.go b/internal/controller/reconciler/legacy_overrides.go new file mode 100644 index 00000000..e6bb73a8 --- /dev/null +++ b/internal/controller/reconciler/legacy_overrides.go @@ -0,0 +1,77 @@ +package reconciler + +import ( + "context" + "sort" + + apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/logx" + serverManifest "github.com/wandb/operator/pkg/wandb/manifest" + corev1 "k8s.io/api/core/v1" +) + +// validateLegacyOverrides logs legacyOverrides keys that are neither "global" +// nor a manifest application. The spec is left untouched — unknown keys are +// simply never applied. +func validateLegacyOverrides(ctx context.Context, wandb *apiv2.WeightsAndBiases, manifest serverManifest.Manifest) { + if len(wandb.Spec.Wandb.LegacyOverrides) == 0 { + return + } + logger := logx.GetSlog(ctx) + + keys := make([]string, 0, len(wandb.Spec.Wandb.LegacyOverrides)) + for key := range wandb.Spec.Wandb.LegacyOverrides { + keys = append(keys, key) + } + sort.Strings(keys) + + for _, key := range keys { + if key == apiv2.LegacyOverridesGlobalKey { + continue + } + if _, ok := manifest.Applications[key]; ok { + continue + } + logger.Warn("legacy override section does not map to any application in the server manifest; ignoring", + "section", key, "version", wandb.Spec.Wandb.Version) + } +} + +// applyLegacyOverrideEnv layers global then per-app overrides (per-app wins) +// onto a fully built env list, so they beat manifest and injected vars — as +// in v1, where user env displaced chart-computed env. +func applyLegacyOverrideEnv(ctx context.Context, wandb *apiv2.WeightsAndBiases, appName string, envVars []corev1.EnvVar) []corev1.EnvVar { + overrides := wandb.Spec.Wandb.LegacyOverrides + if len(overrides) == 0 { + return envVars + } + envVars = overrideEnvVars(ctx, envVars, overrides[apiv2.LegacyOverridesGlobalKey].Env) + envVars = overrideEnvVars(ctx, envVars, overrides[appName].Env) + return envVars +} + +// overrideEnvVars replaces same-named vars in place and appends the rest — +// the inverse of appendMissingEnvVars. Empty names skip with a log; a later +// duplicate wins. +func overrideEnvVars(ctx context.Context, base []corev1.EnvVar, overrides []corev1.EnvVar) []corev1.EnvVar { + if len(overrides) == 0 { + return base + } + index := make(map[string]int, len(base)) + for i, envVar := range base { + index[envVar.Name] = i + } + for _, envVar := range overrides { + if envVar.Name == "" { + logx.GetSlog(ctx).Warn("skipping legacy override env var with empty name") + continue + } + if i, ok := index[envVar.Name]; ok { + base[i] = envVar + continue + } + index[envVar.Name] = len(base) + base = append(base, envVar) + } + return base +} diff --git a/internal/controller/reconciler/legacy_overrides_test.go b/internal/controller/reconciler/legacy_overrides_test.go new file mode 100644 index 00000000..d518d757 --- /dev/null +++ b/internal/controller/reconciler/legacy_overrides_test.go @@ -0,0 +1,181 @@ +package reconciler + +import ( + "context" + "testing" + + apiv2 "github.com/wandb/operator/api/v2" + serverManifest "github.com/wandb/operator/pkg/wandb/manifest" + corev1 "k8s.io/api/core/v1" +) + +func envNamesValues(envVars []corev1.EnvVar) map[string]string { + out := make(map[string]string, len(envVars)) + for _, envVar := range envVars { + out[envVar.Name] = envVar.Value + } + return out +} + +func TestOverrideEnvVarsReplacesInPlaceAndAppends(t *testing.T) { + base := []corev1.EnvVar{ + {Name: "A", Value: "base-a"}, + {Name: "B", Value: "base-b"}, + } + result := overrideEnvVars(context.Background(), base, []corev1.EnvVar{ + {Name: "B", Value: "override-b"}, + {Name: "C", Value: "override-c"}, + }) + + if len(result) != 3 { + t.Fatalf("expected 3 env vars, got %d: %v", len(result), result) + } + // Replaced in place: order preserved for existing names. + if result[1].Name != "B" || result[1].Value != "override-b" { + t.Errorf("expected B replaced in place, got %v", result[1]) + } + if result[2].Name != "C" || result[2].Value != "override-c" { + t.Errorf("expected C appended, got %v", result[2]) + } + if result[0].Value != "base-a" { + t.Errorf("expected A untouched, got %v", result[0]) + } +} + +func TestOverrideEnvVarsReplacesValueFrom(t *testing.T) { + base := []corev1.EnvVar{ + {Name: "SECRET", ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "old"}, + Key: "k", + }, + }}, + } + result := overrideEnvVars(context.Background(), base, []corev1.EnvVar{ + {Name: "SECRET", Value: "literal-now"}, + }) + + if result[0].ValueFrom != nil || result[0].Value != "literal-now" { + t.Errorf("expected override to fully replace the var, got %+v", result[0]) + } +} + +func TestOverrideEnvVarsHandAuthoredEdgeCases(t *testing.T) { + base := []corev1.EnvVar{{Name: "A", Value: "base-a"}} + result := overrideEnvVars(context.Background(), base, []corev1.EnvVar{ + {Name: "", Value: "skipped"}, + {Name: "DUP", Value: "first"}, + {Name: "DUP", Value: "last-wins"}, + }) + + values := envNamesValues(result) + if _, ok := values[""]; ok { + t.Error("empty-name entry should be skipped") + } + if values["DUP"] != "last-wins" { + t.Errorf("expected later duplicate to win, got %q", values["DUP"]) + } + if len(result) != 2 { + t.Fatalf("expected 2 env vars, got %d: %v", len(result), result) + } +} + +func TestOverrideEnvVarsNoOverridesNoOp(t *testing.T) { + base := []corev1.EnvVar{{Name: "A", Value: "base-a"}} + result := overrideEnvVars(context.Background(), base, nil) + if len(result) != 1 || result[0].Value != "base-a" { + t.Errorf("expected base unchanged, got %v", result) + } +} + +func TestApplyLegacyOverrideEnvPrecedence(t *testing.T) { + wandb := testWeightsAndBiases() + wandb.Spec.Wandb.LegacyOverrides = map[string]apiv2.LegacyOverrides{ + apiv2.LegacyOverridesGlobalKey: {Env: []corev1.EnvVar{ + {Name: "GLOBAL_ONLY", Value: "global"}, + {Name: "BOTH", Value: "from-global"}, + {Name: "MANIFEST_VAR", Value: "global-override"}, + }}, + "api": {Env: []corev1.EnvVar{ + {Name: "BOTH", Value: "from-app"}, + {Name: "APP_ONLY", Value: "app"}, + }}, + } + + base := []corev1.EnvVar{{Name: "MANIFEST_VAR", Value: "manifest"}} + + result := applyLegacyOverrideEnv(context.Background(), wandb, "api", base) + values := envNamesValues(result) + + if values["MANIFEST_VAR"] != "global-override" { + t.Errorf("expected override to beat manifest env, got %q", values["MANIFEST_VAR"]) + } + if values["BOTH"] != "from-app" { + t.Errorf("expected per-app to beat global, got %q", values["BOTH"]) + } + if values["GLOBAL_ONLY"] != "global" || values["APP_ONLY"] != "app" { + t.Errorf("expected both layers present, got %v", values) + } +} + +func TestApplyLegacyOverrideEnvAppWithoutEntryGetsGlobal(t *testing.T) { + wandb := testWeightsAndBiases() + wandb.Spec.Wandb.LegacyOverrides = map[string]apiv2.LegacyOverrides{ + apiv2.LegacyOverridesGlobalKey: {Env: []corev1.EnvVar{{Name: "HTTP_PROXY", Value: "http://proxy"}}}, + "parquet": {Env: []corev1.EnvVar{{Name: "PARQUET_VAR", Value: "x"}}}, + } + + result := applyLegacyOverrideEnv(context.Background(), wandb, "weave", nil) + values := envNamesValues(result) + + if values["HTTP_PROXY"] != "http://proxy" { + t.Errorf("expected global env applied, got %v", values) + } + if _, ok := values["PARQUET_VAR"]; ok { + t.Error("another app's overrides must not apply") + } +} + +func TestApplyLegacyOverrideEnvNoOverrides(t *testing.T) { + base := []corev1.EnvVar{{Name: "A", Value: "a"}} + result := applyLegacyOverrideEnv(context.Background(), testWeightsAndBiases(), "api", base) + if len(result) != 1 || result[0].Value != "a" { + t.Errorf("expected base unchanged, got %v", result) + } +} + +func TestValidateLegacyOverridesDoesNotMutateSpec(t *testing.T) { + wandb := testWeightsAndBiases() + wandb.Spec.Wandb.LegacyOverrides = map[string]apiv2.LegacyOverrides{ + apiv2.LegacyOverridesGlobalKey: {Env: []corev1.EnvVar{{Name: "A", Value: "1"}}}, + "api": {Env: []corev1.EnvVar{{Name: "B", Value: "2"}}}, + "console": {Env: []corev1.EnvVar{{Name: "C", Value: "3"}}}, + "app": {Env: []corev1.EnvVar{{Name: "D", Value: "4"}}}, + } + manifest := serverManifest.Manifest{ + Applications: map[string]serverManifest.Application{ + "api": {Name: "api"}, + }, + } + + validateLegacyOverrides(context.Background(), wandb, manifest) + + // Unmapped keys (console, app) are logged but must remain in the spec. + if len(wandb.Spec.Wandb.LegacyOverrides) != 4 { + t.Fatalf("expected spec untouched, got %v", wandb.Spec.Wandb.LegacyOverrides) + } + for _, key := range []string{apiv2.LegacyOverridesGlobalKey, "api", "console", "app"} { + if _, ok := wandb.Spec.Wandb.LegacyOverrides[key]; !ok { + t.Errorf("expected key %q to remain in spec", key) + } + } +} + +func TestValidateLegacyOverridesEmptyInputs(t *testing.T) { + // Must not panic with nil overrides or an empty manifest. + validateLegacyOverrides(context.Background(), testWeightsAndBiases(), serverManifest.Manifest{}) + + wandb := testWeightsAndBiases() + wandb.Spec.Wandb.LegacyOverrides = map[string]apiv2.LegacyOverrides{"api": {}} + validateLegacyOverrides(context.Background(), wandb, serverManifest.Manifest{}) +} diff --git a/internal/controller/reconciler/migrate_legacy.go b/internal/controller/reconciler/migrate_legacy.go index 17dc1594..c3557bd5 100644 --- a/internal/controller/reconciler/migrate_legacy.go +++ b/internal/controller/reconciler/migrate_legacy.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "net/url" "strconv" "strings" "time" @@ -31,6 +32,7 @@ import ( apiv1 "github.com/wandb/operator/api/v1" apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/infra/objectstore" ) // migrateLegacyAnnotations drains `legacy.operator.wandb.com/*-pending` @@ -57,7 +59,11 @@ func migrateLegacyAnnotations( if err != nil { return ctrl.Result{}, err } - if !mysqlChanged && !redisChanged && !bucketChanged && !oidcChanged { + clickHouseChanged, err := migrateLegacyClickHouse(ctx, c, wandb) + if err != nil { + return ctrl.Result{}, err + } + if !mysqlChanged && !redisChanged && !bucketChanged && !oidcChanged && !clickHouseChanged { return ctrl.Result{}, nil } @@ -99,7 +105,7 @@ func migrateLegacyMySQL( } secretName := fmt.Sprintf("%s-mysql-converted", wandb.Name) - conn := wandb.Spec.MySQL.ExternalMysql + conn := wandb.Spec.MySQL[apiv2.DefaultInstanceName].ExternalMysql if conn == nil { conn = &apiv2.MysqlConnection{} } @@ -124,7 +130,7 @@ func migrateLegacyMySQL( return false, err } - wandb.Spec.MySQL.ExternalMysql = conn + setExternalInstance(&wandb.Spec.MySQL, func(s *apiv2.MySQLSpec) { s.ExternalMysql = conn }) delete(wandb.Annotations, apiv1.MySQLPendingAnnotation) return true, nil } @@ -159,7 +165,7 @@ func migrateLegacyRedis( } secretName := fmt.Sprintf("%s-redis-converted", wandb.Name) - conn := wandb.Spec.Redis.ExternalRedis + conn := wandb.Spec.Redis[apiv2.DefaultInstanceName].ExternalRedis if conn == nil { conn = &apiv2.RedisConnection{} } @@ -183,15 +189,76 @@ func migrateLegacyRedis( return false, err } - wandb.Spec.Redis.ExternalRedis = conn + setExternalInstance(&wandb.Spec.Redis, func(s *apiv2.RedisSpec) { s.ExternalRedis = conn }) delete(wandb.Annotations, apiv1.RedisPendingAnnotation) return true, nil } +// legacyClickHousePayload is the literal-string subset the webhook couldn't +// turn into typed selectors. Port is `any` to accept JSON number or string. +type legacyClickHousePayload struct { + Host string `json:"host,omitempty"` + Port any `json:"port,omitempty"` + Database string `json:"database,omitempty"` + User string `json:"user,omitempty"` + Password string `json:"password,omitempty"` +} + +// migrateLegacyClickHouse drains the clickhouse-pending annotation into a +// Secret + externalClickhouse selectors (v1 `port` fills HTTPPort). +func migrateLegacyClickHouse( + ctx context.Context, + c ctrlClient.Client, + wandb *apiv2.WeightsAndBiases, +) (bool, error) { + raw, ok := wandb.Annotations[apiv1.ClickHousePendingAnnotation] + if !ok { + return false, nil + } + + dec := json.NewDecoder(strings.NewReader(raw)) + dec.UseNumber() + var payload legacyClickHousePayload + if err := dec.Decode(&payload); err != nil { + return false, fmt.Errorf("decode %s: %w", apiv1.ClickHousePendingAnnotation, err) + } + + secretName := fmt.Sprintf("%s-clickhouse-converted", wandb.Name) + conn := wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ExternalClickHouse + if conn == nil { + conn = &apiv2.ClickHouseConnection{} + } + + data := map[string][]byte{} + fill := func(target *corev1.SecretKeySelector, dataKey, value string) { + if target.Name != "" || value == "" { + return + } + data[dataKey] = []byte(value) + *target = secretSelector(secretName, dataKey) + } + + fill(&conn.Host, "host", payload.Host) + fill(&conn.HTTPPort, "httpPort", normalizePort(payload.Port)) + fill(&conn.Database, "database", payload.Database) + fill(&conn.Username, "username", payload.User) + fill(&conn.Password, "password", payload.Password) + + if err := materializeConvertedSecret(ctx, c, wandb, secretName, data); err != nil { + return false, err + } + + setExternalInstance(&wandb.Spec.ClickHouse, func(s *apiv2.ClickHouseSpec) { s.ExternalClickHouse = conn }) + delete(wandb.Annotations, apiv1.ClickHousePendingAnnotation) + return true, nil +} + // legacyBucketPayload is the flat literal subset from the webhook's -// bucket+defaultBucket merge. provider/path/kmsKey have no v2 home; ignored. +// bucket+defaultBucket merge. kmsKey has no v2 home; ignored. type legacyBucketPayload struct { + Provider string `json:"provider,omitempty"` Name string `json:"name,omitempty"` + Path string `json:"path,omitempty"` Region string `json:"region,omitempty"` AccessKey string `json:"accessKey,omitempty"` SecretKey string `json:"secretKey,omitempty"` @@ -215,12 +282,19 @@ func migrateLegacyBucket( } secretName := fmt.Sprintf("%s-bucket-converted", wandb.Name) - conn := wandb.Spec.ObjectStore.ExternalObjectStore + conn := wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ExternalObjectStore if conn == nil { conn = &apiv2.ObjectStoreConnection{} } - endpoint, port, bucket := parseBucketName(payload.Name) + name, path, query := splitBucketQuery(payload.Name, payload.Path) + endpoint, port, bucket, path := objectstore.ParseLegacyBucket(payload.Provider, name, path) + // Query param beats the region field, matching gorilla's precedence. + region := payload.Region + if v := query.Get("region"); v != "" { + region = v + } + forcePathStyle, tlsEnabled := deriveBucketAddressing(payload.Provider, endpoint, query) data := map[string][]byte{} fill := func(target *corev1.SecretKeySelector, dataKey, value string) { @@ -234,33 +308,71 @@ func migrateLegacyBucket( fill(&conn.Endpoint, "endpoint", endpoint) fill(&conn.Port, "port", port) fill(&conn.Bucket, "bucket", bucket) - fill(&conn.Region, "region", payload.Region) + fill(&conn.Path, "path", path) + fill(&conn.Region, "region", region) fill(&conn.AccessKey, "accessKey", payload.AccessKey) fill(&conn.SecretKey, "secretKey", payload.SecretKey) + fill(&conn.ForcePathStyle, "forcePathStyle", forcePathStyle) + fill(&conn.TlsEnabled, "tlsEnabled", tlsEnabled) if err := materializeConvertedSecret(ctx, c, wandb, secretName, data); err != nil { return false, err } - wandb.Spec.ObjectStore.ExternalObjectStore = conn + setExternalInstance(&wandb.Spec.ObjectStore, func(s *apiv2.ObjectStoreSpec) { s.ExternalObjectStore = conn }) delete(wandb.Annotations, apiv1.BucketPendingAnnotation) return true, nil } -// parseBucketName splits v1's bucket.name. A "/" indicates the embedded -// "host[:port]/bucket" form (S3 bucket names can't contain "/"); otherwise -// the whole string is the bucket name. -func parseBucketName(name string) (endpoint, port, bucket string) { - if name == "" || !strings.Contains(name, "/") { - return "", "", name +// setExternalInstance applies fn to the default instance of an infra map, +// creating the map and/or default entry when absent. +func setExternalInstance[T any](m *map[string]T, fn func(*T)) { + if *m == nil { + *m = map[string]T{} + } + instance := (*m)[apiv2.DefaultInstanceName] + fn(&instance) + (*m)[apiv2.DefaultInstanceName] = instance +} + +// splitBucketQuery strips the ?tls=/?forcePathStyle=/?region= overrides gorilla +// accepted on v1 bucket URLs; they rode in bucket.name or bucket.path. +func splitBucketQuery(name, path string) (cleanName, cleanPath string, q url.Values) { + raw := "" + if i := strings.IndexByte(path, '?'); i >= 0 { + path, raw = path[:i], path[i+1:] + } + if i := strings.IndexByte(name, '?'); i >= 0 { + name, raw = name[:i], name[i+1:] + } + query, err := url.ParseQuery(raw) + if err != nil { + return name, path, url.Values{} + } + return name, path, query +} + +// deriveBucketAddressing decides forcePathStyle/tlsEnabled for a drained v1 bucket: +// explicit ?forcePathStyle=/?tls= win, else any embedded endpoint means path-style over +// http (prefixes belong in bucket.path, so a host in bucket.name is always an endpoint). +func deriveBucketAddressing(provider, endpoint string, query url.Values) (forcePathStyle, tlsEnabled string) { + if !objectstore.S3Compatible(provider) { + return "", "" + } + fps := provider != "cw" && objectstore.RequiresPathStyle(endpoint) + if v, err := strconv.ParseBool(query.Get("forcePathStyle")); err == nil { + fps = v + } + forcePathStyle = strconv.FormatBool(fps) + if endpoint == "" { + return forcePathStyle, "" } - slash := strings.IndexByte(name, '/') - host := name[:slash] - bucket = name[slash+1:] - if colon := strings.IndexByte(host, ':'); colon >= 0 { - return host[:colon], host[colon+1:], bucket + // gorilla defaulted S3-compatible endpoints to http, CoreWeave to https. + tls := provider == "cw" + if v, err := strconv.ParseBool(query.Get("tls")); err == nil { + tls = v } - return host, "", bucket + return forcePathStyle, strconv.FormatBool(tls) } // legacyOIDCPayload is the literal-string subset the webhook couldn't turn diff --git a/internal/controller/reconciler/migrate_legacy_test.go b/internal/controller/reconciler/migrate_legacy_test.go index 9d52ed84..6a16d569 100644 --- a/internal/controller/reconciler/migrate_legacy_test.go +++ b/internal/controller/reconciler/migrate_legacy_test.go @@ -88,12 +88,19 @@ func getOIDCConvertedSecret(t *testing.T, c ctrlClient.Client) (*corev1.Secret, return &secret, err } +func getClickHouseConvertedSecret(t *testing.T, c ctrlClient.Client) (*corev1.Secret, error) { + t.Helper() + var secret corev1.Secret + err := c.Get(context.Background(), types.NamespacedName{Name: "wandb-clickhouse-converted", Namespace: "default"}, &secret) + return &secret, err +} + func TestMigrateLegacyAnnotations_NoAnnotation(t *testing.T) { client, wandb := newMigrationFixture(t, nil, nil) res, err := migrateLegacyAnnotations(context.Background(), client, wandb) require.NoError(t, err) require.Zero(t, res.RequeueAfter) - require.Nil(t, wandb.Spec.MySQL.ExternalMysql) + require.Nil(t, wandb.Spec.MySQL[apiv2.DefaultInstanceName].ExternalMysql) _, err = getConvertedSecret(t, client) require.True(t, apiErrors.IsNotFound(err), "expected no converted Secret, got err=%v", err) @@ -122,8 +129,8 @@ func TestMigrateLegacyMySQL_FullLiteralPayload(t *testing.T) { var fresh apiv2.WeightsAndBiases require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: "wandb", Namespace: "default"}, &fresh)) require.NotContains(t, fresh.Annotations, apiv1.MySQLPendingAnnotation) - require.NotNil(t, fresh.Spec.MySQL.ExternalMysql) - conn := fresh.Spec.MySQL.ExternalMysql + require.NotNil(t, fresh.Spec.MySQL[apiv2.DefaultInstanceName].ExternalMysql) + conn := fresh.Spec.MySQL[apiv2.DefaultInstanceName].ExternalMysql require.Equal(t, "wandb-mysql-converted", conn.Host.Name) require.Equal(t, "host", conn.Host.Key) require.Equal(t, "port", conn.Port.Key) @@ -156,7 +163,7 @@ func TestMigrateLegacyMySQL_PartialPayload(t *testing.T) { require.NotContains(t, secret.Data, "username") require.NotContains(t, secret.Data, "sslCa") - conn := wandb.Spec.MySQL.ExternalMysql + conn := wandb.Spec.MySQL[apiv2.DefaultInstanceName].ExternalMysql require.NotNil(t, conn) require.Equal(t, "host", conn.Host.Key) require.Equal(t, "password", conn.Password.Key) @@ -171,10 +178,14 @@ func TestMigrateLegacyMySQL_PreSetFieldsAreRespected(t *testing.T) { client, wandb := newMigrationFixture(t, map[string]string{ apiv1.MySQLPendingAnnotation: payload, }, func(w *apiv2.WeightsAndBiases) { - w.Spec.MySQL.ExternalMysql = &apiv2.MysqlConnection{ - Host: corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "preset-secret"}, - Key: "preset-host-key", + w.Spec.MySQL = map[string]apiv2.MySQLSpec{ + apiv2.DefaultInstanceName: { + ExternalMysql: &apiv2.MysqlConnection{ + Host: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "preset-secret"}, + Key: "preset-host-key", + }, + }, }, } }) @@ -189,7 +200,7 @@ func TestMigrateLegacyMySQL_PreSetFieldsAreRespected(t *testing.T) { require.Contains(t, secret.Data, "port") require.Contains(t, secret.Data, "database") - conn := wandb.Spec.MySQL.ExternalMysql + conn := wandb.Spec.MySQL[apiv2.DefaultInstanceName].ExternalMysql require.Equal(t, "preset-secret", conn.Host.Name) require.Equal(t, "preset-host-key", conn.Host.Key) require.Equal(t, "wandb-mysql-converted", conn.Port.Name) @@ -201,10 +212,14 @@ func TestMigrateLegacyMySQL_AllPreSetEmptyAnnotationPayload(t *testing.T) { client, wandb := newMigrationFixture(t, map[string]string{ apiv1.MySQLPendingAnnotation: payload, }, func(w *apiv2.WeightsAndBiases) { - w.Spec.MySQL.ExternalMysql = &apiv2.MysqlConnection{ - Host: corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "preset"}, - Key: "host", + w.Spec.MySQL = map[string]apiv2.MySQLSpec{ + apiv2.DefaultInstanceName: { + ExternalMysql: &apiv2.MysqlConnection{ + Host: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "preset"}, + Key: "host", + }, + }, }, } }) @@ -219,7 +234,7 @@ func TestMigrateLegacyMySQL_AllPreSetEmptyAnnotationPayload(t *testing.T) { var fresh apiv2.WeightsAndBiases require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: "wandb", Namespace: "default"}, &fresh)) require.NotContains(t, fresh.Annotations, apiv1.MySQLPendingAnnotation) - require.Equal(t, "preset", fresh.Spec.MySQL.ExternalMysql.Host.Name) + require.Equal(t, "preset", fresh.Spec.MySQL[apiv2.DefaultInstanceName].ExternalMysql.Host.Name) } func TestMigrateLegacyMySQL_PreExistingSecretOverwritten(t *testing.T) { @@ -254,7 +269,7 @@ func TestMigrateLegacyMySQL_MalformedJSON(t *testing.T) { require.Error(t, err) require.Contains(t, wandb.Annotations, apiv1.MySQLPendingAnnotation) - require.Nil(t, wandb.Spec.MySQL.ExternalMysql) + require.Nil(t, wandb.Spec.MySQL[apiv2.DefaultInstanceName].ExternalMysql) } func TestMigrateLegacyMySQL_EmptyAnnotation(t *testing.T) { @@ -301,7 +316,7 @@ func TestMigrateLegacyRedis_FullLiteralPayload(t *testing.T) { var fresh apiv2.WeightsAndBiases require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: "wandb", Namespace: "default"}, &fresh)) require.NotContains(t, fresh.Annotations, apiv1.RedisPendingAnnotation) - conn := fresh.Spec.Redis.ExternalRedis + conn := fresh.Spec.Redis[apiv2.DefaultInstanceName].ExternalRedis require.NotNil(t, conn) require.Equal(t, "wandb-redis-converted", conn.Host.Name) require.Equal(t, "host", conn.Host.Key) @@ -317,10 +332,14 @@ func TestMigrateLegacyRedis_PreSetFieldsAreRespected(t *testing.T) { client, wandb := newMigrationFixture(t, map[string]string{ apiv1.RedisPendingAnnotation: payload, }, func(w *apiv2.WeightsAndBiases) { - w.Spec.Redis.ExternalRedis = &apiv2.RedisConnection{ - Host: corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "preset-secret"}, - Key: "preset-host-key", + w.Spec.Redis = map[string]apiv2.RedisSpec{ + apiv2.DefaultInstanceName: { + ExternalRedis: &apiv2.RedisConnection{ + Host: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "preset-secret"}, + Key: "preset-host-key", + }, + }, }, } }) @@ -333,7 +352,7 @@ func TestMigrateLegacyRedis_PreSetFieldsAreRespected(t *testing.T) { require.NotContains(t, secret.Data, "host") require.Contains(t, secret.Data, "port") - conn := wandb.Spec.Redis.ExternalRedis + conn := wandb.Spec.Redis[apiv2.DefaultInstanceName].ExternalRedis require.Equal(t, "preset-secret", conn.Host.Name) require.Equal(t, "preset-host-key", conn.Host.Key) require.Equal(t, "wandb-redis-converted", conn.Port.Name) @@ -347,7 +366,7 @@ func TestMigrateLegacyRedis_MalformedJSON(t *testing.T) { require.Error(t, err) require.Contains(t, wandb.Annotations, apiv1.RedisPendingAnnotation) - require.Nil(t, wandb.Spec.Redis.ExternalRedis) + require.Nil(t, wandb.Spec.Redis[apiv2.DefaultInstanceName].ExternalRedis) } func TestMigrateLegacyAnnotations_MySQLAndRedisInOneCall(t *testing.T) { @@ -364,8 +383,8 @@ func TestMigrateLegacyAnnotations_MySQLAndRedisInOneCall(t *testing.T) { require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: "wandb", Namespace: "default"}, &fresh)) require.NotContains(t, fresh.Annotations, apiv1.MySQLPendingAnnotation) require.NotContains(t, fresh.Annotations, apiv1.RedisPendingAnnotation) - require.NotNil(t, fresh.Spec.MySQL.ExternalMysql) - require.NotNil(t, fresh.Spec.Redis.ExternalRedis) + require.NotNil(t, fresh.Spec.MySQL[apiv2.DefaultInstanceName].ExternalMysql) + require.NotNil(t, fresh.Spec.Redis[apiv2.DefaultInstanceName].ExternalRedis) mysqlSecret, err := getConvertedSecret(t, client) require.NoError(t, err) @@ -390,28 +409,6 @@ func TestMigrateLegacyMySQL_PortStringValueAccepted(t *testing.T) { require.Equal(t, []byte("3308"), secret.Data["port"]) } -func TestParseBucketName(t *testing.T) { - cases := []struct { - name string - endpoint, port, bkt string - }{ - {"", "", "", ""}, - {"my-bucket", "", "", "my-bucket"}, - {"minio.example.com/wandb", "minio.example.com", "", "wandb"}, - {"minio.example.com:9000/wandb", "minio.example.com", "9000", "wandb"}, - {"minio:9000/wandb", "minio", "9000", "wandb"}, - {"minio.minio.svc.cluster.local:9000/bucket", "minio.minio.svc.cluster.local", "9000", "bucket"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - e, p, b := parseBucketName(tc.name) - require.Equal(t, tc.endpoint, e) - require.Equal(t, tc.port, p) - require.Equal(t, tc.bkt, b) - }) - } -} - func TestMigrateLegacyBucket_BareBucketName(t *testing.T) { payload := `{"name":"my-bucket","region":"us-east-1","accessKey":"AKIA","secretKey":"shh"}` client, wandb := newMigrationFixture(t, map[string]string{ @@ -430,8 +427,10 @@ func TestMigrateLegacyBucket_BareBucketName(t *testing.T) { require.Equal(t, []byte("shh"), secret.Data["secretKey"]) require.NotContains(t, secret.Data, "endpoint") require.NotContains(t, secret.Data, "port") + require.Equal(t, []byte("false"), secret.Data["forcePathStyle"], "bare name means native AWS, virtual-hosted") + require.NotContains(t, secret.Data, "tlsEnabled", "tls is meaningless without a custom endpoint") - conn := wandb.Spec.ObjectStore.ExternalObjectStore + conn := wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ExternalObjectStore require.NotNil(t, conn) require.Equal(t, "bucket", conn.Bucket.Key) require.Equal(t, "region", conn.Region.Key) @@ -439,6 +438,7 @@ func TestMigrateLegacyBucket_BareBucketName(t *testing.T) { require.Equal(t, "secretKey", conn.SecretKey.Key) require.Empty(t, conn.Endpoint.Name) require.Empty(t, conn.Port.Name) + require.Equal(t, "forcePathStyle", conn.ForcePathStyle.Key) } func TestMigrateLegacyBucket_EmbeddedEndpoint(t *testing.T) { @@ -455,11 +455,278 @@ func TestMigrateLegacyBucket_EmbeddedEndpoint(t *testing.T) { require.Equal(t, []byte("minio.minio.svc"), secret.Data["endpoint"]) require.Equal(t, []byte("9000"), secret.Data["port"]) require.Equal(t, []byte("wandb-bucket"), secret.Data["bucket"]) + require.Equal(t, []byte("true"), secret.Data["forcePathStyle"], "custom endpoint requires path-style") + require.Equal(t, []byte("false"), secret.Data["tlsEnabled"], "gorilla defaulted custom endpoints to http") - conn := wandb.Spec.ObjectStore.ExternalObjectStore + conn := wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ExternalObjectStore require.Equal(t, "endpoint", conn.Endpoint.Key) require.Equal(t, "port", conn.Port.Key) require.Equal(t, "bucket", conn.Bucket.Key) + require.Equal(t, "forcePathStyle", conn.ForcePathStyle.Key) + require.Equal(t, "tlsEnabled", conn.TlsEnabled.Key) +} + +func TestMigrateLegacyBucket_HostPortEndpointWithBucketInPath(t *testing.T) { + payload := `{ + "provider": "s3", + "name": "minio.minio.svc.cluster.local:9000", + "path": "lsahu-minio-bucket", + "region": "us-east-1" +}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, func(w *apiv2.WeightsAndBiases) { + w.Spec.ObjectStore = map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: { + ExternalObjectStore: &apiv2.ObjectStoreConnection{ + AccessKey: secretSelector("wandb-minio", "ACCESS_KEY"), + SecretKey: secretSelector("wandb-minio", "SECRET_KEY"), + }, + }, + } + }) + + res, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + require.NotZero(t, res.RequeueAfter) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("minio.minio.svc.cluster.local"), secret.Data["endpoint"]) + require.Equal(t, []byte("9000"), secret.Data["port"]) + require.Equal(t, []byte("lsahu-minio-bucket"), secret.Data["bucket"]) + require.Equal(t, []byte("us-east-1"), secret.Data["region"]) + require.Equal(t, []byte("true"), secret.Data["forcePathStyle"]) + require.Equal(t, []byte("false"), secret.Data["tlsEnabled"]) + require.NotContains(t, secret.Data, "path") + require.NotContains(t, secret.Data, "accessKey") + require.NotContains(t, secret.Data, "secretKey") + + var fresh apiv2.WeightsAndBiases + require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: "wandb", Namespace: "default"}, &fresh)) + require.NotContains(t, fresh.Annotations, apiv1.BucketPendingAnnotation) + + conn := fresh.Spec.ObjectStore[apiv2.DefaultInstanceName].ExternalObjectStore + require.Equal(t, secretSelector("wandb-bucket-converted", "endpoint"), conn.Endpoint) + require.Equal(t, secretSelector("wandb-bucket-converted", "port"), conn.Port) + require.Equal(t, secretSelector("wandb-bucket-converted", "bucket"), conn.Bucket) + require.Equal(t, secretSelector("wandb-bucket-converted", "region"), conn.Region) + require.Equal(t, secretSelector("wandb-bucket-converted", "forcePathStyle"), conn.ForcePathStyle) + require.Equal(t, secretSelector("wandb-bucket-converted", "tlsEnabled"), conn.TlsEnabled) + require.Empty(t, conn.Path.Name) + require.Equal(t, secretSelector("wandb-minio", "ACCESS_KEY"), conn.AccessKey) + require.Equal(t, secretSelector("wandb-minio", "SECRET_KEY"), conn.SecretKey) +} + +func TestMigrateLegacyBucket_HostPortEndpointWithBucketAndPrefixInPath(t *testing.T) { + payload := `{ + "provider": "s3", + "name": "minio.minio.svc.cluster.local:9000", + "path": "/lsahu-minio-bucket/team/project/", + "region": "us-east-1" +}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("minio.minio.svc.cluster.local"), secret.Data["endpoint"]) + require.Equal(t, []byte("9000"), secret.Data["port"]) + require.Equal(t, []byte("lsahu-minio-bucket"), secret.Data["bucket"]) + require.Equal(t, []byte("team/project"), secret.Data["path"]) + + conn := wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ExternalObjectStore + require.Equal(t, "bucket", conn.Bucket.Key) + require.Equal(t, "path", conn.Path.Key) +} + +func TestMigrateLegacyBucket_AWSBucketWithPathIsNotEndpoint(t *testing.T) { + payload := `{"provider":"s3","name":"my-aws-bucket","path":"prefix","region":"us-east-1"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("my-aws-bucket"), secret.Data["bucket"]) + require.Equal(t, []byte("prefix"), secret.Data["path"]) + require.NotContains(t, secret.Data, "endpoint") + require.NotContains(t, secret.Data, "port") + require.Equal(t, []byte("false"), secret.Data["forcePathStyle"]) + require.NotContains(t, secret.Data, "tlsEnabled") +} + +func TestMigrateLegacyBucket_HostPortEndpointNoProvider(t *testing.T) { + payload := `{"name":"minio.minio.svc:9000","path":"wandb-bucket","region":"us-east-1"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("minio.minio.svc"), secret.Data["endpoint"], "host:port name is an endpoint even without a provider") + require.Equal(t, []byte("9000"), secret.Data["port"]) + require.Equal(t, []byte("wandb-bucket"), secret.Data["bucket"]) + require.Equal(t, []byte("true"), secret.Data["forcePathStyle"]) + require.Equal(t, []byte("false"), secret.Data["tlsEnabled"]) + require.NotContains(t, secret.Data, "path") +} + +func TestMigrateLegacyBucket_HostPortEndpointQueryInPath(t *testing.T) { + payload := `{"provider":"s3","name":"minio.example.com:9000","path":"wandb-bucket/team?tls=true"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("minio.example.com"), secret.Data["endpoint"]) + require.Equal(t, []byte("9000"), secret.Data["port"]) + require.Equal(t, []byte("wandb-bucket"), secret.Data["bucket"]) + require.Equal(t, []byte("team"), secret.Data["path"], "query is stripped, the prefix survives") + require.Equal(t, []byte("true"), secret.Data["tlsEnabled"], "?tls= on the path still wins") +} + +func TestMigrateLegacyBucket_HostPortEndpointIPv6(t *testing.T) { + payload := `{"provider":"s3","name":"[fd00::1]:9000","path":"wandb-bucket"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("fd00::1"), secret.Data["endpoint"]) + require.Equal(t, []byte("9000"), secret.Data["port"]) + require.Equal(t, []byte("wandb-bucket"), secret.Data["bucket"]) +} + +func TestMigrateLegacyBucket_QueryParamOverrides(t *testing.T) { + payload := `{"provider":"s3","name":"minio.example.com:9000/wandb","region":"us-east-1","path":"prefix?tls=true&forcePathStyle=false®ion=eu-west-1"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("minio.example.com"), secret.Data["endpoint"]) + require.Equal(t, []byte("wandb"), secret.Data["bucket"]) + require.Equal(t, []byte("false"), secret.Data["forcePathStyle"], "explicit ?forcePathStyle= wins over the endpoint rule") + require.Equal(t, []byte("true"), secret.Data["tlsEnabled"], "explicit ?tls= wins over the http default") + require.Equal(t, []byte("eu-west-1"), secret.Data["region"], "?region= beats the region field, matching gorilla") + require.Equal(t, []byte("prefix"), secret.Data["path"], "the prefix survives with its query stripped") +} + +func TestMigrateLegacyBucket_PathPrefix(t *testing.T) { + payload := `{"name":"minio.example.com:9000/wandb","path":"wandb-files/"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("wandb-files"), secret.Data["path"]) + + conn := wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ExternalObjectStore + require.Equal(t, "path", conn.Path.Key) +} + +func TestMigrateLegacyBucket_QueryOnlyPath(t *testing.T) { + payload := `{"name":"my-bucket","path":"?forcePathStyle=true"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.NotContains(t, secret.Data, "path", "a query-only path carries no prefix") + require.Equal(t, []byte("true"), secret.Data["forcePathStyle"], "explicit override applies even without an endpoint") +} + +func TestMigrateLegacyBucket_QueryInName(t *testing.T) { + payload := `{"name":"minio.example.com/wandb?tls=true"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("wandb"), secret.Data["bucket"], "query must be stripped from the bucket name") + require.Equal(t, []byte("true"), secret.Data["forcePathStyle"]) + require.Equal(t, []byte("true"), secret.Data["tlsEnabled"]) +} + +func TestMigrateLegacyBucket_AwsEndpointInName(t *testing.T) { + payload := `{"name":"s3.us-east-1.amazonaws.com/my-bucket"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("s3.us-east-1.amazonaws.com"), secret.Data["endpoint"]) + require.Equal(t, []byte("true"), secret.Data["forcePathStyle"], + "a host in bucket.name is always an endpoint (prefixes belong in bucket.path) and gets path-style") +} + +func TestMigrateLegacyBucket_CoreWeaveProvider(t *testing.T) { + payload := `{"provider":"cw","name":"cwobject.com/my-bucket"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("false"), secret.Data["forcePathStyle"], "CoreWeave object storage is virtual-hosted") + require.Equal(t, []byte("true"), secret.Data["tlsEnabled"], "CoreWeave object storage is https") +} + +func TestMigrateLegacyBucket_NonS3ProviderSkipsAddressing(t *testing.T) { + payload := `{"provider":"gcs","name":"my-gcs-bucket"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.BucketPendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getBucketConvertedSecret(t, client) + require.NoError(t, err) + require.NotContains(t, secret.Data, "forcePathStyle") + require.NotContains(t, secret.Data, "tlsEnabled") } func TestMigrateLegacyBucket_PreSetCredentialsRespected(t *testing.T) { @@ -467,14 +734,18 @@ func TestMigrateLegacyBucket_PreSetCredentialsRespected(t *testing.T) { client, wandb := newMigrationFixture(t, map[string]string{ apiv1.BucketPendingAnnotation: payload, }, func(w *apiv2.WeightsAndBiases) { - w.Spec.ObjectStore.ExternalObjectStore = &apiv2.ObjectStoreConnection{ - AccessKey: corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "preset"}, - Key: "ACCESS_KEY", - }, - SecretKey: corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "preset"}, - Key: "SECRET_KEY", + w.Spec.ObjectStore = map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: { + ExternalObjectStore: &apiv2.ObjectStoreConnection{ + AccessKey: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "preset"}, + Key: "ACCESS_KEY", + }, + SecretKey: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "preset"}, + Key: "SECRET_KEY", + }, + }, }, } }) @@ -488,7 +759,7 @@ func TestMigrateLegacyBucket_PreSetCredentialsRespected(t *testing.T) { require.NotContains(t, secret.Data, "secretKey", "webhook-set SecretKey must not be overwritten") require.Contains(t, secret.Data, "bucket") - conn := wandb.Spec.ObjectStore.ExternalObjectStore + conn := wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ExternalObjectStore require.Equal(t, "preset", conn.AccessKey.Name) require.Equal(t, "preset", conn.SecretKey.Name) require.Equal(t, "wandb-bucket-converted", conn.Bucket.Name) @@ -506,7 +777,7 @@ func TestMigrateLegacyBucket_UnknownFieldsIgnored(t *testing.T) { secret, err := getBucketConvertedSecret(t, client) require.NoError(t, err) require.NotContains(t, secret.Data, "provider") - require.NotContains(t, secret.Data, "path") + require.Equal(t, []byte("sub/path"), secret.Data["path"]) require.NotContains(t, secret.Data, "kmsKey") } @@ -517,7 +788,7 @@ func TestMigrateLegacyBucket_MalformedJSON(t *testing.T) { _, err := migrateLegacyAnnotations(context.Background(), client, wandb) require.Error(t, err) require.Contains(t, wandb.Annotations, apiv1.BucketPendingAnnotation) - require.Nil(t, wandb.Spec.ObjectStore.ExternalObjectStore) + require.Nil(t, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ExternalObjectStore) } func TestMigrateLegacyOIDC_AllLiterals(t *testing.T) { @@ -581,3 +852,120 @@ func TestMigrateLegacyOIDC_MalformedJSON(t *testing.T) { require.Error(t, err) require.Contains(t, wandb.Annotations, apiv1.OIDCPendingAnnotation) } + +func TestMigrateLegacyClickHouse_FullLiteralPayload(t *testing.T) { + payload := `{"host":"clickhouse.example.com","port":8123,"database":"weave","user":"weave","password":"shh"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.ClickHousePendingAnnotation: payload, + }, nil) + + res, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + require.NotZero(t, res.RequeueAfter) + + secret, err := getClickHouseConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, corev1.SecretTypeOpaque, secret.Type) + require.Equal(t, []byte("clickhouse.example.com"), secret.Data["host"]) + require.Equal(t, []byte("8123"), secret.Data["httpPort"]) + require.Equal(t, []byte("weave"), secret.Data["database"]) + require.Equal(t, []byte("weave"), secret.Data["username"]) + require.Equal(t, []byte("shh"), secret.Data["password"]) + + var fresh apiv2.WeightsAndBiases + require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: "wandb", Namespace: "default"}, &fresh)) + require.NotContains(t, fresh.Annotations, apiv1.ClickHousePendingAnnotation) + conn := fresh.Spec.ClickHouse[apiv2.DefaultInstanceName].ExternalClickHouse + require.NotNil(t, conn) + require.Nil(t, fresh.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse) + require.Equal(t, "wandb-clickhouse-converted", conn.Host.Name) + require.Equal(t, "host", conn.Host.Key) + require.Equal(t, "httpPort", conn.HTTPPort.Key) + require.Equal(t, "database", conn.Database.Key) + require.Equal(t, "username", conn.Username.Key) + require.Equal(t, "password", conn.Password.Key) + require.Empty(t, conn.TCPPort.Name) + require.Empty(t, conn.URL.Name) +} + +func TestMigrateLegacyClickHouse_PartialPayload(t *testing.T) { + payload := `{"host":"clickhouse.example.com","password":"shh"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.ClickHousePendingAnnotation: payload, + }, nil) + + res, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + require.NotZero(t, res.RequeueAfter) + + secret, err := getClickHouseConvertedSecret(t, client) + require.NoError(t, err) + require.Contains(t, secret.Data, "host") + require.Contains(t, secret.Data, "password") + require.NotContains(t, secret.Data, "httpPort") + require.NotContains(t, secret.Data, "database") + require.NotContains(t, secret.Data, "username") + + conn := wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ExternalClickHouse + require.NotNil(t, conn) + require.Equal(t, "host", conn.Host.Key) + require.Equal(t, "password", conn.Password.Key) + require.Empty(t, conn.HTTPPort.Name) + require.Empty(t, conn.Database.Name) + require.Empty(t, conn.Username.Name) +} + +// TestMigrateLegacyClickHouse_PreSetFieldsAreRespected: preset selectors are +// not overwritten by the annotation drain. +func TestMigrateLegacyClickHouse_PreSetFieldsAreRespected(t *testing.T) { + payload := `{"host":"clickhouse.example.com","password":"shh"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.ClickHousePendingAnnotation: payload, + }, func(w *apiv2.WeightsAndBiases) { + w.Spec.ClickHouse = map[string]apiv2.ClickHouseSpec{ + apiv2.DefaultInstanceName: { + ExternalClickHouse: &apiv2.ClickHouseConnection{ + Password: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "preset-ch"}, + Key: "PRESET", + }, + }, + }, + } + }) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getClickHouseConvertedSecret(t, client) + require.NoError(t, err) + require.Contains(t, secret.Data, "host") + require.NotContains(t, secret.Data, "password", "preset password selector must be respected") + + conn := wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ExternalClickHouse + require.Equal(t, "preset-ch", conn.Password.Name) + require.Equal(t, "PRESET", conn.Password.Key) +} + +func TestMigrateLegacyClickHouse_PortStringified(t *testing.T) { + payload := `{"port":"8123"}` + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.ClickHousePendingAnnotation: payload, + }, nil) + + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.NoError(t, err) + + secret, err := getClickHouseConvertedSecret(t, client) + require.NoError(t, err) + require.Equal(t, []byte("8123"), secret.Data["httpPort"]) +} + +func TestMigrateLegacyClickHouse_MalformedJSON(t *testing.T) { + client, wandb := newMigrationFixture(t, map[string]string{ + apiv1.ClickHousePendingAnnotation: "{not json", + }, nil) + _, err := migrateLegacyAnnotations(context.Background(), client, wandb) + require.Error(t, err) + require.Contains(t, wandb.Annotations, apiv1.ClickHousePendingAnnotation) +} diff --git a/internal/controller/reconciler/mysql.go b/internal/controller/reconciler/mysql.go index 7b4a09a9..4e36848c 100644 --- a/internal/controller/reconciler/mysql.go +++ b/internal/controller/reconciler/mysql.go @@ -28,29 +28,38 @@ func mysqlWriteState( client client.Client, wandb *apiv2.WeightsAndBiases, mfst manifest.Manifest, -) []metav1.Condition { - if wandb.Spec.MySQL.ManagedMysql != nil { - return managedMysqlWriteState(ctx, client, wandb, mfst) - } - if wandb.Spec.MySQL.ExternalMysql != nil { - return externalMysqlWriteState(ctx, client, wandb) +) map[string][]metav1.Condition { + out := map[string][]metav1.Condition{} + for key, spec := range wandb.Spec.MySQL { + switch { + case spec.ManagedMysql != nil: + out[key] = managedMysqlWriteState(ctx, client, wandb, spec.ManagedMysql, mfst) + case spec.ExternalMysql != nil: + out[key] = externalmysql.WriteState(ctx, client, wandb, key, spec.ExternalMysql) + } } - return nil + return out } func mysqlReadState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, - newConditions []metav1.Condition, -) ([]metav1.Condition, *apiv2.MysqlConnection) { - if wandb.Spec.MySQL.ManagedMysql != nil { - return managedMysqlReadState(ctx, client, wandb, newConditions) - } - if wandb.Spec.MySQL.ExternalMysql != nil { - return externalMysqlReadState(ctx, client, wandb, newConditions) + conditions map[string][]metav1.Condition, +) (map[string][]metav1.Condition, map[string]*apiv2.MysqlConnection) { + outConds := map[string][]metav1.Condition{} + outConns := map[string]*apiv2.MysqlConnection{} + for key, spec := range wandb.Spec.MySQL { + switch { + case spec.ManagedMysql != nil: + outConds[key], outConns[key] = managedMysqlReadState(ctx, client, wandb, spec.ManagedMysql, conditions[key]) + case spec.ExternalMysql != nil: + outConds[key], outConns[key] = externalmysql.ReadState(ctx, client, wandb, key, conditions[key]) + default: + outConds[key] = conditions[key] + } } - return newConditions, nil + return outConds, outConns } func mysqlInferStatus( @@ -58,30 +67,64 @@ func mysqlInferStatus( client client.Client, recorder record.EventRecorder, wandb *apiv2.WeightsAndBiases, - newConditions []metav1.Condition, - newInfraConn *apiv2.MysqlConnection, + conditions map[string][]metav1.Condition, + infraConns map[string]*apiv2.MysqlConnection, ) (ctrl.Result, error) { - if wandb.Spec.MySQL.ManagedMysql != nil { - return managedMysqlInferStatus(ctx, client, recorder, wandb, newConditions, newInfraConn) + if wandb.Status.MySQLStatus == nil { + wandb.Status.MySQLStatus = map[string]apiv2.MysqlInfraStatus{} + } + var results []ctrl.Result + var firstErr error + for key, spec := range wandb.Spec.MySQL { + var res ctrl.Result + var err error + switch { + case spec.ManagedMysql != nil: + res, err = managedMysqlInferStatus(ctx, client, recorder, wandb, key, conditions[key], infraConns[key]) + case spec.ExternalMysql != nil: + res, err = externalMysqlInferStatus(ctx, client, wandb, key, conditions[key], infraConns[key]) + } + results = append(results, res) + if err != nil && firstErr == nil { + firstErr = err + } } - if wandb.Spec.MySQL.ExternalMysql != nil { - return externalMysqlInferStatus(ctx, client, wandb, newConditions, newInfraConn) + return consolidateResults(results), firstErr +} + +// runMysqlRetentionFinalizer applies the configured retention policy for a +// single MySQL instance during deletion. +func runMysqlRetentionFinalizer(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string, spec apiv2.MySQLSpec) error { + switch wandb.GetRetentionPolicy(mysqlInstanceInfraSpec(spec)).OnDelete { + case apiv2.PurgeOnDelete: + return mysqlPurgeFinalizer(ctx, c, wandb, key, spec) + case apiv2.DetachOnDelete: + return mysqlDetachFinalizer(ctx, c, wandb, key, spec) } - return ctrl.Result{}, nil + return nil +} + +func mysqlInstanceInfraSpec(spec apiv2.MySQLSpec) apiv2.ManagedInfraSpec { + if spec.ManagedMysql != nil { + return spec.ManagedMysql.ManagedInfraSpec + } + return apiv2.ManagedInfraSpec{} } func mysqlPurgeFinalizer( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + key string, + spec apiv2.MySQLSpec, ) error { - if spec := wandb.Spec.MySQL.ManagedMysql; spec != nil { - specNamespacedName := managedMysqlSpecNamespacedName(spec) - onDeleteRule := moco.ToMysqlOnDeleteRule(wandb, wandb.GetRetentionPolicy(spec.ManagedInfraSpec)) + if managed := spec.ManagedMysql; managed != nil { + specNamespacedName := managedMysqlSpecNamespacedName(managed) + onDeleteRule := moco.ToMysqlOnDeleteRule(wandb, wandb.GetRetentionPolicy(managed.ManagedInfraSpec)) return moco.PurgeFinalizer(ctx, client, specNamespacedName, onDeleteRule) } - if wandb.Spec.MySQL.ExternalMysql != nil { - return externalmysql.DeleteConnectionSecret(ctx, client, wandb) + if spec.ExternalMysql != nil { + return externalmysql.DeleteConnectionSecret(ctx, client, wandb, key) } return nil } @@ -90,12 +133,14 @@ func mysqlDetachFinalizer( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + _ string, + spec apiv2.MySQLSpec, ) error { - spec := wandb.Spec.MySQL.ManagedMysql - if spec == nil { + managed := spec.ManagedMysql + if managed == nil { return nil } - specNamespacedName := managedMysqlSpecNamespacedName(spec) + specNamespacedName := managedMysqlSpecNamespacedName(managed) return moco.DetachFinalizer(ctx, client, specNamespacedName, wandb) } @@ -105,10 +150,9 @@ func managedMysqlWriteState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedMysqlSpec, mfst manifest.Manifest, ) []metav1.Condition { - spec := wandb.Spec.MySQL.ManagedMysql - var specNamespacedName = managedMysqlSpecNamespacedName(spec) logger := ctrl.LoggerFrom(ctx) @@ -194,9 +238,9 @@ func managedMysqlReadState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedMysqlSpec, newConditions []metav1.Condition, ) ([]metav1.Condition, *apiv2.MysqlConnection) { - spec := wandb.Spec.MySQL.ManagedMysql specNamespacedName := managedMysqlSpecNamespacedName(spec) readConditions, newInfraConn := moco.ReadState(ctx, client, specNamespacedName, wandb, moco.ToMysqlOnDeleteRule(wandb, wandb.GetRetentionPolicy(spec.ManagedInfraSpec))) @@ -209,12 +253,15 @@ func managedMysqlInferStatus( client client.Client, recorder record.EventRecorder, wandb *apiv2.WeightsAndBiases, + key string, newConditions []metav1.Condition, newInfraConn *apiv2.MysqlConnection, ) (ctrl.Result, error) { + statusBefore := wandb.DeepCopy().Status enabled := true - oldConditions := wandb.Status.MySQLStatus.Conditions - oldInfraConn := wandb.Status.MySQLStatus.Connection + oldStatus := wandb.Status.MySQLStatus[key] + oldConditions := oldStatus.Conditions + oldInfraConn := oldStatus.Connection updatedStatus, events, ctrlResult := moco.ComputeStatus( ctx, @@ -228,32 +275,26 @@ func managedMysqlInferStatus( for _, e := range events { recorder.Event(wandb, e.Type, e.Reason, e.Message) } - wandb.Status.MySQLStatus = updatedStatus - err := client.Status().Update(ctx, wandb) + wandb.Status.MySQLStatus[key] = updatedStatus + err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore) return ctrlResult, err } // external -func externalMysqlWriteState(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases) []metav1.Condition { - return externalmysql.WriteState(ctx, c, wandb) -} - -func externalMysqlReadState(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, newConditions []metav1.Condition) ([]metav1.Condition, *apiv2.MysqlConnection) { - return externalmysql.ReadState(ctx, c, wandb, newConditions) -} - -func externalMysqlInferStatus(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, newConditions []metav1.Condition, newInfraConn *apiv2.MysqlConnection) (ctrl.Result, error) { - oldInfraConn := wandb.Status.MySQLStatus.Connection - state, ready, updatedConditions := external.InferExternalStatus(wandb.Status.MySQLStatus.Conditions, newConditions, wandb.Generation, newInfraConn != nil) +func externalMysqlInferStatus(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string, newConditions []metav1.Condition, newInfraConn *apiv2.MysqlConnection) (ctrl.Result, error) { + statusBefore := wandb.DeepCopy().Status + oldStatus := wandb.Status.MySQLStatus[key] + oldInfraConn := oldStatus.Connection + state, ready, updatedConditions := external.InferExternalStatus(oldStatus.Conditions, newConditions, wandb.Generation, newInfraConn != nil) conn := utils.Coalesce(newInfraConn, &oldInfraConn) - wandb.Status.MySQLStatus = apiv2.MysqlInfraStatus{ + wandb.Status.MySQLStatus[key] = apiv2.MysqlInfraStatus{ WBInfraStatus: apiv2.WBInfraStatus{Ready: ready, State: state, Conditions: updatedConditions}, Connection: *conn, } - return ctrl.Result{}, c.Status().Update(ctx, wandb) + return ctrl.Result{}, updateWandbStatusIfChanged(ctx, c, wandb, statusBefore) } // helpers @@ -265,18 +306,56 @@ func managedMysqlSpecNamespacedName(spec *apiv2.ManagedMysqlSpec) types.Namespac } } -func runMysqlInitJob(ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, manifest manifest.Manifest) (ctrl.Result, error) { - if wandb.Spec.MySQL.ManagedMysql == nil { - return ctrl.Result{}, nil +// allMysqlInitSucceeded reports whether every managed MySQL instance has a +// successful database-initialization job. +func allMysqlInitSucceeded(wandb *apiv2.WeightsAndBiases) bool { + for key, spec := range wandb.Spec.MySQL { + if spec.ManagedMysql == nil { + continue + } + if !wandb.Status.Wandb.MySQLInit[key].Succeeded { + return false + } + } + return true +} + +// mysqlManifestConfig returns the manifest infra config for the instance key, +// falling back to the manifest "default" entry. +func mysqlManifestConfig(mfst manifest.Manifest, key string) manifest.InfraConfig { + cfg, _ := infraSizingConfig(mfst.Mysql, key) + return cfg +} + +func runMysqlInitJob(ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, mfst manifest.Manifest) (ctrl.Result, error) { + if wandb.Status.Wandb.MySQLInit == nil { + wandb.Status.Wandb.MySQLInit = map[string]apiv2.MigrationJobStatus{} } - if wandb.Status.Wandb.MySQLInit.Succeeded { + var results []ctrl.Result + for key, spec := range wandb.Spec.MySQL { + if spec.ManagedMysql == nil { + continue + } + res, err := runMysqlInitJobInstance(ctx, client, wandb, key, spec.ManagedMysql, mfst) + if err != nil { + return ctrl.Result{}, err + } + results = append(results, res) + } + return consolidateResults(results), nil +} + +func runMysqlInitJobInstance(ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, key string, spec *apiv2.ManagedMysqlSpec, mfst manifest.Manifest) (ctrl.Result, error) { + if wandb.Status.Wandb.MySQLInit[key].Succeeded { return ctrl.Result{}, nil } + statusBefore := wandb.DeepCopy().Status - logger := ctrl.LoggerFrom(ctx).WithName("mysqlInit") + logger := ctrl.LoggerFrom(ctx).WithName("mysqlInit").WithValues("instance", key) - jobName := fmt.Sprintf("%s-moco-init", wandb.Name) + specNamespacedName := managedMysqlSpecNamespacedName(spec) + jobName := fmt.Sprintf("%s-moco-init", specNamespacedName.Name) logger.Info("Checking for MySQL init job", "job", jobName) job := &v1.Job{} err := client.Get(ctx, types.NamespacedName{Name: jobName, Namespace: wandb.Namespace}, job) @@ -288,7 +367,6 @@ func runMysqlInitJob(ctx context.Context, client client.Client, wandb *apiv2.Wei if errors.IsNotFound(err) { logger.Info("Creating MySQL init job") - specNamespacedName := managedMysqlSpecNamespacedName(wandb.Spec.MySQL.ManagedMysql) connSecretName := fmt.Sprintf("%s-connection", specNamespacedName.Name) // moco-writable has DDL/DML privileges on all non-system databases, @@ -309,24 +387,22 @@ func runMysqlInitJob(ctx context.Context, client client.Client, wandb *apiv2.Wei } } + initJobLabels := common.StandardLabels(wandb, "moco-init", common.RoleMigration, "") job = &v1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: jobName, Namespace: wandb.Namespace, - Labels: map[string]string{ - "app.kubernetes.io/managed-by": "wandb-operator", - "app.kubernetes.io/instance": wandb.Name, - "app.kubernetes.io/component": "moco-init", - }, + Labels: initJobLabels, }, Spec: v1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: initJobLabels}, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, Containers: []corev1.Container{ { Name: "moco-init", - Image: moco.MocoMySQLImage(manifest.Mysql["default"].Images["mysql"], wandb.Spec.Global.ImageRegistry), + Image: moco.MocoMySQLImage(mysqlManifestConfig(mfst, key).Images["mysql"], wandb.Spec.Global.ImageRegistry), Command: []string{"/bin/sh", "-c", mysqlCmd}, Env: []corev1.EnvVar{ envFromConn("MYSQL_HOST", "Host"), @@ -350,9 +426,12 @@ func runMysqlInitJob(ctx context.Context, client client.Client, wandb *apiv2.Wei return ctrl.Result{}, err } - wandb.Status.Wandb.MySQLInit.Name = jobName - wandb.Status.Wandb.MySQLInit.Succeeded = false - if err := client.Status().Update(ctx, wandb); err != nil { + wandb.Status.Wandb.MySQLInit[key] = apiv2.MigrationJobStatus{ + Name: jobName, + Phase: migrationPhaseRunning, + Reason: "JobCreated", + } + if err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore); err != nil { return ctrl.Result{}, err } @@ -361,8 +440,13 @@ func runMysqlInitJob(ctx context.Context, client client.Client, wandb *apiv2.Wei if job.Status.Succeeded > 0 { logger.Info("MySQL init job succeeded") - wandb.Status.Wandb.MySQLInit.Succeeded = true - if err := client.Status().Update(ctx, wandb); err != nil { + wandb.Status.Wandb.MySQLInit[key] = apiv2.MigrationJobStatus{ + Name: jobName, + Succeeded: true, + Phase: migrationPhaseSucceeded, + Reason: "JobSucceeded", + } + if err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore); err != nil { return ctrl.Result{}, err } return ctrl.Result{}, nil @@ -370,8 +454,13 @@ func runMysqlInitJob(ctx context.Context, client client.Client, wandb *apiv2.Wei if job.Status.Failed > 0 { logger.Info("MySQL init job failed") - wandb.Status.Wandb.MySQLInit.Failed = true - if err := client.Status().Update(ctx, wandb); err != nil { + wandb.Status.Wandb.MySQLInit[key] = apiv2.MigrationJobStatus{ + Name: jobName, + Failed: true, + Phase: migrationPhaseFailed, + Reason: "JobFailed", + } + if err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore); err != nil { return ctrl.Result{}, err } // We might want to return an error or just requeue diff --git a/internal/controller/reconciler/networking_route_builders_test.go b/internal/controller/reconciler/networking_route_builders_test.go index 677546ae..e86ebf43 100644 --- a/internal/controller/reconciler/networking_route_builders_test.go +++ b/internal/controller/reconciler/networking_route_builders_test.go @@ -15,8 +15,10 @@ var _ = Describe("Networking Route Builders", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "wandb-ns"}, Spec: apiv2.WeightsAndBiasesSpec{ - ObjectStore: apiv2.ObjectStoreSpec{ - ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{Namespace: "wandb-ns"}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: { + ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{Namespace: "wandb-ns"}, + }, }, }, } @@ -32,8 +34,10 @@ var _ = Describe("Networking Route Builders", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "wandb-ns"}, Spec: apiv2.WeightsAndBiasesSpec{ - ObjectStore: apiv2.ObjectStoreSpec{ - ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{Namespace: "infra-ns"}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: { + ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{Namespace: "infra-ns"}, + }, }, }, } diff --git a/internal/controller/reconciler/objectstore.go b/internal/controller/reconciler/objectstore.go index ce082a8e..84d0bae3 100644 --- a/internal/controller/reconciler/objectstore.go +++ b/internal/controller/reconciler/objectstore.go @@ -22,29 +22,38 @@ func objectStoreWriteState( client client.Client, wandb *apiv2.WeightsAndBiases, mfst manifest.Manifest, -) ([]metav1.Condition, *apiv2.ObjectStoreConnection) { - if wandb.Spec.ObjectStore.ManagedObjectStore != nil { - return managedObjectStoreWriteState(ctx, client, wandb, mfst) - } - if wandb.Spec.ObjectStore.ExternalObjectStore != nil { - return externalObjectStoreWriteState(ctx, client, wandb) +) (map[string][]metav1.Condition, map[string]*apiv2.ObjectStoreConnection) { + outConds := map[string][]metav1.Condition{} + outConns := map[string]*apiv2.ObjectStoreConnection{} + for key, spec := range wandb.Spec.ObjectStore { + switch { + case spec.ManagedObjectStore != nil: + outConds[key], outConns[key] = managedObjectStoreWriteState(ctx, client, wandb, key, spec.ManagedObjectStore, mfst) + case spec.ExternalObjectStore != nil: + outConds[key], outConns[key] = externalobjectstore.WriteState(ctx, client, wandb, key, spec.ExternalObjectStore) + } } - return nil, nil + return outConds, outConns } func objectStoreReadState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, - newConditions []metav1.Condition, -) []metav1.Condition { - if wandb.Spec.ObjectStore.ManagedObjectStore != nil { - return managedObjectStoreReadState(ctx, client, wandb, newConditions) + conditions map[string][]metav1.Condition, +) map[string][]metav1.Condition { + out := map[string][]metav1.Condition{} + for key, spec := range wandb.Spec.ObjectStore { + switch { + case spec.ManagedObjectStore != nil: + out[key] = managedObjectStoreReadState(ctx, client, wandb, spec.ManagedObjectStore, conditions[key]) + case spec.ExternalObjectStore != nil: + out[key] = externalobjectstore.ReadState(ctx, client, wandb, key, conditions[key]) + default: + out[key] = conditions[key] + } } - if wandb.Spec.ObjectStore.ExternalObjectStore != nil { - return externalObjectStoreReadState(ctx, client, wandb, newConditions) - } - return newConditions + return out } func objectStoreInferStatus( @@ -52,34 +61,70 @@ func objectStoreInferStatus( client client.Client, recorder record.EventRecorder, wandb *apiv2.WeightsAndBiases, - newConditions []metav1.Condition, - newInfraConn *apiv2.ObjectStoreConnection, + conditions map[string][]metav1.Condition, + infraConns map[string]*apiv2.ObjectStoreConnection, ) (ctrl.Result, error) { - if wandb.Spec.ObjectStore.ManagedObjectStore != nil { - return managedObjectStoreInferStatus(ctx, client, recorder, wandb, newConditions, newInfraConn) + if wandb.Status.ObjectStoreStatus == nil { + wandb.Status.ObjectStoreStatus = map[string]apiv2.ObjectStoreInfraStatus{} } - if wandb.Spec.ObjectStore.ExternalObjectStore != nil { - return externalObjectStoreInferStatus(ctx, client, wandb, newConditions, newInfraConn) + var results []ctrl.Result + var firstErr error + for key, spec := range wandb.Spec.ObjectStore { + var res ctrl.Result + var err error + switch { + case spec.ManagedObjectStore != nil: + res, err = managedObjectStoreInferStatus(ctx, client, recorder, wandb, key, conditions[key], infraConns[key]) + case spec.ExternalObjectStore != nil: + res, err = externalObjectStoreInferStatus(ctx, client, wandb, key, conditions[key], infraConns[key]) + } + results = append(results, res) + if err != nil && firstErr == nil { + firstErr = err + } + } + return consolidateResults(results), firstErr +} + +func runObjectStoreRetentionFinalizer(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string, spec apiv2.ObjectStoreSpec) error { + switch wandb.GetRetentionPolicy(objectStoreInstanceInfraSpec(spec)).OnDelete { + case apiv2.PurgeOnDelete: + return objectStorePurgeFinalizer(ctx, c, wandb, key, spec) + case apiv2.DetachOnDelete: + return objectStoreDetachFinalizer(ctx, c, wandb, key, spec) + } + return nil +} + +func objectStoreInstanceInfraSpec(spec apiv2.ObjectStoreSpec) apiv2.ManagedInfraSpec { + if spec.ManagedObjectStore != nil { + return spec.ManagedObjectStore.ManagedInfraSpec } - return ctrl.Result{}, nil + return apiv2.ManagedInfraSpec{} } func objectStorePurgeFinalizer( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + key string, + spec apiv2.ObjectStoreSpec, ) error { - if spec := wandb.Spec.ObjectStore.ManagedObjectStore; spec != nil { - onDeleteRule := seaweedfs.ToObjectStoreOnDeleteRule(wandb, wandb.GetRetentionPolicy(spec.ManagedInfraSpec)) - _ = seaweedfs.CleanupLegacyMinio( - ctx, client, wandb.Name, wandb.Namespace, wandb.GetUID(), - true, onDeleteRule.Selector, - ) - specNamespacedName := managedObjectStoreSpecNamespacedName(spec) + if managed := spec.ManagedObjectStore; managed != nil { + onDeleteRule := seaweedfs.ToObjectStoreOnDeleteRule(wandb, wandb.GetRetentionPolicy(managed.ManagedInfraSpec)) + // Legacy MinIO predates multi-instance and is scoped to the CR, so only + // the default instance triggers its cleanup. + if key == apiv2.DefaultInstanceName { + _ = seaweedfs.CleanupLegacyMinio( + ctx, client, wandb.Name, wandb.Namespace, wandb.GetUID(), + true, onDeleteRule.Selector, + ) + } + specNamespacedName := managedObjectStoreSpecNamespacedName(managed) return seaweedfs.PurgeFinalizer(ctx, client, specNamespacedName, onDeleteRule) } - if wandb.Spec.ObjectStore.ExternalObjectStore != nil { - return externalobjectstore.DeleteConnectionSecret(ctx, client, wandb) + if spec.ExternalObjectStore != nil { + return externalobjectstore.DeleteConnectionSecret(ctx, client, wandb, key) } return nil } @@ -88,16 +133,20 @@ func objectStoreDetachFinalizer( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + key string, + spec apiv2.ObjectStoreSpec, ) error { - spec := wandb.Spec.ObjectStore.ManagedObjectStore - if spec == nil { + managed := spec.ManagedObjectStore + if managed == nil { return nil } - _ = seaweedfs.CleanupLegacyMinio( - ctx, client, wandb.Name, wandb.Namespace, wandb.GetUID(), - false, nil, - ) - specNamespacedName := managedObjectStoreSpecNamespacedName(spec) + if key == apiv2.DefaultInstanceName { + _ = seaweedfs.CleanupLegacyMinio( + ctx, client, wandb.Name, wandb.Namespace, wandb.GetUID(), + false, nil, + ) + } + specNamespacedName := managedObjectStoreSpecNamespacedName(managed) return seaweedfs.DetachFinalizer(ctx, client, specNamespacedName, wandb) } @@ -107,29 +156,31 @@ func managedObjectStoreWriteState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + key string, + spec *apiv2.ManagedObjectStoreSpec, mfst manifest.Manifest, ) ([]metav1.Condition, *apiv2.ObjectStoreConnection) { - spec := wandb.Spec.ObjectStore.ManagedObjectStore - log := ctrl.LoggerFrom(ctx) var specNamespacedName = managedObjectStoreSpecNamespacedName(spec) retentionPolicy := wandb.GetRetentionPolicy(spec.ManagedInfraSpec) onDeleteRule := seaweedfs.ToObjectStoreOnDeleteRule(wandb, retentionPolicy) - if err := seaweedfs.CleanupLegacyMinio( - ctx, client, - wandb.Name, wandb.Namespace, wandb.GetUID(), - onDeleteRule.Policy == common.Purge, - onDeleteRule.Selector, - ); err != nil { - log.Error(err, "failed to clean up legacy MinIO resources") + if key == apiv2.DefaultInstanceName { + if err := seaweedfs.CleanupLegacyMinio( + ctx, client, + wandb.Name, wandb.Namespace, wandb.GetUID(), + onDeleteRule.Policy == common.Purge, + onDeleteRule.Selector, + ); err != nil { + log.Error(err, "failed to clean up legacy MinIO resources") + } } if conditions := seaweedfs.CheckDetached(ctx, client, specNamespacedName, wandb.GetUID(), spec.Replicas); conditions != nil { return conditions, nil } - desiredCr, err := seaweedfs.ToObjectStoreVendorSpec(ctx, wandb, client.Scheme(), mfst) + desiredCr, err := seaweedfs.ToObjectStoreVendorSpec(ctx, wandb, spec, client.Scheme(), mfst) if err != nil { log.Error(err, "failed to translate object store spec to vendor spec") return []metav1.Condition{ @@ -161,10 +212,9 @@ func managedObjectStoreReadState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedObjectStoreSpec, newConditions []metav1.Condition, ) []metav1.Condition { - spec := wandb.Spec.ObjectStore.ManagedObjectStore - specNamespacedName := managedObjectStoreSpecNamespacedName(spec) retentionPolicy := wandb.GetRetentionPolicy(spec.ManagedInfraSpec) readConditions := seaweedfs.ReadState( @@ -182,12 +232,15 @@ func managedObjectStoreInferStatus( client client.Client, recorder record.EventRecorder, wandb *apiv2.WeightsAndBiases, + key string, newConditions []metav1.Condition, newInfraConn *apiv2.ObjectStoreConnection, ) (ctrl.Result, error) { + statusBefore := wandb.DeepCopy().Status enabled := true - oldConditions := wandb.Status.ObjectStoreStatus.Conditions - oldInfraConn := wandb.Status.ObjectStoreStatus.Connection + oldStatus := wandb.Status.ObjectStoreStatus[key] + oldConditions := oldStatus.Conditions + oldInfraConn := oldStatus.Connection updatedStatus, events, ctrlResult := seaweedfs.ComputeStatus( ctx, @@ -200,32 +253,26 @@ func managedObjectStoreInferStatus( for _, e := range events { recorder.Event(wandb, e.Type, e.Reason, e.Message) } - wandb.Status.ObjectStoreStatus = updatedStatus - err := client.Status().Update(ctx, wandb) + wandb.Status.ObjectStoreStatus[key] = updatedStatus + err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore) return ctrlResult, err } // external -func externalObjectStoreWriteState(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases) ([]metav1.Condition, *apiv2.ObjectStoreConnection) { - return externalobjectstore.WriteState(ctx, c, wandb) -} - -func externalObjectStoreReadState(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, newConditions []metav1.Condition) []metav1.Condition { - return externalobjectstore.ReadState(ctx, c, wandb, newConditions) -} - -func externalObjectStoreInferStatus(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, newConditions []metav1.Condition, newInfraConn *apiv2.ObjectStoreConnection) (ctrl.Result, error) { - oldInfraConn := wandb.Status.ObjectStoreStatus.Connection - state, ready, updatedConditions := external.InferExternalStatus(wandb.Status.ObjectStoreStatus.Conditions, newConditions, wandb.Generation, newInfraConn != nil) +func externalObjectStoreInferStatus(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string, newConditions []metav1.Condition, newInfraConn *apiv2.ObjectStoreConnection) (ctrl.Result, error) { + statusBefore := wandb.DeepCopy().Status + oldStatus := wandb.Status.ObjectStoreStatus[key] + oldInfraConn := oldStatus.Connection + state, ready, updatedConditions := external.InferExternalStatus(oldStatus.Conditions, newConditions, wandb.Generation, newInfraConn != nil) conn := utils.Coalesce(newInfraConn, &oldInfraConn) - wandb.Status.ObjectStoreStatus = apiv2.ObjectStoreInfraStatus{ + wandb.Status.ObjectStoreStatus[key] = apiv2.ObjectStoreInfraStatus{ WBInfraStatus: apiv2.WBInfraStatus{Ready: ready, State: state, Conditions: updatedConditions}, Connection: *conn, } - return ctrl.Result{}, c.Status().Update(ctx, wandb) + return ctrl.Result{}, updateWandbStatusIfChanged(ctx, c, wandb, statusBefore) } // helpers diff --git a/internal/controller/reconciler/pods.go b/internal/controller/reconciler/pods.go index ca477049..ec03d584 100644 --- a/internal/controller/reconciler/pods.go +++ b/internal/controller/reconciler/pods.go @@ -146,6 +146,15 @@ func resolveEnvvars(ctx context.Context, client ctrlClient.Client, wandb *v2.Wei } } + for _, env := range envs { + for i, combinedEnv := range combinedEnvs { + if combinedEnv.Name == env.Name { + combinedEnvs = append(combinedEnvs[:i], combinedEnvs[i+1:]...) + break + } + } + } + combinedEnvs = append(combinedEnvs, envs...) var envVars []v1.EnvVar @@ -198,20 +207,34 @@ func resolveEnvvars(ctx context.Context, client ctrlClient.Client, wandb *v2.Wei addSecretComponent(sel, idx) } case "mysql": - // MySQL connection URL as a secret ref - selector := wandb.Status.MySQLStatus.Connection.URL + // MySQL connection URL as a secret ref. src.Name selects the + // instance, falling back to the default instance when empty or + // when the named instance has no status yet. + status, ok := v2.ResolveInstance(wandb.Status.MySQLStatus, src.Name) + if !ok { + continue + } + selector := status.Connection.URL // Record for potential direct assignment case singleSecretSelector = selector secretOnlyCount++ addSecretComponent(selector, idx) case "redis": - selector := wandb.Status.RedisStatus.Connection.URL + status, ok := v2.ResolveInstance(wandb.Status.RedisStatus, src.Name) + if !ok { + continue + } + selector := status.Connection.URL singleSecretSelector = selector secretOnlyCount++ addSecretComponent(selector, idx) case "bucket": + status, ok := v2.ResolveInstance(wandb.Status.ObjectStoreStatus, src.Name) + if !ok { + continue + } selector := v1.SecretKeySelector{ - LocalObjectReference: wandb.Status.ObjectStoreStatus.Connection.URL.LocalObjectReference, + LocalObjectReference: status.Connection.URL.LocalObjectReference, } switch src.Field { case "host": @@ -234,20 +257,28 @@ func resolveEnvvars(ctx context.Context, client ctrlClient.Client, wandb *v2.Wei addSecretComponent(selector, idx) case "clickhouse": // clickhouse fields are provided as separate keys in the same secret + status, ok := v2.ResolveInstance(wandb.Status.ClickHouseStatus, src.Name) + if !ok { + continue + } selector := v1.SecretKeySelector{ - LocalObjectReference: wandb.Status.ClickHouseStatus.Connection.URL.LocalObjectReference, + LocalObjectReference: status.Connection.URL.LocalObjectReference, } switch src.Field { case "host": selector.Key = "Host" - case "port": - selector.Key = "Port" + case "http-port": + selector.Key = "HTTPPort" + case "tcp-port": + selector.Key = "TCPPort" case "user": selector.Key = "User" case "password": selector.Key = "Password" case "database": selector.Key = "Database" + case "url": + selector.Key = "url" default: // Unrecognized field; skip continue @@ -337,7 +368,7 @@ func resolveEnvvars(ctx context.Context, client ctrlClient.Client, wandb *v2.Wei // applications status and reading from there is probably more correct // Prefer deterministic manifest-derived service resolution to avoid startup races // where the Service object has not been created yet. - if resolved, ok := manifest.ResolveServiceURL(src); ok { + if resolved, ok := manifest.ResolveServiceURL(src, wandb.Namespace); ok { components = append(components, resolved) continue } @@ -371,7 +402,13 @@ func resolveEnvvars(ctx context.Context, client ctrlClient.Client, wandb *v2.Wei } } } - components = append(components, fmt.Sprintf("%s%s:%d%s", proto, serviceList.Items[0].Name, selectedPort, src.Path)) + // Fully-qualified host (see ResolveServiceURL) so NO_PROXY cluster + // suffixes cover it; the Service was just listed InNamespace(wandb.Namespace). + svcHost := serviceList.Items[0].Name + if wandb.Namespace != "" { + svcHost = fmt.Sprintf("%s.%s.svc.cluster.local", serviceList.Items[0].Name, wandb.Namespace) + } + components = append(components, fmt.Sprintf("%s%s:%d%s", proto, svcHost, selectedPort, src.Path)) case "jwt-issuer-map": if wandb.Spec.Wandb.InternalServiceAuth.Enabled != nil && *wandb.Spec.Wandb.InternalServiceAuth.Enabled { diff --git a/internal/controller/reconciler/pods_instance_test.go b/internal/controller/reconciler/pods_instance_test.go new file mode 100644 index 00000000..90ed00b0 --- /dev/null +++ b/internal/controller/reconciler/pods_instance_test.go @@ -0,0 +1,80 @@ +package reconciler + +import ( + "context" + "testing" + + apiv2 "github.com/wandb/operator/api/v2" + serverManifest "github.com/wandb/operator/pkg/wandb/manifest" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func mysqlURLSelector(secretName string) corev1.SecretKeySelector { + return corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, + Key: "url", + } +} + +func wandbWithTwoMysqlInstances() *apiv2.WeightsAndBiases { + return &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wb", Namespace: "default"}, + Status: apiv2.WeightsAndBiasesStatus{ + MySQLStatus: map[string]apiv2.MysqlInfraStatus{ + apiv2.DefaultInstanceName: {Connection: apiv2.MysqlConnection{URL: mysqlURLSelector("default-conn")}}, + "analytics": {Connection: apiv2.MysqlConnection{URL: mysqlURLSelector("analytics-conn")}}, + }, + }, + } +} + +func resolveSingleMysqlEnv(t *testing.T, wandb *apiv2.WeightsAndBiases, instance string) corev1.EnvVar { + t.Helper() + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("failed adding corev1 to scheme: %v", err) + } + client := fake.NewClientBuilder().WithScheme(scheme).Build() + + envs := []serverManifest.EnvVar{ + {Name: "MYSQL", Sources: []serverManifest.EnvSource{{Type: "mysql", Name: instance}}}, + } + resolved, err := resolveEnvvars(context.Background(), client, wandb, serverManifest.Manifest{}, nil, envs) + if err != nil { + t.Fatalf("resolveEnvvars returned error: %v", err) + } + return mustFindEnvVar(t, resolved, "MYSQL") +} + +func TestResolveEnvvarsMysqlRoutesToNamedInstance(t *testing.T) { + env := resolveSingleMysqlEnv(t, wandbWithTwoMysqlInstances(), "analytics") + if env.ValueFrom == nil || env.ValueFrom.SecretKeyRef == nil { + t.Fatalf("expected secret-backed env var, got %+v", env) + } + if got := env.ValueFrom.SecretKeyRef.Name; got != "analytics-conn" { + t.Fatalf("expected analytics-conn, got %q", got) + } +} + +func TestResolveEnvvarsMysqlEmptyInstanceUsesDefault(t *testing.T) { + env := resolveSingleMysqlEnv(t, wandbWithTwoMysqlInstances(), "") + if env.ValueFrom == nil || env.ValueFrom.SecretKeyRef == nil { + t.Fatalf("expected secret-backed env var, got %+v", env) + } + if got := env.ValueFrom.SecretKeyRef.Name; got != "default-conn" { + t.Fatalf("expected default-conn, got %q", got) + } +} + +func TestResolveEnvvarsMysqlMissingInstanceFallsBackToDefault(t *testing.T) { + env := resolveSingleMysqlEnv(t, wandbWithTwoMysqlInstances(), "does-not-exist") + if env.ValueFrom == nil || env.ValueFrom.SecretKeyRef == nil { + t.Fatalf("expected secret-backed env var, got %+v", env) + } + if got := env.ValueFrom.SecretKeyRef.Name; got != "default-conn" { + t.Fatalf("expected fallback to default-conn, got %q", got) + } +} diff --git a/internal/controller/reconciler/proxy_env.go b/internal/controller/reconciler/proxy_env.go new file mode 100644 index 00000000..440132ac --- /dev/null +++ b/internal/controller/reconciler/proxy_env.go @@ -0,0 +1,113 @@ +package reconciler + +import ( + "os" + "strings" + + corev1 "k8s.io/api/core/v1" + + apiv2 "github.com/wandb/operator/api/v2" +) + +// proxyNoProxyStatic is the always-present in-cluster NO_PROXY baseline. Both +// .svc and the full suffix are emitted because suffix-match rules differ across +// stacks (Go matches name+subdomains; Python urllib does plain string-suffix). +// Datastore and app service hosts are NOT enumerated here: the operator emits +// FQDNs (..svc.cluster.local) for every wired service, so the +// .svc.cluster.local suffix already covers the whole in-cluster HTTP mesh. +var proxyNoProxyStatic = []string{ + "localhost", "127.0.0.1", "::1", + ".svc", ".svc.cluster.local", ".cluster.local", + "kubernetes.default.svc", +} + +// computeNoProxy builds the NO_PROXY value the app workloads receive: the static +// in-cluster baseline plus the API-server ClusterIP (an IP literal no dot-suffix +// rule covers) plus the user's extra entries, deduplicated with order preserved. +// The API-server IP comes from the operator pod's own $KUBERNETES_SERVICE_HOST, +// which is the same in-cluster API endpoint every app pod sees. +func computeNoProxy(extras []string) string { + entries := append([]string{}, proxyNoProxyStatic...) + if apiHost := os.Getenv("KUBERNETES_SERVICE_HOST"); apiHost != "" { + entries = append(entries, apiHost) + } + for _, e := range extras { + if e != "" { + entries = append(entries, e) + } + } + return joinNoProxy(entries) +} + +// joinNoProxy deduplicates entries (order preserved) and comma-joins them. +func joinNoProxy(entries []string) string { + seen := map[string]struct{}{} + out := make([]string, 0, len(entries)) + for _, e := range entries { + if _, ok := seen[e]; ok { + continue + } + seen[e] = struct{}{} + out = append(out, e) + } + return strings.Join(out, ",") +} + +// proxyValueEnvVars turns one ProxyValue into the upper/lower env-var pair for +// the given base name. A literal value becomes a literal env var; a valueFrom +// becomes a SecretKeyRef env source (both casings reference the same key) so +// credential-bearing URLs stay in the Secret and never land in the workload +// spec. Returns nil when the value is unset. +func proxyValueEnvVars(upper, lower string, pv *apiv2.ProxyValue) []corev1.EnvVar { + if pv == nil { + return nil + } + switch { + case pv.Value != "": + return []corev1.EnvVar{ + {Name: upper, Value: pv.Value}, + {Name: lower, Value: pv.Value}, + } + case pv.ValueFrom != nil && pv.ValueFrom.SecretKeyRef != nil: + return []corev1.EnvVar{ + {Name: upper, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: pv.ValueFrom.SecretKeyRef.DeepCopy()}}, + {Name: lower, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: pv.ValueFrom.SecretKeyRef.DeepCopy()}}, + } + default: + return nil + } +} + +// proxyEnvVars builds the full six-variable proxy env set for spec.global.proxy: +// HTTP_PROXY/HTTPS_PROXY plus the operator-computed NO_PROXY, and each one's +// lowercase twin (Go honors both casings; many libraries read only lowercase). +// NO_PROXY is emitted whenever any proxy URL is set — the computed baseline is +// never empty. +func proxyEnvVars(proxy *apiv2.ProxySpec) []corev1.EnvVar { + if proxy == nil { + return nil + } + var envVars []corev1.EnvVar + envVars = append(envVars, proxyValueEnvVars("HTTP_PROXY", "http_proxy", proxy.HTTPProxy)...) + envVars = append(envVars, proxyValueEnvVars("HTTPS_PROXY", "https_proxy", proxy.HTTPSProxy)...) + if proxy.HTTPProxy != nil || proxy.HTTPSProxy != nil { + noProxy := computeNoProxy(proxy.NoProxy) + envVars = append(envVars, + corev1.EnvVar{Name: "NO_PROXY", Value: noProxy}, + corev1.EnvVar{Name: "no_proxy", Value: noProxy}, + ) + } + return envVars +} + +// applyProxyToWorkload appends the spec.global.proxy env vars to a workload's +// env, skipping any name already present. Injected AFTER customCACerts env and +// BEFORE applyLegacyOverrideEnv, so a legacyOverrides entry can still override +// or blank any proxy var per-app (the deliberate escape hatch). No-op when +// spec.global.proxy is unset. +func applyProxyToWorkload(wandb *apiv2.WeightsAndBiases, envVars []corev1.EnvVar) []corev1.EnvVar { + if wandb.Spec.Global.Proxy == nil { + return envVars + } + return appendMissingEnvVars(envVars, proxyEnvVars(wandb.Spec.Global.Proxy)) +} diff --git a/internal/controller/reconciler/proxy_env_test.go b/internal/controller/reconciler/proxy_env_test.go new file mode 100644 index 00000000..06d16e44 --- /dev/null +++ b/internal/controller/reconciler/proxy_env_test.go @@ -0,0 +1,154 @@ +package reconciler + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + + apiv2 "github.com/wandb/operator/api/v2" +) + +func envByName(vars []corev1.EnvVar, name string) (corev1.EnvVar, bool) { + for _, v := range vars { + if v.Name == name { + return v, true + } + } + return corev1.EnvVar{}, false +} + +func TestComputeNoProxyBaselineAndExtras(t *testing.T) { + t.Setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1") + got := computeNoProxy([]string{"internal.example.com", "127.0.0.1"}) + parts := strings.Split(got, ",") + for _, want := range []string{ + "localhost", "127.0.0.1", "::1", ".svc", ".svc.cluster.local", + ".cluster.local", "kubernetes.default.svc", "10.96.0.1", "internal.example.com", + } { + found := false + for _, p := range parts { + if p == want { + found = true + } + } + if !found { + t.Errorf("NO_PROXY missing %q: %s", want, got) + } + } + // 127.0.0.1 appears once despite being both baseline and an extra. + count := 0 + for _, p := range parts { + if p == "127.0.0.1" { + count++ + } + } + if count != 1 { + t.Errorf("127.0.0.1 duplicated (%d): %s", count, got) + } +} + +func TestComputeNoProxyNoAPIServerHost(t *testing.T) { + t.Setenv("KUBERNETES_SERVICE_HOST", "") + got := computeNoProxy(nil) + if strings.Contains(got, ",,") || strings.HasPrefix(got, ",") || strings.HasSuffix(got, ",") { + t.Errorf("blank entry in NO_PROXY: %q", got) + } + if !strings.Contains(got, ".svc.cluster.local") { + t.Errorf("baseline missing suffix: %q", got) + } +} + +func TestProxyEnvVarsLiteral(t *testing.T) { + t.Setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1") + proxy := &apiv2.ProxySpec{ + HTTPProxy: &apiv2.ProxyValue{Value: "http://proxy:3128"}, + HTTPSProxy: &apiv2.ProxyValue{Value: "http://proxy:3128"}, + NoProxy: []string{"wandb.localhost"}, + } + vars := proxyEnvVars(proxy) + // Six vars: HTTP_PROXY/HTTPS_PROXY/NO_PROXY + lowercase. + for _, name := range []string{"HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"} { + v, ok := envByName(vars, name) + if !ok { + t.Fatalf("missing env var %q", name) + } + if v.ValueFrom != nil { + t.Errorf("%s should be a literal, got ValueFrom", name) + } + } + if v, _ := envByName(vars, "HTTP_PROXY"); v.Value != "http://proxy:3128" { + t.Errorf("HTTP_PROXY = %q", v.Value) + } + np, _ := envByName(vars, "NO_PROXY") + if !strings.Contains(np.Value, "wandb.localhost") || !strings.Contains(np.Value, "10.96.0.1") || !strings.Contains(np.Value, ".svc.cluster.local") { + t.Errorf("NO_PROXY missing computed baseline/extras: %q", np.Value) + } +} + +func TestProxyEnvVarsValueFromStaysSecretRef(t *testing.T) { + proxy := &apiv2.ProxySpec{ + HTTPSProxy: &apiv2.ProxyValue{ + ValueFrom: &apiv2.ProxyValueSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "egress-proxy"}, + Key: "httpsProxy", + }, + }, + }, + } + vars := proxyEnvVars(proxy) + // httpsProxy only: no HTTP_PROXY pair, but NO_PROXY still emitted. + if _, ok := envByName(vars, "HTTP_PROXY"); ok { + t.Errorf("HTTP_PROXY should be absent when only httpsProxy is set") + } + for _, name := range []string{"HTTPS_PROXY", "https_proxy"} { + v, ok := envByName(vars, name) + if !ok { + t.Fatalf("missing %q", name) + } + // Credential-bearing values stay a SecretKeyRef, never a literal. + if v.Value != "" || v.ValueFrom == nil || v.ValueFrom.SecretKeyRef == nil { + t.Errorf("%s should be a SecretKeyRef, got %+v", name, v) + } + if v.ValueFrom.SecretKeyRef.Name != "egress-proxy" || v.ValueFrom.SecretKeyRef.Key != "httpsProxy" { + t.Errorf("%s secret ref wrong: %+v", name, v.ValueFrom.SecretKeyRef) + } + } + if _, ok := envByName(vars, "NO_PROXY"); !ok { + t.Errorf("NO_PROXY should be emitted whenever any proxy URL is set") + } +} + +func TestProxyEnvVarsNil(t *testing.T) { + if proxyEnvVars(nil) != nil { + t.Errorf("nil proxy should yield no env vars") + } + // A ProxySpec with no URLs set emits nothing (NO_PROXY only rides with a proxy). + if got := proxyEnvVars(&apiv2.ProxySpec{}); len(got) != 0 { + t.Errorf("empty proxy spec should yield no env vars, got %v", got) + } +} + +func TestApplyProxyToWorkload(t *testing.T) { + t.Setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1") + base := []corev1.EnvVar{{Name: "EXISTING", Value: "x"}, {Name: "HTTP_PROXY", Value: "manifest-value"}} + + // No proxy spec: unchanged. + wandbNoProxy := &apiv2.WeightsAndBiases{} + if got := applyProxyToWorkload(wandbNoProxy, base); len(got) != len(base) { + t.Fatalf("no-proxy workload should be unchanged, got %v", got) + } + + // With proxy: appends missing vars, does not clobber an existing HTTP_PROXY + // (appendMissing semantics — legacy/manifest precedence handled elsewhere). + wandb := &apiv2.WeightsAndBiases{} + wandb.Spec.Global.Proxy = &apiv2.ProxySpec{HTTPProxy: &apiv2.ProxyValue{Value: "http://proxy:3128"}} + got := applyProxyToWorkload(wandb, base) + if v, _ := envByName(got, "HTTP_PROXY"); v.Value != "manifest-value" { + t.Errorf("existing HTTP_PROXY should be preserved by appendMissing, got %q", v.Value) + } + if _, ok := envByName(got, "NO_PROXY"); !ok { + t.Errorf("NO_PROXY should have been appended") + } +} diff --git a/internal/controller/reconciler/rbac.go b/internal/controller/reconciler/rbac.go index 66dcd298..4196869b 100644 --- a/internal/controller/reconciler/rbac.go +++ b/internal/controller/reconciler/rbac.go @@ -29,10 +29,14 @@ func createOrUpdateServiceAccount( ObjectMeta: v3.ObjectMeta{ Name: serviceAccountName, Namespace: wandb.Namespace, + // The shared ServiceAccount spans all services, so it is exempt from + // name/component per docs/pod-labeling-standards.md; it still carries + // the descriptive identity labels. Labels: map[string]string{ "app.kubernetes.io/managed-by": "wandb-operator", "app.kubernetes.io/instance": wandb.Name, "app.kubernetes.io/part-of": "wandb", + "app.kubernetes.io/version": wandb.Spec.Wandb.Version, }, Annotations: wandb.Spec.Wandb.ServiceAccount.Annotations, }, diff --git a/internal/controller/reconciler/readiness.go b/internal/controller/reconciler/readiness.go new file mode 100644 index 00000000..b7ef38d5 --- /dev/null +++ b/internal/controller/reconciler/readiness.go @@ -0,0 +1,140 @@ +package reconciler + +import ( + "context" + "fmt" + "sort" + "strings" + + apiv2 "github.com/wandb/operator/api/v2" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + readyConditionType = "Ready" + + migrationPhaseRunning = "Running" + migrationPhaseFailed = "Failed" + migrationPhaseSucceeded = "Succeeded" + migrationPhaseUnknown = "Unknown" +) + +func setReadyStatus(wandb *apiv2.WeightsAndBiases, ready bool, reason, message string) { + wandb.Status.Ready = ready + status := metav1.ConditionFalse + if ready { + status = metav1.ConditionTrue + } + apimeta.SetStatusCondition(&wandb.Status.Conditions, metav1.Condition{ + Type: readyConditionType, + Status: status, + ObservedGeneration: wandb.Generation, + Reason: reason, + Message: message, + }) +} + +func updateReadyStatus( + ctx context.Context, + c ctrlclient.Client, + wandb *apiv2.WeightsAndBiases, + statusBefore apiv2.WeightsAndBiasesStatus, + ready bool, + reason string, + message string, +) error { + setReadyStatus(wandb, ready, reason, message) + return updateWandbStatusIfChanged(ctx, c, wandb, statusBefore) +} + +func infrastructureBlockers(wandb *apiv2.WeightsAndBiases) []string { + var blockers []string + for key := range wandb.Spec.Redis { + if !wandb.Status.RedisStatus[key].Ready { + blockers = append(blockers, "redis/"+key) + } + } + for key := range wandb.Spec.MySQL { + if !wandb.Status.MySQLStatus[key].Ready { + blockers = append(blockers, "mysql/"+key) + } + } + if wandb.Spec.Kafka.ManagedKafka != nil && !wandb.Status.KafkaStatus.Ready { + blockers = append(blockers, "kafka") + } + for key := range wandb.Spec.ObjectStore { + if !wandb.Status.ObjectStoreStatus[key].Ready { + blockers = append(blockers, "objectStore/"+key) + } + } + for key := range wandb.Spec.ClickHouse { + if !wandb.Status.ClickHouseStatus[key].Ready { + blockers = append(blockers, "clickhouse/"+key) + } + } + sort.Strings(blockers) + return blockers +} + +func mysqlInitializationReadiness(wandb *apiv2.WeightsAndBiases) (string, string) { + var failed []string + var pending []string + for key, spec := range wandb.Spec.MySQL { + if spec.ManagedMysql == nil { + continue + } + status := wandb.Status.Wandb.MySQLInit[key] + switch { + case status.Succeeded: + case status.Failed: + failed = append(failed, key) + default: + pending = append(pending, key) + } + } + sort.Strings(failed) + sort.Strings(pending) + if len(failed) > 0 { + return "MySQLInitializationFailed", "MySQL initialization jobs failed: " + strings.Join(failed, ", ") + } + return "MySQLInitializationPending", "waiting for MySQL initialization jobs: " + strings.Join(pending, ", ") +} + +func migrationReadiness(wandb *apiv2.WeightsAndBiases) (string, string) { + status := wandb.Status.Wandb.Migration + if status.Phase != migrationPhaseFailed && status.Reason != migrationPhaseFailed { + phase := status.Phase + if phase == "" { + phase = status.Reason + } + if phase == "" { + phase = migrationPhaseUnknown + } + return "MigrationPending", fmt.Sprintf("migration phase is %s for version %s", phase, status.Version) + } + + var failures []string + for name, job := range status.Jobs { + if !job.Failed && job.Phase != migrationPhaseFailed { + continue + } + detail := job.Name + if detail == "" { + detail = name + } + switch { + case job.Message != "": + detail += ": " + job.Message + case job.Reason != "": + detail += ": " + job.Reason + } + failures = append(failures, detail) + } + sort.Strings(failures) + if len(failures) == 0 { + return "MigrationFailed", "one or more migration jobs failed" + } + return "MigrationFailed", "migration jobs failed: " + strings.Join(failures, "; ") +} diff --git a/internal/controller/reconciler/readiness_test.go b/internal/controller/reconciler/readiness_test.go new file mode 100644 index 00000000..6b771b24 --- /dev/null +++ b/internal/controller/reconciler/readiness_test.go @@ -0,0 +1,155 @@ +package reconciler + +import ( + "context" + "testing" + + apiv2 "github.com/wandb/operator/api/v2" + servermanifest "github.com/wandb/operator/pkg/wandb/manifest" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestInferStateBlocksOnExternalInfrastructure(t *testing.T) { + scheme := runtime.NewScheme() + if err := apiv2.AddToScheme(scheme); err != nil { + t.Fatalf("add W&B API to scheme: %v", err) + } + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "default", Generation: 2}, + Spec: apiv2.WeightsAndBiasesSpec{ + Redis: map[string]apiv2.RedisSpec{ + apiv2.DefaultInstanceName: {ExternalRedis: &apiv2.RedisConnection{}}, + }, + }, + Status: apiv2.WeightsAndBiasesStatus{ + Ready: true, + ObservedGeneration: 2, + RedisStatus: map[string]apiv2.RedisInfraStatus{ + apiv2.DefaultInstanceName: {}, + }, + }, + } + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&apiv2.WeightsAndBiases{}). + WithObjects(wandb). + Build() + + if err := inferState(context.Background(), c, wandb); err != nil { + t.Fatalf("infer state: %v", err) + } + + actual := &apiv2.WeightsAndBiases{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(wandb), actual); err != nil { + t.Fatalf("get updated W&B resource: %v", err) + } + if actual.Status.Ready { + t.Fatal("external Redis must block overall readiness") + } + condition := apimeta.FindStatusCondition(actual.Status.Conditions, readyConditionType) + if condition == nil { + t.Fatal("Ready condition was not written") + } + if condition.Status != metav1.ConditionFalse || condition.Reason != "DependenciesNotReady" { + t.Fatalf("unexpected Ready condition: %#v", condition) + } + if condition.ObservedGeneration != 2 { + t.Fatalf("observed generation = %d, want 2", condition.ObservedGeneration) + } +} + +func TestSetReadyStatusKeepsBooleanAndConditionConsistent(t *testing.T) { + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Generation: 4}, + } + + setReadyStatus(wandb, true, "ReconciliationSucceeded", "ready") + + if !wandb.Status.Ready { + t.Fatal("status.ready was not set") + } + condition := apimeta.FindStatusCondition(wandb.Status.Conditions, readyConditionType) + if condition == nil || condition.Status != metav1.ConditionTrue { + t.Fatalf("unexpected Ready condition: %#v", condition) + } + if condition.ObservedGeneration != 4 { + t.Fatalf("observed generation = %d, want 4", condition.ObservedGeneration) + } +} + +func TestRunMigrationsSurfacesFailedJobPhaseAndReason(t *testing.T) { + scheme := runtime.NewScheme() + if err := apiv2.AddToScheme(scheme); err != nil { + t.Fatalf("add W&B API to scheme: %v", err) + } + if err := batchv1.AddToScheme(scheme); err != nil { + t.Fatalf("add batch API to scheme: %v", err) + } + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "default"}, + Spec: apiv2.WeightsAndBiasesSpec{ + Wandb: apiv2.WandbAppSpec{Version: "0.82.2"}, + }, + Status: apiv2.WeightsAndBiasesStatus{ + Wandb: apiv2.WandbStatus{ + Migration: apiv2.WandbMigrationStatus{ + Version: "0.82.2", + Jobs: map[string]apiv2.MigrationJobStatus{}, + }, + }, + }, + } + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb-weave-trace", Namespace: "default"}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{{ + Type: batchv1.JobFailed, + Status: corev1.ConditionTrue, + Reason: "BackoffLimitExceeded", + Message: "migration exited after detecting a partial version", + }}, + }, + } + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&apiv2.WeightsAndBiases{}). + WithObjects(wandb, job). + Build() + manifest := servermanifest.Manifest{ + Migrations: map[string]servermanifest.MigrationJob{ + "weave-trace": {}, + }, + } + + result, err := runMigrations(context.Background(), c, wandb, manifest) + if err != nil { + t.Fatalf("run migrations: %v", err) + } + if result.RequeueAfter == 0 { + t.Fatal("failed migration should requeue") + } + if wandb.Status.Wandb.Migration.Phase != migrationPhaseFailed { + t.Fatalf("migration phase = %q, want %q", wandb.Status.Wandb.Migration.Phase, migrationPhaseFailed) + } + jobStatus := wandb.Status.Wandb.Migration.Jobs["weave-trace"] + if jobStatus.Phase != migrationPhaseFailed || jobStatus.Reason != "BackoffLimitExceeded" { + t.Fatalf("unexpected migration job status: %#v", jobStatus) + } + if jobStatus.Message == "" { + t.Fatal("migration failure message was not surfaced") + } + + reason, message := migrationReadiness(wandb) + if reason != "MigrationFailed" { + t.Fatalf("readiness reason = %q, want MigrationFailed", reason) + } + if message == "" { + t.Fatal("readiness message should identify the failed migration") + } +} diff --git a/internal/controller/reconciler/reconcile_v2.go b/internal/controller/reconciler/reconcile_v2.go index 7737443b..7317058b 100644 --- a/internal/controller/reconciler/reconcile_v2.go +++ b/internal/controller/reconciler/reconcile_v2.go @@ -35,6 +35,7 @@ import ( serverManifest "github.com/wandb/operator/pkg/wandb/manifest" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -226,48 +227,31 @@ func Reconcile( if isFlaggedForDeletion && !wandb.GetDeletionTimestamp().IsZero() { if ctrlqueue.ContainsString(wandb.GetFinalizers(), CleanupFinalizer) { - if wandb.Spec.ObjectStore.ManagedObjectStore != nil { - if err = runRetentionFinalizer(ctx, client, wandb, wandb.Spec.ObjectStore.ManagedObjectStore.ManagedInfraSpec, objectStorePurgeFinalizer, objectStoreDetachFinalizer); err != nil { + // Multi-instance infra: the per-type retention dispatcher applies the + // configured policy to each managed or external instance. + for key, spec := range wandb.Spec.ObjectStore { + if err = runObjectStoreRetentionFinalizer(ctx, client, wandb, key, spec); err != nil { return ctrl.Result{}, err } } - if wandb.Spec.MySQL.ManagedMysql != nil { - if err = runRetentionFinalizer(ctx, client, wandb, wandb.Spec.MySQL.ManagedMysql.ManagedInfraSpec, mysqlPurgeFinalizer, mysqlDetachFinalizer); err != nil { + for key, spec := range wandb.Spec.MySQL { + if err = runMysqlRetentionFinalizer(ctx, client, wandb, key, spec); err != nil { return ctrl.Result{}, err } } - if wandb.Spec.Redis.ManagedRedis != nil { - if err = runRetentionFinalizer(ctx, client, wandb, wandb.Spec.Redis.ManagedRedis.ManagedInfraSpec, redisPurgeFinalizer, redisDetachFinalizer); err != nil { + for key, spec := range wandb.Spec.Redis { + if err = runRedisRetentionFinalizer(ctx, client, wandb, key, spec); err != nil { return ctrl.Result{}, err } } - if wandb.Spec.Kafka.ManagedKafka != nil { - if err = runRetentionFinalizer(ctx, client, wandb, wandb.Spec.Kafka.ManagedKafka.ManagedInfraSpec, kafkaPurgeFinalizer, kafkaDetachFinalizer); err != nil { - return ctrl.Result{}, err - } - } - if wandb.Spec.ClickHouse.ManagedClickHouse != nil { - if err = runRetentionFinalizer(ctx, client, wandb, wandb.Spec.ClickHouse.ManagedClickHouse.ManagedInfraSpec, clickHousePurgeFinalizer, clickHouseDetachFinalizer); err != nil { - return ctrl.Result{}, err - } - } - if wandb.Spec.ObjectStore.ExternalObjectStore != nil && wandb.Spec.RetentionPolicy.OnDelete == apiv2.PurgeOnDelete { - if err = objectStorePurgeFinalizer(ctx, client, wandb); err != nil { - return ctrl.Result{}, err - } - } - if wandb.Spec.MySQL.ExternalMysql != nil && wandb.Spec.RetentionPolicy.OnDelete == apiv2.PurgeOnDelete { - if err = mysqlPurgeFinalizer(ctx, client, wandb); err != nil { - return ctrl.Result{}, err - } - } - if wandb.Spec.Redis.ExternalRedis != nil && wandb.Spec.RetentionPolicy.OnDelete == apiv2.PurgeOnDelete { - if err = redisPurgeFinalizer(ctx, client, wandb); err != nil { + for key, spec := range wandb.Spec.ClickHouse { + if err = runClickHouseRetentionFinalizer(ctx, client, wandb, key, spec); err != nil { return ctrl.Result{}, err } } - if wandb.Spec.ClickHouse.ExternalClickHouse != nil && wandb.Spec.RetentionPolicy.OnDelete == apiv2.PurgeOnDelete { - if err = clickHousePurgeFinalizer(ctx, client, wandb); err != nil { + // Kafka remains single-instance. + if wandb.Spec.Kafka.ManagedKafka != nil { + if err = runRetentionFinalizer(ctx, client, wandb, wandb.Spec.Kafka.ManagedKafka.ManagedInfraSpec, kafkaPurgeFinalizer, kafkaDetachFinalizer); err != nil { return ctrl.Result{}, err } } @@ -322,8 +306,8 @@ func Reconcile( // Write Infra State redisConditions := redisWriteState(ctx, client, wandb, manifest) mysqlConditions := mysqlWriteState(ctx, client, wandb, manifest) - kafkaConditions := kafkaWriteState(ctx, client, wandb, manifest) objectStoreConditions, objectStoreConnection := objectStoreWriteState(ctx, client, wandb, manifest) + kafkaConditions := kafkaWriteState(ctx, client, wandb, manifest) clickHouseConditions := clickHouseWriteState(ctx, client, wandb, manifest) ///////////////////////// @@ -377,11 +361,11 @@ func Reconcile( return ctrl.Result{}, err } - redisReady := wandb.Status.RedisStatus.Ready - mysqlReady := wandb.Status.MySQLStatus.Ready + redisReady := redisAllReady(wandb) + mysqlReady := mysqlAllReady(wandb) kafkaReady := wandb.Status.KafkaStatus.Ready - objectStoreReady := wandb.Status.ObjectStoreStatus.Ready - clickHouseReady := wandb.Status.ClickHouseStatus.Ready + objectStoreReady := objectStoreAllReady(wandb) + clickHouseReady := clickHouseAllReady(wandb) if !redisReady || !mysqlReady || !kafkaReady || !objectStoreReady || !clickHouseReady { log := ctrl.LoggerFrom(ctx) @@ -428,20 +412,36 @@ func ReconcileWandbManifest( var result ctrl.Result var err error - redisReady := wandb.Status.RedisStatus.Ready - mysqlReady := wandb.Status.MySQLStatus.Ready + statusBefore := wandb.DeepCopy().Status + + redisReady := redisAllReady(wandb) + mysqlReady := mysqlAllReady(wandb) kafkaReady := wandb.Status.KafkaStatus.Ready - objectStoreReady := wandb.Status.ObjectStoreStatus.Ready - clickHouseReady := wandb.Status.ClickHouseStatus.Ready + objectStoreReady := objectStoreAllReady(wandb) + clickHouseReady := clickHouseAllReady(wandb) if !redisReady || !mysqlReady || !kafkaReady || !objectStoreReady || !clickHouseReady { logger.Info("Infra components not ready yet, requeuing for reconciliation", "redis", redisReady, "moco", mysqlReady, "kafka", kafkaReady, "objectStore", objectStoreReady, "clickhouse", clickHouseReady) + blockers := infrastructureBlockers(wandb) + if err := updateReadyStatus( + ctx, + client, + wandb, + statusBefore, + false, + "DependenciesNotReady", + "dependencies not ready: "+strings.Join(blockers, ", "), + ); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: defaultRequeueDuration}, nil } logger.Info("Manifest Features", "features", manifest.Features) + validateLegacyOverrides(ctx, wandb, manifest) + result, err = generateSecrets(ctx, client, wandb, manifest) if err != nil { return result, err @@ -457,8 +457,12 @@ func ReconcileWandbManifest( return result, err } - if wandb.Spec.MySQL.ManagedMysql != nil && !wandb.Status.Wandb.MySQLInit.Succeeded { - logger.Info("Mysql init not yet successful", "Message", wandb.Status.Wandb.MySQLInit.Message) + if !allMysqlInitSucceeded(wandb) { + logger.Info("Mysql init not yet successful") + reason, message := mysqlInitializationReadiness(wandb) + if err := updateReadyStatus(ctx, client, wandb, statusBefore, false, reason, message); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } @@ -487,6 +491,11 @@ func ReconcileWandbManifest( } resetInactiveNetworkingStatus(wandb) + if err := reconcileCustomCACerts(ctx, client, wandb); err != nil { + logger.Error(err, "Failed to reconcile custom CA certificates") + return ctrl.Result{}, err + } + if wandb.Spec.Networking.Mode == apiv2.NetworkingModeGatewayAPI { wandb.Status.GatewayStatus = nil if err := reconcileGateway(ctx, client, wandb); err != nil { @@ -502,6 +511,10 @@ func ReconcileWandbManifest( if !wandb.Status.Wandb.Migration.Ready { logger.Info("Migration not yet successful for version", "version", wandb.Spec.Wandb.Version, "reason", wandb.Status.Wandb.Migration.Reason) + reason, message := migrationReadiness(wandb) + if err := updateReadyStatus(ctx, client, wandb, statusBefore, false, reason, message); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } @@ -510,11 +523,18 @@ func ReconcileWandbManifest( return result, err } - if appsHealthy(wandb.Status.Wandb.Applications, buildDesiredAppNames(manifest)) { + // Gate on live Deployment readiness, not status.wandb.applications: the + // copied status map can be a stale snapshot (it only refreshes when this + // reconciler runs), and a frozen mid-rollout entry would block cleanup forever. + applicationsHealthy, notReady := deploymentsHealthy(ctx, client, wandb.Namespace, buildDesiredAppNames(manifest)) + if applicationsHealthy { if err := cleanupLegacyV1Deployments(ctx, client, wandb); err != nil { logger.Error(err, "Failed to clean up legacy v1 deployments") return ctrl.Result{}, err } + } else { + logger.Info("Deferring legacy v1 deployment cleanup until all application Deployments are ready", + "notReady", notReady) } if wandb.Spec.Networking.Mode == apiv2.NetworkingModeGatewayAPI { @@ -524,7 +544,26 @@ func ReconcileWandbManifest( } } - return ctrl.Result{}, nil + if applicationsHealthy { + setReadyStatus( + wandb, + true, + "ReconciliationSucceeded", + "all dependencies, migrations, and application deployments are ready", + ) + } else { + message := "waiting for application deployments: " + strings.Join(notReady, ", ") + if len(notReady) == 0 { + message = "no desired application deployments were found" + } + setReadyStatus(wandb, false, "ApplicationsNotReady", message) + } + + if err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore); err != nil { + return ctrl.Result{}, err + } + + return result, nil } func reconcileApplications( @@ -588,12 +627,26 @@ func reconcileApplications( volumes, volumeMounts = resolveJWTTokens(app, volumes, volumeMounts) } + var caChecksum string + envVars, volumes, volumeMounts, caChecksum, err = applyCustomCACertsToWorkload(ctx, client, wandb, envVars, volumes, volumeMounts) + if err != nil { + return ctrl.Result{}, err + } + + // spec.global.proxy env: after CA (so both are present) and before legacy + // overrides (so legacyOverrides can still override/blank any proxy var). + envVars = applyProxyToWorkload(wandb, envVars) + + // Applied last so legacy overrides beat manifest and injected env, as in v1. + envVars = applyLegacyOverrideEnv(ctx, wandb, app.Name, envVars) + containers := resolveContainers(app, wandb, envVars, volumeMounts) initContainers := resolveInitContainers(app, wandb, envVars, volumeMounts) application := &apiv2.Application{} err = client.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: wandb.Namespace}, application) + before := application.DeepCopy() if err != nil { if apiErrors.IsNotFound(err) { application.SetName(app.Name) @@ -604,6 +657,17 @@ func reconcileApplications( } application.Spec.Kind = "Deployment" + + // Label the workload with both families: the descriptive app.kubernetes.io/* + // set (for tooling/NetworkPolicies) and the operator/ownership set (which + // backs the immutable pod selector). See docs/pod-labeling-standards.md. + standardLabels := common.StandardLabels(wandb, app.Name, common.AppComponentRole(app.Name), wandb.Spec.Wandb.Version) + operatorLabels := common.BuildWandbLabels(wandb, app.Name) + application.Spec.MetaTemplate.Labels = oputils.MergeMapsStringString(application.Spec.MetaTemplate.Labels, standardLabels) + application.Spec.PodTemplate.Labels = oputils.MergeMapsStringString( + application.Spec.PodTemplate.Labels, standardLabels, operatorLabels, + ) + application.Spec.PodTemplate.Spec.Containers = containers // Replace volumes entirely on each reconcile to avoid accumulating duplicates // across updates (e.g., duplicate "files-inline" volume names). @@ -612,6 +676,7 @@ func reconcileApplications( application.Spec.PodTemplate.Spec.SecurityContext = resolvePodSecurityContext() application.Spec.PodTemplate.Spec.Affinity = wandb.Spec.Affinity application.Spec.PodTemplate.Spec.Tolerations = *wandb.Spec.Tolerations + setCustomCACertsChecksumAnnotation(&application.Spec.PodTemplate, caChecksum) application.Spec.HpaTemplate = ResolveAutoscaling(app, wandb) @@ -623,10 +688,16 @@ func reconcileApplications( // change to port numbers, names, or protocols is propagated on each // reconcile. If no service ports are declared, clear the ServiceTemplate. if app.Service != nil && len(app.Service.Ports) > 0 { + // Copy + normalize: the CRD schema defaults ports[].protocol, so an + // un-normalized template never round-trips equal and the update gate + // below would fire on every reconcile, churning the Application. + ports := make([]corev1.ServicePort, len(app.Service.Ports)) + copy(ports, app.Service.Ports) + common.NormalizeServicePorts(ports) application.Spec.ServiceTemplate = &corev1.ServiceSpec{ - Type: app.Service.Type, + Type: app.Service.Type, + Ports: ports, } - application.Spec.ServiceTemplate.Ports = app.Service.Ports } else { // No service declared in manifest; ensure we clear any previous template application.Spec.ServiceTemplate = nil @@ -639,6 +710,9 @@ func reconcileApplications( application.Spec.HTTPRouteTemplate = nil } + // A plain owner ref (not a controller ref) so multiple CRs can share a + // namespace; the parent's Owns(Application) watch uses MatchEveryOwner + // to still enqueue on app status changes. err = controllerutil.SetOwnerReference(wandb, application, client.Scheme()) if err != nil { return ctrl.Result{}, err @@ -648,7 +722,7 @@ func reconcileApplications( if err = client.Create(ctx, application); err != nil { return ctrl.Result{}, err } - } else { + } else if !applicationManagedFieldsEqual(before, application) { if err = client.Update(ctx, application); err != nil { return ctrl.Result{}, err } @@ -723,13 +797,22 @@ func reconcileApplications( } } - if err := client.Status().Update(ctx, wandb); err != nil { - return ctrl.Result{}, err - } + // Every application now carries this generation's spec, so consumers can + // gate on observedGeneration == generation plus workload rollout. Earlier + // exits (infra, mysql-init, migrations) must not advance it: their specs + // haven't reached the workloads yet. + wandb.Status.ObservedGeneration = wandb.GetGeneration() return ctrl.Result{}, nil } +func applicationManagedFieldsEqual(before, after *apiv2.Application) bool { + return apiequality.Semantic.DeepEqual(before.Spec, after.Spec) && + apiequality.Semantic.DeepEqual(before.Labels, after.Labels) && + apiequality.Semantic.DeepEqual(before.Annotations, after.Annotations) && + apiequality.Semantic.DeepEqual(before.OwnerReferences, after.OwnerReferences) +} + func buildHTTPRouteTemplate(wandb *apiv2.WeightsAndBiases, app serverManifest.Application) *apiv2.HTTPRouteTemplateSpec { gwConfig := wandb.Spec.Networking.GatewayAPI @@ -874,23 +957,6 @@ func appendResolvedManagedTelemetryEnvvars( return appendMissingEnvVars(envVars, telemetryEnvVars), nil } -// This function binds the secret name if `secretName` is empty for telemetry. -func bindTelemetrySecretName(base []serverManifest.EnvVar, secretName string) []serverManifest.EnvVar { - output := make([]serverManifest.EnvVar, len(base)) - for i, envVar := range base { - sources := make([]serverManifest.EnvSource, len(envVar.Sources)) - for j, source := range envVar.Sources { - if source.Type == "telemetry" && source.Name == "" { - source.Name = secretName - } - sources[j] = source - } - envVar.Sources = sources - output[i] = envVar - } - return output -} - func shouldInjectManagedWorkloadTelemetry(appName string) bool { _, ok := managedWorkloadTelemetryApplications[appName] return ok @@ -1118,9 +1184,21 @@ func resolveInlineFiles(ctx context.Context, client ctrlClient.Client, wandb *ap } func runMigrations(ctx context.Context, client ctrlClient.Client, wandb *apiv2.WeightsAndBiases, manifest serverManifest.Manifest) (ctrl.Result, error) { + statusBefore := wandb.DeepCopy().Status version := wandb.Spec.Wandb.Version if wandb.Status.Wandb.Migration.Ready && wandb.Status.Wandb.Migration.Version == version { + wandb.Status.Wandb.Migration.Phase = migrationPhaseSucceeded + if wandb.Status.Wandb.Migration.Reason == "" { + wandb.Status.Wandb.Migration.Reason = "Complete" + } + for name, jobStatus := range wandb.Status.Wandb.Migration.Jobs { + if jobStatus.Succeeded && jobStatus.Phase == "" { + jobStatus.Phase = migrationPhaseSucceeded + jobStatus.Reason = "JobSucceeded" + wandb.Status.Wandb.Migration.Jobs[name] = jobStatus + } + } for name := range manifest.Migrations { jobName := fmt.Sprintf("%s-%s", wandb.Name, name) job := &batchv1.Job{ @@ -1138,24 +1216,30 @@ func runMigrations(ctx context.Context, client ctrlClient.Client, wandb *apiv2.W } } } + if err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{}, nil } if wandb.Status.Wandb.Migration.Version != version { wandb.Status.Wandb.Migration.Version = version wandb.Status.Wandb.Migration.Ready = false + wandb.Status.Wandb.Migration.Phase = migrationPhaseRunning wandb.Status.Wandb.Migration.Reason = "Running" wandb.Status.Wandb.Migration.Jobs = make(map[string]apiv2.MigrationJobStatus) - if err := client.Status().Update(ctx, wandb); err != nil { + if err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore); err != nil { return ctrl.Result{}, err } + statusBefore = wandb.DeepCopy().Status } if len(manifest.Migrations) == 0 { wandb.Status.Wandb.Migration.Ready = true + wandb.Status.Wandb.Migration.Phase = migrationPhaseSucceeded wandb.Status.Wandb.Migration.Reason = "Complete" wandb.Status.Wandb.Migration.LastSuccessVersion = version - if err := client.Status().Update(ctx, wandb); err != nil { + if err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore); err != nil { return ctrl.Result{}, err } return ctrl.Result{}, nil @@ -1194,34 +1278,48 @@ func runMigrations(ctx context.Context, client ctrlClient.Client, wandb *apiv2.W return ctrl.Result{}, err } + var caChecksum string + envVars, volumes, volumeMounts, caChecksum, err = applyCustomCACertsToWorkload(ctx, client, wandb, envVars, volumes, volumeMounts) + if err != nil { + return ctrl.Result{}, err + } + + // spec.global.proxy env (migration Jobs egress too — v1 parity); before + // legacy overrides so the escape hatch still wins. + envVars = applyProxyToWorkload(wandb, envVars) + + // v1's global env reached job pods too (e.g. HTTP_PROXY); per-app entries don't apply here. + envVars = overrideEnvVars(ctx, envVars, wandb.Spec.Wandb.LegacyOverrides[apiv2.LegacyOverridesGlobalKey].Env) + + migrationJobLabels := common.StandardLabels(wandb, "migration", common.RoleMigration, wandb.Spec.Wandb.Version) + podTemplate := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: migrationJobLabels}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + Containers: []corev1.Container{ + { + Name: "migrate", + Image: migrationTask.Image.GetImage(""), + Args: migrationTask.Args, + Command: migrationTask.Command, + Env: envVars, + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + ServiceAccountName: wandb.Spec.Wandb.ServiceAccount.ServiceAccountName, + }, + } + setCustomCACertsChecksumAnnotation(&podTemplate, caChecksum) + job = &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: jobName, Namespace: wandb.Namespace, - Labels: map[string]string{ - "app.kubernetes.io/managed-by": "wandb-operator", - "app.kubernetes.io/instance": wandb.Name, - "app.kubernetes.io/component": "migration", - }, + Labels: migrationJobLabels, }, Spec: batchv1.JobSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyOnFailure, - Containers: []corev1.Container{ - { - Name: "migrate", - Image: migrationTask.Image.GetImage(""), - Args: migrationTask.Args, - Command: migrationTask.Command, - Env: envVars, - VolumeMounts: volumeMounts, - }, - }, - Volumes: volumes, - ServiceAccountName: wandb.Spec.Wandb.ServiceAccount.ServiceAccountName, - }, - }, + Template: podTemplate, }, } @@ -1234,10 +1332,13 @@ func runMigrations(ctx context.Context, client ctrlClient.Client, wandb *apiv2.W } jobStatus.Succeeded = false + jobStatus.Phase = migrationPhaseRunning + jobStatus.Reason = "JobCreated" wandb.Status.Wandb.Migration.Jobs[name] = jobStatus + wandb.Status.Wandb.Migration.Phase = migrationPhaseRunning wandb.Status.Wandb.Migration.Reason = "Running" wandb.Status.Wandb.Migration.Ready = false - if err := client.Status().Update(ctx, wandb); err != nil { + if err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore); err != nil { return ctrl.Result{}, err } @@ -1246,11 +1347,27 @@ func runMigrations(ctx context.Context, client ctrlClient.Client, wandb *apiv2.W if job.Status.Succeeded > 0 { jobStatus.Succeeded = true + jobStatus.Phase = migrationPhaseSucceeded + jobStatus.Reason = "JobSucceeded" + for _, cond := range job.Status.Conditions { + if cond.Type == batchv1.JobComplete && cond.Status == corev1.ConditionTrue { + if cond.Reason != "" { + jobStatus.Reason = cond.Reason + } + jobStatus.Message = cond.Message + break + } + } } else { allSucceeded = false for _, cond := range job.Status.Conditions { if cond.Type == batchv1.JobFailed && cond.Status == corev1.ConditionTrue { jobStatus.Failed = true + jobStatus.Phase = migrationPhaseFailed + jobStatus.Reason = cond.Reason + if jobStatus.Reason == "" { + jobStatus.Reason = "JobFailed" + } jobStatus.Message = cond.Message anyFailed = true break @@ -1258,6 +1375,12 @@ func runMigrations(ctx context.Context, client ctrlClient.Client, wandb *apiv2.W } if !jobStatus.Failed { anyRunning = true + jobStatus.Phase = migrationPhaseRunning + if job.Status.Active > 0 { + jobStatus.Reason = "JobRunning" + } else { + jobStatus.Reason = "JobPending" + } } } @@ -1265,30 +1388,30 @@ func runMigrations(ctx context.Context, client ctrlClient.Client, wandb *apiv2.W } if anyFailed { + wandb.Status.Wandb.Migration.Phase = migrationPhaseFailed wandb.Status.Wandb.Migration.Reason = "Failed" wandb.Status.Wandb.Migration.Ready = false } else if anyRunning || !allSucceeded { + wandb.Status.Wandb.Migration.Phase = migrationPhaseRunning wandb.Status.Wandb.Migration.Reason = "Running" wandb.Status.Wandb.Migration.Ready = false } else if allSucceeded { + wandb.Status.Wandb.Migration.Phase = migrationPhaseSucceeded wandb.Status.Wandb.Migration.Reason = "Complete" wandb.Status.Wandb.Migration.Ready = true if wandb.Status.Wandb.Migration.LastSuccessVersion != version { wandb.Status.Wandb.Migration.LastSuccessVersion = version } } else { + wandb.Status.Wandb.Migration.Phase = migrationPhaseUnknown wandb.Status.Wandb.Migration.Reason = "Unknown" wandb.Status.Wandb.Migration.Ready = false } - if err := client.Status().Update(ctx, wandb); err != nil { + if err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore); err != nil { return ctrl.Result{}, err } - if anyFailed { - return ctrl.Result{}, fmt.Errorf("one or more migration jobs failed") - } - if allSucceeded { return ctrl.Result{}, nil } @@ -1297,6 +1420,7 @@ func runMigrations(ctx context.Context, client ctrlClient.Client, wandb *apiv2.W } func generateSecrets(ctx context.Context, client ctrlClient.Client, wandb *apiv2.WeightsAndBiases, manifest serverManifest.Manifest) (ctrl.Result, error) { + statusBefore := wandb.DeepCopy().Status // Ensure any manifest-declared generated secrets exist and capture their selectors in status if wandb.Status.GeneratedSecrets == nil { wandb.Status.GeneratedSecrets = map[string]corev1.SecretKeySelector{} @@ -1326,11 +1450,9 @@ func generateSecrets(ctx context.Context, client ctrlClient.Client, wandb *apiv2 ObjectMeta: metav1.ObjectMeta{ Name: secretName, Namespace: wandb.Namespace, - Labels: map[string]string{ - "app.kubernetes.io/managed-by": "wandb-operator", - "app.kubernetes.io/instance": wandb.Name, - "app.kubernetes.io/part-of": "wandb", - }, + // Generated secrets are role-less, so component is omitted + // per docs/pod-labeling-standards.md (non-pod resources). + Labels: common.StandardLabels(wandb, gs.Name, "", ""), }, StringData: map[string]string{keyName: pw}, Type: corev1.SecretTypeOpaque, @@ -1372,7 +1494,7 @@ func generateSecrets(ctx context.Context, client ctrlClient.Client, wandb *apiv2 } } // Persist status after updating generated secret selectors - if err := client.Status().Update(ctx, wandb); err != nil { + if err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore); err != nil { return ctrl.Result{}, err } return ctrl.Result{}, nil @@ -1443,29 +1565,67 @@ func resolveCRFieldSecretSelector(obj any, path string) (corev1.SecretKeySelecto return sel, true } +// allInstancesReady reports whether every instance (managed or external) has a +// ready status. +func allInstancesReady[S any, T any](specs map[string]S, statuses map[string]T, ready func(T) bool) bool { + for key := range specs { + if !ready(statuses[key]) { + return false + } + } + return true +} + +func redisAllReady(wandb *apiv2.WeightsAndBiases) bool { + return allInstancesReady(wandb.Spec.Redis, wandb.Status.RedisStatus, func(s apiv2.RedisInfraStatus) bool { return s.Ready }) +} + +func mysqlAllReady(wandb *apiv2.WeightsAndBiases) bool { + return allInstancesReady(wandb.Spec.MySQL, wandb.Status.MySQLStatus, func(s apiv2.MysqlInfraStatus) bool { return s.Ready }) +} + +func objectStoreAllReady(wandb *apiv2.WeightsAndBiases) bool { + return allInstancesReady(wandb.Spec.ObjectStore, wandb.Status.ObjectStoreStatus, func(s apiv2.ObjectStoreInfraStatus) bool { return s.Ready }) +} + +func clickHouseAllReady(wandb *apiv2.WeightsAndBiases) bool { + return allInstancesReady(wandb.Spec.ClickHouse, wandb.Status.ClickHouseStatus, func(s apiv2.ClickHouseInfraStatus) bool { return s.Ready }) +} + func inferState( ctx context.Context, client ctrlClient.Client, wandb *apiv2.WeightsAndBiases, ) error { log := ctrl.LoggerFrom(ctx) - - redisOk := wandb.Spec.Redis.ManagedRedis == nil || wandb.Status.RedisStatus.Ready - objectStoreOk := wandb.Spec.ObjectStore.ManagedObjectStore == nil || wandb.Status.ObjectStoreStatus.Ready - mysqlOk := wandb.Spec.MySQL.ManagedMysql == nil || wandb.Status.MySQLStatus.Ready - clickHouseOk := wandb.Spec.ClickHouse.ManagedClickHouse == nil || wandb.Status.ClickHouseStatus.Ready - kafkaOk := wandb.Spec.Kafka.ManagedKafka == nil || wandb.Status.KafkaStatus.Ready - - if redisOk && objectStoreOk && mysqlOk && clickHouseOk && kafkaOk { - wandb.Status.Ready = true - } else { - wandb.Status.Ready = false - } - - log.Info("About to update status", "apiVersion", wandb.APIVersion, "kind", wandb.Kind) - if err := client.Status().Update(ctx, wandb); err != nil { + statusBefore := wandb.DeepCopy().Status + + blockers := infrastructureBlockers(wandb) + switch { + case len(blockers) > 0: + setReadyStatus( + wandb, + false, + "DependenciesNotReady", + "dependencies not ready: "+strings.Join(blockers, ", "), + ) + case wandb.Status.ObservedGeneration != wandb.Generation: + setReadyStatus( + wandb, + false, + "Reconciling", + fmt.Sprintf( + "observed generation %d does not match generation %d", + wandb.Status.ObservedGeneration, + wandb.Generation, + ), + ) + default: + return nil + } + + if err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore); err != nil { log.Error(err, "Failed to update status") return err } - log.Info("Status update successful") return nil } diff --git a/internal/controller/reconciler/reconcile_v2_infra_sizing_test.go b/internal/controller/reconciler/reconcile_v2_infra_sizing_test.go index 1f8f28bf..7ba1cbf7 100644 --- a/internal/controller/reconciler/reconcile_v2_infra_sizing_test.go +++ b/internal/controller/reconciler/reconcile_v2_infra_sizing_test.go @@ -99,7 +99,7 @@ var _ = Describe("Infra Sizing", func() { wandb := &apiv2.WeightsAndBiases{ Spec: apiv2.WeightsAndBiasesSpec{ Size: "small", - MySQL: apiv2.MySQLSpec{ManagedMysql: &apiv2.ManagedMysqlSpec{}}, + MySQL: map[string]apiv2.MySQLSpec{apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{}}}, }, } manifest := serverManifest.Manifest{ @@ -120,18 +120,20 @@ var _ = Describe("Infra Sizing", func() { }, } v2.ApplyInfraSizing(wandb, manifest) - Expect(wandb.Spec.MySQL.ManagedMysql.Replicas).To(Equal(int32(3))) - Expect(wandb.Spec.MySQL.ManagedMysql.Config.Resources.Requests.Cpu().String()).To(Equal("2")) + Expect(wandb.Spec.MySQL[apiv2.DefaultInstanceName].ManagedMysql.Replicas).To(Equal(int32(3))) + Expect(wandb.Spec.MySQL[apiv2.DefaultInstanceName].ManagedMysql.Config.Resources.Requests.Cpu().String()).To(Equal("2")) }) It("should not override user-specified spec fields", func() { wandb := &apiv2.WeightsAndBiases{ Spec: apiv2.WeightsAndBiasesSpec{ Size: "small", - MySQL: apiv2.MySQLSpec{ - ManagedMysql: &apiv2.ManagedMysqlSpec{ - Replicas: 5, - StorageSize: "50Gi", + MySQL: map[string]apiv2.MySQLSpec{ + apiv2.DefaultInstanceName: { + ManagedMysql: &apiv2.ManagedMysqlSpec{ + Replicas: 5, + StorageSize: "50Gi", + }, }, }, }, @@ -149,8 +151,83 @@ var _ = Describe("Infra Sizing", func() { }, } v2.ApplyInfraSizing(wandb, manifest) - Expect(wandb.Spec.MySQL.ManagedMysql.Replicas).To(Equal(int32(5))) - Expect(wandb.Spec.MySQL.ManagedMysql.StorageSize).To(Equal("50Gi")) + Expect(wandb.Spec.MySQL[apiv2.DefaultInstanceName].ManagedMysql.Replicas).To(Equal(int32(5))) + Expect(wandb.Spec.MySQL[apiv2.DefaultInstanceName].ManagedMysql.StorageSize).To(Equal("50Gi")) + }) + + It("should default object store copies from the manifest, treating CR values as overrides", func() { + wandb := &apiv2.WeightsAndBiases{ + Spec: apiv2.WeightsAndBiasesSpec{ + Size: "small", + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: {ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}, + }, + }, + } + manifest := serverManifest.Manifest{ + Bucket: map[string]serverManifest.InfraConfig{ + "default": { + Sizing: map[apiv2.Size]serverManifest.SizingConfig{ + "default": {Replicas: 1}, + "small": {Replicas: 3, Copies: 2}, + }, + }, + }, + } + v2.ApplyInfraSizing(wandb, manifest) + Expect(wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Copies).To(Equal(int32(2))) + }) + + It("should not override a CR-specified object store copies value", func() { + wandb := &apiv2.WeightsAndBiases{ + Spec: apiv2.WeightsAndBiasesSpec{ + Size: "small", + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: {ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{Copies: 1}}, + }, + }, + } + manifest := serverManifest.Manifest{ + Bucket: map[string]serverManifest.InfraConfig{ + "default": { + Sizing: map[apiv2.Size]serverManifest.SizingConfig{ + "small": {Replicas: 3, Copies: 2}, + }, + }, + }, + } + v2.ApplyInfraSizing(wandb, manifest) + Expect(wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Copies).To(Equal(int32(1))) + }) + + It("should apply keeper sizing from the clickhouseKeeper block, treating CR values as overrides", func() { + wandb := &apiv2.WeightsAndBiases{ + Spec: apiv2.WeightsAndBiasesSpec{ + Size: "small", + ClickHouse: map[string]apiv2.ClickHouseSpec{ + apiv2.DefaultInstanceName: { + ManagedClickHouse: &apiv2.ManagedClickHouseSpec{ + // User explicitly set keeper storage; the manifest must not override it. + Keeper: apiv2.ClickHouseKeeperSpec{StorageSize: "20Gi"}, + }, + }, + }, + }, + } + manifest := serverManifest.Manifest{ + ClickhouseKeeper: map[string]serverManifest.InfraConfig{ + "default": { + Sizing: map[apiv2.Size]serverManifest.SizingConfig{ + "default": {Replicas: 1, VolumeSize: "10Gi"}, + "small": {Replicas: 3}, + }, + }, + }, + } + v2.ApplyInfraSizing(wandb, manifest) + keeper := wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.Keeper + Expect(keeper.Replicas).To(Equal(int32(3))) // from manifest small tier + Expect(keeper.StorageSize).To(Equal("20Gi")) // user override preserved }) }) diff --git a/internal/controller/reconciler/reconcile_v2_objectstore_sizing_test.go b/internal/controller/reconciler/reconcile_v2_objectstore_sizing_test.go new file mode 100644 index 00000000..db6d87b2 --- /dev/null +++ b/internal/controller/reconciler/reconcile_v2_objectstore_sizing_test.go @@ -0,0 +1,112 @@ +package reconciler_test + +import ( + "context" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/infra/managed/objectstore/seaweedfs" + v2 "github.com/wandb/operator/internal/controller/reconciler" + seaweedv1 "github.com/wandb/operator/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1" + serverManifest "github.com/wandb/operator/pkg/wandb/manifest" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// localManifestVersion is the checked-in server manifest used for local dev; the +// sizing assertions below mirror its bucket.default.sizing block. +const localManifestVersion = "0.83.0-clickhouse-keeper.2" + +func objectStoreScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + Expect(apiv2.AddToScheme(scheme)).To(Succeed()) + Expect(seaweedv1.AddToScheme(scheme)).To(Succeed()) + return scheme +} + +func objectStoreWandb(size apiv2.Size) *apiv2.WeightsAndBiases { + tolerations := []corev1.Toleration{} + return &apiv2.WeightsAndBiases{ + TypeMeta: metav1.TypeMeta{APIVersion: apiv2.GroupVersion.String(), Kind: "WeightsAndBiases"}, + ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "wandb"}, + Spec: apiv2.WeightsAndBiasesSpec{ + Size: size, + Tolerations: &tolerations, + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: { + ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{ + Name: "object-store", + Namespace: "wandb", + Config: apiv2.ObjectStoreConfig{AccessKey: "admin"}, + }, + }, + }, + }, + } +} + +var _ = Describe("ObjectStore sizing per tier", func() { + var mfst serverManifest.Manifest + + BeforeEach(func() { + repoRoot, err := filepath.Abs(filepath.Join("..", "..", "..")) + Expect(err).NotTo(HaveOccurred()) + repository := "file://" + filepath.Join(repoRoot, "hack", "testing-manifests", "server-manifest") + mfst, err = serverManifest.GetServerManifest(context.Background(), repository, localManifestVersion) + Expect(err).NotTo(HaveOccurred()) + Expect(mfst.Bucket).To(HaveKey("default")) + }) + + // Values mirror bucket.default.sizing in the local manifest. Every tier must + // keep the same hardening regardless of size: 1024MB rollover, a writable + // volume count sized to the disk, and a small fixed filer disk. + DescribeTable("renders a healthy Seaweed spec for each size", + func(size apiv2.Size, wantReplicas int32, wantVolumeSize, wantReplication string, wantCPU string, wantFilerSize string) { + wandb := objectStoreWandb(size) + v2.ApplyInfraSizing(wandb, mfst) + + seaweed, err := seaweedfs.ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, objectStoreScheme(), mfst) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed).NotTo(BeNil()) + + Expect(seaweed.Spec.Volume.Replicas).To(Equal(wantReplicas)) + Expect(seaweed.Spec.Volume.Requests[corev1.ResourceStorage]).To(Equal(resource.MustParse(wantVolumeSize))) + Expect(seaweed.Spec.Master.DefaultReplication).NotTo(BeNil()) + Expect(*seaweed.Spec.Master.DefaultReplication).To(Equal(wantReplication)) + + // Hardening that must hold for every tier. + Expect(*seaweed.Spec.Master.VolumeSizeLimitMB).To(Equal(int32(1024))) + Expect(seaweed.Spec.Volume.MaxVolumeCounts).NotTo(BeNil()) + Expect(*seaweed.Spec.Volume.MaxVolumeCounts).To(BeNumerically(">", int32(0))) + // Filer disk follows the manifest's metadataVolumeSize per tier, falling back to 20Gi. + Expect(seaweed.Spec.Filer.Persistence.Resources.Requests[corev1.ResourceStorage]).To(Equal(resource.MustParse(wantFilerSize))) + + if wantCPU == "" { + Expect(seaweed.Spec.Volume.Requests).NotTo(HaveKey(corev1.ResourceCPU)) + } else { + Expect(seaweed.Spec.Volume.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse(wantCPU))) + } + }, + Entry("dev", apiv2.Size("dev"), int32(1), "10Gi", "000", "", "20Gi"), + Entry("micro", apiv2.Size("micro"), int32(3), "50Gi", "001", "1", "20Gi"), + Entry("small", apiv2.Size("small"), int32(3), "100Gi", "001", "2", "20Gi"), + Entry("medium", apiv2.Size("medium"), int32(3), "100Gi", "001", "4", "20Gi"), + Entry("large", apiv2.Size("large"), int32(3), "200Gi", "001", "8", "40Gi"), + Entry("xlarge", apiv2.Size("xlarge"), int32(3), "200Gi", "001", "8", "20Gi"), + Entry("2xlarge", apiv2.Size("2xlarge"), int32(3), "200Gi", "001", "8", "20Gi"), + ) + + It("lets a CR filer size override the manifest metadataVolumeSize", func() { + wandb := objectStoreWandb(apiv2.Size("large")) + wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.SeaweedObjectStoreSpec.FilerStorageSize = "100Gi" + v2.ApplyInfraSizing(wandb, mfst) + + seaweed, err := seaweedfs.ToObjectStoreVendorSpec(context.Background(), wandb, wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore, objectStoreScheme(), mfst) + Expect(err).NotTo(HaveOccurred()) + Expect(seaweed.Spec.Filer.Persistence.Resources.Requests[corev1.ResourceStorage]).To(Equal(resource.MustParse("100Gi"))) + }) +}) diff --git a/internal/controller/reconciler/reconcile_v2_sizing_test.go b/internal/controller/reconciler/reconcile_v2_sizing_test.go index d7364e98..f1a766fe 100644 --- a/internal/controller/reconciler/reconcile_v2_sizing_test.go +++ b/internal/controller/reconciler/reconcile_v2_sizing_test.go @@ -125,6 +125,117 @@ var _ = Describe("ReconcileV2 Sizing", func() { Expect(res.Requests.Cpu().String()).To(Equal("100m")) Expect(res.Limits).To(BeNil()) }) + + It("should apply legacy overrides over sizing-derived resources", func() { + app := serverManifest.Application{ + Name: "api", + Sizing: map[apiv2.Size]serverManifest.SizingConfig{ + "default": { + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + }, + }, + }, + }, + } + wandb := &apiv2.WeightsAndBiases{ + Spec: apiv2.WeightsAndBiasesSpec{ + Size: "small", + RequireLimits: true, + Wandb: apiv2.WandbAppSpec{ + LegacyOverrides: map[string]apiv2.LegacyOverrides{ + "api": { + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + }, + }, + }, + }, + }, + } + + res := v2.ResolveResources(app, wandb, nil) + Expect(res).NotTo(BeNil()) + // Override wins per resource name; untouched fields survive. + Expect(res.Requests.Cpu().String()).To(Equal("2")) + Expect(res.Requests.Memory().String()).To(Equal("128Mi")) + Expect(res.Limits.Cpu().String()).To(Equal("4")) + }) + + It("should strip legacy override limits when RequireLimits is false", func() { + app := serverManifest.Application{Name: "api"} + wandb := &apiv2.WeightsAndBiases{ + Spec: apiv2.WeightsAndBiasesSpec{ + Size: "small", + RequireLimits: false, + Wandb: apiv2.WandbAppSpec{ + LegacyOverrides: map[string]apiv2.LegacyOverrides{ + "api": { + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + }, + }, + }, + }, + }, + } + + res := v2.ResolveResources(app, wandb, nil) + Expect(res).NotTo(BeNil()) + Expect(res.Requests.Cpu().String()).To(Equal("2")) + Expect(res.Limits).To(BeNil()) + }) + + It("should not apply another application's legacy override", func() { + app := serverManifest.Application{ + Name: "weave", + Sizing: map[apiv2.Size]serverManifest.SizingConfig{ + "default": { + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }, + }, + } + wandb := &apiv2.WeightsAndBiases{ + Spec: apiv2.WeightsAndBiasesSpec{ + Size: "small", + RequireLimits: true, + Wandb: apiv2.WandbAppSpec{ + LegacyOverrides: map[string]apiv2.LegacyOverrides{ + "api": { + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }, + }, + }, + }, + }, + } + + res := v2.ResolveResources(app, wandb, nil) + Expect(res).NotTo(BeNil()) + Expect(res.Requests.Cpu().String()).To(Equal("100m")) + }) }) Context("ResolveAutoscaling", func() { diff --git a/internal/controller/reconciler/redis.go b/internal/controller/reconciler/redis.go index 5faea8b1..781498cc 100644 --- a/internal/controller/reconciler/redis.go +++ b/internal/controller/reconciler/redis.go @@ -22,29 +22,38 @@ func redisWriteState( client client.Client, wandb *apiv2.WeightsAndBiases, mfst manifest.Manifest, -) []metav1.Condition { - if wandb.Spec.Redis.ManagedRedis != nil { - return managedRedisWriteState(ctx, client, wandb, mfst) - } - if wandb.Spec.Redis.ExternalRedis != nil { - return externalRedisWriteState(ctx, client, wandb) +) map[string][]metav1.Condition { + out := map[string][]metav1.Condition{} + for key, spec := range wandb.Spec.Redis { + switch { + case spec.ManagedRedis != nil: + out[key] = managedRedisWriteState(ctx, client, wandb, spec.ManagedRedis, mfst) + case spec.ExternalRedis != nil: + out[key] = externalredis.WriteState(ctx, client, wandb, key, spec.ExternalRedis) + } } - return nil + return out } func redisReadState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, - newConditions []metav1.Condition, -) ([]metav1.Condition, *apiv2.RedisConnection) { - if wandb.Spec.Redis.ManagedRedis != nil { - return managedRedisReadState(ctx, client, wandb, newConditions) - } - if wandb.Spec.Redis.ExternalRedis != nil { - return externalRedisReadState(ctx, client, wandb, newConditions) + conditions map[string][]metav1.Condition, +) (map[string][]metav1.Condition, map[string]*apiv2.RedisConnection) { + outConds := map[string][]metav1.Condition{} + outConns := map[string]*apiv2.RedisConnection{} + for key, spec := range wandb.Spec.Redis { + switch { + case spec.ManagedRedis != nil: + outConds[key], outConns[key] = managedRedisReadState(ctx, client, wandb, spec.ManagedRedis, conditions[key]) + case spec.ExternalRedis != nil: + outConds[key], outConns[key] = externalredis.ReadState(ctx, client, wandb, key, conditions[key]) + default: + outConds[key] = conditions[key] + } } - return newConditions, nil + return outConds, outConns } func redisInferStatus( @@ -52,30 +61,62 @@ func redisInferStatus( client client.Client, recorder record.EventRecorder, wandb *apiv2.WeightsAndBiases, - newConditions []metav1.Condition, - newInfraConn *apiv2.RedisConnection, + conditions map[string][]metav1.Condition, + infraConns map[string]*apiv2.RedisConnection, ) (ctrl.Result, error) { - if wandb.Spec.Redis.ManagedRedis != nil { - return managedRedisInferStatus(ctx, client, recorder, wandb, newConditions, newInfraConn) + if wandb.Status.RedisStatus == nil { + wandb.Status.RedisStatus = map[string]apiv2.RedisInfraStatus{} + } + var results []ctrl.Result + var firstErr error + for key, spec := range wandb.Spec.Redis { + var res ctrl.Result + var err error + switch { + case spec.ManagedRedis != nil: + res, err = managedRedisInferStatus(ctx, client, recorder, wandb, key, conditions[key], infraConns[key]) + case spec.ExternalRedis != nil: + res, err = externalRedisInferStatus(ctx, client, wandb, key, conditions[key], infraConns[key]) + } + results = append(results, res) + if err != nil && firstErr == nil { + firstErr = err + } } - if wandb.Spec.Redis.ExternalRedis != nil { - return externalRedisInferStatus(ctx, client, wandb, newConditions, newInfraConn) + return consolidateResults(results), firstErr +} + +func runRedisRetentionFinalizer(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string, spec apiv2.RedisSpec) error { + switch wandb.GetRetentionPolicy(redisInstanceInfraSpec(spec)).OnDelete { + case apiv2.PurgeOnDelete: + return redisPurgeFinalizer(ctx, c, wandb, key, spec) + case apiv2.DetachOnDelete: + return redisDetachFinalizer(ctx, c, wandb, key, spec) } - return ctrl.Result{}, nil + return nil +} + +func redisInstanceInfraSpec(spec apiv2.RedisSpec) apiv2.ManagedInfraSpec { + if spec.ManagedRedis != nil { + return spec.ManagedRedis.ManagedInfraSpec + } + return apiv2.ManagedInfraSpec{} } func redisPurgeFinalizer( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + key string, + spec apiv2.RedisSpec, ) error { - if spec := wandb.Spec.Redis.ManagedRedis; spec != nil { - specNamespacedName := managedRedisSpecNamespacedName(spec) - onDeleteRule := opstree.ToRedisOnDeleteRule(wandb, wandb.GetRetentionPolicy(spec.ManagedInfraSpec)) + if managed := spec.ManagedRedis; managed != nil { + specNamespacedName := managedRedisSpecNamespacedName(managed) + onDeleteRule := opstree.ToRedisOnDeleteRule(wandb, wandb.GetRetentionPolicy(managed.ManagedInfraSpec)) return opstree.PurgeFinalizer(ctx, client, specNamespacedName, onDeleteRule) } - if wandb.Spec.Redis.ExternalRedis != nil { - return externalredis.DeleteConnectionSecret(ctx, client, wandb) + if spec.ExternalRedis != nil { + return externalredis.DeleteConnectionSecret(ctx, client, wandb, key) } return nil } @@ -84,12 +125,14 @@ func redisDetachFinalizer( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + _ string, + spec apiv2.RedisSpec, ) error { - spec := wandb.Spec.Redis.ManagedRedis - if spec == nil { + managed := spec.ManagedRedis + if managed == nil { return nil } - specNamespacedName := managedRedisSpecNamespacedName(spec) + specNamespacedName := managedRedisSpecNamespacedName(managed) return opstree.DetachFinalizer(ctx, client, specNamespacedName, wandb) } @@ -99,14 +142,13 @@ func managedRedisWriteState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedRedisSpec, mfst manifest.Manifest, ) []metav1.Condition { - spec := wandb.Spec.Redis.ManagedRedis - log := ctrl.LoggerFrom(ctx) var specNamespacedName = managedRedisSpecNamespacedName(spec) - standaloneDesired, err := opstree.ToRedisStandaloneVendorSpec(ctx, wandb, client.Scheme(), mfst) + standaloneDesired, err := opstree.ToRedisStandaloneVendorSpec(ctx, wandb, spec, client.Scheme(), mfst) if err != nil { log.Error(err, "failed to translate redis standalone spec") return []metav1.Condition{ @@ -118,7 +160,7 @@ func managedRedisWriteState( } } - sentinelDesired, err := opstree.ToRedisSentinelVendorSpec(ctx, wandb, client.Scheme(), mfst) + sentinelDesired, err := opstree.ToRedisSentinelVendorSpec(ctx, wandb, spec, client.Scheme(), mfst) if err != nil { log.Error(err, "failed to translate redis sentinel spec") return []metav1.Condition{ @@ -130,7 +172,7 @@ func managedRedisWriteState( } } - replicationDesired, err := opstree.ToRedisReplicationVendorSpec(ctx, wandb, client.Scheme(), mfst) + replicationDesired, err := opstree.ToRedisReplicationVendorSpec(ctx, wandb, spec, client.Scheme(), mfst) if err != nil { log.Error(err, "failed to translate redis replication spec") return []metav1.Condition{ @@ -154,10 +196,9 @@ func managedRedisReadState( ctx context.Context, client client.Client, wandb *apiv2.WeightsAndBiases, + spec *apiv2.ManagedRedisSpec, newConditions []metav1.Condition, ) ([]metav1.Condition, *apiv2.RedisConnection) { - spec := wandb.Spec.Redis.ManagedRedis - specNamespacedName := managedRedisSpecNamespacedName(spec) onDeleteRule := opstree.ToRedisOnDeleteRule(wandb, wandb.GetRetentionPolicy(spec.ManagedInfraSpec)) readConditions, newInfraConn := opstree.ReadState(ctx, client, specNamespacedName, wandb, onDeleteRule) @@ -170,12 +211,15 @@ func managedRedisInferStatus( client client.Client, recorder record.EventRecorder, wandb *apiv2.WeightsAndBiases, + key string, newConditions []metav1.Condition, newInfraConn *apiv2.RedisConnection, ) (ctrl.Result, error) { + statusBefore := wandb.DeepCopy().Status enabled := true - oldConditions := wandb.Status.RedisStatus.Conditions - oldInfraConn := wandb.Status.RedisStatus.Connection + oldStatus := wandb.Status.RedisStatus[key] + oldConditions := oldStatus.Conditions + oldInfraConn := oldStatus.Connection updatedStatus, events, ctrlResult := opstree.ComputeStatus( ctx, @@ -188,32 +232,26 @@ func managedRedisInferStatus( for _, e := range events { recorder.Event(wandb, e.Type, e.Reason, e.Message) } - wandb.Status.RedisStatus = updatedStatus - err := client.Status().Update(ctx, wandb) + wandb.Status.RedisStatus[key] = updatedStatus + err := updateWandbStatusIfChanged(ctx, client, wandb, statusBefore) return ctrlResult, err } // external -func externalRedisWriteState(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases) []metav1.Condition { - return externalredis.WriteState(ctx, c, wandb) -} - -func externalRedisReadState(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, newConditions []metav1.Condition) ([]metav1.Condition, *apiv2.RedisConnection) { - return externalredis.ReadState(ctx, c, wandb, newConditions) -} - -func externalRedisInferStatus(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, newConditions []metav1.Condition, newInfraConn *apiv2.RedisConnection) (ctrl.Result, error) { - oldInfraConn := wandb.Status.RedisStatus.Connection - state, ready, updatedConditions := external.InferExternalStatus(wandb.Status.RedisStatus.Conditions, newConditions, wandb.Generation, newInfraConn != nil) +func externalRedisInferStatus(ctx context.Context, c client.Client, wandb *apiv2.WeightsAndBiases, key string, newConditions []metav1.Condition, newInfraConn *apiv2.RedisConnection) (ctrl.Result, error) { + statusBefore := wandb.DeepCopy().Status + oldStatus := wandb.Status.RedisStatus[key] + oldInfraConn := oldStatus.Connection + state, ready, updatedConditions := external.InferExternalStatus(oldStatus.Conditions, newConditions, wandb.Generation, newInfraConn != nil) conn := utils.Coalesce(newInfraConn, &oldInfraConn) - wandb.Status.RedisStatus = apiv2.RedisInfraStatus{ + wandb.Status.RedisStatus[key] = apiv2.RedisInfraStatus{ WBInfraStatus: apiv2.WBInfraStatus{Ready: ready, State: state, Conditions: updatedConditions}, Connection: *conn, } - return ctrl.Result{}, c.Status().Update(ctx, wandb) + return ctrl.Result{}, updateWandbStatusIfChanged(ctx, c, wandb, statusBefore) } // helpers diff --git a/internal/controller/reconciler/sizing.go b/internal/controller/reconciler/sizing.go index 1ae2d71c..b1039305 100644 --- a/internal/controller/reconciler/sizing.go +++ b/internal/controller/reconciler/sizing.go @@ -23,6 +23,11 @@ func ResolveResources(app manifest.Application, wandb *v2.WeightsAndBiases, cont // check if the container has a resource and if so apply those settings resources = mergeResources(resources, containerResources, wandb.Spec.RequireLimits) + // Legacy override wins over sizing/container resources; limits stay gated by requireLimits. + if lo, ok := wandb.Spec.Wandb.LegacyOverrides[app.Name]; ok && lo.Resources != nil { + resources = mergeResources(resources, lo.Resources, wandb.Spec.RequireLimits) + } + if resources == nil { return nil } @@ -128,7 +133,9 @@ func ResolveInfraSizing(sizing map[v2.Size]manifest.SizingConfig, size v2.Size, if defaultSizing, ok := sizing["default"]; ok { result.Replicas = defaultSizing.Replicas result.Shards = defaultSizing.Shards + result.Copies = defaultSizing.Copies result.VolumeSize = defaultSizing.VolumeSize + result.MetadataVolumeSize = defaultSizing.MetadataVolumeSize if defaultSizing.Resources != nil { result.Resources = defaultSizing.Resources.DeepCopy() } @@ -142,9 +149,15 @@ func ResolveInfraSizing(sizing map[v2.Size]manifest.SizingConfig, size v2.Size, if sizeSizing.Shards != 0 { result.Shards = sizeSizing.Shards } + if sizeSizing.Copies != 0 { + result.Copies = sizeSizing.Copies + } if sizeSizing.VolumeSize != "" { result.VolumeSize = sizeSizing.VolumeSize } + if sizeSizing.MetadataVolumeSize != "" { + result.MetadataVolumeSize = sizeSizing.MetadataVolumeSize + } result.Resources = mergeResources(result.Resources, sizeSizing.Resources, requireLimits) } @@ -199,71 +212,126 @@ func ResolveKafkaSizing(sizing map[v2.Size]manifest.KafkaSizingConfig, size v2.S // ApplyInfraSizing applies manifest-derived sizing to the wandb spec's infra // components. Values from the manifest are only applied when the corresponding // spec field has not been explicitly set by the user (i.e., is zero-valued). +// infraSizingConfig returns the manifest sizing config for an instance key, +// falling back to the manifest "default" config when the key has no entry. +func infraSizingConfig[T any](m map[string]T, key string) (T, bool) { + if cfg, ok := m[key]; ok { + return cfg, true + } + cfg, ok := m[v2.DefaultInstanceName] + return cfg, ok +} + func ApplyInfraSizing(wandb *v2.WeightsAndBiases, manifest manifest.Manifest) { size := wandb.Spec.Size - // Default MySQL - if wandb.Spec.MySQL.ManagedMysql != nil { - if mysqlConfig, ok := manifest.Mysql["default"]; ok { - sizing := ResolveInfraSizing(mysqlConfig.Sizing, size, wandb.Spec.RequireLimits) - spec := wandb.Spec.MySQL.ManagedMysql - if spec.Replicas == 0 && sizing.Replicas != 0 { - spec.Replicas = sizing.Replicas - } - if spec.StorageSize == "" && sizing.VolumeSize != "" { - spec.StorageSize = sizing.VolumeSize - } - if sizing.Resources != nil && len(spec.Config.Resources.Requests) == 0 && len(spec.Config.Resources.Limits) == 0 { - spec.Config.Resources = *sizing.Resources - } + // MySQL: size each managed instance, preferring a manifest sizing config + // matching the instance key and falling back to the manifest "default". + for key, instance := range wandb.Spec.MySQL { + spec := instance.ManagedMysql + if spec == nil { + continue + } + mysqlConfig, ok := infraSizingConfig(manifest.Mysql, key) + if !ok { + continue + } + sizing := ResolveInfraSizing(mysqlConfig.Sizing, size, wandb.Spec.RequireLimits) + if spec.Replicas == 0 && sizing.Replicas != 0 { + spec.Replicas = sizing.Replicas + } + if spec.StorageSize == "" && sizing.VolumeSize != "" { + spec.StorageSize = sizing.VolumeSize + } + if sizing.Resources != nil && len(spec.Config.Resources.Requests) == 0 && len(spec.Config.Resources.Limits) == 0 { + spec.Config.Resources = *sizing.Resources } } - // Default Redis - if wandb.Spec.Redis.ManagedRedis != nil { - if redisConfig, ok := manifest.Redis["default"]; ok { - sizing := ResolveInfraSizing(redisConfig.Sizing, size, wandb.Spec.RequireLimits) - spec := wandb.Spec.Redis.ManagedRedis - if spec.StorageSize == "" && sizing.VolumeSize != "" { - spec.StorageSize = sizing.VolumeSize - } - if sizing.Resources != nil && len(spec.Config.Resources.Requests) == 0 && len(spec.Config.Resources.Limits) == 0 { - spec.Config.Resources = *sizing.Resources - } + // Redis + for key, instance := range wandb.Spec.Redis { + spec := instance.ManagedRedis + if spec == nil { + continue + } + redisConfig, ok := infraSizingConfig(manifest.Redis, key) + if !ok { + continue + } + sizing := ResolveInfraSizing(redisConfig.Sizing, size, wandb.Spec.RequireLimits) + if spec.StorageSize == "" && sizing.VolumeSize != "" { + spec.StorageSize = sizing.VolumeSize + } + if sizing.Resources != nil && len(spec.Config.Resources.Requests) == 0 && len(spec.Config.Resources.Limits) == 0 { + spec.Config.Resources = *sizing.Resources } } - // Default ClickHouse - if wandb.Spec.ClickHouse.ManagedClickHouse != nil { - if clickhouseConfig, ok := manifest.Clickhouse["default"]; ok { - sizing := ResolveInfraSizing(clickhouseConfig.Sizing, size, wandb.Spec.RequireLimits) - spec := wandb.Spec.ClickHouse.ManagedClickHouse - if spec.Replicas == 0 && sizing.Replicas != 0 { - spec.Replicas = sizing.Replicas + // ClickHouse + for key, instance := range wandb.Spec.ClickHouse { + spec := instance.ManagedClickHouse + if spec == nil { + continue + } + + // Keeper sizing comes from the manifest's clickhouseKeeper block + // (independent of the clickhouse block); CR values are treated as user + // overrides. + if keeperConfig, ok := infraSizingConfig(manifest.ClickhouseKeeper, key); ok { + keeperSizing := ResolveInfraSizing(keeperConfig.Sizing, size, wandb.Spec.RequireLimits) + if spec.Keeper.Replicas == 0 && keeperSizing.Replicas != 0 { + spec.Keeper.Replicas = keeperSizing.Replicas } - if spec.StorageSize == "" && sizing.VolumeSize != "" { - spec.StorageSize = sizing.VolumeSize + if spec.Keeper.StorageSize == "" && keeperSizing.VolumeSize != "" { + spec.Keeper.StorageSize = keeperSizing.VolumeSize } - if sizing.Resources != nil && len(spec.Config.Resources.Requests) == 0 && len(spec.Config.Resources.Limits) == 0 { - spec.Config.Resources = *sizing.Resources + if keeperSizing.Resources != nil && len(spec.Keeper.Config.Resources.Requests) == 0 && len(spec.Keeper.Config.Resources.Limits) == 0 { + spec.Keeper.Config.Resources = *keeperSizing.Resources } } + + clickhouseConfig, ok := infraSizingConfig(manifest.Clickhouse, key) + if !ok { + continue + } + sizing := ResolveInfraSizing(clickhouseConfig.Sizing, size, wandb.Spec.RequireLimits) + if spec.Replicas == 0 && sizing.Replicas != 0 { + spec.Replicas = sizing.Replicas + } + if spec.StorageSize == "" && sizing.VolumeSize != "" { + spec.StorageSize = sizing.VolumeSize + } + if sizing.Resources != nil && len(spec.Config.Resources.Requests) == 0 && len(spec.Config.Resources.Limits) == 0 { + spec.Config.Resources = *sizing.Resources + } } - // Default ObjectStore (bucket) - if wandb.Spec.ObjectStore.ManagedObjectStore != nil { - if objectStoreConfig, ok := manifest.Bucket["default"]; ok { - sizing := ResolveInfraSizing(objectStoreConfig.Sizing, size, wandb.Spec.RequireLimits) - spec := wandb.Spec.ObjectStore.ManagedObjectStore - if spec.Replicas == 0 && sizing.Replicas != 0 { - spec.Replicas = sizing.Replicas - } - if spec.StorageSize == "" && sizing.VolumeSize != "" { - spec.StorageSize = sizing.VolumeSize - } - if sizing.Resources != nil && len(spec.Config.Resources.Requests) == 0 && len(spec.Config.Resources.Limits) == 0 { - spec.Config.Resources = *sizing.Resources - } + // ObjectStore (bucket) + for key, instance := range wandb.Spec.ObjectStore { + spec := instance.ManagedObjectStore + if spec == nil { + continue + } + objectStoreConfig, ok := infraSizingConfig(manifest.Bucket, key) + if !ok { + continue + } + sizing := ResolveInfraSizing(objectStoreConfig.Sizing, size, wandb.Spec.RequireLimits) + if spec.Replicas == 0 && sizing.Replicas != 0 { + spec.Replicas = sizing.Replicas + } + if spec.Copies == 0 && sizing.Copies != 0 { + spec.Copies = sizing.Copies + } + if spec.StorageSize == "" && sizing.VolumeSize != "" { + spec.StorageSize = sizing.VolumeSize + } + // Neutral manifest value maps to the SeaweedFS-specific filer disk; CR override wins. + if spec.SeaweedObjectStoreSpec.FilerStorageSize == "" && sizing.MetadataVolumeSize != "" { + spec.SeaweedObjectStoreSpec.FilerStorageSize = sizing.MetadataVolumeSize + } + if sizing.Resources != nil && len(spec.Config.Resources.Requests) == 0 && len(spec.Config.Resources.Limits) == 0 { + spec.Config.Resources = *sizing.Resources } } diff --git a/internal/controller/reconciler/status_update.go b/internal/controller/reconciler/status_update.go new file mode 100644 index 00000000..8e7fb6d5 --- /dev/null +++ b/internal/controller/reconciler/status_update.go @@ -0,0 +1,21 @@ +package reconciler + +import ( + "context" + + apiv2 "github.com/wandb/operator/api/v2" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func updateWandbStatusIfChanged( + ctx context.Context, + c client.Client, + wandb *apiv2.WeightsAndBiases, + statusBefore apiv2.WeightsAndBiasesStatus, +) error { + if apiequality.Semantic.DeepEqual(statusBefore, wandb.Status) { + return nil + } + return c.Status().Update(ctx, wandb) +} diff --git a/internal/controller/reconciler/status_update_test.go b/internal/controller/reconciler/status_update_test.go new file mode 100644 index 00000000..62d58d56 --- /dev/null +++ b/internal/controller/reconciler/status_update_test.go @@ -0,0 +1,71 @@ +package reconciler + +import ( + "context" + "testing" + + apiv2 "github.com/wandb/operator/api/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestUpdateWandbStatusIfChangedSkipsEqualStatus(t *testing.T) { + t.Parallel() + + wandb := &apiv2.WeightsAndBiases{Status: apiv2.WeightsAndBiasesStatus{Ready: true}} + if err := updateWandbStatusIfChanged(context.Background(), nil, wandb, wandb.DeepCopy().Status); err != nil { + t.Fatalf("unchanged status returned an error: %v", err) + } +} + +func TestUpdateWandbStatusIfChangedPersistsChange(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := apiv2.AddToScheme(scheme); err != nil { + t.Fatalf("add API to scheme: %v", err) + } + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "default"}, + } + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&apiv2.WeightsAndBiases{}). + WithObjects(wandb). + Build() + + statusBefore := wandb.DeepCopy().Status + wandb.Status.Ready = true + if err := updateWandbStatusIfChanged(context.Background(), c, wandb, statusBefore); err != nil { + t.Fatalf("update status: %v", err) + } + + actual := &apiv2.WeightsAndBiases{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(wandb), actual); err != nil { + t.Fatalf("get updated resource: %v", err) + } + if !actual.Status.Ready { + t.Fatal("status change was not persisted") + } +} + +func TestApplicationManagedFieldsEqual(t *testing.T) { + t.Parallel() + + before := &apiv2.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "default"}, + Spec: apiv2.ApplicationSpec{Kind: "Deployment"}, + } + after := before.DeepCopy() + after.Status.Ready = true + if !applicationManagedFieldsEqual(before, after) { + t.Fatal("status-only changes should not rewrite the Application") + } + + after.Spec.Kind = "StatefulSet" + if applicationManagedFieldsEqual(before, after) { + t.Fatal("spec changes must update the Application") + } +} diff --git a/internal/controller/reconciler/telemetry_test.go b/internal/controller/reconciler/telemetry_test.go index f10a9968..6ffb602b 100644 --- a/internal/controller/reconciler/telemetry_test.go +++ b/internal/controller/reconciler/telemetry_test.go @@ -953,7 +953,7 @@ func TestResolveEnvvarsServiceSourceFromManifest(t *testing.T) { } sweepProvider := mustFindEnvVar(t, resolved, "GORILLA_SWEEP_PROVIDER") - if sweepProvider.Value != "http://anaconda2:8080" { + if sweepProvider.Value != "http://anaconda2.default.svc.cluster.local:8080" { t.Fatalf("unexpected sweep provider value: %s", sweepProvider.Value) } } @@ -994,7 +994,7 @@ func TestResolveEnvvarsServiceSourcePortNameFromManifest(t *testing.T) { } historyStore := mustFindEnvVar(t, resolved, "GORILLA_HISTORY_STORE") - if historyStore.Value != "http://parquet:9000/_goRPC_" { + if historyStore.Value != "http://parquet.default.svc.cluster.local:9000/_goRPC_" { t.Fatalf("unexpected history store value: %s", historyStore.Value) } } diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 185d77bc..977aed74 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -29,6 +29,7 @@ import ( apiv1 "github.com/wandb/operator/api/v1" apiv2 "github.com/wandb/operator/api/v2" webhookv2 "github.com/wandb/operator/internal/webhook/v2" + clickhousekeeperv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1" clickhousev1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" argov1alpha1 "github.com/wandb/operator/pkg/vendored/argo-rollouts/argoproj.io.rollouts/v1alpha1" redisv1beta2 "github.com/wandb/operator/pkg/vendored/redis-operator/redis/v1beta2" @@ -81,6 +82,9 @@ var _ = BeforeSuite(func() { err = clickhousev1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = clickhousekeeperv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = seaweedv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) diff --git a/internal/controller/weightsandbiases_controller.go b/internal/controller/weightsandbiases_controller.go index 1162af05..965264ee 100644 --- a/internal/controller/weightsandbiases_controller.go +++ b/internal/controller/weightsandbiases_controller.go @@ -66,6 +66,8 @@ type WeightsAndBiasesReconciler struct { //+kubebuilder:rbac:groups=batch,resources=cronjobs;jobs,verbs=get;list;watch;create;delete;update;patch //+kubebuilder:rbac:groups=clickhouse.altinity.com,resources=clickhouseinstallations,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=clickhouse.altinity.com,resources=clickhouseinstallations/status,verbs=get +//+kubebuilder:rbac:groups=clickhouse-keeper.altinity.com,resources=clickhousekeeperinstallations,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=clickhouse-keeper.altinity.com,resources=clickhousekeeperinstallations/status,verbs=get //+kubebuilder:rbac:groups=cloud.google.com,resources=backendconfigs,verbs=update;delete;get;list;patch;create;watch //+kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=list;watch //+kubebuilder:rbac:groups=grafana.integreatly.org,resources=grafanas;grafanadashboards;grafanadatasources,verbs=get;list;watch @@ -86,6 +88,7 @@ type WeightsAndBiasesReconciler struct { //+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings;clusterroles;clusterrolebindings,verbs=update;delete;get;list;patch;create;watch //+kubebuilder:rbac:groups=redis.redis.opstreelabs.in,resources=redis;redissentinels;redisreplications,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=redis.redis.opstreelabs.in,resources=redis/status,verbs=get +//+kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,resourceNames=nonroot-v2,verbs=use //+kubebuilder:rbac:urls=/metrics,verbs=get // Deprecated/Erroneously required RBAC rules @@ -132,7 +135,10 @@ func (r *WeightsAndBiasesReconciler) Delete(e event.DeleteEvent) bool { func (r *WeightsAndBiasesReconciler) SetupWithManager(mgr ctrl.Manager) error { var b = ctrl.NewControllerManagedBy(mgr). For(&apiv2.WeightsAndBiases{}). - Owns(&apiv2.Application{}). + // Applications carry plain (non-controller) owner refs; without + // MatchEveryOwner this watch never fires and app status changes stop + // refreshing status.wandb.applications once the estate settles. + Owns(&apiv2.Application{}, builder.MatchEveryOwner). Owns(&batchv1.Job{}). Owns(&corev1.Secret{}). Owns(&corev1.ConfigMap{}). diff --git a/internal/controller/weightsandbiases_controller_networking_test.go b/internal/controller/weightsandbiases_controller_networking_test.go index 15a99286..e8da2ee3 100644 --- a/internal/controller/weightsandbiases_controller_networking_test.go +++ b/internal/controller/weightsandbiases_controller_networking_test.go @@ -172,7 +172,7 @@ var _ = Describe("WeightsAndBiases Networking", func() { wandbName := "network-ingress" ingressClassName := "nginx" - wandb, _ := newNetworkingWandb(wandbName, "") + wandb, service := newNetworkingWandb(wandbName, "") wandb.Spec.Networking = apiv2.NetworkingSpec{ Mode: apiv2.NetworkingModeIngress, Ingress: &apiv2.IngressConfig{ @@ -186,6 +186,7 @@ var _ = Describe("WeightsAndBiases Networking", func() { }, } Expect(k8sClient.Create(ctx, wandb)).To(Succeed()) + Expect(k8sClient.Create(ctx, service)).To(Succeed()) DeferCleanup(deleteIfPresent, ctx, wandb) wandb = markWandbReadyForNetworking(ctx, wandbName, wandbNamespace) @@ -236,7 +237,7 @@ func newNetworkingWandb(name string, infraNamespace string) (*apiv2.WeightsAndBi Hostname: "http://localhost", Features: map[string]bool{}, ManifestRepository: manifestsRepository, - Version: "0.78.0", + Version: "0.83.0-clickhouse-keeper.2", InternalServiceAuth: apiv2.InternalServiceAuth{ Enabled: &internalServiceAuthEnabled, }, @@ -244,22 +245,30 @@ func newNetworkingWandb(name string, infraNamespace string) (*apiv2.WeightsAndBi RetentionPolicy: apiv2.RetentionPolicy{ OnDelete: apiv2.DetachOnDelete, }, - MySQL: apiv2.MySQLSpec{ - ManagedMysql: &apiv2.ManagedMysqlSpec{}, + MySQL: map[string]apiv2.MySQLSpec{ + apiv2.DefaultInstanceName: { + ManagedMysql: &apiv2.ManagedMysqlSpec{}, + }, }, - Redis: apiv2.RedisSpec{ - ManagedRedis: &apiv2.ManagedRedisSpec{}, + Redis: map[string]apiv2.RedisSpec{ + apiv2.DefaultInstanceName: { + ManagedRedis: &apiv2.ManagedRedisSpec{}, + }, }, Kafka: apiv2.KafkaSpec{ ManagedKafka: &apiv2.ManagedKafkaSpec{}, }, - ObjectStore: apiv2.ObjectStoreSpec{ - ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{ - Namespace: infraNamespace, + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: { + ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{ + Namespace: infraNamespace, + }, }, }, - ClickHouse: apiv2.ClickHouseSpec{ - ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}, + ClickHouse: map[string]apiv2.ClickHouseSpec{ + apiv2.DefaultInstanceName: { + ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}, + }, }, }, } @@ -282,24 +291,46 @@ func newNetworkingWandb(name string, infraNamespace string) (*apiv2.WeightsAndBi func markWandbReadyForNetworking(ctx context.Context, name, namespace string) *apiv2.WeightsAndBiases { wandb := getWandb(ctx, name, namespace) - wandb.Status.MySQLStatus.Ready = true - wandb.Status.RedisStatus.Ready = true + wandb.Status.MySQLStatus = map[string]apiv2.MysqlInfraStatus{ + apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + Connection: apiv2.MysqlConnection{ + URL: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: name}, + Key: "mysql-url", + }, + }, + }, + } + wandb.Status.RedisStatus = map[string]apiv2.RedisInfraStatus{ + apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + }, + } wandb.Status.KafkaStatus.Ready = true - wandb.Status.ObjectStoreStatus.Ready = true - wandb.Status.ClickHouseStatus.Ready = true - wandb.Status.MySQLStatus.Connection.URL = corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: name}, - Key: "mysql-url", + wandb.Status.ObjectStoreStatus = map[string]apiv2.ObjectStoreInfraStatus{ + apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + }, } - wandb.Status.ClickHouseStatus.Connection.URL = corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: name}, - Key: "clickhouse-url", + wandb.Status.ClickHouseStatus = map[string]apiv2.ClickHouseInfraStatus{ + apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + Connection: apiv2.ClickHouseConnection{ + URL: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: name}, + Key: "clickhouse-url", + }, + }, + }, } wandb.Status.Wandb.Migration.Version = wandb.Spec.Wandb.Version wandb.Status.Wandb.Migration.LastSuccessVersion = wandb.Spec.Wandb.Version wandb.Status.Wandb.Migration.Ready = true wandb.Status.Wandb.Migration.Reason = "Complete" - wandb.Status.Wandb.MySQLInit.Succeeded = true + wandb.Status.Wandb.MySQLInit = map[string]apiv2.MigrationJobStatus{ + apiv2.DefaultInstanceName: {Succeeded: true}, + } Expect(k8sClient.Status().Update(ctx, wandb)).To(Succeed()) return getWandb(ctx, name, namespace) diff --git a/internal/controller/weightsandbiases_controller_test.go b/internal/controller/weightsandbiases_controller_test.go index 691832ee..93c2a457 100644 --- a/internal/controller/weightsandbiases_controller_test.go +++ b/internal/controller/weightsandbiases_controller_test.go @@ -12,6 +12,7 @@ import ( v2 "github.com/wandb/operator/internal/controller/reconciler" "github.com/wandb/operator/pkg/utils" "github.com/wandb/operator/pkg/wandb/manifest" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -92,16 +93,20 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { "proxy": true, }, ManifestRepository: manifestsRepository, - Version: "0.78.0", + Version: "0.83.0-clickhouse-keeper.2", }, - MySQL: apiv2.MySQLSpec{ - ManagedMysql: &apiv2.ManagedMysqlSpec{ - StorageSize: "10Gi", + MySQL: map[string]apiv2.MySQLSpec{ + apiv2.DefaultInstanceName: { + ManagedMysql: &apiv2.ManagedMysqlSpec{ + StorageSize: "10Gi", + }, }, }, - Redis: apiv2.RedisSpec{ - ManagedRedis: &apiv2.ManagedRedisSpec{ - StorageSize: "10Gi", + Redis: map[string]apiv2.RedisSpec{ + apiv2.DefaultInstanceName: { + ManagedRedis: &apiv2.ManagedRedisSpec{ + StorageSize: "10Gi", + }, }, }, Kafka: apiv2.KafkaSpec{ @@ -109,13 +114,17 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { StorageSize: "10Gi", }, }, - ObjectStore: apiv2.ObjectStoreSpec{ - ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{ - StorageSize: "10Gi", + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: { + ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{ + StorageSize: "10Gi", + }, }, }, - ClickHouse: apiv2.ClickHouseSpec{ - ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}, + ClickHouse: map[string]apiv2.ClickHouseSpec{ + apiv2.DefaultInstanceName: { + ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}, + }, }, }, Status: apiv2.WeightsAndBiasesStatus{}, @@ -160,25 +169,25 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { Hostname: "http://localhost", Features: map[string]bool{}, ManifestRepository: manifestsRepository, - Version: "0.78.0", + Version: "0.83.0-clickhouse-keeper.2", }, - MySQL: apiv2.MySQLSpec{ - ManagedMysql: &apiv2.ManagedMysqlSpec{}, + MySQL: map[string]apiv2.MySQLSpec{ + apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{}}, }, - Redis: apiv2.RedisSpec{ManagedRedis: &apiv2.ManagedRedisSpec{}}, + Redis: map[string]apiv2.RedisSpec{apiv2.DefaultInstanceName: {ManagedRedis: &apiv2.ManagedRedisSpec{}}}, Kafka: apiv2.KafkaSpec{ManagedKafka: &apiv2.ManagedKafkaSpec{}}, - ObjectStore: apiv2.ObjectStoreSpec{ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}, - ClickHouse: apiv2.ClickHouseSpec{ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{apiv2.DefaultInstanceName: {ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}}, + ClickHouse: map[string]apiv2.ClickHouseSpec{apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}}, }, } Expect(k8sClient.Create(ctx, wandb)).Should(Succeed()) By("Setting infra to ready") - wandb.Status.MySQLStatus.Ready = true - wandb.Status.RedisStatus.Ready = true + wandb.Status.MySQLStatus = map[string]apiv2.MysqlInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.RedisStatus = map[string]apiv2.RedisInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} wandb.Status.KafkaStatus.Ready = true - wandb.Status.ObjectStoreStatus.Ready = true - wandb.Status.ClickHouseStatus.Ready = true + wandb.Status.ObjectStoreStatus = map[string]apiv2.ObjectStoreInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.ClickHouseStatus = map[string]apiv2.ClickHouseInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} Expect(k8sClient.Status().Update(ctx, wandb)).Should(Succeed()) By("Creating the db-password secret") @@ -213,13 +222,17 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { By("Setting infrastructure status to ready") Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) - wandb.Status.MySQLStatus.Ready = true - wandb.Status.RedisStatus.Ready = true + wandb.Status.MySQLStatus = map[string]apiv2.MysqlInfraStatus{apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + Connection: apiv2.MysqlConnection{URL: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"}}, + }} + wandb.Status.RedisStatus = map[string]apiv2.RedisInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} wandb.Status.KafkaStatus.Ready = true - wandb.Status.ObjectStoreStatus.Ready = true - wandb.Status.ClickHouseStatus.Ready = true - wandb.Status.MySQLStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} - wandb.Status.ClickHouseStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} + wandb.Status.ObjectStoreStatus = map[string]apiv2.ObjectStoreInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.ClickHouseStatus = map[string]apiv2.ClickHouseInfraStatus{apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + Connection: apiv2.ClickHouseConnection{URL: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"}}, + }} Expect(k8sClient.Status().Update(ctx, wandb)).Should(Succeed()) @@ -230,9 +243,11 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { Expect(err).Should(Succeed()) By("Checking if the MySQL init job was created") + // The init job is now per managed MySQL instance, named after the + // instance's resource name (default instance: "-mysql"). job := &batchv1.Job{} Eventually(func() error { - return k8sClient.Get(ctx, types.NamespacedName{Name: wandbName + "-moco-init", Namespace: WandbNamespace}, job) + return k8sClient.Get(ctx, types.NamespacedName{Name: wandbName + "-mysql-moco-init", Namespace: WandbNamespace}, job) }, timeout, interval).Should(Succeed()) Expect(job.Spec.Template.Spec.Containers[0].Name).To(Equal("moco-init")) @@ -252,22 +267,22 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { Hostname: "http://localhost", Features: map[string]bool{}, ManifestRepository: manifestsRepository, - Version: "0.78.0", + Version: "0.83.0-clickhouse-keeper.2", }, - MySQL: apiv2.MySQLSpec{ - ManagedMysql: &apiv2.ManagedMysqlSpec{}, + MySQL: map[string]apiv2.MySQLSpec{ + apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{}}, }, - Redis: apiv2.RedisSpec{ - ManagedRedis: &apiv2.ManagedRedisSpec{}, + Redis: map[string]apiv2.RedisSpec{ + apiv2.DefaultInstanceName: {ManagedRedis: &apiv2.ManagedRedisSpec{}}, }, Kafka: apiv2.KafkaSpec{ ManagedKafka: &apiv2.ManagedKafkaSpec{}, }, - ObjectStore: apiv2.ObjectStoreSpec{ - ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: {ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}, }, - ClickHouse: apiv2.ClickHouseSpec{ - ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}, + ClickHouse: map[string]apiv2.ClickHouseSpec{ + apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}, }, }, } @@ -292,13 +307,17 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { By("Setting infrastructure status to ready") Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) - wandb.Status.MySQLStatus.Ready = true - wandb.Status.RedisStatus.Ready = true + wandb.Status.MySQLStatus = map[string]apiv2.MysqlInfraStatus{apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + Connection: apiv2.MysqlConnection{URL: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"}}, + }} + wandb.Status.RedisStatus = map[string]apiv2.RedisInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} wandb.Status.KafkaStatus.Ready = true - wandb.Status.ObjectStoreStatus.Ready = true - wandb.Status.ClickHouseStatus.Ready = true - wandb.Status.MySQLStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} - wandb.Status.ClickHouseStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} + wandb.Status.ObjectStoreStatus = map[string]apiv2.ObjectStoreInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.ClickHouseStatus = map[string]apiv2.ClickHouseInfraStatus{apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + Connection: apiv2.ClickHouseConnection{URL: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"}}, + }} Expect(k8sClient.Status().Update(ctx, wandb)).Should(Succeed()) @@ -319,7 +338,7 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { wandb.Status.Wandb.Migration.LastSuccessVersion = wandb.Spec.Wandb.Version wandb.Status.Wandb.Migration.Ready = true wandb.Status.Wandb.Migration.Reason = "Complete" - wandb.Status.Wandb.MySQLInit.Succeeded = true + wandb.Status.Wandb.MySQLInit = map[string]apiv2.MigrationJobStatus{apiv2.DefaultInstanceName: {Succeeded: true}} Expect(k8sClient.Status().Update(ctx, wandb)).Should(Succeed()) // For now test by calling ReconcileWandbManifest directly, but this will get refactored into the reconciler later @@ -339,7 +358,234 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { // The 0.76.1.yaml manifest should have some applications defined. // We expect them to be created as Application CRs. - Expect(len(appList.Items)).Should(BeNumerically("==", len(wandbManifest.Applications)-2), "Expected all non-feature flagged applications to be created") + Expect(len(appList.Items)).Should(BeNumerically("==", len(wandbManifest.Applications)-1), "Expected all non-feature flagged applications to be created") + }) + + It("Should advance status.observedGeneration only once applications are reconciled for a generation", func() { + By("Creating a new WeightsAndBiases v2 object at the initial version") + ctx := context.Background() + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{ + Name: WandbName, + Namespace: WandbNamespace, + }, + Spec: apiv2.WeightsAndBiasesSpec{ + Size: apiv2.SizeDev, + Wandb: apiv2.WandbAppSpec{ + Hostname: "http://localhost", + Features: map[string]bool{}, + ManifestRepository: manifestsRepository, + Version: "0.83.0-clickhouse-keeper.1", + }, + MySQL: map[string]apiv2.MySQLSpec{apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{}}}, + Redis: map[string]apiv2.RedisSpec{apiv2.DefaultInstanceName: {ManagedRedis: &apiv2.ManagedRedisSpec{}}}, + Kafka: apiv2.KafkaSpec{ManagedKafka: &apiv2.ManagedKafkaSpec{}}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{apiv2.DefaultInstanceName: {ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}}, + ClickHouse: map[string]apiv2.ClickHouseSpec{apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}}, + }, + } + Expect(k8sClient.Create(ctx, wandb)).Should(Succeed()) + + wandbLookupKey := types.NamespacedName{Name: wandb.Name, Namespace: wandb.Namespace} + Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) + Expect(wandb.Status.ObservedGeneration).Should(BeZero()) + + By("Marking infrastructure, mysql init, and migrations ready") + wandb.Status.MySQLStatus = map[string]apiv2.MysqlInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.RedisStatus = map[string]apiv2.RedisInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.KafkaStatus.Ready = true + wandb.Status.ObjectStoreStatus = map[string]apiv2.ObjectStoreInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.ClickHouseStatus = map[string]apiv2.ClickHouseInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + mysqlStatus := wandb.Status.MySQLStatus[apiv2.DefaultInstanceName] + mysqlStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} + wandb.Status.MySQLStatus[apiv2.DefaultInstanceName] = mysqlStatus + clickHouseStatus := wandb.Status.ClickHouseStatus[apiv2.DefaultInstanceName] + clickHouseStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} + wandb.Status.ClickHouseStatus[apiv2.DefaultInstanceName] = clickHouseStatus + wandb.Status.Wandb.Migration.Version = wandb.Spec.Wandb.Version + wandb.Status.Wandb.Migration.LastSuccessVersion = wandb.Spec.Wandb.Version + wandb.Status.Wandb.Migration.Ready = true + wandb.Status.Wandb.Migration.Reason = "Complete" + wandb.Status.Wandb.MySQLInit = map[string]apiv2.MigrationJobStatus{apiv2.DefaultInstanceName: {Succeeded: true}} + Expect(k8sClient.Status().Update(ctx, wandb)).Should(Succeed()) + + By("Reconciling the manifest to completion for the initial generation") + wandbManifest, err := manifest.GetServerManifest(ctx, wandb.Spec.Wandb.ManifestRepository, wandb.Spec.Wandb.Version) + Expect(err).Should(Succeed()) + ctrlResult, err := v2.ReconcileWandbManifest(ctx, k8sClient, wandb, wandbManifest, v2.DefaultTelemetryRuntimeConfig()) + Expect(err).Should(Succeed()) + Expect(ctrlResult.RequeueAfter).Should(BeZero()) + + Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) + initialGeneration := wandb.Generation + Expect(initialGeneration).Should(BeNumerically(">", 0)) + Expect(wandb.Status.ObservedGeneration).Should(Equal(initialGeneration)) + + By("Upgrading spec.wandb.version to bump the generation") + wandb.Spec.Wandb.Version = "0.83.0-clickhouse-keeper.2" + Expect(k8sClient.Update(ctx, wandb)).Should(Succeed()) + Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) + Expect(wandb.Generation).Should(BeNumerically(">", initialGeneration)) + Expect(wandb.Status.ObservedGeneration).Should(Equal(initialGeneration)) + + By("Reconciling while the new version's migration is still pending") + wandbManifest, err = manifest.GetServerManifest(ctx, wandb.Spec.Wandb.ManifestRepository, wandb.Spec.Wandb.Version) + Expect(err).Should(Succeed()) + ctrlResult, err = v2.ReconcileWandbManifest(ctx, k8sClient, wandb, wandbManifest, v2.DefaultTelemetryRuntimeConfig()) + Expect(err).Should(Succeed()) + Expect(ctrlResult.RequeueAfter).Should(BeNumerically(">", 0)) + + Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) + Expect(wandb.Status.ObservedGeneration).Should(Equal(initialGeneration), + "observedGeneration must not advance before applications carry the new generation's spec") + + By("Completing the migration and reconciling to completion") + wandb.Status.Wandb.Migration.Version = wandb.Spec.Wandb.Version + wandb.Status.Wandb.Migration.LastSuccessVersion = wandb.Spec.Wandb.Version + wandb.Status.Wandb.Migration.Ready = true + wandb.Status.Wandb.Migration.Reason = "Complete" + Expect(k8sClient.Status().Update(ctx, wandb)).Should(Succeed()) + + ctrlResult, err = v2.ReconcileWandbManifest(ctx, k8sClient, wandb, wandbManifest, v2.DefaultTelemetryRuntimeConfig()) + Expect(err).Should(Succeed()) + Expect(ctrlResult.RequeueAfter).Should(BeZero()) + + Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) + Expect(wandb.Status.ObservedGeneration).Should(Equal(wandb.Generation)) + Expect(wandb.Status.ObservedGeneration).Should(BeNumerically(">", initialGeneration)) + }) + + It("Should clean up legacy v1 deployments once live Deployments are ready, even with a stale status map", func() { + By("Creating a new WeightsAndBiases v2 object") + ctx := context.Background() + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{ + Name: WandbName, + Namespace: WandbNamespace, + }, + Spec: apiv2.WeightsAndBiasesSpec{ + Size: apiv2.SizeDev, + Wandb: apiv2.WandbAppSpec{ + Hostname: "http://localhost", + Features: map[string]bool{}, + ManifestRepository: manifestsRepository, + Version: "0.83.0-clickhouse-keeper.1", + }, + MySQL: map[string]apiv2.MySQLSpec{apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{}}}, + Redis: map[string]apiv2.RedisSpec{apiv2.DefaultInstanceName: {ManagedRedis: &apiv2.ManagedRedisSpec{}}}, + Kafka: apiv2.KafkaSpec{ManagedKafka: &apiv2.ManagedKafkaSpec{}}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{apiv2.DefaultInstanceName: {ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}}, + ClickHouse: map[string]apiv2.ClickHouseSpec{apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}}, + }, + } + Expect(k8sClient.Create(ctx, wandb)).Should(Succeed()) + wandbLookupKey := types.NamespacedName{Name: wandb.Name, Namespace: wandb.Namespace} + + By("Creating a legacy v1 helm Deployment left over from the upgrade") + legacy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: WandbName + "-app-bc", Namespace: WandbNamespace}, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "legacy"}}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "legacy"}}, + Spec: v1.PodSpec{Containers: []v1.Container{{Name: "app", Image: "wandb/local:latest"}}}, + }, + }, + } + Expect(k8sClient.Create(ctx, legacy)).Should(Succeed()) + DeferCleanup(func() { + _ = k8sClient.Delete(ctx, legacy) + }) + + By("Marking infrastructure, mysql init, and migrations ready") + Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) + wandb.Status.MySQLStatus = map[string]apiv2.MysqlInfraStatus{apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + Connection: apiv2.MysqlConnection{URL: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"}}, + }} + wandb.Status.RedisStatus = map[string]apiv2.RedisInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.KafkaStatus.Ready = true + wandb.Status.ObjectStoreStatus = map[string]apiv2.ObjectStoreInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.ClickHouseStatus = map[string]apiv2.ClickHouseInfraStatus{apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + Connection: apiv2.ClickHouseConnection{URL: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"}}, + }} + wandb.Status.Wandb.Migration.Version = wandb.Spec.Wandb.Version + wandb.Status.Wandb.Migration.LastSuccessVersion = wandb.Spec.Wandb.Version + wandb.Status.Wandb.Migration.Ready = true + wandb.Status.Wandb.Migration.Reason = "Complete" + wandb.Status.Wandb.MySQLInit = map[string]apiv2.MigrationJobStatus{apiv2.DefaultInstanceName: {Succeeded: true}} + Expect(k8sClient.Status().Update(ctx, wandb)).Should(Succeed()) + + By("Reconciling the manifest to create the Applications") + wandbManifest, err := manifest.GetServerManifest(ctx, wandb.Spec.Wandb.ManifestRepository, wandb.Spec.Wandb.Version) + Expect(err).Should(Succeed()) + _, err = v2.ReconcileWandbManifest(ctx, k8sClient, wandb, wandbManifest, v2.DefaultTelemetryRuntimeConfig()) + Expect(err).Should(Succeed()) + + appList := &apiv2.ApplicationList{} + Expect(k8sClient.List(ctx, appList, client.InNamespace(WandbNamespace))).Should(Succeed()) + Expect(appList.Items).ShouldNot(BeEmpty()) + + By("Verifying Applications carry a WeightsAndBiases owner reference for the MatchEveryOwner watch") + var ownerKinds []string + for _, ref := range appList.Items[0].OwnerReferences { + ownerKinds = append(ownerKinds, ref.Kind) + } + Expect(ownerKinds).To(ContainElement("WeightsAndBiases"), + "the parent's Owns(Application, MatchEveryOwner) watch maps events through this owner ref") + + By("Verifying cleanup is deferred while application Deployments are absent") + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: legacy.Name, Namespace: WandbNamespace}, &appsv1.Deployment{})).Should(Succeed(), + "legacy deployment must survive until the estate is ready") + + By("Simulating the Application controller: rolled-out Deployments while the status map stays stale-false") + for i := range appList.Items { + app := appList.Items[i] + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: app.Name, Namespace: WandbNamespace}, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": app.Name}}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": app.Name}}, + Spec: v1.PodSpec{Containers: []v1.Container{{Name: "app", Image: "wandb/local:latest"}}}, + }, + }, + } + Expect(k8sClient.Create(ctx, dep)).Should(Succeed()) + DeferCleanup(func() { + _ = k8sClient.Delete(ctx, dep) + }) + dep.Status = appsv1.DeploymentStatus{ + ObservedGeneration: dep.Generation, + Replicas: 1, + ReadyReplicas: 1, + } + Expect(k8sClient.Status().Update(ctx, dep)).Should(Succeed()) + } + + By("Reconciling again: the gate must pass on live Deployments even though the status map says not-ready") + Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) + _, err = v2.ReconcileWandbManifest(ctx, k8sClient, wandb, wandbManifest, v2.DefaultTelemetryRuntimeConfig()) + Expect(err).Should(Succeed()) + + err = k8sClient.Get(ctx, types.NamespacedName{Name: legacy.Name, Namespace: WandbNamespace}, &appsv1.Deployment{}) + Expect(errors.IsNotFound(err)).To(BeTrue(), + "legacy -bc deployment must be deleted once live Deployments are ready") + + By("Verifying the status map refreshes from live Application status on the next pass") + refreshed := &apiv2.Application{} + appName := appList.Items[0].Name + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: appName, Namespace: WandbNamespace}, refreshed)).Should(Succeed()) + refreshed.Status.Ready = true + Expect(k8sClient.Status().Update(ctx, refreshed)).Should(Succeed()) + + Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) + _, err = v2.ReconcileWandbManifest(ctx, k8sClient, wandb, wandbManifest, v2.DefaultTelemetryRuntimeConfig()) + Expect(err).Should(Succeed()) + Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) + Expect(wandb.Status.Wandb.Applications[appName].Ready).To(BeTrue(), + "the parent status map must reflect the Application's current status") }) It("Should handle various migration states correctly", func() { @@ -356,13 +602,13 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { Hostname: "http://localhost", Features: map[string]bool{}, ManifestRepository: manifestsRepository, - Version: "0.78.0", + Version: "0.83.0-clickhouse-keeper.2", }, - MySQL: apiv2.MySQLSpec{ManagedMysql: &apiv2.ManagedMysqlSpec{}}, - Redis: apiv2.RedisSpec{ManagedRedis: &apiv2.ManagedRedisSpec{}}, + MySQL: map[string]apiv2.MySQLSpec{apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{}}}, + Redis: map[string]apiv2.RedisSpec{apiv2.DefaultInstanceName: {ManagedRedis: &apiv2.ManagedRedisSpec{}}}, Kafka: apiv2.KafkaSpec{ManagedKafka: &apiv2.ManagedKafkaSpec{}}, - ObjectStore: apiv2.ObjectStoreSpec{ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}, - ClickHouse: apiv2.ClickHouseSpec{ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{apiv2.DefaultInstanceName: {ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}}, + ClickHouse: map[string]apiv2.ClickHouseSpec{apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}}, }, } Expect(k8sClient.Create(ctx, wandb)).Should(Succeed()) @@ -371,11 +617,11 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) // Mark infra as ready - wandb.Status.MySQLStatus.Ready = true - wandb.Status.RedisStatus.Ready = true + wandb.Status.MySQLStatus = map[string]apiv2.MysqlInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.RedisStatus = map[string]apiv2.RedisInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} wandb.Status.KafkaStatus.Ready = true - wandb.Status.ObjectStoreStatus.Ready = true - wandb.Status.ClickHouseStatus.Ready = true + wandb.Status.ObjectStoreStatus = map[string]apiv2.ObjectStoreInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.ClickHouseStatus = map[string]apiv2.ClickHouseInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} Expect(k8sClient.Status().Update(ctx, wandb)).Should(Succeed()) wandbManifest, err := manifest.GetServerManifest(ctx, wandb.Spec.Wandb.ManifestRepository, wandb.Spec.Wandb.Version) @@ -386,8 +632,12 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { wandb.Status.Wandb.Migration.Version = wandb.Spec.Wandb.Version wandb.Status.Wandb.Migration.Ready = false wandb.Status.Wandb.Migration.Reason = "Running" - wandb.Status.MySQLStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} - wandb.Status.ClickHouseStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} + mysqlStatus := wandb.Status.MySQLStatus[apiv2.DefaultInstanceName] + mysqlStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} + wandb.Status.MySQLStatus[apiv2.DefaultInstanceName] = mysqlStatus + clickHouseStatus := wandb.Status.ClickHouseStatus[apiv2.DefaultInstanceName] + clickHouseStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} + wandb.Status.ClickHouseStatus[apiv2.DefaultInstanceName] = clickHouseStatus Expect(k8sClient.Status().Update(ctx, wandb)).Should(Succeed()) ctrlResult, err := v2.ReconcileWandbManifest(ctx, k8sClient, wandb, wandbManifest, v2.DefaultTelemetryRuntimeConfig()) @@ -409,7 +659,7 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { wandb.Status.Wandb.Migration.Ready = true wandb.Status.Wandb.Migration.Reason = "Complete" wandb.Status.Wandb.Migration.LastSuccessVersion = wandb.Spec.Wandb.Version - wandb.Status.Wandb.MySQLInit.Succeeded = true + wandb.Status.Wandb.MySQLInit = map[string]apiv2.MigrationJobStatus{apiv2.DefaultInstanceName: {Succeeded: true}} Expect(k8sClient.Status().Update(ctx, wandb)).Should(Succeed()) ctrlResult, err = v2.ReconcileWandbManifest(ctx, k8sClient, wandb, wandbManifest, v2.DefaultTelemetryRuntimeConfig()) @@ -420,8 +670,8 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { It("Should trigger new migrations on version upgrade", func() { By("Creating a new WeightsAndBiases v2 object with an old version") ctx := context.Background() - oldVersion := "0.78.0-pre" - newVersion := "0.78.0" + oldVersion := "0.83.0-clickhouse-keeper.1" + newVersion := "0.83.0-clickhouse-keeper.2" wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{ Name: WandbName, @@ -435,11 +685,11 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { ManifestRepository: manifestsRepository, Version: oldVersion, }, - MySQL: apiv2.MySQLSpec{ManagedMysql: &apiv2.ManagedMysqlSpec{}}, - Redis: apiv2.RedisSpec{ManagedRedis: &apiv2.ManagedRedisSpec{}}, + MySQL: map[string]apiv2.MySQLSpec{apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{}}}, + Redis: map[string]apiv2.RedisSpec{apiv2.DefaultInstanceName: {ManagedRedis: &apiv2.ManagedRedisSpec{}}}, Kafka: apiv2.KafkaSpec{ManagedKafka: &apiv2.ManagedKafkaSpec{}}, - ObjectStore: apiv2.ObjectStoreSpec{ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}, - ClickHouse: apiv2.ClickHouseSpec{ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{apiv2.DefaultInstanceName: {ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}}, + ClickHouse: map[string]apiv2.ClickHouseSpec{apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}}, }, } Expect(k8sClient.Create(ctx, wandb)).Should(Succeed()) @@ -448,18 +698,22 @@ var _ = Describe("WeightsAndBiases Controller V2", func() { Expect(k8sClient.Get(ctx, wandbLookupKey, wandb)).Should(Succeed()) // Mark infra as ready and migration as complete for old version - wandb.Status.MySQLStatus.Ready = true - wandb.Status.RedisStatus.Ready = true + wandb.Status.MySQLStatus = map[string]apiv2.MysqlInfraStatus{apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + Connection: apiv2.MysqlConnection{URL: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"}}, + }} + wandb.Status.RedisStatus = map[string]apiv2.RedisInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} wandb.Status.KafkaStatus.Ready = true - wandb.Status.ObjectStoreStatus.Ready = true - wandb.Status.ClickHouseStatus.Ready = true - wandb.Status.MySQLStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} - wandb.Status.ClickHouseStatus.Connection.URL = v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"} + wandb.Status.ObjectStoreStatus = map[string]apiv2.ObjectStoreInfraStatus{apiv2.DefaultInstanceName: {WBInfraStatus: apiv2.WBInfraStatus{Ready: true}}} + wandb.Status.ClickHouseStatus = map[string]apiv2.ClickHouseInfraStatus{apiv2.DefaultInstanceName: { + WBInfraStatus: apiv2.WBInfraStatus{Ready: true}, + Connection: apiv2.ClickHouseConnection{URL: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: WandbName}, Key: "test"}}, + }} wandb.Status.Wandb.Migration.Version = oldVersion wandb.Status.Wandb.Migration.LastSuccessVersion = oldVersion wandb.Status.Wandb.Migration.Ready = true wandb.Status.Wandb.Migration.Reason = "Complete" - wandb.Status.Wandb.MySQLInit.Succeeded = true + wandb.Status.Wandb.MySQLInit = map[string]apiv2.MigrationJobStatus{apiv2.DefaultInstanceName: {Succeeded: true}} Expect(k8sClient.Status().Update(ctx, wandb)).Should(Succeed()) By("Upgrading the version in the spec") diff --git a/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml b/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml index b6897d27..4ee11815 100644 --- a/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml +++ b/internal/crdinstaller/crds/operator/apps.wandb.com_weightsandbiases.yaml @@ -51,21 +51,24 @@ spec: - jsonPath: .status.ready name: Ready type: boolean - - jsonPath: .status.mysqlStatus.state + - jsonPath: .status.mysqlStatus.default.state name: MySQL type: string - - jsonPath: .status.redisStatus.state + - jsonPath: .status.redisStatus.default.state name: Redis type: string - jsonPath: .status.kafkaStatus.state name: Kafka type: string - - jsonPath: .status.objectStoreStatus.state + - jsonPath: .status.objectStoreStatus.default.state name: ObjectStore type: string - - jsonPath: .status.clickhouseStatus.state + - jsonPath: .status.clickhouseStatus.default.state name: ClickHouse type: string + - jsonPath: .status.wandb.migration.phase + name: Migration + type: string name: v2 schema: openAPIV3Schema: @@ -518,324 +521,759 @@ spec: type: object type: object clickhouse: - properties: - externalClickhouse: - properties: - database: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - host: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - username: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - managedClickhouse: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: + additionalProperties: + properties: + externalClickhouse: + properties: + database: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + host: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + httpPort: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tcpPort: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + managedClickhouse: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: properties: - preference: - properties: - matchExpressions: - items: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + matchLabels: + additionalProperties: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - operator: + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + config: + properties: + resources: + properties: + claims: + items: + properties: + name: + type: string + request: type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + keeper: + properties: + config: + properties: + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object type: array - x-kubernetes-list-type: atomic - namespaceSelector: + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + replicas: + format: int32 + type: integer + storageSize: + type: string + type: object + name: + type: string + namespace: + type: string + objectStorage: + properties: + insecure: + type: boolean + prefix: + type: string + type: object + replicas: + format: int32 + type: integer + retentionPolicy: + properties: + onDelete: + default: detach + type: string + required: + - onDelete + type: object + serviceAccount: + properties: + annotations: + additionalProperties: + type: string + type: object + create: + type: boolean + serviceAccountName: + type: string + type: object + storageSize: + type: string + telemetry: + properties: + enabled: + default: true + type: boolean + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + version: + type: string + type: object + type: object + type: object + global: + properties: + caCertsConfigMap: + type: string + customCACerts: + items: + type: string + type: array + imageRegistry: + type: string + proxy: + properties: + httpProxy: + properties: + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + httpsProxy: + properties: + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + noProxy: + items: + type: string + type: array + type: object + type: object + kafka: + properties: + managedKafka: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: properties: matchExpressions: items: @@ -855,26 +1293,86 @@ spec: type: object type: array x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic type: object x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string + weight: + format: int32 + type: integer required: - - topologyKey + - preference + - weight type: object type: array x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic type: object - podAntiAffinity: + podAffinity: properties: preferredDuringSchedulingIgnoredDuringExecution: items: @@ -1040,370 +1538,7 @@ spec: type: array x-kubernetes-list-type: atomic type: object - type: object - config: - properties: - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - type: object - name: - type: string - namespace: - type: string - replicas: - format: int32 - type: integer - retentionPolicy: - properties: - onDelete: - default: detach - type: string - required: - - onDelete - type: object - storageSize: - type: string - telemetry: - properties: - enabled: - default: true - type: boolean - required: - - enabled - type: object - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - version: - type: string - type: object - type: object - global: - properties: - imageRegistry: - type: string - type: object - kafka: - properties: - managedKafka: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: + podAntiAffinity: properties: preferredDuringSchedulingIgnoredDuringExecution: items: @@ -1639,6 +1774,17 @@ spec: required: - onDelete type: object + serviceAccount: + properties: + annotations: + additionalProperties: + type: string + type: object + create: + type: boolean + serviceAccountName: + type: string + type: object skipDataRecovery: type: boolean storageSize: @@ -1670,659 +1816,661 @@ spec: type: object type: object mysql: - properties: - externalMysql: - properties: - database: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - host: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslCa: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslCert: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslKey: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - tls: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - username: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - managedMysql: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: + additionalProperties: + properties: + externalMysql: + properties: + database: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + host: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCa: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCert: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tls: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + managedMysql: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: properties: - preference: - properties: - matchExpressions: - items: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + matchLabels: + additionalProperties: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + operator: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - config: - properties: - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + config: + properties: + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - type: object - name: - type: string - namespace: - type: string - replicas: - format: int32 - type: integer - retentionPolicy: - properties: - onDelete: - default: detach - type: string - required: - - onDelete - type: object - storageSize: - type: string - telemetry: - properties: - enabled: - default: true - type: boolean - required: - - enabled - type: object - tolerations: - items: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + name: + type: string + namespace: + type: string + replicas: + format: int32 + type: integer + retentionPolicy: properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: + onDelete: + default: detach type: string + required: + - onDelete type: object - type: array - type: object + storageSize: + type: string + telemetry: + properties: + enabled: + default: true + type: boolean + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object type: object networking: properties: @@ -2417,1929 +2565,1523 @@ spec: type: object type: object objectStore: - properties: - externalObjectStore: - properties: - accessKey: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - bucket: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - endpoint: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - provider: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - region: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secretKey: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - managedObjectStore: - properties: - SeaweedObjectStoreSpec: - properties: - tlsEnabled: - type: boolean - type: object - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + additionalProperties: + properties: + externalObjectStore: + properties: + accessKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + bucket: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpoint: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + forcePathStyle: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + path: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + provider: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + region: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secretKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tlsEnabled: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + managedObjectStore: + properties: + SeaweedObjectStoreSpec: + properties: + filerStorageSize: + type: string + tlsEnabled: + type: boolean + type: object + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic required: - - preference - - weight + - nodeSelectorTerms type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - operator: + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + config: + properties: + accessKey: + type: string + minioBrowserSetting: + type: string + resources: + properties: + claims: + items: + properties: + name: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: + request: type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - config: - properties: - accessKey: - type: string - minioBrowserSetting: - type: string - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - rootUser: - type: string - type: object - name: - type: string - namespace: - type: string - replicas: - format: int32 - type: integer - retentionPolicy: - properties: - onDelete: - default: detach - type: string - required: - - onDelete - type: object - storageSize: - type: string - telemetry: - properties: - enabled: - default: true - type: boolean - required: - - enabled - type: object - tolerations: - items: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + rootUser: + type: string + type: object + copies: + format: int32 + type: integer + name: + type: string + namespace: + type: string + replicas: + format: int32 + type: integer + retentionPolicy: properties: - effect: + onDelete: + default: detach type: string + required: + - onDelete + type: object + storageSize: + type: string + telemetry: + properties: + enabled: + default: true + type: boolean + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + type: object + redis: + additionalProperties: + properties: + externalRedis: + properties: + host: + properties: key: type: string - operator: + name: + default: "" type: string - tolerationSeconds: - format: int64 - type: integer - value: + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" type: string + optional: + type: boolean + required: + - key type: object - type: array - type: object - type: object - redis: - properties: - externalRedis: - properties: - host: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslCa: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - tls: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - managedRedis: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCa: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tls: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + managedRedis: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + operator: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - format: int32 - type: integer + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic required: - - preference - - weight + - nodeSelectorTerms type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: properties: - key: - type: string - operator: - type: string - values: + matchExpressions: items: - type: string + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object type: array x-kubernetes-list-type: atomic - required: - - key - - operator + matchLabels: + additionalProperties: + type: string + type: object type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - items: + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - items: + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + config: + properties: + resources: + properties: + claims: + items: + properties: + name: type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - items: + request: type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - config: - properties: - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - type: object - name: - type: string - namespace: - type: string - retentionPolicy: - properties: - onDelete: - default: detach - type: string - required: - - onDelete - type: object - sentinel: - properties: - config: - properties: - masterName: - type: string - resources: - properties: - claims: - items: - properties: - name: - type: string - request: - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: + required: - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true type: object - type: object - type: object - enabled: - type: boolean - required: - - enabled - type: object - storageSize: - type: string - telemetry: - properties: - enabled: - default: true - type: boolean - required: - - enabled - type: object - tolerations: - items: + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + name: + type: string + namespace: + type: string + retentionPolicy: properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: + onDelete: + default: detach type: string + required: + - onDelete type: object - type: array - type: object - type: object - requireLimits: - type: boolean - retentionPolicy: - properties: - onDelete: - default: detach - type: string - required: - - onDelete - type: object - size: - enum: - - dev - - micro - - small - - medium - - large - - xlarge - - xxlarge - type: string - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - wandb: - properties: - additionalHostnames: - items: - type: string - type: array - features: - additionalProperties: - type: boolean - type: object - hostname: - type: string - internalServiceAuth: - properties: - enabled: - type: boolean - oidcIssuer: - type: string - type: object - license: - type: string - manifestRepository: - type: string - oidc: - properties: - authMethod: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - clientId: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - clientSecret: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - issuerUrl: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sessionLength: - type: string - type: object - probes: - properties: - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - startupProbe: - properties: - exec: - properties: - command: - items: + sentinel: + properties: + config: + properties: + masterName: type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - format: int32 - type: integer - grpc: - properties: - port: - format: int32 - type: integer - service: - default: "" - type: string - required: - - port - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - type: object - serviceAccount: - properties: - annotations: - additionalProperties: - type: string - type: object - create: - default: true - type: boolean - serviceAccountName: - default: wandb - type: string - required: - - create - type: object - version: - type: string - required: - - features - - hostname - - version - type: object - required: - - retentionPolicy - type: object - status: - properties: - clickhouseStatus: - properties: - conditions: - items: - properties: - lastTransitionTime: - format: date-time - type: string - message: - maxLength: 32768 - type: string - observedGeneration: - format: int64 - minimum: 0 - type: integer - reason: - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - enum: - - "True" - - "False" - - Unknown - type: string - type: - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - connection: - properties: - database: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - host: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - username: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - ready: - type: boolean - state: - type: string - required: - - ready + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + enabled: + type: boolean + required: + - enabled + type: object + storageSize: + type: string + telemetry: + properties: + enabled: + default: true + type: boolean + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object type: object - gatewayStatus: + requireLimits: + type: boolean + retentionPolicy: properties: - addresses: - items: - type: string - type: array - gatewayRef: - properties: - name: - type: string - namespace: - type: string - required: - - name - type: object - name: + onDelete: + default: detach type: string - ready: - type: boolean + required: + - onDelete type: object - generatedSecrets: - additionalProperties: + size: + enum: + - dev + - micro + - small + - medium + - large + - xlarge + - xxlarge + type: string + tolerations: + items: properties: + effect: + type: string key: type: string - name: - default: "" + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: type: string - optional: - type: boolean - required: - - key type: object - x-kubernetes-map-type: atomic - type: object - ingressStatus: + type: array + wandb: properties: - loadBalancerIngress: + additionalHostnames: items: + type: string + type: array + bucketProxy: + type: boolean + features: + additionalProperties: + type: boolean + type: object + hostname: + type: string + internalServiceAuth: + properties: + enabled: + type: boolean + oidcIssuer: + type: string + type: object + legacyOverrides: + additionalProperties: properties: - hostname: - type: string - ip: - type: string - ipMode: - type: string - ports: + env: items: properties: - error: - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + name: type: string - port: - format: int32 - type: integer - protocol: + value: type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object required: - - error - - port - - protocol + - name type: object type: array - x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object type: object - type: array - name: + type: object + license: type: string - type: object - kafkaStatus: - properties: - conditions: - items: - properties: - lastTransitionTime: - format: date-time - type: string - message: - maxLength: 32768 - type: string - observedGeneration: - format: int64 - minimum: 0 - type: integer - reason: - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - enum: - - "True" - - "False" - - Unknown - type: string - type: - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - connection: - properties: - brokerEndpoint: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - clusterID: + manifestRepository: + type: string + oidc: + properties: + authMethod: properties: key: type: string @@ -4352,7 +4094,7 @@ spec: - key type: object x-kubernetes-map-type: atomic - host: + clientId: properties: key: type: string @@ -4365,7 +4107,7 @@ spec: - key type: object x-kubernetes-map-type: atomic - port: + clientSecret: properties: key: type: string @@ -4378,7 +4120,7 @@ spec: - key type: object x-kubernetes-map-type: atomic - url: + issuerUrl: properties: key: type: string @@ -4390,194 +4132,535 @@ spec: required: - key type: object - x-kubernetes-map-type: atomic + x-kubernetes-map-type: atomic + sessionLength: + type: string + type: object + probes: + properties: + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object type: object - ready: - type: boolean - state: + serviceAccount: + properties: + annotations: + additionalProperties: + type: string + type: object + create: + default: true + type: boolean + serviceAccountName: + default: wandb + type: string + required: + - create + type: object + version: type: string required: - - ready + - bucketProxy + - features + - hostname + - version type: object - mysqlStatus: - properties: - conditions: - items: - properties: - lastTransitionTime: - format: date-time - type: string - message: - maxLength: 32768 - type: string - observedGeneration: - format: int64 - minimum: 0 - type: integer - reason: - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - enum: - - "True" - - "False" - - Unknown - type: string - type: - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - connection: - properties: - database: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - host: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslCa: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslCert: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - sslKey: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - tls: + required: + - retentionPolicy + type: object + status: + properties: + clickhouseStatus: + additionalProperties: + properties: + conditions: + items: properties: - key: + lastTransitionTime: + format: date-time type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: + message: + maxLength: 32768 type: string - name: - default: "" + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - username: - properties: - key: + status: + enum: + - "True" + - "False" + - Unknown type: string - name: - default: "" + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string - optional: - type: boolean required: - - key + - lastTransitionTime + - message + - reason + - status + - type type: object - x-kubernetes-map-type: atomic + type: array + connection: + properties: + database: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + host: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + httpPort: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tcpPort: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + ready: + type: boolean + state: + type: string + required: + - ready + type: object + type: object + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + gatewayStatus: + properties: + addresses: + items: + type: string + type: array + gatewayRef: + properties: + name: + type: string + namespace: + type: string + required: + - name type: object + name: + type: string ready: type: boolean - state: + type: object + generatedSecrets: + additionalProperties: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + ingressStatus: + properties: + loadBalancerIngress: + items: + properties: + hostname: + type: string + ip: + type: string + ipMode: + type: string + ports: + items: + properties: + error: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - error + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + name: type: string - required: - - ready type: object - objectStoreStatus: + kafkaStatus: properties: conditions: items: @@ -4617,7 +4700,7 @@ spec: type: array connection: properties: - accessKey: + brokerEndpoint: properties: key: type: string @@ -4630,7 +4713,7 @@ spec: - key type: object x-kubernetes-map-type: atomic - bucket: + clusterID: properties: key: type: string @@ -4643,7 +4726,7 @@ spec: - key type: object x-kubernetes-map-type: atomic - endpoint: + host: properties: key: type: string @@ -4669,45 +4752,6 @@ spec: - key type: object x-kubernetes-map-type: atomic - provider: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - region: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - secretKey: - properties: - key: - type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic url: properties: key: @@ -4729,136 +4773,511 @@ spec: required: - ready type: object - observedGeneration: - format: int64 - type: integer - ready: - type: boolean - redisStatus: - properties: - conditions: - items: + mysqlStatus: + additionalProperties: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + connection: properties: - lastTransitionTime: - format: date-time - type: string - message: - maxLength: 32768 - type: string - observedGeneration: - format: int64 - minimum: 0 - type: integer - reason: - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - enum: - - "True" - - "False" - - Unknown - type: string - type: - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - connection: - properties: - host: + database: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + host: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCa: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCert: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tls: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + ready: + type: boolean + state: + type: string + required: + - ready + type: object + type: object + objectStoreStatus: + additionalProperties: + properties: + conditions: + items: properties: - key: - type: string - name: - default: "" + lastTransitionTime: + format: date-time type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - password: - properties: - key: + message: + maxLength: 32768 type: string - name: - default: "" + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - port: - properties: - key: + status: + enum: + - "True" + - "False" + - Unknown type: string - name: - default: "" + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string - optional: - type: boolean required: - - key + - lastTransitionTime + - message + - reason + - status + - type type: object - x-kubernetes-map-type: atomic - sslCa: + type: array + connection: + properties: + accessKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + bucket: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpoint: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + forcePathStyle: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + path: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + provider: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + region: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secretKey: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tlsEnabled: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + ready: + type: boolean + state: + type: string + required: + - ready + type: object + type: object + observedGeneration: + format: int64 + type: integer + ready: + type: boolean + redisStatus: + additionalProperties: + properties: + conditions: + items: properties: - key: + lastTransitionTime: + format: date-time type: string - name: - default: "" - type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - tls: - properties: - key: + message: + maxLength: 32768 type: string - name: - default: "" + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string - optional: - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - url: - properties: - key: + status: + enum: + - "True" + - "False" + - Unknown type: string - name: - default: "" + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string - optional: - type: boolean required: - - key + - lastTransitionTime + - message + - reason + - status + - type type: object - x-kubernetes-map-type: atomic - type: object - ready: - type: boolean - state: - type: string - required: - - ready + type: array + connection: + properties: + host: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + port: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + sslCa: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + tls: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + url: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + ready: + type: boolean + state: + type: string + required: + - ready + type: object type: object telemetryStatus: properties: @@ -5897,12 +6316,18 @@ spec: type: string name: type: string + phase: + type: string + reason: + type: string succeeded: type: boolean type: object type: object lastSuccessVersion: type: string + phase: + type: string ready: type: boolean reason: @@ -5911,16 +6336,22 @@ spec: type: string type: object mysqlInit: + additionalProperties: + properties: + failed: + type: boolean + message: + type: string + name: + type: string + phase: + type: string + reason: + type: string + succeeded: + type: boolean + type: object default: {} - properties: - failed: - type: boolean - message: - type: string - name: - type: string - succeeded: - type: boolean type: object required: - hostname diff --git a/internal/webhook/v2/weightsandbiases_defaulter_clickhouse_test.go b/internal/webhook/v2/weightsandbiases_defaulter_clickhouse_test.go index 3018728b..2250ab68 100644 --- a/internal/webhook/v2/weightsandbiases_defaulter_clickhouse_test.go +++ b/internal/webhook/v2/weightsandbiases_defaulter_clickhouse_test.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/ginkgo/v2" g "github.com/onsi/gomega" apiv2 "github.com/wandb/operator/api/v2" + "github.com/wandb/operator/internal/controller/infra/managed/clickhouse/altinity" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -23,36 +24,38 @@ var _ = Describe("WeightsAndBiasesCustomDefaulter - ClickHouse", func() { It("defaults ClickHouse namespace to the parent namespace", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, - Spec: apiv2.WeightsAndBiasesSpec{ClickHouse: apiv2.ClickHouseSpec{ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}}, + Spec: apiv2.WeightsAndBiasesSpec{ClickHouse: map[string]apiv2.ClickHouseSpec{apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}}}, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.ClickHouse.ManagedClickHouse.Namespace).To(g.Equal("test-namespace")) + g.Expect(wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.Namespace).To(g.Equal("test-namespace")) }) It("preserves a custom ClickHouse namespace", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - ClickHouse: apiv2.ClickHouseSpec{ManagedClickHouse: &apiv2.ManagedClickHouseSpec{Namespace: "custom-clickhouse-namespace"}}, + ClickHouse: map[string]apiv2.ClickHouseSpec{apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{Namespace: "custom-clickhouse-namespace"}}}, }, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.ClickHouse.ManagedClickHouse.Namespace).To(g.Equal("custom-clickhouse-namespace")) + g.Expect(wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.Namespace).To(g.Equal("custom-clickhouse-namespace")) }) It("does not mutate unrelated ClickHouse fields", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - ClickHouse: apiv2.ClickHouseSpec{ - ManagedClickHouse: &apiv2.ManagedClickHouseSpec{ - StorageSize: "100Gi", - Replicas: 2, - Version: "24.1", + ClickHouse: map[string]apiv2.ClickHouseSpec{ + apiv2.DefaultInstanceName: { + ManagedClickHouse: &apiv2.ManagedClickHouseSpec{ + StorageSize: "100Gi", + Replicas: 2, + Version: "24.1", + }, }, }, }, @@ -60,23 +63,90 @@ var _ = Describe("WeightsAndBiasesCustomDefaulter - ClickHouse", func() { err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.ClickHouse.ManagedClickHouse.StorageSize).To(g.Equal("100Gi")) - g.Expect(wandb.Spec.ClickHouse.ManagedClickHouse.Replicas).To(g.Equal(int32(2))) - g.Expect(wandb.Spec.ClickHouse.ManagedClickHouse.Version).To(g.Equal("24.1")) + g.Expect(wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.StorageSize).To(g.Equal("100Gi")) + g.Expect(wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.Replicas).To(g.Equal(int32(2))) + g.Expect(wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.Version).To(g.Equal("24.1")) + }) + + It("defaults the plain '-chi' name for CR names that fit", func() { + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, + Spec: apiv2.WeightsAndBiasesSpec{ClickHouse: map[string]apiv2.ClickHouseSpec{apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}}}, + } + + g.Expect(defaulter.Default(ctx, wandb)).To(g.Succeed()) + g.Expect(wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.Name).To(g.Equal("test-wandb-chi")) + g.Expect(wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.ServiceAccount.Create).ToNot(g.BeNil()) + g.Expect(*wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.ServiceAccount.Create).To(g.BeTrue()) + g.Expect(wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.ServiceAccount.ServiceAccountName).To(g.Equal("test-wandb-chi")) + }) + + It("keys non-default instance names before the suffix", func() { + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, + Spec: apiv2.WeightsAndBiasesSpec{ClickHouse: map[string]apiv2.ClickHouseSpec{ + apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}, + "analytics": {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}, + }}, + } + + g.Expect(defaulter.Default(ctx, wandb)).To(g.Succeed()) + g.Expect(wandb.Spec.ClickHouse["analytics"].ManagedClickHouse.Name).To(g.Equal("test-wandb-analytics-chi")) + }) + + It("defaults a deployable name for CR names the plain default would wedge", func() { + // 32 chars: "-chi" would overflow the derived per-host volume names + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb-integration-environments-2", Namespace: "test-namespace"}, + Spec: apiv2.WeightsAndBiasesSpec{ClickHouse: map[string]apiv2.ClickHouseSpec{apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}}}, + } + + g.Expect(defaulter.Default(ctx, wandb)).To(g.Succeed()) + + managed := wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse + g.Expect(managed.Name).To(g.HaveSuffix("-chi")) + g.Expect(len(managed.Name)).To(g.BeNumerically("<=", altinity.MaxSpecNameLength())) + g.Expect(altinity.ValidateDerivedNames(managed)).To(g.Succeed()) + + // persisted in the spec, so it must be deterministic + again := &apiv2.WeightsAndBiases{ + ObjectMeta: wandb.ObjectMeta, + Spec: apiv2.WeightsAndBiasesSpec{ClickHouse: map[string]apiv2.ClickHouseSpec{apiv2.DefaultInstanceName: {ManagedClickHouse: &apiv2.ManagedClickHouseSpec{}}}}, + } + g.Expect(defaulter.Default(ctx, again)).To(g.Succeed()) + g.Expect(again.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse.Name).To(g.Equal(managed.Name)) + }) + + It("keeps plain default names for the other infra at CR lengths only ClickHouse would break", func() { + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wandb-legacy-overrides-v1", Namespace: "test-namespace"}, + Spec: apiv2.WeightsAndBiasesSpec{ + MySQL: map[string]apiv2.MySQLSpec{apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{}}}, + Redis: map[string]apiv2.RedisSpec{apiv2.DefaultInstanceName: {ManagedRedis: &apiv2.ManagedRedisSpec{}}}, + Kafka: apiv2.KafkaSpec{ManagedKafka: &apiv2.ManagedKafkaSpec{}}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{apiv2.DefaultInstanceName: {ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}}, + }, + } + + g.Expect(defaulter.Default(ctx, wandb)).To(g.Succeed()) + g.Expect(wandb.Spec.MySQL[apiv2.DefaultInstanceName].ManagedMysql.Name).To(g.Equal("wandb-legacy-overrides-v1-mysql")) + g.Expect(wandb.Spec.Redis[apiv2.DefaultInstanceName].ManagedRedis.Name).To(g.Equal("wandb-legacy-overrides-v1-redis")) + g.Expect(wandb.Spec.Kafka.ManagedKafka.Name).To(g.Equal("wandb-legacy-overrides-v1-kafka")) + g.Expect(wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Name).To(g.Equal("wandb-legacy-overrides-v1-seaweedfs")) }) It("does not apply defaults when ExternalClickhouse is present", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - ClickHouse: apiv2.ClickHouseSpec{ - ExternalClickHouse: &apiv2.ClickHouseConnection{}, + ClickHouse: map[string]apiv2.ClickHouseSpec{ + apiv2.DefaultInstanceName: {ExternalClickHouse: &apiv2.ClickHouseConnection{}}, }, }, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.ClickHouse.ManagedClickHouse).To(g.BeNil()) + g.Expect(wandb.Spec.ClickHouse[apiv2.DefaultInstanceName].ManagedClickHouse).To(g.BeNil()) }) }) diff --git a/internal/webhook/v2/weightsandbiases_defaulter_kafka_test.go b/internal/webhook/v2/weightsandbiases_defaulter_kafka_test.go index 74c1d862..fcc93ffe 100644 --- a/internal/webhook/v2/weightsandbiases_defaulter_kafka_test.go +++ b/internal/webhook/v2/weightsandbiases_defaulter_kafka_test.go @@ -69,5 +69,8 @@ var _ = Describe("WeightsAndBiasesCustomDefaulter - Kafka", func() { g.Expect(wandb.Spec.Kafka.ManagedKafka).ToNot(g.BeNil()) g.Expect(wandb.Spec.Kafka.ManagedKafka.Name).To(g.Equal("test-wandb-kafka")) g.Expect(wandb.Spec.Kafka.ManagedKafka.Namespace).To(g.Equal("test-namespace")) + g.Expect(wandb.Spec.Kafka.ManagedKafka.ServiceAccount.Create).ToNot(g.BeNil()) + g.Expect(*wandb.Spec.Kafka.ManagedKafka.ServiceAccount.Create).To(g.BeTrue()) + g.Expect(wandb.Spec.Kafka.ManagedKafka.ServiceAccount.ServiceAccountName).To(g.Equal("test-wandb-kafka")) }) }) diff --git a/internal/webhook/v2/weightsandbiases_defaulter_mysql_test.go b/internal/webhook/v2/weightsandbiases_defaulter_mysql_test.go index aa012455..4e208878 100644 --- a/internal/webhook/v2/weightsandbiases_defaulter_mysql_test.go +++ b/internal/webhook/v2/weightsandbiases_defaulter_mysql_test.go @@ -23,21 +23,23 @@ var _ = Describe("WeightsAndBiasesCustomDefaulter - MySQL", func() { It("defaults MySQL namespace", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, - Spec: apiv2.WeightsAndBiasesSpec{MySQL: apiv2.MySQLSpec{ManagedMysql: &apiv2.ManagedMysqlSpec{}}}, + Spec: apiv2.WeightsAndBiasesSpec{MySQL: map[string]apiv2.MySQLSpec{apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{}}}}, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.MySQL.ManagedMysql.Namespace).To(g.Equal("test-namespace")) + g.Expect(wandb.Spec.MySQL[apiv2.DefaultInstanceName].ManagedMysql.Namespace).To(g.Equal("test-namespace")) }) It("preserves custom MySQL namespace", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - MySQL: apiv2.MySQLSpec{ - ManagedMysql: &apiv2.ManagedMysqlSpec{ - Namespace: "custom-moco-namespace", + MySQL: map[string]apiv2.MySQLSpec{ + apiv2.DefaultInstanceName: { + ManagedMysql: &apiv2.ManagedMysqlSpec{ + Namespace: "custom-moco-namespace", + }, }, }, }, @@ -45,34 +47,34 @@ var _ = Describe("WeightsAndBiasesCustomDefaulter - MySQL", func() { err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.MySQL.ManagedMysql.Namespace).To(g.Equal("custom-moco-namespace")) + g.Expect(wandb.Spec.MySQL[apiv2.DefaultInstanceName].ManagedMysql.Namespace).To(g.Equal("custom-moco-namespace")) }) It("does not mutate unrelated MySQL fields", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - MySQL: apiv2.MySQLSpec{ManagedMysql: &apiv2.ManagedMysqlSpec{StorageSize: "50Gi"}}, + MySQL: map[string]apiv2.MySQLSpec{apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{StorageSize: "50Gi"}}}, }, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.MySQL.ManagedMysql.StorageSize).To(g.Equal("50Gi")) + g.Expect(wandb.Spec.MySQL[apiv2.DefaultInstanceName].ManagedMysql.StorageSize).To(g.Equal("50Gi")) }) It("does not apply defaults when ExternalMysql is present", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - MySQL: apiv2.MySQLSpec{ - ExternalMysql: &apiv2.MysqlConnection{}, + MySQL: map[string]apiv2.MySQLSpec{ + apiv2.DefaultInstanceName: {ExternalMysql: &apiv2.MysqlConnection{}}, }, }, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.MySQL.ManagedMysql).To(g.BeNil()) + g.Expect(wandb.Spec.MySQL[apiv2.DefaultInstanceName].ManagedMysql).To(g.BeNil()) }) }) diff --git a/internal/webhook/v2/weightsandbiases_defaulter_objectstore_test.go b/internal/webhook/v2/weightsandbiases_defaulter_objectstore_test.go index 666f2e45..ad96f480 100644 --- a/internal/webhook/v2/weightsandbiases_defaulter_objectstore_test.go +++ b/internal/webhook/v2/weightsandbiases_defaulter_objectstore_test.go @@ -23,36 +23,38 @@ var _ = Describe("WeightsAndBiasesCustomDefaulter - ObjectStore", func() { It("defaults ObjectStore namespace to the parent namespace", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, - Spec: apiv2.WeightsAndBiasesSpec{ObjectStore: apiv2.ObjectStoreSpec{ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}}, + Spec: apiv2.WeightsAndBiasesSpec{ObjectStore: map[string]apiv2.ObjectStoreSpec{apiv2.DefaultInstanceName: {ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{}}}}, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.ObjectStore.ManagedObjectStore.Namespace).To(g.Equal("test-namespace")) + g.Expect(wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Namespace).To(g.Equal("test-namespace")) }) It("preserves a custom ObjectStore namespace", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - ObjectStore: apiv2.ObjectStoreSpec{ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{Namespace: "custom-objectstore-namespace"}}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{apiv2.DefaultInstanceName: {ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{Namespace: "custom-objectstore-namespace"}}}, }, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.ObjectStore.ManagedObjectStore.Namespace).To(g.Equal("custom-objectstore-namespace")) + g.Expect(wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Namespace).To(g.Equal("custom-objectstore-namespace")) }) It("does not mutate unrelated ObjectStore fields", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - ObjectStore: apiv2.ObjectStoreSpec{ - ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{ - StorageSize: "50Gi", - Replicas: 4, - Config: apiv2.ObjectStoreConfig{AccessKey: "custom-admin"}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: { + ManagedObjectStore: &apiv2.ManagedObjectStoreSpec{ + StorageSize: "50Gi", + Replicas: 4, + Config: apiv2.ObjectStoreConfig{AccessKey: "custom-admin"}, + }, }, }, }, @@ -60,23 +62,23 @@ var _ = Describe("WeightsAndBiasesCustomDefaulter - ObjectStore", func() { err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.ObjectStore.ManagedObjectStore.StorageSize).To(g.Equal("50Gi")) - g.Expect(wandb.Spec.ObjectStore.ManagedObjectStore.Replicas).To(g.Equal(int32(4))) - g.Expect(wandb.Spec.ObjectStore.ManagedObjectStore.Config.AccessKey).To(g.Equal("custom-admin")) + g.Expect(wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.StorageSize).To(g.Equal("50Gi")) + g.Expect(wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Replicas).To(g.Equal(int32(4))) + g.Expect(wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore.Config.AccessKey).To(g.Equal("custom-admin")) }) It("does not apply defaults when ExternalObjectStore is present", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - ObjectStore: apiv2.ObjectStoreSpec{ - ExternalObjectStore: &apiv2.ObjectStoreConnection{}, + ObjectStore: map[string]apiv2.ObjectStoreSpec{ + apiv2.DefaultInstanceName: {ExternalObjectStore: &apiv2.ObjectStoreConnection{}}, }, }, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.ObjectStore.ManagedObjectStore).To(g.BeNil()) + g.Expect(wandb.Spec.ObjectStore[apiv2.DefaultInstanceName].ManagedObjectStore).To(g.BeNil()) }) }) diff --git a/internal/webhook/v2/weightsandbiases_defaulter_redis_test.go b/internal/webhook/v2/weightsandbiases_defaulter_redis_test.go index 6608763e..45eb16b9 100644 --- a/internal/webhook/v2/weightsandbiases_defaulter_redis_test.go +++ b/internal/webhook/v2/weightsandbiases_defaulter_redis_test.go @@ -23,53 +23,53 @@ var _ = Describe("WeightsAndBiasesCustomDefaulter - Redis", func() { It("defaults Redis namespace to the parent namespace", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, - Spec: apiv2.WeightsAndBiasesSpec{Redis: apiv2.RedisSpec{ManagedRedis: &apiv2.ManagedRedisSpec{}}}, + Spec: apiv2.WeightsAndBiasesSpec{Redis: map[string]apiv2.RedisSpec{apiv2.DefaultInstanceName: {ManagedRedis: &apiv2.ManagedRedisSpec{}}}}, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.Redis.ManagedRedis.Namespace).To(g.Equal("test-namespace")) + g.Expect(wandb.Spec.Redis[apiv2.DefaultInstanceName].ManagedRedis.Namespace).To(g.Equal("test-namespace")) }) It("preserves a custom Redis namespace", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - Redis: apiv2.RedisSpec{ManagedRedis: &apiv2.ManagedRedisSpec{Namespace: "custom-redis-namespace"}}, + Redis: map[string]apiv2.RedisSpec{apiv2.DefaultInstanceName: {ManagedRedis: &apiv2.ManagedRedisSpec{Namespace: "custom-redis-namespace"}}}, }, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.Redis.ManagedRedis.Namespace).To(g.Equal("custom-redis-namespace")) + g.Expect(wandb.Spec.Redis[apiv2.DefaultInstanceName].ManagedRedis.Namespace).To(g.Equal("custom-redis-namespace")) }) It("does not mutate unrelated Redis fields", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - Redis: apiv2.RedisSpec{ManagedRedis: &apiv2.ManagedRedisSpec{StorageSize: "20Gi", Sentinel: apiv2.RedisSentinelSpec{Enabled: true}}}, + Redis: map[string]apiv2.RedisSpec{apiv2.DefaultInstanceName: {ManagedRedis: &apiv2.ManagedRedisSpec{StorageSize: "20Gi", Sentinel: apiv2.RedisSentinelSpec{Enabled: true}}}}, }, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.Redis.ManagedRedis.StorageSize).To(g.Equal("20Gi")) - g.Expect(wandb.Spec.Redis.ManagedRedis.Sentinel.Enabled).To(g.BeTrue()) + g.Expect(wandb.Spec.Redis[apiv2.DefaultInstanceName].ManagedRedis.StorageSize).To(g.Equal("20Gi")) + g.Expect(wandb.Spec.Redis[apiv2.DefaultInstanceName].ManagedRedis.Sentinel.Enabled).To(g.BeTrue()) }) It("does not apply defaults when External is present", func() { wandb := &apiv2.WeightsAndBiases{ ObjectMeta: metav1.ObjectMeta{Name: "test-wandb", Namespace: "test-namespace"}, Spec: apiv2.WeightsAndBiasesSpec{ - Redis: apiv2.RedisSpec{ - ExternalRedis: &apiv2.RedisConnection{}, + Redis: map[string]apiv2.RedisSpec{ + apiv2.DefaultInstanceName: {ExternalRedis: &apiv2.RedisConnection{}}, }, }, } err := defaulter.Default(ctx, wandb) g.Expect(err).ToNot(g.HaveOccurred()) - g.Expect(wandb.Spec.Redis.ManagedRedis).To(g.BeNil()) + g.Expect(wandb.Spec.Redis[apiv2.DefaultInstanceName].ManagedRedis).To(g.BeNil()) }) }) diff --git a/internal/webhook/v2/weightsandbiases_multi_instance_test.go b/internal/webhook/v2/weightsandbiases_multi_instance_test.go new file mode 100644 index 00000000..5fc44694 --- /dev/null +++ b/internal/webhook/v2/weightsandbiases_multi_instance_test.go @@ -0,0 +1,86 @@ +package v2 + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + g "github.com/onsi/gomega" + apiv2 "github.com/wandb/operator/api/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("WeightsAndBiases multi-instance infra", func() { + var ( + ctx context.Context + defaulter WeightsAndBiasesCustomDefaulter + validator WeightsAndBiasesCustomValidator + ) + + BeforeEach(func() { + ctx = context.Background() + defaulter = WeightsAndBiasesCustomDefaulter{} + validator = WeightsAndBiasesCustomValidator{} + }) + + It("seeds a managed default instance when the map is empty", func() { + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wb", Namespace: "ns"}, + } + + g.Expect(defaulter.Default(ctx, wandb)).To(g.Succeed()) + + mysql, ok := wandb.Spec.MySQL[apiv2.DefaultInstanceName] + g.Expect(ok).To(g.BeTrue()) + g.Expect(mysql.ManagedMysql).ToNot(g.BeNil()) + g.Expect(mysql.ManagedMysql.Name).To(g.Equal("wb-mysql")) + }) + + It("names the default instance plainly and keys other instances before the suffix", func() { + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wb", Namespace: "ns"}, + Spec: apiv2.WeightsAndBiasesSpec{ + MySQL: map[string]apiv2.MySQLSpec{ + apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{}}, + "analytics": {ManagedMysql: &apiv2.ManagedMysqlSpec{}}, + }, + }, + } + + g.Expect(defaulter.Default(ctx, wandb)).To(g.Succeed()) + + g.Expect(wandb.Spec.MySQL[apiv2.DefaultInstanceName].ManagedMysql.Name).To(g.Equal("wb-mysql")) + g.Expect(wandb.Spec.MySQL["analytics"].ManagedMysql.Name).To(g.Equal("wb-analytics-mysql")) + g.Expect(wandb.Spec.MySQL["analytics"].ManagedMysql.Namespace).To(g.Equal("ns")) + }) + + It("rejects instances defined without a default instance", func() { + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wb", Namespace: "ns"}, + Spec: apiv2.WeightsAndBiasesSpec{ + MySQL: map[string]apiv2.MySQLSpec{ + "analytics": {ManagedMysql: &apiv2.ManagedMysqlSpec{Name: "wb-mysql-analytics", Namespace: "ns"}}, + }, + }, + } + + _, err := validator.ValidateCreate(ctx, wandb) + g.Expect(err).To(g.HaveOccurred()) + g.Expect(err.Error()).To(g.ContainSubstring("default")) + }) + + It("accepts multiple instances when a default is present", func() { + wandb := &apiv2.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{Name: "wb", Namespace: "ns"}, + Spec: apiv2.WeightsAndBiasesSpec{ + Wandb: apiv2.WandbAppSpec{Hostname: "https://wandb.example.com"}, + MySQL: map[string]apiv2.MySQLSpec{ + apiv2.DefaultInstanceName: {ManagedMysql: &apiv2.ManagedMysqlSpec{Name: "wb-mysql", Namespace: "ns"}}, + "analytics": {ManagedMysql: &apiv2.ManagedMysqlSpec{Name: "wb-mysql-analytics", Namespace: "ns"}}, + }, + }, + } + + _, err := validator.ValidateCreate(ctx, wandb) + g.Expect(err).ToNot(g.HaveOccurred()) + }) +}) diff --git a/internal/webhook/v2/weightsandbiases_proxy_test.go b/internal/webhook/v2/weightsandbiases_proxy_test.go new file mode 100644 index 00000000..8a7f4162 --- /dev/null +++ b/internal/webhook/v2/weightsandbiases_proxy_test.go @@ -0,0 +1,54 @@ +package v2 + +import ( + "strings" + "testing" + + appsv2 "github.com/wandb/operator/api/v2" + corev1 "k8s.io/api/core/v1" +) + +func wandbWithProxy(proxy *appsv2.ProxySpec) *appsv2.WeightsAndBiases { + wandb := &appsv2.WeightsAndBiases{} + wandb.Spec.Global.Proxy = proxy + return wandb +} + +func TestValidateProxySpec(t *testing.T) { + secretRef := &appsv2.ProxyValueSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "egress-proxy"}, + Key: "httpsProxy", + }, + } + cases := []struct { + name string + proxy *appsv2.ProxySpec + wantErr string // substring; "" = accept + }{ + {"nil proxy", nil, ""}, + {"literal http url", &appsv2.ProxySpec{HTTPProxy: &appsv2.ProxyValue{Value: "http://proxy.corp:3128"}}, ""}, + {"secret-backed https", &appsv2.ProxySpec{HTTPSProxy: &appsv2.ProxyValue{ValueFrom: secretRef}}, ""}, + {"noProxy extras ok", &appsv2.ProxySpec{HTTPProxy: &appsv2.ProxyValue{Value: "http://p:3128"}, NoProxy: []string{"internal.example.com", "10.0.0.0/8"}}, ""}, + {"both value and valueFrom", &appsv2.ProxySpec{HTTPProxy: &appsv2.ProxyValue{Value: "http://p:3128", ValueFrom: secretRef}}, "exactly one"}, + {"neither value nor valueFrom", &appsv2.ProxySpec{HTTPProxy: &appsv2.ProxyValue{}}, "one of value or valueFrom is required"}, + {"userinfo in literal", &appsv2.ProxySpec{HTTPProxy: &appsv2.ProxyValue{Value: "http://user:pass@proxy:3128"}}, "must not contain credentials"}, + {"bad scheme", &appsv2.ProxySpec{HTTPProxy: &appsv2.ProxyValue{Value: "socks5://proxy:1080"}}, "scheme must be http or https"}, + {"comma in noProxy", &appsv2.ProxySpec{HTTPProxy: &appsv2.ProxyValue{Value: "http://p:3128"}, NoProxy: []string{"a,b"}}, "must not contain commas"}, + {"empty noProxy entry", &appsv2.ProxySpec{HTTPProxy: &appsv2.ProxyValue{Value: "http://p:3128"}, NoProxy: []string{""}}, "must not be empty"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + errs := validateProxySpec(wandbWithProxy(tc.proxy)) + if tc.wantErr == "" { + if len(errs) != 0 { + t.Fatalf("expected no errors, got %v", errs) + } + return + } + if len(errs) == 0 || !strings.Contains(errs.ToAggregate().Error(), tc.wantErr) { + t.Fatalf("expected error containing %q, got %v", tc.wantErr, errs) + } + }) + } +} diff --git a/internal/webhook/v2/weightsandbiases_webhook.go b/internal/webhook/v2/weightsandbiases_webhook.go index a951fb2f..fd229ca4 100644 --- a/internal/webhook/v2/weightsandbiases_webhook.go +++ b/internal/webhook/v2/weightsandbiases_webhook.go @@ -19,14 +19,22 @@ package v2 import ( "context" "fmt" + "net/url" "strings" + v1 "github.com/wandb/operator/api/v1" + "github.com/wandb/operator/internal/controller/infra/managed/clickhouse/altinity" + "github.com/wandb/operator/internal/controller/infra/managed/kafka/bufstream" + "github.com/wandb/operator/internal/controller/infra/managed/mysql/moco" + "github.com/wandb/operator/internal/controller/infra/managed/objectstore/seaweedfs" + "github.com/wandb/operator/internal/controller/infra/managed/redis/opstree" "github.com/wandb/operator/internal/logx" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -90,7 +98,7 @@ func (d *WeightsAndBiasesCustomDefaulter) Default(ctx context.Context, obj runti } if wandb.Spec.Wandb.ManifestRepository == "" { - wandb.Spec.Wandb.ManifestRepository = "oci://us-docker.pkg.dev/wandb-production/public/wandb/server-manifest" + wandb.Spec.Wandb.ManifestRepository = appsv2.DefaultManifestRepository } if !strings.Contains(wandb.Spec.Wandb.ManifestRepository, "://") { @@ -125,6 +133,10 @@ func (d *WeightsAndBiasesCustomDefaulter) Default(ctx context.Context, obj runti applyClickHouseDefaults(wandb) applyProbeDefaults(wandb) + if defaultStore, ok := wandb.Spec.ObjectStore["default"]; ok && defaultStore.ManagedObjectStore != nil { + wandb.Spec.Wandb.BucketProxy = true + } + return nil } @@ -152,7 +164,7 @@ func (v *WeightsAndBiasesCustomValidator) ValidateCreate(ctx context.Context, ob } log.Info("Validation for WeightsAndBiases upon creation", "name", wandb.GetName()) - return validateSpec(ctx, wandb) + return validateSpec(ctx, wandb, nil) } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type WeightsAndBiases. @@ -173,7 +185,7 @@ func (v *WeightsAndBiasesCustomValidator) ValidateUpdate(ctx context.Context, ol log.Info("validate V2 update", "name", newWandb.Name) - if specWarnings, err = validateSpec(ctx, newWandb); err != nil { + if specWarnings, err = validateSpec(ctx, newWandb, oldWandb); err != nil { return specWarnings, err } changeWarnings, err = validateChanges(ctx, newWandb, oldWandb) @@ -195,44 +207,55 @@ func (v *WeightsAndBiasesCustomValidator) ValidateDelete(ctx context.Context, ob } func applyMySQLDefaults(wandb *appsv2.WeightsAndBiases) { - if wandb.Spec.MySQL.ManagedMysql == nil { - if wandb.Spec.MySQL.ExternalMysql != nil { - return - } - wandb.Spec.MySQL.ManagedMysql = &appsv2.ManagedMysqlSpec{} + if wandb.Spec.MySQL == nil { + wandb.Spec.MySQL = map[string]appsv2.MySQLSpec{} } - - spec := wandb.Spec.MySQL.ManagedMysql - - if spec.Name == "" { - spec.Name = fmt.Sprintf("%s-mysql", wandb.Name) + if len(wandb.Spec.MySQL) == 0 { + wandb.Spec.MySQL[appsv2.DefaultInstanceName] = appsv2.MySQLSpec{ManagedMysql: &appsv2.ManagedMysqlSpec{}} } - if spec.Namespace == "" { - spec.Namespace = wandb.Namespace + for key, spec := range wandb.Spec.MySQL { + if spec.ExternalMysql != nil { + continue + } + if spec.ManagedMysql == nil { + spec.ManagedMysql = &appsv2.ManagedMysqlSpec{} + } + if spec.ManagedMysql.Name == "" { + spec.ManagedMysql.Name = moco.DefaultSpecName(wandb.Name, key) + } + if spec.ManagedMysql.Namespace == "" { + spec.ManagedMysql.Namespace = wandb.Namespace + } + wandb.Spec.MySQL[key] = spec } } func applyRedisDefaults(wandb *appsv2.WeightsAndBiases) { - if wandb.Spec.Redis.ManagedRedis == nil { - if wandb.Spec.Redis.ExternalRedis != nil { - return - } - wandb.Spec.Redis.ManagedRedis = &appsv2.ManagedRedisSpec{} + if wandb.Spec.Redis == nil { + wandb.Spec.Redis = map[string]appsv2.RedisSpec{} } - - spec := wandb.Spec.Redis.ManagedRedis - - if spec.Name == "" { - spec.Name = fmt.Sprintf("%s-redis", wandb.Name) + if len(wandb.Spec.Redis) == 0 { + wandb.Spec.Redis[appsv2.DefaultInstanceName] = appsv2.RedisSpec{ManagedRedis: &appsv2.ManagedRedisSpec{}} } - if spec.Namespace == "" { - spec.Namespace = wandb.Namespace - } - - if wandb.Spec.Size != appsv2.SizeDev { - spec.Sentinel.Enabled = true + for key, spec := range wandb.Spec.Redis { + if spec.ExternalRedis != nil { + continue + } + if spec.ManagedRedis == nil { + spec.ManagedRedis = &appsv2.ManagedRedisSpec{} + } + if spec.ManagedRedis.Name == "" { + spec.ManagedRedis.Name = opstree.DefaultSpecName(wandb.Name, key) + } + if spec.ManagedRedis.Namespace == "" { + spec.ManagedRedis.Namespace = wandb.Namespace + } + if wandb.Spec.Size != appsv2.SizeDev { + spec.ManagedRedis.Sentinel.Enabled = true + } + wandb.Spec.Redis[key] = spec } } @@ -244,71 +267,99 @@ func applyKafkaDefaults(wandb *appsv2.WeightsAndBiases) { spec := wandb.Spec.Kafka.ManagedKafka if spec.Name == "" { - spec.Name = fmt.Sprintf("%s-kafka", wandb.Name) + spec.Name = bufstream.DefaultSpecName(wandb.Name, appsv2.DefaultInstanceName) } if spec.Namespace == "" { spec.Namespace = wandb.Namespace } + + applyManagedServiceAccountDefaults(&spec.ServiceAccount, spec.Name) } func applyObjectStoreDefaults(wandb *appsv2.WeightsAndBiases) { - - if wandb.Spec.ObjectStore.ManagedObjectStore == nil { - if wandb.Spec.ObjectStore.ExternalObjectStore != nil { - return - } - wandb.Spec.ObjectStore.ManagedObjectStore = &appsv2.ManagedObjectStoreSpec{} + if wandb.Spec.ObjectStore == nil { + wandb.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{} } - - spec := wandb.Spec.ObjectStore.ManagedObjectStore - - if spec.Name == "" { - spec.Name = fmt.Sprintf("%s-seaweedfs", wandb.Name) + if len(wandb.Spec.ObjectStore) == 0 { + wandb.Spec.ObjectStore[appsv2.DefaultInstanceName] = appsv2.ObjectStoreSpec{ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{}} } - if spec.Namespace == "" { - spec.Namespace = wandb.Namespace + for key, spec := range wandb.Spec.ObjectStore { + if spec.ExternalObjectStore != nil { + continue + } + if spec.ManagedObjectStore == nil { + spec.ManagedObjectStore = &appsv2.ManagedObjectStoreSpec{} + } + managed := spec.ManagedObjectStore + if managed.Name == "" { + managed.Name = seaweedfs.DefaultSpecName(wandb.Name, key) + } + if managed.Namespace == "" { + managed.Namespace = wandb.Namespace + } + if managed.Config.AccessKey == "" && managed.Config.RootUser != "" { //nolint:staticcheck + managed.Config.AccessKey = managed.Config.RootUser //nolint:staticcheck + } + if managed.Config.AccessKey == "" { + managed.Config.AccessKey = "admin" + } + wandb.Spec.ObjectStore[key] = spec } +} - if spec.Config.AccessKey == "" && spec.Config.RootUser != "" { //nolint:staticcheck - spec.Config.AccessKey = spec.Config.RootUser //nolint:staticcheck +func applyClickHouseDefaults(wandb *appsv2.WeightsAndBiases) { + if wandb.Spec.ClickHouse == nil { + wandb.Spec.ClickHouse = map[string]appsv2.ClickHouseSpec{} } - if spec.Config.AccessKey == "" { - spec.Config.AccessKey = "admin" + if len(wandb.Spec.ClickHouse) == 0 { + wandb.Spec.ClickHouse[appsv2.DefaultInstanceName] = appsv2.ClickHouseSpec{ManagedClickHouse: &appsv2.ManagedClickHouseSpec{}} } -} -func applyClickHouseDefaults(wandb *appsv2.WeightsAndBiases) { - if wandb.Spec.ClickHouse.ManagedClickHouse == nil { - if wandb.Spec.ClickHouse.ExternalClickHouse != nil { - return + for key, spec := range wandb.Spec.ClickHouse { + if spec.ExternalClickHouse != nil { + continue + } + if spec.ManagedClickHouse == nil { + spec.ManagedClickHouse = &appsv2.ManagedClickHouseSpec{} + } + if spec.ManagedClickHouse.Name == "" { + spec.ManagedClickHouse.Name = altinity.DefaultSpecName(wandb.Name, key) } - wandb.Spec.ClickHouse.ManagedClickHouse = &appsv2.ManagedClickHouseSpec{} + if spec.ManagedClickHouse.Namespace == "" { + spec.ManagedClickHouse.Namespace = wandb.Namespace + } + applyManagedServiceAccountDefaults(&spec.ManagedClickHouse.ServiceAccount, spec.ManagedClickHouse.Name) + wandb.Spec.ClickHouse[key] = spec } +} - spec := wandb.Spec.ClickHouse.ManagedClickHouse - - if spec.Name == "" { - spec.Name = fmt.Sprintf("%s-clickhouse", wandb.Name) +func applyManagedServiceAccountDefaults(serviceAccount *appsv2.ManagedServiceAccountSpec, defaultName string) { + if serviceAccount.Create == nil { + serviceAccount.Create = ptr.To(true) } - - if spec.Namespace == "" { - spec.Namespace = wandb.Namespace + if serviceAccount.ServiceAccountName == "" { + serviceAccount.ServiceAccountName = defaultName } } -func validateSpec(_ context.Context, newWandb *appsv2.WeightsAndBiases) (admission.Warnings, error) { +// validateSpec validates the (already defaulted) spec. oldWandb is nil on +// create; update rules use it to skip values unchanged from the stored object. +func validateSpec(_ context.Context, newWandb, oldWandb *appsv2.WeightsAndBiases) (admission.Warnings, error) { var allErrors field.ErrorList var warnings admission.Warnings + allErrors = append(allErrors, validateWandbSpec(newWandb)...) allErrors = append(allErrors, validateMySQLSpec(newWandb)...) allErrors = append(allErrors, validateRedisSpec(newWandb)...) allErrors = append(allErrors, validateObjectStoreSpec(newWandb)...) allErrors = append(allErrors, validateClickHouseSpec(newWandb)...) + allErrors = append(allErrors, validateInfraNames(newWandb, oldWandb)...) networkingErrors, networkingWarnings := validateNetworkingSpec(newWandb) allErrors = append(allErrors, networkingErrors...) warnings = append(warnings, networkingWarnings...) + allErrors = append(allErrors, validateProxySpec(newWandb)...) if len(allErrors) == 0 { return warnings, nil @@ -339,6 +390,22 @@ func validateChanges(_ context.Context, newWandb *appsv2.WeightsAndBiases, oldWa ) } +// validateHasDefaultInstance reports an error when a multi-instance infra type +// defines at least one instance but is missing the reserved default key, which +// the env-var fallback relies on. +func validateHasDefaultInstance[T any](m map[string]T, path *field.Path) field.ErrorList { + if len(m) == 0 { + return nil + } + if _, ok := m[appsv2.DefaultInstanceName]; ok { + return nil + } + return field.ErrorList{field.Required( + path.Key(appsv2.DefaultInstanceName), + fmt.Sprintf("a %q instance is required when other instances are defined", appsv2.DefaultInstanceName), + )} +} + // validateMySQLChanges rejects an update that lowers an explicitly-set replica // count. Moco does not support in-place replica reduction, so catch it at // admission for immediate feedback. A size-driven change leaves replicas unset @@ -346,19 +413,38 @@ func validateChanges(_ context.Context, newWandb *appsv2.WeightsAndBiases, oldWa // directly-edited count. func validateMySQLChanges(newWandb, oldWandb *appsv2.WeightsAndBiases) field.ErrorList { var errors field.ErrorList - mysqlPath := field.NewPath("spec").Child("mysql").Child("managedMysql") - newSpec := newWandb.Spec.MySQL.ManagedMysql - oldSpec := oldWandb.Spec.MySQL.ManagedMysql + mysqlPath := field.NewPath("spec").Child("mysql") - if newSpec == nil || oldSpec == nil { - return errors + for key, newInstance := range newWandb.Spec.MySQL { + oldInstance, ok := oldWandb.Spec.MySQL[key] + if !ok { + continue + } + newSpec := newInstance.ManagedMysql + oldSpec := oldInstance.ManagedMysql + if newSpec == nil || oldSpec == nil { + continue + } + + if oldSpec.Replicas != 0 && newSpec.Replicas != 0 && newSpec.Replicas < oldSpec.Replicas { + errors = append(errors, field.Invalid( + mysqlPath.Key(key).Child("managedMysql").Child("replicas"), + newSpec.Replicas, + "replicas cannot be decreased; Moco does not support in-place replica reduction (use its manual stop-clustering procedure)", + )) + } } - if oldSpec.Replicas != 0 && newSpec.Replicas != 0 && newSpec.Replicas < oldSpec.Replicas { - errors = append(errors, field.Invalid( - mysqlPath.Child("replicas"), - newSpec.Replicas, - "replicas cannot be decreased; Moco does not support in-place replica reduction (use its manual stop-clustering procedure)", + return errors +} + +func validateWandbSpec(wandb *appsv2.WeightsAndBiases) field.ErrorList { + var errors field.ErrorList + + if strings.TrimSpace(wandb.Spec.Wandb.Hostname) == "" { + errors = append(errors, field.Required( + field.NewPath("spec").Child("wandb").Child("hostname"), + "hostname is required", )) } @@ -369,21 +455,26 @@ func validateMySQLSpec(wandb *appsv2.WeightsAndBiases) field.ErrorList { var errors field.ErrorList mysqlPath := field.NewPath("spec").Child("mysql") - if wandb.Spec.MySQL.ManagedMysql != nil && wandb.Spec.MySQL.ExternalMysql != nil { - errors = append(errors, field.Invalid( - mysqlPath, - "", - "managedMysql and externalMysql are mutually exclusive", - )) - } - if spec := wandb.Spec.MySQL.ManagedMysql; spec != nil { - if spec.Replicas != 0 && !appsv2.ValidMysqlReplicaCount(spec.Replicas) { + errors = append(errors, validateHasDefaultInstance(wandb.Spec.MySQL, mysqlPath)...) + + for key, spec := range wandb.Spec.MySQL { + instancePath := mysqlPath.Key(key) + if spec.ManagedMysql != nil && spec.ExternalMysql != nil { errors = append(errors, field.Invalid( - mysqlPath.Child("managedMysql").Child("replicas"), - spec.Replicas, - "replicas must be an odd number (Moco enforces quorum-based replication)", + instancePath, + "", + "managedMysql and externalMysql are mutually exclusive", )) } + if managed := spec.ManagedMysql; managed != nil { + if managed.Replicas != 0 && !appsv2.ValidMysqlReplicaCount(managed.Replicas) { + errors = append(errors, field.Invalid( + instancePath.Child("managedMysql").Child("replicas"), + managed.Replicas, + "replicas must be an odd number (Moco enforces quorum-based replication)", + )) + } + } } return errors @@ -392,54 +483,113 @@ func validateMySQLSpec(wandb *appsv2.WeightsAndBiases) field.ErrorList { func validateRedisSpec(wandb *appsv2.WeightsAndBiases) field.ErrorList { var errors field.ErrorList redisPath := field.NewPath("spec").Child("redis") + _, hasPendingLegacyRedis := wandb.Annotations[v1.RedisPendingAnnotation] - if wandb.Spec.Redis.ManagedRedis != nil && wandb.Spec.Redis.ExternalRedis != nil { - errors = append(errors, field.Invalid( - redisPath, - "", - "managedRedis and externalRedis are mutually exclusive", - )) - } + errors = append(errors, validateHasDefaultInstance(wandb.Spec.Redis, redisPath)...) - spec := wandb.Spec.Redis.ManagedRedis - if spec == nil { - return errors - } - - if spec.StorageSize != "" { - if _, err := resource.ParseQuantity(spec.StorageSize); err != nil { + for key, spec := range wandb.Spec.Redis { + instancePath := redisPath.Key(key) + if spec.ManagedRedis != nil && spec.ExternalRedis != nil { errors = append(errors, field.Invalid( - redisPath.Child("managedRedis").Child("storageSize"), - spec.StorageSize, - "must be a valid resource quantity (e.g., '10Gi')", + instancePath, + "", + "managedRedis and externalRedis are mutually exclusive", )) } + + if externalRedis := spec.ExternalRedis; externalRedis != nil && !hasPendingLegacyRedis { + externalPath := instancePath.Child("externalRedis") + errors = append(errors, validateRequiredSecretSelector(externalRedis.Host, externalPath.Child("host"))...) + errors = append(errors, validateRequiredSecretSelector(externalRedis.Port, externalPath.Child("port"))...) + } + + if spec.ManagedRedis == nil { + continue + } + + if spec.ManagedRedis.StorageSize != "" { + if _, err := resource.ParseQuantity(spec.ManagedRedis.StorageSize); err != nil { + errors = append(errors, field.Invalid( + instancePath.Child("managedRedis").Child("storageSize"), + spec.ManagedRedis.StorageSize, + "must be a valid resource quantity (e.g., '10Gi')", + )) + } + } } return errors } +func validateRequiredSecretSelector(selector corev1.SecretKeySelector, path *field.Path) field.ErrorList { + var errors field.ErrorList + if selector.Name == "" { + errors = append(errors, field.Required(path.Child("name"), "secret name is required")) + } + if selector.Key == "" { + errors = append(errors, field.Required(path.Child("key"), "secret key is required")) + } + return errors +} + func validateObjectStoreSpec(wandb *appsv2.WeightsAndBiases) field.ErrorList { var errors field.ErrorList objectStorePath := field.NewPath("spec").Child("objectStore") - if wandb.Spec.ObjectStore.ManagedObjectStore != nil && wandb.Spec.ObjectStore.ExternalObjectStore != nil { - errors = append(errors, field.Invalid( - objectStorePath, - "", - "managedObjectStore and externalObjectStore are mutually exclusive", - )) - } + errors = append(errors, validateHasDefaultInstance(wandb.Spec.ObjectStore, objectStorePath)...) - if ext := wandb.Spec.ObjectStore.ExternalObjectStore; ext != nil { - extPath := objectStorePath.Child("externalObjectStore") - // provider is sourced from a secret key, so it is resolved and defaulted at reconcile time, not here. - if ext.Bucket.Name == "" { - errors = append(errors, field.Required( - extPath.Child("bucket"), - "externalObjectStore requires a bucket secret reference", + for key, spec := range wandb.Spec.ObjectStore { + if spec.ManagedObjectStore != nil && spec.ExternalObjectStore != nil { + errors = append(errors, field.Invalid( + objectStorePath.Key(key), + "", + "managedObjectStore and externalObjectStore are mutually exclusive", )) } + + if mgd := spec.ManagedObjectStore; mgd != nil && mgd.StorageSize != "" { + // Reject non-positive too: "0"/"-5Gi" parse fine but only fail later at PVC creation. + if q, err := resource.ParseQuantity(mgd.StorageSize); err != nil || q.Sign() <= 0 { + errors = append(errors, field.Invalid( + objectStorePath.Key(key).Child("managedObjectStore").Child("storageSize"), + mgd.StorageSize, + "must be a positive resource quantity (e.g., '10Gi')", + )) + } + } + + if mgd := spec.ManagedObjectStore; mgd != nil && mgd.SeaweedObjectStoreSpec.FilerStorageSize != "" { + if q, err := resource.ParseQuantity(mgd.SeaweedObjectStoreSpec.FilerStorageSize); err != nil || q.Sign() <= 0 { + errors = append(errors, field.Invalid( + objectStorePath.Key(key).Child("managedObjectStore").Child("SeaweedObjectStoreSpec").Child("filerStorageSize"), + mgd.SeaweedObjectStoreSpec.FilerStorageSize, + "must be a positive resource quantity (e.g., '10Gi')", + )) + } + } + + if mgd := spec.ManagedObjectStore; mgd != nil { + // Only check the copies/replicas relationship when the user pinned replicas; + // otherwise the manifest supplies it at reconcile and seaweedReplication clamps it. + if mgd.Copies < 0 || (mgd.Replicas > 0 && mgd.Copies > mgd.Replicas-1) { + errors = append(errors, field.Invalid( + objectStorePath.Key(key).Child("managedObjectStore").Child("copies"), + mgd.Copies, + "copies cannot be negative or exceed replicas-1 (one copy per other data node)", + )) + } + } + + if ext := spec.ExternalObjectStore; ext != nil { + extPath := objectStorePath.Key(key).Child("externalObjectStore") + // provider is sourced from a secret key, so it is resolved and defaulted at reconcile time, not here. + if _, ok := wandb.GetAnnotations()[v1.BucketPendingAnnotation]; !ok && ext.Bucket.Name == "" { + errors = append(errors, field.Required( + extPath.Child("bucket"), + "externalObjectStore requires a bucket secret reference", + )) + } + } } return errors @@ -449,32 +599,208 @@ func validateClickHouseSpec(wandb *appsv2.WeightsAndBiases) field.ErrorList { var errors field.ErrorList chPath := field.NewPath("spec").Child("clickhouse") - if wandb.Spec.ClickHouse.ManagedClickHouse != nil && wandb.Spec.ClickHouse.ExternalClickHouse != nil { + errors = append(errors, validateHasDefaultInstance(wandb.Spec.ClickHouse, chPath)...) + + for key, spec := range wandb.Spec.ClickHouse { + instancePath := chPath.Key(key) + if spec.ManagedClickHouse != nil && spec.ExternalClickHouse != nil { + errors = append(errors, field.Invalid( + instancePath, + "", + "managedClickhouse and externalClickhouse are mutually exclusive", + )) + } + + managed := spec.ManagedClickHouse + if managed == nil { + continue + } + + // Managed ClickHouse stores table data in the object store, so one must be configured. + if len(wandb.Spec.ObjectStore) == 0 { + errors = append(errors, field.Invalid( + instancePath.Child("managedClickhouse"), + "", + "managed ClickHouse stores data in the object store; configure spec.objectStore (managed or external)", + )) + } + + // Keeper requires an odd number of replicas to form a quorum. + if managed.Keeper.Replicas != 0 && managed.Keeper.Replicas%2 == 0 { + errors = append(errors, field.Invalid( + instancePath.Child("managedClickhouse").Child("keeper").Child("replicas"), + managed.Keeper.Replicas, + "replicas must be an odd number so the Keeper ensemble can form a quorum", + )) + } + + for _, sz := range []struct { + value string + path *field.Path + }{ + {managed.StorageSize, instancePath.Child("managedClickhouse").Child("storageSize")}, + {managed.Keeper.StorageSize, instancePath.Child("managedClickhouse").Child("keeper").Child("storageSize")}, + } { + if sz.value == "" { + continue + } + if _, err := resource.ParseQuantity(sz.value); err != nil { + errors = append(errors, field.Invalid(sz.path, sz.value, "must be a valid resource quantity (e.g., '10Gi')")) + } + } + } + + return errors +} + +// validateInfraNames rejects managed infra names whose derived object names +// cannot be deployed (vendor operators wedge silently past DNS-1123 limits). +// Empty names are the defaulter's to fill; on update only changed names are +// checked (per instance key), so pre-existing CRs stay updatable and deletable. +func validateInfraNames(newWandb, oldWandb *appsv2.WeightsAndBiases) field.ErrorList { + var errors field.ErrorList + + changed := func(oldName, name string) bool { + if name == "" { + return false + } + return oldWandb == nil || oldName != name + } + + for key, spec := range newWandb.Spec.ClickHouse { + managed := spec.ManagedClickHouse + if managed == nil { + continue + } + oldName := "" + if oldWandb != nil { + if old, ok := oldWandb.Spec.ClickHouse[key]; ok && old.ManagedClickHouse != nil { + oldName = old.ManagedClickHouse.Name + } + } + if changed(oldName, managed.Name) { + if err := altinity.ValidateDerivedNames(managed); err != nil { + errors = append(errors, field.Invalid( + field.NewPath("spec").Child("clickhouse").Key(key).Child("managedClickhouse").Child("name"), + managed.Name, err.Error(), + )) + } + } + } + + for key, spec := range newWandb.Spec.MySQL { + managed := spec.ManagedMysql + if managed == nil { + continue + } + oldName := "" + if oldWandb != nil { + if old, ok := oldWandb.Spec.MySQL[key]; ok && old.ManagedMysql != nil { + oldName = old.ManagedMysql.Name + } + } + if changed(oldName, managed.Name) { + errors = append(errors, validateInfraName( + field.NewPath("spec").Child("mysql").Key(key).Child("managedMysql").Child("name"), + managed.Name, moco.MaxClusterNameLength, + "Moco rejects MySQLCluster names longer than 40 characters", + )...) + } + } + + for key, spec := range newWandb.Spec.Redis { + managed := spec.ManagedRedis + if managed == nil { + continue + } + oldName := "" + if oldWandb != nil { + if old, ok := oldWandb.Spec.Redis[key]; ok && old.ManagedRedis != nil { + oldName = old.ManagedRedis.Name + } + } + if changed(oldName, managed.Name) { + errors = append(errors, validateInfraName( + field.NewPath("spec").Child("redis").Key(key).Child("managedRedis").Child("name"), + managed.Name, opstree.MaxSpecNameLength, + "derived Redis workload and Service names must fit 63-character DNS-1123 labels", + )...) + } + } + + if spec := newWandb.Spec.Kafka.ManagedKafka; spec != nil { + oldName := "" + if oldWandb != nil && oldWandb.Spec.Kafka.ManagedKafka != nil { + oldName = oldWandb.Spec.Kafka.ManagedKafka.Name + } + if changed(oldName, spec.Name) { + errors = append(errors, validateInfraName( + field.NewPath("spec").Child("kafka").Child("managedKafka").Child("name"), + spec.Name, bufstream.MaxSpecNameLength(), + "derived Kafka/etcd pod names must fit 63-character DNS-1123 labels", + )...) + } + } + + for key, spec := range newWandb.Spec.ObjectStore { + managed := spec.ManagedObjectStore + if managed == nil { + continue + } + oldName := "" + if oldWandb != nil { + if old, ok := oldWandb.Spec.ObjectStore[key]; ok && old.ManagedObjectStore != nil { + oldName = old.ManagedObjectStore.Name + } + } + if changed(oldName, managed.Name) { + errors = append(errors, validateInfraName( + field.NewPath("spec").Child("objectStore").Key(key).Child("managedObjectStore").Child("name"), + managed.Name, seaweedfs.MaxSpecNameLength, + "derived SeaweedFS workload and Service names must fit 63-character DNS-1123 labels", + )...) + } + } + + return errors +} + +func validateInfraName(path *field.Path, name string, budget int, why string) field.ErrorList { + var errors field.ErrorList + if labelErrs := validation.IsDNS1123Label(name); len(labelErrs) > 0 { + errors = append(errors, field.Invalid( + path, name, + fmt.Sprintf("must be a valid DNS-1123 label: %s", strings.Join(labelErrs, "; ")), + )) + } else if len(name) > budget { errors = append(errors, field.Invalid( - chPath, - "", - "managedClickhouse and externalClickhouse are mutually exclusive", + path, name, + fmt.Sprintf("must be at most %d characters: %s", budget, why), )) } - return errors } func validateRedisChanges(newWandb, oldWandb *appsv2.WeightsAndBiases) field.ErrorList { var errors field.ErrorList - redisPath := field.NewPath("spec").Child("redis").Child("managedRedis") - newSpec := newWandb.Spec.Redis.ManagedRedis - oldSpec := oldWandb.Spec.Redis.ManagedRedis + redisPath := field.NewPath("spec").Child("redis") - if newSpec == nil { - return errors - } + for key, newInstance := range newWandb.Spec.Redis { + newSpec := newInstance.ManagedRedis + if newSpec == nil { + continue + } + oldInstance, ok := oldWandb.Spec.Redis[key] + if !ok || oldInstance.ManagedRedis == nil { + continue + } + oldSpec := oldInstance.ManagedRedis + instancePath := redisPath.Key(key).Child("managedRedis") - if oldSpec != nil { if oldSpec.StorageSize != "" && oldSpec.StorageSize != newSpec.StorageSize { errors = append(errors, field.Invalid( - redisPath.Child("storageSize"), + instancePath.Child("storageSize"), newSpec.StorageSize, "storageSize may not be changed", )) @@ -482,7 +808,7 @@ func validateRedisChanges(newWandb, oldWandb *appsv2.WeightsAndBiases) field.Err if oldSpec.Namespace != newSpec.Namespace { errors = append(errors, field.Invalid( - redisPath.Child("namespace"), + instancePath.Child("namespace"), newSpec.Namespace, "namespace may not be changed", )) @@ -490,7 +816,7 @@ func validateRedisChanges(newWandb, oldWandb *appsv2.WeightsAndBiases) field.Err if oldSpec.Sentinel.Enabled != newSpec.Sentinel.Enabled { errors = append(errors, field.Invalid( - redisPath.Child("sentinel").Child("enabled"), + instancePath.Child("sentinel").Child("enabled"), newSpec.Sentinel.Enabled, "Redis Sentinel cannot be toggled between enabled and disabled (yet)", )) @@ -500,6 +826,67 @@ func validateRedisChanges(newWandb, oldWandb *appsv2.WeightsAndBiases) field.Err return errors } +// validateProxySpec validates spec.global.proxy: each proxy value sets exactly +// one of value|valueFrom, a literal value parses as an http(s) URL with no +// userinfo (credentials must use valueFrom so they never land in the CR), and +// noProxy entries are non-empty and comma-free (the operator owns the join). +func validateProxySpec(wandb *appsv2.WeightsAndBiases) field.ErrorList { + var errors field.ErrorList + if wandb.Spec.Global.Proxy == nil { + return errors + } + proxy := wandb.Spec.Global.Proxy + base := field.NewPath("spec").Child("global").Child("proxy") + + validateValue := func(pv *appsv2.ProxyValue, child string) { + if pv == nil { + return + } + p := base.Child(child) + hasValue := pv.Value != "" + hasValueFrom := pv.ValueFrom != nil + switch { + case hasValue && hasValueFrom: + errors = append(errors, field.Invalid(p, pv, "set exactly one of value or valueFrom, not both")) + return + case !hasValue && !hasValueFrom: + errors = append(errors, field.Required(p, "one of value or valueFrom is required")) + return + } + if hasValueFrom && pv.ValueFrom.SecretKeyRef == nil { + errors = append(errors, field.Required(p.Child("valueFrom").Child("secretKeyRef"), + "valueFrom requires secretKeyRef")) + } + if hasValue { + parsed, err := url.Parse(pv.Value) + if err != nil { + errors = append(errors, field.Invalid(p.Child("value"), pv.Value, "must be a valid URL")) + return + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + errors = append(errors, field.Invalid(p.Child("value"), pv.Value, "scheme must be http or https")) + } + if parsed.User != nil { + errors = append(errors, field.Invalid(p.Child("value"), "[redacted]", + "must not contain credentials (userinfo); use valueFrom with a Secret for authenticated proxies")) + } + } + } + validateValue(proxy.HTTPProxy, "httpProxy") + validateValue(proxy.HTTPSProxy, "httpsProxy") + + for i, entry := range proxy.NoProxy { + p := base.Child("noProxy").Index(i) + if strings.TrimSpace(entry) == "" { + errors = append(errors, field.Invalid(p, entry, "must not be empty")) + } + if strings.Contains(entry, ",") { + errors = append(errors, field.Invalid(p, entry, "must not contain commas; use separate list entries")) + } + } + return errors +} + func validateNetworkingSpec(wandb *appsv2.WeightsAndBiases) (field.ErrorList, admission.Warnings) { var errors field.ErrorList var warnings admission.Warnings diff --git a/internal/webhook/v2/weightsandbiases_webhook_test.go b/internal/webhook/v2/weightsandbiases_webhook_test.go index 444f787b..8bb0d2ae 100644 --- a/internal/webhook/v2/weightsandbiases_webhook_test.go +++ b/internal/webhook/v2/weightsandbiases_webhook_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "github.com/wandb/operator/api/v1" appsv2 "github.com/wandb/operator/api/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,7 +40,9 @@ var _ = Describe("WeightsAndBiases Webhook", func() { BeforeEach(func() { ctx = context.Background() obj = &appsv2.WeightsAndBiases{ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "test-ns"}} + obj.Spec.Wandb.Hostname = "https://wandb.example.com" oldObj = &appsv2.WeightsAndBiases{ObjectMeta: metav1.ObjectMeta{Name: "wandb", Namespace: "test-ns"}} + oldObj.Spec.Wandb.Hostname = "https://wandb.example.com" validator = WeightsAndBiasesCustomValidator{} defaulter = WeightsAndBiasesCustomDefaulter{} }) @@ -48,10 +51,10 @@ var _ = Describe("WeightsAndBiases Webhook", func() { It("sets webhook defaults and preserves user-provided values", func() { obj.Spec.RetentionPolicy.OnDelete = "" obj.Spec.Wandb.ManifestRepository = "example.com/wandb/server-manifest" - obj.Spec.MySQL.ManagedMysql = &appsv2.ManagedMysqlSpec{} - obj.Spec.Redis.ManagedRedis = &appsv2.ManagedRedisSpec{} + obj.Spec.MySQL = map[string]appsv2.MySQLSpec{appsv2.DefaultInstanceName: {ManagedMysql: &appsv2.ManagedMysqlSpec{}}} + obj.Spec.Redis = map[string]appsv2.RedisSpec{appsv2.DefaultInstanceName: {ManagedRedis: &appsv2.ManagedRedisSpec{}}} obj.Spec.Kafka.ManagedKafka = &appsv2.ManagedKafkaSpec{} - obj.Spec.ObjectStore.ManagedObjectStore = &appsv2.ManagedObjectStoreSpec{} + obj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{}}} Expect(defaulter.Default(ctx, obj)).To(Succeed()) Expect(obj.Spec.Size).To(Equal(appsv2.SizeDev)) @@ -66,10 +69,10 @@ var _ = Describe("WeightsAndBiases Webhook", func() { Expect(*obj.Spec.Wandb.ServiceAccount.Create).To(BeTrue()) Expect(obj.Spec.Wandb.ServiceAccount.ServiceAccountName).To(Equal("wandb-app")) Expect(obj.Status.Wandb.Applications).ToNot(BeNil()) - Expect(obj.Spec.MySQL.ManagedMysql.Namespace).To(Equal("test-ns")) - Expect(obj.Spec.Redis.ManagedRedis.Namespace).To(Equal("test-ns")) + Expect(obj.Spec.MySQL[appsv2.DefaultInstanceName].ManagedMysql.Namespace).To(Equal("test-ns")) + Expect(obj.Spec.Redis[appsv2.DefaultInstanceName].ManagedRedis.Namespace).To(Equal("test-ns")) Expect(obj.Spec.Kafka.ManagedKafka.Namespace).To(Equal("test-ns")) - Expect(obj.Spec.ObjectStore.ManagedObjectStore.Namespace).To(Equal("test-ns")) + Expect(obj.Spec.ObjectStore[appsv2.DefaultInstanceName].ManagedObjectStore.Namespace).To(Equal("test-ns")) }) It("does not override already set values", func() { @@ -84,12 +87,12 @@ var _ = Describe("WeightsAndBiases Webhook", func() { obj.Spec.Wandb.InternalServiceAuth.OIDCIssuer = "https://issuer.example.com" obj.Spec.Wandb.ServiceAccount.Create = boolPtr(false) obj.Spec.Wandb.ServiceAccount.ServiceAccountName = "custom-sa" - obj.Spec.MySQL.ManagedMysql = &appsv2.ManagedMysqlSpec{ + obj.Spec.MySQL = map[string]appsv2.MySQLSpec{appsv2.DefaultInstanceName: {ManagedMysql: &appsv2.ManagedMysqlSpec{ Namespace: "custom-moco", - } - obj.Spec.Redis.ManagedRedis = &appsv2.ManagedRedisSpec{Namespace: "custom-redis"} + }}} + obj.Spec.Redis = map[string]appsv2.RedisSpec{appsv2.DefaultInstanceName: {ManagedRedis: &appsv2.ManagedRedisSpec{Namespace: "custom-redis"}}} obj.Spec.Kafka.ManagedKafka = &appsv2.ManagedKafkaSpec{Namespace: "custom-kafka"} - obj.Spec.ObjectStore.ManagedObjectStore = &appsv2.ManagedObjectStoreSpec{Namespace: "custom-objectstore"} + obj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{Namespace: "custom-objectstore"}}} obj.Status.Wandb.Applications = map[string]appsv2.ApplicationStatus{"api": {}} Expect(defaulter.Default(ctx, obj)).To(Succeed()) @@ -103,10 +106,10 @@ var _ = Describe("WeightsAndBiases Webhook", func() { Expect(obj.Spec.Wandb.InternalServiceAuth.OIDCIssuer).To(Equal("https://issuer.example.com")) Expect(*obj.Spec.Wandb.ServiceAccount.Create).To(BeFalse()) Expect(obj.Spec.Wandb.ServiceAccount.ServiceAccountName).To(Equal("custom-sa")) - Expect(obj.Spec.MySQL.ManagedMysql.Namespace).To(Equal("custom-moco")) - Expect(obj.Spec.Redis.ManagedRedis.Namespace).To(Equal("custom-redis")) + Expect(obj.Spec.MySQL[appsv2.DefaultInstanceName].ManagedMysql.Namespace).To(Equal("custom-moco")) + Expect(obj.Spec.Redis[appsv2.DefaultInstanceName].ManagedRedis.Namespace).To(Equal("custom-redis")) Expect(obj.Spec.Kafka.ManagedKafka.Namespace).To(Equal("custom-kafka")) - Expect(obj.Spec.ObjectStore.ManagedObjectStore.Namespace).To(Equal("custom-objectstore")) + Expect(obj.Spec.ObjectStore[appsv2.DefaultInstanceName].ManagedObjectStore.Namespace).To(Equal("custom-objectstore")) Expect(obj.Status.Wandb.Applications).To(HaveKey("api")) }) @@ -120,12 +123,80 @@ var _ = Describe("WeightsAndBiases Webhook", func() { Context("When creating or updating WeightsAndBiases under Validating Webhook", func() { It("allows create when ManagedRedis is nil", func() { warnings, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) }) + + It("rejects external Redis without host and port selectors", func() { + obj.Spec.Redis = map[string]appsv2.RedisSpec{ + appsv2.DefaultInstanceName: {ExternalRedis: &appsv2.RedisConnection{}}, + } + + _, err := validator.ValidateCreate(ctx, obj) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("externalRedis.host.name")) + Expect(err.Error()).To(ContainSubstring("externalRedis.host.key")) + Expect(err.Error()).To(ContainSubstring("externalRedis.port.name")) + Expect(err.Error()).To(ContainSubstring("externalRedis.port.key")) + }) + + It("allows external Redis with host and port selectors", func() { + obj.Spec.Redis = map[string]appsv2.RedisSpec{ + appsv2.DefaultInstanceName: { + ExternalRedis: &appsv2.RedisConnection{ + Host: secretKeySelector("redis", "host"), + Port: secretKeySelector("redis", "port"), + }, + }, + } + + warnings, err := validator.ValidateCreate(ctx, obj) + + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + }) + + It("allows external Redis while v1 literal values are pending materialization", func() { + obj.Annotations = map[string]string{ + appsv1.RedisPendingAnnotation: `{"host":"redis.example.com","port":"6379"}`, + } + obj.Spec.Redis = map[string]appsv2.RedisSpec{ + appsv2.DefaultInstanceName: {ExternalRedis: &appsv2.RedisConnection{}}, + } + + warnings, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) Expect(warnings).To(BeEmpty()) }) + It("rejects create when hostname is missing", func() { + obj.Spec.Wandb.Hostname = "" + + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("spec.wandb.hostname")) + Expect(err.Error()).To(ContainSubstring("hostname is required")) + }) + + It("rejects create when hostname is only whitespace", func() { + obj.Spec.Wandb.Hostname = " " + + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("spec.wandb.hostname")) + }) + + It("rejects update when hostname is missing", func() { + obj.Spec.Wandb.Hostname = "" + + _, err := validator.ValidateUpdate(ctx, oldObj, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("spec.wandb.hostname")) + }) + It("rejects create when Redis storage size is invalid", func() { - obj.Spec.Redis.ManagedRedis = &appsv2.ManagedRedisSpec{StorageSize: "bad-size"} + obj.Spec.Redis = map[string]appsv2.RedisSpec{appsv2.DefaultInstanceName: {ManagedRedis: &appsv2.ManagedRedisSpec{StorageSize: "bad-size"}}} _, err := validator.ValidateCreate(ctx, obj) Expect(err).To(HaveOccurred()) @@ -133,8 +204,8 @@ var _ = Describe("WeightsAndBiases Webhook", func() { }) It("rejects redis namespace changes on update", func() { - oldObj.Spec.Redis.ManagedRedis = &appsv2.ManagedRedisSpec{Namespace: "redis-a"} - obj.Spec.Redis.ManagedRedis = &appsv2.ManagedRedisSpec{Namespace: "redis-b"} + oldObj.Spec.Redis = map[string]appsv2.RedisSpec{appsv2.DefaultInstanceName: {ManagedRedis: &appsv2.ManagedRedisSpec{Namespace: "redis-a"}}} + obj.Spec.Redis = map[string]appsv2.RedisSpec{appsv2.DefaultInstanceName: {ManagedRedis: &appsv2.ManagedRedisSpec{Namespace: "redis-b"}}} _, err := validator.ValidateUpdate(ctx, oldObj, obj) Expect(err).To(HaveOccurred()) @@ -142,8 +213,8 @@ var _ = Describe("WeightsAndBiases Webhook", func() { }) It("rejects redis storage size changes when already set", func() { - oldObj.Spec.Redis.ManagedRedis = &appsv2.ManagedRedisSpec{Namespace: "redis", StorageSize: "10Gi"} - obj.Spec.Redis.ManagedRedis = &appsv2.ManagedRedisSpec{Namespace: "redis", StorageSize: "20Gi"} + oldObj.Spec.Redis = map[string]appsv2.RedisSpec{appsv2.DefaultInstanceName: {ManagedRedis: &appsv2.ManagedRedisSpec{Namespace: "redis", StorageSize: "10Gi"}}} + obj.Spec.Redis = map[string]appsv2.RedisSpec{appsv2.DefaultInstanceName: {ManagedRedis: &appsv2.ManagedRedisSpec{Namespace: "redis", StorageSize: "20Gi"}}} _, err := validator.ValidateUpdate(ctx, oldObj, obj) Expect(err).To(HaveOccurred()) @@ -151,8 +222,8 @@ var _ = Describe("WeightsAndBiases Webhook", func() { }) It("allows redis storage size to be initially set on update", func() { - oldObj.Spec.Redis.ManagedRedis = &appsv2.ManagedRedisSpec{Namespace: "redis", StorageSize: ""} - obj.Spec.Redis.ManagedRedis = &appsv2.ManagedRedisSpec{Namespace: "redis", StorageSize: "20Gi"} + oldObj.Spec.Redis = map[string]appsv2.RedisSpec{appsv2.DefaultInstanceName: {ManagedRedis: &appsv2.ManagedRedisSpec{Namespace: "redis", StorageSize: ""}}} + obj.Spec.Redis = map[string]appsv2.RedisSpec{appsv2.DefaultInstanceName: {ManagedRedis: &appsv2.ManagedRedisSpec{Namespace: "redis", StorageSize: "20Gi"}}} warnings, err := validator.ValidateUpdate(ctx, oldObj, obj) Expect(err).NotTo(HaveOccurred()) @@ -160,8 +231,8 @@ var _ = Describe("WeightsAndBiases Webhook", func() { }) It("rejects decreasing managed MySQL replicas on update", func() { - oldObj.Spec.MySQL.ManagedMysql = &appsv2.ManagedMysqlSpec{Replicas: 3} - obj.Spec.MySQL.ManagedMysql = &appsv2.ManagedMysqlSpec{Replicas: 1} + oldObj.Spec.MySQL = map[string]appsv2.MySQLSpec{appsv2.DefaultInstanceName: {ManagedMysql: &appsv2.ManagedMysqlSpec{Replicas: 3}}} + obj.Spec.MySQL = map[string]appsv2.MySQLSpec{appsv2.DefaultInstanceName: {ManagedMysql: &appsv2.ManagedMysqlSpec{Replicas: 1}}} _, err := validator.ValidateUpdate(ctx, oldObj, obj) Expect(err).To(HaveOccurred()) @@ -169,14 +240,117 @@ var _ = Describe("WeightsAndBiases Webhook", func() { }) It("allows increasing managed MySQL replicas on update", func() { - oldObj.Spec.MySQL.ManagedMysql = &appsv2.ManagedMysqlSpec{Replicas: 1} - obj.Spec.MySQL.ManagedMysql = &appsv2.ManagedMysqlSpec{Replicas: 3} + oldObj.Spec.MySQL = map[string]appsv2.MySQLSpec{appsv2.DefaultInstanceName: {ManagedMysql: &appsv2.ManagedMysqlSpec{Replicas: 1}}} + obj.Spec.MySQL = map[string]appsv2.MySQLSpec{appsv2.DefaultInstanceName: {ManagedMysql: &appsv2.ManagedMysqlSpec{Replicas: 3}}} + + warnings, err := validator.ValidateUpdate(ctx, oldObj, obj) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + }) + + It("rejects managed ClickHouse when no object store is configured", func() { + obj.Spec.ClickHouse = map[string]appsv2.ClickHouseSpec{appsv2.DefaultInstanceName: {ManagedClickHouse: &appsv2.ManagedClickHouseSpec{}}} + + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("object store")) + }) + + It("allows managed ClickHouse when an object store is configured", func() { + obj.Spec.ClickHouse = map[string]appsv2.ClickHouseSpec{appsv2.DefaultInstanceName: {ManagedClickHouse: &appsv2.ManagedClickHouseSpec{}}} + obj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{}}} + + warnings, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + }) + + It("rejects a managed ClickHouse name whose derived object names cannot be deployed", func() { + obj.Spec.ClickHouse = map[string]appsv2.ClickHouseSpec{appsv2.DefaultInstanceName: {ManagedClickHouse: &appsv2.ManagedClickHouseSpec{Name: "wandb-legacy-overrides-v1-clickhouse"}}} + obj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{}}} + + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("cannot be deployed")) + Expect(err.Error()).To(ContainSubstring("deploy-confd")) + }) + + It("rejects a managed MySQL name beyond Moco's cluster-name cap", func() { + obj.Spec.MySQL = map[string]appsv2.MySQLSpec{appsv2.DefaultInstanceName: {ManagedMysql: &appsv2.ManagedMysqlSpec{ + Name: "a-managed-mysql-name-that-is-well-past-forty-characters", + }}} + + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Moco")) + }) + + It("grandfathers an over-budget name that is unchanged on update", func() { + longName := "wandb-legacy-overrides-v1-clickhouse" + oldObj.Spec.ClickHouse = map[string]appsv2.ClickHouseSpec{appsv2.DefaultInstanceName: {ManagedClickHouse: &appsv2.ManagedClickHouseSpec{Name: longName}}} + oldObj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{}}} + obj.Spec.ClickHouse = map[string]appsv2.ClickHouseSpec{appsv2.DefaultInstanceName: {ManagedClickHouse: &appsv2.ManagedClickHouseSpec{Name: longName}}} + obj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{}}} warnings, err := validator.ValidateUpdate(ctx, oldObj, obj) Expect(err).NotTo(HaveOccurred()) Expect(warnings).To(BeEmpty()) }) + It("rejects changing a name to one that cannot be deployed on update", func() { + oldObj.Spec.ClickHouse = map[string]appsv2.ClickHouseSpec{appsv2.DefaultInstanceName: {ManagedClickHouse: &appsv2.ManagedClickHouseSpec{Name: "wandb-chi"}}} + oldObj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{}}} + obj.Spec.ClickHouse = map[string]appsv2.ClickHouseSpec{appsv2.DefaultInstanceName: {ManagedClickHouse: &appsv2.ManagedClickHouseSpec{Name: "wandb-legacy-overrides-v1-clickhouse"}}} + obj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{}}} + + _, err := validator.ValidateUpdate(ctx, oldObj, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("cannot be deployed")) + }) + + It("allows object store copies within the data-node count", func() { + obj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{Replicas: 3, Copies: 2}}} + + warnings, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + }) + + It("rejects object store copies that exceed replicas-1", func() { + obj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{Replicas: 3, Copies: 3}}} + + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("copies")) + }) + + It("rejects negative object store copies", func() { + obj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{Copies: -1}}} + + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("copies")) + }) + + It("allows object store copies when replicas is unset (deferred to reconcile)", func() { + obj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ManagedObjectStore: &appsv2.ManagedObjectStoreSpec{Copies: 2}}} + + warnings, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + }) + + It("rejects even Keeper replica counts", func() { + obj.Spec.ClickHouse = map[string]appsv2.ClickHouseSpec{appsv2.DefaultInstanceName: {ManagedClickHouse: &appsv2.ManagedClickHouseSpec{ + Keeper: appsv2.ClickHouseKeeperSpec{Replicas: 2}, + }}} + obj.Spec.ObjectStore = map[string]appsv2.ObjectStoreSpec{appsv2.DefaultInstanceName: {ExternalObjectStore: &appsv2.ObjectStoreConnection{}}} + + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("odd number")) + }) + It("rejects gatewayAPI config when mode is ingress", func() { obj.Spec.Networking.Mode = appsv2.NetworkingModeIngress obj.Spec.Networking.GatewayAPI = &appsv2.GatewayAPIConfig{ @@ -236,3 +410,10 @@ var _ = Describe("WeightsAndBiases Webhook", func() { func boolPtr(v bool) *bool { return &v } + +func secretKeySelector(name, key string) corev1.SecretKeySelector { + return corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: name}, + Key: key, + } +} diff --git a/pkg/utils/connection_secrets.go b/pkg/utils/connection_secrets.go new file mode 100644 index 00000000..00911cb6 --- /dev/null +++ b/pkg/utils/connection_secrets.go @@ -0,0 +1,38 @@ +package utils + +import ( + "context" + "fmt" + "strings" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// connSecretResolver resolves SecretKeySelectors from an ObjectStoreConnection, +// caching each referenced secret so a connection spanning multiple secrets is +// fetched once per secret. +type ConnSecretResolver struct { + Client client.Client + Namespace string + Cache map[string]*v1.Secret +} + +// value returns the trimmed value the selector points at, or "" if the selector +// is unset or the key is absent. +func (r *ConnSecretResolver) Value(ctx context.Context, sel v1.SecretKeySelector) (string, error) { + if sel.Name == "" || sel.Key == "" { + return "", nil + } + secret, ok := r.Cache[sel.Name] + if !ok { + secret = &v1.Secret{} + key := types.NamespacedName{Namespace: r.Namespace, Name: sel.Name} + if err := r.Client.Get(ctx, key, secret); err != nil { + return "", fmt.Errorf("read object store connection secret %q: %w", key, err) + } + r.Cache[sel.Name] = secret + } + return strings.TrimSpace(string(secret.Data[sel.Key])), nil +} diff --git a/pkg/vendored/altinity-clickhouse/README.md b/pkg/vendored/altinity-clickhouse/README.md index 8360d614..9ff50764 100644 --- a/pkg/vendored/altinity-clickhouse/README.md +++ b/pkg/vendored/altinity-clickhouse/README.md @@ -56,6 +56,18 @@ The following DeepCopyInto methods in `clickhouse.altinity.com/v1/zz_generated.d - **OperatorConfigTemplate** (lines 2034-2038): Removed `*out = *in` shallow copy - **Status** (lines 2656-2747): Removed `*out = *in` shallow copy, explicitly copied all non-mutex fields, commented out `mu` field copy +The same class of fixes was applied to +`clickhouse-keeper.altinity.com/v1/zz_generated.deepcopy.go` (commented out the +`*out = *in` shallow copy and the mutex field copies, explicitly copying the +non-mutex scalar fields for `Status`): + +- **ClickHouseKeeperInstallation** — commented out the shallow copy and the + `statusCreatorMutex` / `runtimeCreatorMutex` copies +- **ClickHouseKeeperInstallationRuntime** — commented out the shallow copy and + the `commonConfigMutex` copy +- **Status** — replaced the shallow copy with explicit non-mutex field copies and + commented out the `mu` (sync.RWMutex) copy + ### Mutex Copy in MergeFrom - **clickhouse.altinity.com/v1/type_configuration_chop.go** (line 816): Changed `mergo.Merge(c, *from, ...)` to `mergo.Merge(c, from, ...)` to pass pointer instead of dereferencing (which would copy mutexes) @@ -106,6 +118,14 @@ CRD files are located in `pkg/vendored/altinity-clickhouse/crds/`: - Generated DeepCopy methods (`zz_generated.deepcopy.go`) - API registration and scheme builder - Configuration helpers +- `clickhouse-keeper.altinity.com/v1/` - ClickHouseKeeperInstallation (CHK) CRD + types, used to provision the ClickHouse Keeper ensemble that backs + ReplicatedMergeTree replication. Same upstream version (release-0.26.3); reuses + many shared types from `clickhouse.altinity.com/v1`. + - `api_group.go` parent package holding `APIGroupName` + (`clickhouse-keeper.altinity.com`) + - Same import-path rewrites as the CHI package (see "Import Path Updates") + - Same DeepCopy mutex fixes applied (see below) ### Supporting Packages - `common/` - Common types and constants shared across the operator diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/api_group.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/api_group.go new file mode 100644 index 00000000..3731453c --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/api_group.go @@ -0,0 +1,20 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clickhouse_keeper_altinity_com + +const ( + // APIGroupName is the group name of the ClickHouse Keeper Operator API. + APIGroupName = "clickhouse-keeper.altinity.com" +) diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/api_register.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/api_register.go new file mode 100644 index 00000000..46157c68 --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/api_register.go @@ -0,0 +1,45 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" + + clickhouse_keeper_altinity_com "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{ + Group: clickhouse_keeper_altinity_com.APIGroupName, + Version: APIVersion, + } + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{ + GroupVersion: SchemeGroupVersion, + } + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +func init() { + SchemeBuilder.Register( + &ClickHouseKeeperInstallation{}, + &ClickHouseKeeperInstallationList{}, + ) +} diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/api_resources.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/api_resources.go new file mode 100644 index 00000000..d13fa3c8 --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/api_resources.go @@ -0,0 +1,24 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +k8s:deepcopy-gen=package,register +// +groupName=clickhouse.altinity.com + +// Package v1 defines version 1 of the API used with ClickHouse Installation Custom Resources. +package v1 + +// Possible kinds of CRDs +const ( + ClickHouseKeeperInstallationCRDResourceKind = "ClickHouseKeeperInstallation" +) diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/api_version.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/api_version.go new file mode 100644 index 00000000..c3c15ba2 --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/api_version.go @@ -0,0 +1,20 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +const ( + // APIVersion is the version of the ClickHouse Keeper Operator API. + APIVersion = "v1" +) diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/doc.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/doc.go new file mode 100644 index 00000000..68e204c7 --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/doc.go @@ -0,0 +1,19 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +k8s:deepcopy-gen=package +// +groupName=clickhouse-keeper.altinity.com + +// Package v1 defines version 1 of the API used with ClickHouseKeeper custom resource. +package v1 diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_chk.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_chk.go new file mode 100644 index 00000000..e5b5d892 --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_chk.go @@ -0,0 +1,715 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "context" + "encoding/json" + "fmt" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/swversion" + + "github.com/imdario/mergo" + "gopkg.in/yaml.v3" + + apiChi "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/common/types" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/util" +) + +func (cr *ClickHouseKeeperInstallation) GetSpec() apiChi.ICRSpec { + return &cr.Spec +} + +func (cr *ClickHouseKeeperInstallation) GetSpecT() *ChkSpec { + return &cr.Spec +} + +func (cr *ClickHouseKeeperInstallation) GetSpecA() any { + return &cr.Spec +} + +func (cr *ClickHouseKeeperInstallation) GetRuntime() apiChi.ICustomResourceRuntime { + return cr.EnsureRuntime() +} + +func (cr *ClickHouseKeeperInstallation) EnsureRuntime() *ClickHouseKeeperInstallationRuntime { + if cr == nil { + return nil + } + + // Assume that most of the time, we'll see a non-nil value. + if cr.runtime != nil { + return cr.runtime + } + + // Otherwise, we need to acquire a lock to initialize the field. + cr.runtimeCreatorMutex.Lock() + defer cr.runtimeCreatorMutex.Unlock() + // Note that we have to check this property again to avoid a TOCTOU bug. + if cr.runtime == nil { + cr.runtime = newClickHouseKeeperInstallationRuntime() + } + return cr.runtime +} + +func (cr *ClickHouseKeeperInstallation) IEnsureStatus() apiChi.IStatus { + return any(cr.EnsureStatus()).(apiChi.IStatus) +} + +// EnsureStatus ensures status +func (cr *ClickHouseKeeperInstallation) EnsureStatus() *Status { + if cr == nil { + return nil + } + + // Assume that most of the time, we'll see a non-nil value. + if cr.Status != nil { + return cr.Status + } + + // Otherwise, we need to acquire a lock to initialize the field. + cr.statusCreatorMutex.Lock() + defer cr.statusCreatorMutex.Unlock() + // Note that we have to check this property again to avoid a TOCTOU bug. + if cr.Status == nil { + cr.Status = &Status{} + } + return cr.Status +} + +// GetStatus gets Status +func (cr *ClickHouseKeeperInstallation) GetStatus() apiChi.IStatus { + if cr == nil { + return (*Status)(nil) + } + return cr.Status +} + +// HasStatus checks whether CHI has Status +func (cr *ClickHouseKeeperInstallation) HasStatus() bool { + if cr == nil { + return false + } + return cr.Status != nil +} + +// HasAncestor checks whether CR has an ancestor +func (cr *ClickHouseKeeperInstallation) HasAncestor() bool { + if !cr.HasStatus() { + return false + } + return cr.Status.HasNormalizedCRCompleted() +} + +// GetAncestor gets ancestor of a CR +func (cr *ClickHouseKeeperInstallation) GetAncestor() apiChi.ICustomResource { + if !cr.HasAncestor() { + return (*ClickHouseKeeperInstallation)(nil) + } + return cr.Status.GetNormalizedCRCompleted() +} + +// GetAncestorT gets ancestor of a CR +func (cr *ClickHouseKeeperInstallation) GetAncestorT() *ClickHouseKeeperInstallation { + if !cr.HasAncestor() { + return nil + } + return cr.Status.GetNormalizedCRCompleted() +} + +// SetAncestor sets ancestor of a CR +func (cr *ClickHouseKeeperInstallation) SetAncestor(a *ClickHouseKeeperInstallation) { + if cr == nil { + return + } + cr.EnsureStatus().NormalizedCRCompleted = a +} + +// HasTarget checks whether CR has a target +func (cr *ClickHouseKeeperInstallation) HasTarget() bool { + if !cr.HasStatus() { + return false + } + return cr.Status.HasNormalizedCR() +} + +// GetTarget gets target of a CR +func (cr *ClickHouseKeeperInstallation) GetTarget() *ClickHouseKeeperInstallation { + if !cr.HasTarget() { + return nil + } + return cr.Status.GetNormalizedCR() +} + +// SetTarget sets target of a CR +func (cr *ClickHouseKeeperInstallation) SetTarget(a *ClickHouseKeeperInstallation) { + if cr == nil { + return + } + cr.EnsureStatus().NormalizedCR = a +} + +func (cr *ClickHouseKeeperInstallation) GetUsedTemplates() []*apiChi.TemplateRef { + return nil +} + +// FillStatus fills .Status +func (cr *ClickHouseKeeperInstallation) FillStatus(endpoints util.Slice[string], pods, fqdns []string, ip string) { + cr.EnsureStatus().Fill(&FillStatusParams{ + CHOpIP: ip, + ClustersCount: cr.ClustersCount(), + ShardsCount: cr.ShardsCount(), + HostsCount: cr.HostsCount(), + TaskID: cr.GetSpecT().GetTaskID().Value(), + HostsUpdatedCount: 0, + HostsAddedCount: 0, + HostsUnchangedCount: 0, + HostsCompletedCount: 0, + HostsDeleteCount: 0, + HostsDeletedCount: 0, + Pods: pods, + FQDNs: fqdns, + Endpoint: endpoints.First(), + Endpoints: append([]string{}, endpoints...), + NormalizedCR: cr.Copy(types.CopyCROptions{ + SkipStatus: true, + SkipManagedFields: true, + }), + }) +} + +func (cr *ClickHouseKeeperInstallation) Fill() { + apiChi.FillCR(cr) +} + +// MergeFrom merges from CHI +func (cr *ClickHouseKeeperInstallation) MergeFrom(from *ClickHouseKeeperInstallation, _type apiChi.MergeType) { + if from == nil { + return + } + + // Merge Meta + switch _type { + case apiChi.MergeTypeFillEmptyValues: + _ = mergo.Merge(&cr.TypeMeta, from.TypeMeta) + _ = mergo.Merge(&cr.ObjectMeta, from.ObjectMeta) + case apiChi.MergeTypeOverrideByNonEmptyValues: + _ = mergo.Merge(&cr.TypeMeta, from.TypeMeta, mergo.WithOverride) + _ = mergo.Merge(&cr.ObjectMeta, from.ObjectMeta, mergo.WithOverride) + } + // Exclude skipped annotations + cr.SetAnnotations( + util.CopyMapFilter( + cr.GetAnnotations(), + nil, + util.ListSkippedAnnotations(), + ), + ) + + // Do actual merge for Spec + cr.GetSpecT().MergeFrom(from.GetSpecT(), _type) + + // Copy service attributes + //cr.ensureRuntime().attributes = from.ensureRuntime().attributes + + cr.EnsureStatus().CopyFrom(from.Status, types.CopyStatusOptions{ + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupInheritable: true, + }, + }) +} + +// FindCluster finds cluster by name or index. +// Expectations: name is expected to be a string, index is expected to be an int. +func (cr *ClickHouseKeeperInstallation) FindCluster(needle interface{}) apiChi.ICluster { + var resultCluster *Cluster + cr.WalkClustersFullPath(func(chk *ClickHouseKeeperInstallation, clusterIndex int, cluster *Cluster) error { + switch v := needle.(type) { + case string: + if cluster.Name == v { + resultCluster = cluster + } + case int: + if clusterIndex == v { + resultCluster = cluster + } + } + return nil + }) + return resultCluster +} + +// FindShard finds shard by name or index +// Expectations: name is expected to be a string, index is expected to be an int. +func (cr *ClickHouseKeeperInstallation) FindShard(needleCluster interface{}, needleShard interface{}) apiChi.IShard { + return cr.FindCluster(needleCluster).FindShard(needleShard) +} + +// FindHost finds shard by name or index +// Expectations: name is expected to be a string, index is expected to be an int. +func (cr *ClickHouseKeeperInstallation) FindHost(needleCluster interface{}, needleShard interface{}, needleHost interface{}) *apiChi.Host { + return cr.FindCluster(needleCluster).FindHost(needleShard, needleHost) +} + +// ClustersCount counts clusters +func (cr *ClickHouseKeeperInstallation) ClustersCount() int { + count := 0 + cr.WalkClusters(func(cluster apiChi.ICluster) error { + count++ + return nil + }) + return count +} + +// ShardsCount counts shards +func (cr *ClickHouseKeeperInstallation) ShardsCount() int { + count := 0 + cr.WalkShards(func(shard *ChkShard) error { + count++ + return nil + }) + return count +} + +// HostsCount counts hosts +func (cr *ClickHouseKeeperInstallation) HostsCount() int { + count := 0 + cr.WalkHosts(func(host *apiChi.Host) error { + count++ + return nil + }) + return count +} + +// HostsWithAttributesCount counts hosts by attributes +func (cr *ClickHouseKeeperInstallation) HostsWithAttributesCount(a *types.ReconcileAttributes) int { + count := 0 + cr.WalkHosts(func(host *apiChi.Host) error { + if host.GetReconcileAttributes().HasIntersectionWith(a) { + count++ + } + return nil + }) + return count +} + +// HasReconcileWork reports whether the CR has any work to reconcile: +// either the ActionPlan has spec/label/finalizer changes, or child resources have drifted. +func (cr *ClickHouseKeeperInstallation) HasReconcileWork() bool { + return cr.EnsureRuntime().ActionPlan.HasActionsToDo() || cr.GetHostsAttributesCounters().HasDrift() +} + +// GetHostsAttributesCounters +func (cr *ClickHouseKeeperInstallation) GetHostsAttributesCounters() *types.ReconcileAttributesCounters { + counters := types.NewReconcileAttributesCounters() + cr.WalkHosts(func(host *apiChi.Host) error { + counters.Add(host.GetReconcileAttributes()) + return nil + }) + return counters +} + +// GetHostTemplate gets HostTemplate by name +func (cr *ClickHouseKeeperInstallation) GetHostTemplate(name string) (*apiChi.HostTemplate, bool) { + if !cr.GetSpecT().GetTemplates().GetHostTemplatesIndex().Has(name) { + return nil, false + } + return cr.GetSpecT().GetTemplates().GetHostTemplatesIndex().Get(name), true +} + +// GetPodTemplate gets PodTemplate by name +func (cr *ClickHouseKeeperInstallation) GetPodTemplate(name string) (*apiChi.PodTemplate, bool) { + if !cr.GetSpecT().GetTemplates().GetPodTemplatesIndex().Has(name) { + return nil, false + } + return cr.GetSpecT().GetTemplates().GetPodTemplatesIndex().Get(name), true +} + +// WalkPodTemplates walks over all PodTemplates +func (cr *ClickHouseKeeperInstallation) WalkPodTemplates(f func(template *apiChi.PodTemplate)) { + cr.GetSpecT().GetTemplates().GetPodTemplatesIndex().Walk(f) +} + +// GetVolumeClaimTemplate gets VolumeClaimTemplate by name +func (cr *ClickHouseKeeperInstallation) GetVolumeClaimTemplate(name string) (*apiChi.VolumeClaimTemplate, bool) { + if cr.GetSpecT().GetTemplates().GetVolumeClaimTemplatesIndex().Has(name) { + return cr.GetSpecT().GetTemplates().GetVolumeClaimTemplatesIndex().Get(name), true + } + return nil, false +} + +// WalkVolumeClaimTemplates walks over all VolumeClaimTemplates +func (cr *ClickHouseKeeperInstallation) WalkVolumeClaimTemplates(f func(template *apiChi.VolumeClaimTemplate)) { + if cr == nil { + return + } + cr.GetSpecT().GetTemplates().GetVolumeClaimTemplatesIndex().Walk(f) +} + +// GetServiceTemplate gets ServiceTemplate by name +func (cr *ClickHouseKeeperInstallation) GetServiceTemplate(name string) (*apiChi.ServiceTemplate, bool) { + if !cr.GetSpecT().GetTemplates().GetServiceTemplatesIndex().Has(name) { + return nil, false + } + return cr.GetSpecT().GetTemplates().GetServiceTemplatesIndex().Get(name), true +} + +// GetServiceTemplates gets ServiceTemplates by name +func (cr *ClickHouseKeeperInstallation) GetServiceTemplates(names ...string) ([]*apiChi.ServiceTemplate, bool) { + if len(names) == 0 { + return nil, false + } + var res []*apiChi.ServiceTemplate + for _, name := range names { + if cr.GetSpecT().GetTemplates().GetServiceTemplatesIndex().Has(name) { + res = append(res, cr.GetSpecT().GetTemplates().GetServiceTemplatesIndex().Get(name)) + } + } + if len(res) == len(names) { + return res, true + } + return nil, false +} + +// GetRootServiceTemplates gets service templates of a CR +func (cr *ClickHouseKeeperInstallation) GetRootServiceTemplates() ([]*apiChi.ServiceTemplate, bool) { + if !cr.GetSpecT().GetDefaults().Templates.HasAnyServiceTemplate() { + return nil, false + } + return cr.GetServiceTemplates(cr.GetSpecT().GetDefaults().Templates.GetAllServiceTemplates()...) +} + +// MatchNamespace matches namespace +func (cr *ClickHouseKeeperInstallation) MatchNamespace(namespace string) bool { + if cr == nil { + return false + } + return cr.Namespace == namespace +} + +// MatchFullName matches full name +func (cr *ClickHouseKeeperInstallation) MatchFullName(namespace, name string) bool { + if cr == nil { + return false + } + return (cr.Namespace == namespace) && (cr.Name == name) +} + +// FoundIn checks whether CHI can be found in haystack +func (cr *ClickHouseKeeperInstallation) FoundIn(haystack []*ClickHouseKeeperInstallation) bool { + if cr == nil { + return false + } + + for _, candidate := range haystack { + if candidate.MatchFullName(cr.Namespace, cr.Name) { + return true + } + } + + return false +} + +// IsAuto checks whether templating policy is auto +func (cr *ClickHouseKeeperInstallation) IsAuto() bool { + return false +} + +// IsStopped checks whether CR is stopped +func (cr *ClickHouseKeeperInstallation) IsStopped() bool { + if cr == nil { + return false + } + return cr.GetSpecT().GetStop().Value() +} + +// IsRollingUpdate checks whether CHI should perform rolling update +func (cr *ClickHouseKeeperInstallation) IsRollingUpdate() bool { + return false +} + +// IsTroubleshoot checks whether CHI is in troubleshoot mode +func (cr *ClickHouseKeeperInstallation) IsTroubleshoot() bool { + return false +} + +// GetReconcile gets reconcile spec +func (cr *ClickHouseKeeperInstallation) GetReconcile() *apiChi.ChiReconcile { + if cr == nil { + return nil + } + return cr.GetSpecT().Reconcile +} + +// Copy makes copy of a CHI, filtering fields according to specified CopyOptions +func (cr *ClickHouseKeeperInstallation) Copy(opts types.CopyCROptions) *ClickHouseKeeperInstallation { + if cr == nil { + return nil + } + jsonBytes, err := json.Marshal(cr) + if err != nil { + return nil + } + + var cr2 *ClickHouseKeeperInstallation + if err := json.Unmarshal(jsonBytes, &cr2); err != nil { + return nil + } + + if opts.SkipStatus { + cr2.Status = nil + } + + if opts.SkipManagedFields { + cr2.SetManagedFields(nil) + } + + return cr2 +} + +// JSON returns JSON string +func (cr *ClickHouseKeeperInstallation) JSON(opts types.CopyCROptions) string { + if cr == nil { + return "" + } + + filtered := cr.Copy(opts) + jsonBytes, err := json.MarshalIndent(filtered, "", " ") + if err != nil { + return fmt.Sprintf("unable to parse. err: %v", err) + } + return string(jsonBytes) + +} + +// YAML return YAML string +func (cr *ClickHouseKeeperInstallation) YAML(opts types.CopyCROptions) string { + if cr == nil { + return "" + } + + filtered := cr.Copy(opts) + yamlBytes, err := yaml.Marshal(filtered) + if err != nil { + return fmt.Sprintf("unable to parse. err: %v", err) + } + return string(yamlBytes) +} + +// FirstHost returns first host of the CR +func (cr *ClickHouseKeeperInstallation) FirstHost() *apiChi.Host { + var result *apiChi.Host + cr.WalkHosts(func(host *apiChi.Host) error { + if result == nil { + result = host + } + return nil + }) + return result +} + +func (cr *ClickHouseKeeperInstallation) GetName() string { + if cr == nil { + return "" + } + return cr.Name +} + +func (cr *ClickHouseKeeperInstallation) GetNamespace() string { + if cr == nil { + return "" + } + return cr.Namespace +} + +func (cr *ClickHouseKeeperInstallation) GetLabels() map[string]string { + if cr == nil { + return nil + } + return cr.Labels +} + +func (cr *ClickHouseKeeperInstallation) GetAnnotations() map[string]string { + if cr == nil { + return nil + } + return cr.Annotations +} + +// WalkClustersFullPath walks clusters with full path +func (cr *ClickHouseKeeperInstallation) WalkClustersFullPath( + f func(chi *ClickHouseKeeperInstallation, clusterIndex int, cluster *Cluster) error, +) []error { + if cr == nil { + return nil + } + res := make([]error, 0) + + for clusterIndex := range cr.GetSpecT().Configuration.Clusters { + res = append(res, f(cr, clusterIndex, cr.GetSpecT().Configuration.Clusters[clusterIndex])) + } + + return res +} + +// WalkClusters walks clusters +func (cr *ClickHouseKeeperInstallation) WalkClusters(f func(i apiChi.ICluster) error) []error { + if cr == nil { + return nil + } + res := make([]error, 0) + + for clusterIndex := range cr.GetSpecT().Configuration.Clusters { + res = append(res, f(cr.GetSpecT().Configuration.Clusters[clusterIndex])) + } + + return res +} + +// WalkShards walks shards +func (cr *ClickHouseKeeperInstallation) WalkShards( + f func( + shard *ChkShard, + ) error, +) []error { + if cr == nil { + return nil + } + res := make([]error, 0) + + for clusterIndex := range cr.GetSpecT().Configuration.Clusters { + cluster := cr.GetSpecT().Configuration.Clusters[clusterIndex] + for shardIndex := range cluster.Layout.Shards { + shard := cluster.Layout.Shards[shardIndex] + res = append(res, f(shard)) + } + } + + return res +} + +// WalkHostsFullPathAndScope walks hosts with full path +func (cr *ClickHouseKeeperInstallation) WalkHostsFullPathAndScope( + crScopeCycleSize int, + clusterScopeCycleSize int, + f apiChi.WalkHostsAddressFn, +) (res []error) { + if cr == nil { + return nil + } + address := types.NewHostScopeAddress(crScopeCycleSize, clusterScopeCycleSize) + for clusterIndex := range cr.GetSpecT().Configuration.Clusters { + cluster := cr.GetSpecT().Configuration.Clusters[clusterIndex] + address.ClusterScopeAddress.Init() + for shardIndex := range cluster.Layout.Shards { + shard := cluster.GetShard(shardIndex) + for replicaIndex, host := range shard.Hosts { + replica := cluster.GetReplica(replicaIndex) + address.ClusterIndex = clusterIndex + address.ShardIndex = shardIndex + address.ReplicaIndex = replicaIndex + res = append(res, f(cr, cluster, shard, replica, host, address)) + address.CRScopeAddress.Inc() + address.ClusterScopeAddress.Inc() + } + } + } + return res +} + +// WalkHostsFullPath walks hosts with a function +func (cr *ClickHouseKeeperInstallation) WalkHostsFullPath(f apiChi.WalkHostsAddressFn) []error { + return cr.WalkHostsFullPathAndScope(0, 0, f) +} + +// WalkHosts walks hosts with a function +func (cr *ClickHouseKeeperInstallation) WalkHosts(f func(host *apiChi.Host) error) []error { + if cr == nil { + return nil + } + res := make([]error, 0) + + for clusterIndex := range cr.GetSpecT().Configuration.Clusters { + cluster := cr.GetSpecT().Configuration.Clusters[clusterIndex] + for shardIndex := range cluster.Layout.Shards { + shard := cluster.Layout.Shards[shardIndex] + for replicaIndex := range shard.Hosts { + host := shard.Hosts[replicaIndex] + res = append(res, f(host)) + } + } + } + + return res +} + +// WalkTillError walks hosts with a function until an error met +func (cr *ClickHouseKeeperInstallation) WalkTillError( + ctx context.Context, + fCRPreliminary func(ctx context.Context, chi *ClickHouseKeeperInstallation) error, + fCluster func(ctx context.Context, cluster *Cluster) error, + fCRFinal func(ctx context.Context, chi *ClickHouseKeeperInstallation) error, +) error { + if err := fCRPreliminary(ctx, cr); err != nil { + return err + } + + for clusterIndex := range cr.GetSpecT().Configuration.Clusters { + cluster := cr.GetSpecT().Configuration.Clusters[clusterIndex] + if err := fCluster(ctx, cluster); err != nil { + return err + } + } + + if err := fCRFinal(ctx, cr); err != nil { + return err + } + + return nil +} + +func (cr *ClickHouseKeeperInstallation) IsZero() bool { + return cr == nil +} + +func (cr *ClickHouseKeeperInstallation) IsNonZero() bool { + return cr != nil +} + +func (cr *ClickHouseKeeperInstallation) NamespaceName() (string, string) { + return util.NamespaceName(cr) +} + +func (cr *ClickHouseKeeperInstallation) FindMinMaxVersions() { + cr.runtime.MinVersion = swversion.MaxVersion() + cr.runtime.MaxVersion = swversion.MinVersion() + cr.WalkHosts(func(host *apiChi.Host) error { + if host.Runtime.Version.Cmp(cr.runtime.MinVersion) < 0 { + cr.runtime.MinVersion = host.Runtime.Version + } + if host.Runtime.Version.Cmp(cr.runtime.MaxVersion) > 0 { + cr.runtime.MaxVersion = host.Runtime.Version + } + return nil + }) +} + +func (cr *ClickHouseKeeperInstallation) GetMinVersion() *swversion.SoftWareVersion { + return cr.runtime.MinVersion +} + +func (cr *ClickHouseKeeperInstallation) GetMaxVersion() *swversion.SoftWareVersion { + return cr.runtime.MaxVersion +} diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_cluster.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_cluster.go new file mode 100644 index 00000000..8f877530 --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_cluster.go @@ -0,0 +1,434 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + apiChi "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/common/types" +) + +// Cluster defines item of a clusters section of .configuration +type Cluster struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Settings *apiChi.Settings `json:"settings,omitempty" yaml:"settings,omitempty"` + Files *apiChi.Settings `json:"files,omitempty" yaml:"files,omitempty"` + Templates *apiChi.TemplatesList `json:"templates,omitempty" yaml:"templates,omitempty"` + PDBManaged *types.StringBool `json:"pdbManaged,omitempty" yaml:"pdbManaged,omitempty"` + PDBMaxUnavailable *types.Int32 `json:"pdbMaxUnavailable,omitempty" yaml:"pdbMaxUnavailable,omitempty"` + Reconcile *apiChi.ClusterReconcile `json:"reconcile,omitempty" yaml:"reconcile,omitempty"` + Layout *ChkClusterLayout `json:"layout,omitempty" yaml:"layout,omitempty"` + + Runtime ChkClusterRuntime `json:"-" yaml:"-"` +} + +type ChkClusterRuntime struct { + Address ChkClusterAddress `json:"-" yaml:"-"` + CHK *ClickHouseKeeperInstallation `json:"-" yaml:"-" testdiff:"ignore"` +} + +func (r *ChkClusterRuntime) GetAddress() apiChi.IClusterAddress { + return &r.Address +} + +func (r ChkClusterRuntime) GetCR() apiChi.ICustomResource { + return r.CHK +} + +func (r *ChkClusterRuntime) SetCR(cr apiChi.ICustomResource) { + r.CHK = cr.(*ClickHouseKeeperInstallation) +} + +// ChkClusterAddress defines address of a cluster within ClickHouseInstallation +type ChkClusterAddress struct { + Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` + CHIName string `json:"chiName,omitempty" yaml:"chiName,omitempty"` + ClusterName string `json:"clusterName,omitempty" yaml:"clusterName,omitempty"` + ClusterIndex int `json:"clusterIndex,omitempty" yaml:"clusterIndex,omitempty"` +} + +func (a *ChkClusterAddress) GetNamespace() string { + return a.Namespace +} + +func (a *ChkClusterAddress) SetNamespace(namespace string) { + a.Namespace = namespace +} + +func (a *ChkClusterAddress) GetCRName() string { + return a.CHIName +} + +func (a *ChkClusterAddress) SetCRName(name string) { + a.CHIName = name +} + +func (a *ChkClusterAddress) GetClusterName() string { + return a.ClusterName +} + +func (a *ChkClusterAddress) SetClusterName(name string) { + a.ClusterName = name +} + +func (a *ChkClusterAddress) GetClusterIndex() int { + return a.ClusterIndex +} + +func (a *ChkClusterAddress) SetClusterIndex(index int) { + a.ClusterIndex = index +} + +func (cluster *Cluster) GetName() string { + return cluster.Name +} + +// HasName checks whether cluster has a name +func (cluster *Cluster) HasName() bool { + if cluster == nil { + return false + } + return len(cluster.GetName()) > 0 +} + +func (c *Cluster) GetZookeeper() *apiChi.ZookeeperConfig { + return nil +} + +func (c *Cluster) GetSchemaPolicy() *apiChi.SchemaPolicy { + return nil +} + +// GetInsecure is a getter +func (cluster *Cluster) GetInsecure() *types.StringBool { + return nil +} + +// GetSecure is a getter +func (cluster *Cluster) GetSecure() *types.StringBool { + return nil +} + +// GetSecret is a getter +func (c *Cluster) GetSecret() *apiChi.ClusterSecret { + return nil +} + +// GetPDBManaged is a getter +func (cluster *Cluster) GetPDBManaged() *types.StringBool { + return cluster.PDBManaged +} + +// GetPDBMaxUnavailable is a getter +func (cluster *Cluster) GetPDBMaxUnavailable() *types.Int32 { + return cluster.PDBMaxUnavailable +} + +// GetReconcile is a getter +func (cluster *Cluster) GetReconcile() *apiChi.ClusterReconcile { + cluster.Reconcile = cluster.Reconcile.Ensure() + return cluster.Reconcile +} + +// GetRuntime is a getter +func (cluster *Cluster) GetRuntime() apiChi.IClusterRuntime { + return &cluster.Runtime +} + +// FillShardsReplicasExplicitlySpecified fills whether shard or replicas are explicitly specified +func (cluster *Cluster) FillShardsReplicasExplicitlySpecified() { + if len(cluster.Layout.Shards) > 0 { + cluster.Layout.ShardsExplicitlySpecified = true + } + if len(cluster.Layout.Replicas) > 0 { + cluster.Layout.ReplicasExplicitlySpecified = true + } +} + +// isShardExplicitlySpecified checks whether shard is explicitly specified +func (cluster *Cluster) isShardExplicitlySpecified() bool { + return cluster.Layout.ShardsExplicitlySpecified +} + +// isReplicaExplicitlySpecified checks whether replica is explicitly specified +func (cluster *Cluster) isReplicaExplicitlySpecified() bool { + return cluster.Layout.ReplicasExplicitlySpecified && !cluster.isShardExplicitlySpecified() +} + +// IsShardSpecified checks whether shard is explicitly specified +func (cluster *Cluster) isShardToBeUsedToInheritSettingsFrom() bool { + if !cluster.isShardExplicitlySpecified() && !cluster.isReplicaExplicitlySpecified() { + return true + } + + return cluster.isShardExplicitlySpecified() +} + +func (cluster *Cluster) SelectSettingsSourceFrom(shard apiChi.IShard, replica apiChi.IReplica) any { + if cluster.isShardToBeUsedToInheritSettingsFrom() { + return shard + } + return replica +} + +// InheritFilesFrom inherits files from CR +func (cluster *Cluster) InheritFilesFrom(chk *ClickHouseKeeperInstallation) { + if chk.GetSpecT().Configuration == nil { + return + } + if chk.GetSpecT().Configuration.Files == nil { + return + } + + // Propagate host section only + cluster.Files = cluster.Files.MergeFromCB(chk.GetSpecT().Configuration.Files, func(path string, _ *apiChi.Setting) bool { + if section, err := apiChi.GetSectionFromPath(path); err == nil { + if section.Equal(apiChi.SectionHost) { + return true + } + } + + return false + }) +} + +// InheritClusterReconcileFrom inherits reconcile runtime from CHI +func (cluster *Cluster) InheritClusterReconcileFrom(chk *ClickHouseKeeperInstallation) { + if chk.Spec.Reconcile == nil { + return + } + reconcile := cluster.GetReconcile() + reconcile.Runtime = reconcile.Runtime.MergeFrom(chk.Spec.Reconcile.Runtime, apiChi.MergeTypeFillEmptyValues) + reconcile.Host = reconcile.Host.MergeFrom(chk.Spec.Reconcile.Host) + cluster.Reconcile = reconcile +} + +// InheritTemplatesFrom inherits templates from CHI +func (cluster *Cluster) InheritTemplatesFrom(chk *ClickHouseKeeperInstallation) { + if chk.GetSpec().GetDefaults() == nil { + return + } + if chk.GetSpec().GetDefaults().Templates == nil { + return + } + cluster.Templates = cluster.Templates.MergeFrom(chk.GetSpec().GetDefaults().Templates, apiChi.MergeTypeFillEmptyValues) + cluster.Templates.HandleDeprecatedFields() +} + +// GetServiceTemplate returns service template, if exists +func (cluster *Cluster) GetServiceTemplate() (*apiChi.ServiceTemplate, bool) { + return nil, false +} + +// GetCR gets parent CR +func (cluster *Cluster) GetCR() *ClickHouseKeeperInstallation { + return cluster.Runtime.CHK +} + +func (cluster *Cluster) GetAncestor() apiChi.ICluster { + return (*Cluster)(nil) +} + +// GetShard gets shard with specified index +func (cluster *Cluster) GetShard(shard int) *ChkShard { + return cluster.Layout.Shards[shard] +} + +// GetOrCreateHost gets or creates host on specified coordinates +func (cluster *Cluster) GetOrCreateHost(shard, replica int) *apiChi.Host { + return cluster.Layout.HostsField.GetOrCreate(shard, replica) +} + +// GetReplica gets replica with specified index +func (cluster *Cluster) GetReplica(replica int) *ChkReplica { + return cluster.Layout.Replicas[replica] +} + +// FindShard finds shard by name or index. +// Expectations: name is expected to be a string, index is expected to be an int. +func (cluster *Cluster) FindShard(needle interface{}) apiChi.IShard { + var resultShard *ChkShard + cluster.WalkShards(func(index int, shard apiChi.IShard) error { + switch v := needle.(type) { + case string: + if shard.GetName() == v { + resultShard = shard.(*ChkShard) + } + case int: + if index == v { + resultShard = shard.(*ChkShard) + } + } + return nil + }) + return resultShard +} + +// FindHost finds host by name or index. +// Expectations: name is expected to be a string, index is expected to be an int. +func (cluster *Cluster) FindHost(needleShard interface{}, needleHost interface{}) *apiChi.Host { + return cluster.FindShard(needleShard).FindHost(needleHost) +} + +// FirstHost finds first host in the cluster +func (cluster *Cluster) FirstHost() *apiChi.Host { + var result *apiChi.Host + cluster.WalkHosts(func(host *apiChi.Host) error { + if result == nil { + result = host + } + return nil + }) + return result +} + +// WalkShards walks shards +func (cluster *Cluster) WalkShards(f func(index int, shard apiChi.IShard) error) []error { + if cluster == nil { + return nil + } + res := make([]error, 0) + + for shardIndex := range cluster.Layout.Shards { + shard := cluster.Layout.Shards[shardIndex] + res = append(res, f(shardIndex, shard)) + } + + return res +} + +// WalkReplicas walks replicas +func (cluster *Cluster) WalkReplicas(f func(index int, replica *ChkReplica) error) []error { + res := make([]error, 0) + + for replicaIndex := range cluster.Layout.Replicas { + replica := cluster.Layout.Replicas[replicaIndex] + res = append(res, f(replicaIndex, replica)) + } + + return res +} + +// WalkHosts walks hosts +func (cluster *Cluster) WalkHosts(f func(host *apiChi.Host) error) []error { + res := make([]error, 0) + + for shardIndex := range cluster.Layout.Shards { + shard := cluster.Layout.Shards[shardIndex] + for replicaIndex := range shard.Hosts { + host := shard.Hosts[replicaIndex] + res = append(res, f(host)) + } + } + + return res +} + +// WalkHostsByShards walks hosts by shards +func (cluster *Cluster) WalkHostsByShards(f func(shard, replica int, host *apiChi.Host) error) []error { + + res := make([]error, 0) + + for shardIndex := range cluster.Layout.Shards { + shard := cluster.Layout.Shards[shardIndex] + for replicaIndex := range shard.Hosts { + host := shard.Hosts[replicaIndex] + res = append(res, f(shardIndex, replicaIndex, host)) + } + } + + return res +} + +func (cluster *Cluster) GetLayout() *ChkClusterLayout { + return cluster.Layout +} + +// WalkHostsByReplicas walks hosts by replicas +func (cluster *Cluster) WalkHostsByReplicas(f func(shard, replica int, host *apiChi.Host) error) []error { + + res := make([]error, 0) + + for replicaIndex := range cluster.Layout.Replicas { + replica := cluster.Layout.Replicas[replicaIndex] + for shardIndex := range replica.Hosts { + host := replica.Hosts[shardIndex] + res = append(res, f(shardIndex, replicaIndex, host)) + } + } + + return res +} + +// HostsCount counts hosts +func (cluster *Cluster) HostsCount() int { + count := 0 + cluster.WalkHosts(func(host *apiChi.Host) error { + count++ + return nil + }) + return count +} + +func (cluster *Cluster) IsZero() bool { + return cluster == nil +} + +func (cluster *Cluster) IsNonZero() bool { + return cluster != nil +} + +// IsStopped checks whether host is stopped +func (cluster *Cluster) IsStopped() bool { + return cluster.GetCR().IsStopped() +} + +func (cluster *Cluster) Ensure(create func() *Cluster) *Cluster { + if cluster == nil { + cluster = create() + } + return cluster +} + +// ChkClusterLayout defines layout section of .spec.configuration.clusters +type ChkClusterLayout struct { + ShardsCount int `json:"shardsCount,omitempty" yaml:"shardsCount,omitempty"` + ReplicasCount int `json:"replicasCount,omitempty" yaml:"replicasCount,omitempty"` + + // TODO refactor into map[string]ChiShard + Shards []*ChkShard `json:"shards,omitempty" yaml:"shards,omitempty"` + Replicas []*ChkReplica `json:"replicas,omitempty" yaml:"replicas,omitempty"` + + // Internal data + // Whether shards or replicas are explicitly specified as Shards []ChiShard or Replicas []ChiReplica + ShardsExplicitlySpecified bool `json:"-" yaml:"-" testdiff:"ignore"` + ReplicasExplicitlySpecified bool `json:"-" yaml:"-" testdiff:"ignore"` + HostsField *apiChi.HostsField `json:"-" yaml:"-" testdiff:"ignore"` +} + +// NewChiClusterLayout creates new cluster layout +func NewChkClusterLayout() *ChkClusterLayout { + return new(ChkClusterLayout) +} + +func (l *ChkClusterLayout) GetReplicasCount() int { + return l.ReplicasCount +} + +func (l *ChkClusterLayout) Ensure() *ChkClusterLayout { + if l == nil { + l = NewChkClusterLayout() + } + return l +} diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_configuration.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_configuration.go new file mode 100644 index 00000000..da7561ab --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_configuration.go @@ -0,0 +1,82 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + apiChi "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" +) + +// Configuration defines configuration section of .spec +type Configuration struct { + Settings *apiChi.Settings `json:"settings,omitempty" yaml:"settings,omitempty"` + Files *apiChi.Settings `json:"files,omitempty" yaml:"files,omitempty"` + Clusters []*Cluster `json:"clusters,omitempty" yaml:"clusters,omitempty"` +} + +// NewConfiguration creates new ChkConfiguration objects +func NewConfiguration() *Configuration { + return new(Configuration) +} + +func (c *Configuration) Ensure() *Configuration { + if c == nil { + c = NewConfiguration() + } + return c +} + +func (c *Configuration) GetUsers() *apiChi.Settings { + return nil +} + +func (c *Configuration) GetProfiles() *apiChi.Settings { + return nil +} + +func (c *Configuration) GetQuotas() *apiChi.Settings { + return nil +} + +func (c *Configuration) GetSettings() *apiChi.Settings { + if c == nil { + return nil + } + return c.Settings +} + +func (c *Configuration) GetFiles() *apiChi.Settings { + if c == nil { + return nil + } + return c.Files +} + +// MergeFrom merges from specified source +func (c *Configuration) MergeFrom(from *Configuration, _type apiChi.MergeType) *Configuration { + if from == nil { + return c + } + + c = c.Ensure() + + c.Settings = c.Settings.MergeFrom(from.Settings) + c.Files = c.Files.MergeFrom(from.Files) + + // TODO merge clusters + // Copy Clusters for now + c.Clusters = from.Clusters + + return c +} diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_replica.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_replica.go new file mode 100644 index 00000000..af7f1705 --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_replica.go @@ -0,0 +1,202 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import apiChi "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + +// ChiReplica defines item of a replica section of .spec.configuration.clusters[n].replicas +// TODO unify with ChiShard based on HostsSet +type ChkReplica struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Settings *apiChi.Settings `json:"settings,omitempty" yaml:"settings,omitempty"` + Files *apiChi.Settings `json:"files,omitempty" yaml:"files,omitempty"` + Templates *apiChi.TemplatesList `json:"templates,omitempty" yaml:"templates,omitempty"` + ShardsCount int `json:"shardsCount,omitempty" yaml:"shardsCount,omitempty"` + // TODO refactor into map[string]Host + Hosts []*apiChi.Host `json:"shards,omitempty" yaml:"shards,omitempty"` + + Runtime ChkReplicaRuntime `json:"-" yaml:"-"` +} + +type ChkReplicaRuntime struct { + Address ChkReplicaAddress `json:"-" yaml:"-"` + CHK *ClickHouseKeeperInstallation `json:"-" yaml:"-" testdiff:"ignore"` +} + +func (r ChkReplicaRuntime) GetAddress() apiChi.IReplicaAddress { + return &r.Address +} + +func (r *ChkReplicaRuntime) SetCR(cr apiChi.ICustomResource) { + r.CHK = cr.(*ClickHouseKeeperInstallation) +} + +func (replica *ChkReplica) GetName() string { + return replica.Name +} + +// InheritSettingsFrom inherits settings from specified cluster +func (replica *ChkReplica) InheritSettingsFrom(cluster *Cluster) { + replica.Settings = replica.Settings.MergeFrom(cluster.Settings) +} + +// InheritFilesFrom inherits files from specified cluster +func (replica *ChkReplica) InheritFilesFrom(cluster *Cluster) { + replica.Files = replica.Files.MergeFrom(cluster.Files) +} + +// InheritTemplatesFrom inherits templates from specified cluster +func (replica *ChkReplica) InheritTemplatesFrom(cluster *Cluster) { + replica.Templates = replica.Templates.MergeFrom(cluster.Templates, apiChi.MergeTypeFillEmptyValues) + replica.Templates.HandleDeprecatedFields() +} + +// GetServiceTemplate gets service template +func (replica *ChkReplica) GetServiceTemplate() (*apiChi.ServiceTemplate, bool) { + if !replica.Templates.HasReplicaServiceTemplate() { + return nil, false + } + name := replica.Templates.GetReplicaServiceTemplate() + return replica.Runtime.CHK.GetServiceTemplate(name) +} + +// HasShardsCount checks whether replica has shards count specified +func (replica *ChkReplica) HasShardsCount() bool { + if replica == nil { + return false + } + + return replica.ShardsCount > 0 +} + +// WalkHosts walks over hosts +func (replica *ChkReplica) WalkHosts(f func(host *apiChi.Host) error) []error { + res := make([]error, 0) + + for shardIndex := range replica.Hosts { + host := replica.Hosts[shardIndex] + res = append(res, f(host)) + } + + return res +} + +// HostsCount returns number of hosts +func (replica *ChkReplica) HostsCount() int { + count := 0 + replica.WalkHosts(func(host *apiChi.Host) error { + count++ + return nil + }) + return count +} + +func (replica *ChkReplica) HasSettings() bool { + return replica.GetSettings() != nil +} + +func (replica *ChkReplica) GetSettings() *apiChi.Settings { + if replica == nil { + return nil + } + return replica.Settings +} + +func (replica *ChkReplica) HasFiles() bool { + return replica.GetFiles() != nil +} + +func (replica *ChkReplica) GetFiles() *apiChi.Settings { + if replica == nil { + return nil + } + return replica.Files +} + +func (replica *ChkReplica) HasTemplates() bool { + return replica.GetTemplates() != nil +} + +func (replica *ChkReplica) GetTemplates() *apiChi.TemplatesList { + if replica == nil { + return nil + } + return replica.Templates +} + +func (replica *ChkReplica) GetRuntime() apiChi.IReplicaRuntime { + if replica == nil { + return (*ChkReplicaRuntime)(nil) + } + return &replica.Runtime +} + +// ChiReplicaAddress defines address of a replica within ClickHouseInstallation +type ChkReplicaAddress struct { + Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` + CHIName string `json:"chiName,omitempty" yaml:"chiName,omitempty"` + ClusterName string `json:"clusterName,omitempty" yaml:"clusterName,omitempty"` + ClusterIndex int `json:"clusterIndex,omitempty" yaml:"clusterIndex,omitempty"` + ReplicaName string `json:"replicaName,omitempty" yaml:"replicaName,omitempty"` + ReplicaIndex int `json:"replicaIndex,omitempty" yaml:"replicaIndex,omitempty"` +} + +func (a *ChkReplicaAddress) GetNamespace() string { + return a.Namespace +} + +func (a *ChkReplicaAddress) SetNamespace(namespace string) { + a.Namespace = namespace +} + +func (a *ChkReplicaAddress) GetCRName() string { + return a.CHIName +} + +func (a *ChkReplicaAddress) SetCRName(name string) { + a.CHIName = name +} + +func (a *ChkReplicaAddress) GetClusterName() string { + return a.ClusterName +} + +func (a *ChkReplicaAddress) SetClusterName(name string) { + a.ClusterName = name +} + +func (a *ChkReplicaAddress) GetClusterIndex() int { + return a.ClusterIndex +} + +func (a *ChkReplicaAddress) SetClusterIndex(index int) { + a.ClusterIndex = index +} + +func (a *ChkReplicaAddress) GetReplicaName() string { + return a.ReplicaName +} + +func (a *ChkReplicaAddress) SetReplicaName(name string) { + a.ReplicaName = name +} + +func (a *ChkReplicaAddress) GetReplicaIndex() int { + return a.ReplicaIndex +} + +func (a *ChkReplicaAddress) SetReplicaIndex(index int) { + a.ReplicaIndex = index +} diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_shard.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_shard.go new file mode 100644 index 00000000..34333e8b --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_shard.go @@ -0,0 +1,310 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + apiChi "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/common/types" +) + +// ChiShard defines item of a shard section of .spec.configuration.clusters[n].shards +// TODO unify with ChiReplica based on HostsSet +type ChkShard struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Weight *int `json:"weight,omitempty" yaml:"weight,omitempty"` + InternalReplication *types.StringBool `json:"internalReplication,omitempty" yaml:"internalReplication,omitempty"` + Settings *apiChi.Settings `json:"settings,omitempty" yaml:"settings,omitempty"` + Files *apiChi.Settings `json:"files,omitempty" yaml:"files,omitempty"` + Templates *apiChi.TemplatesList `json:"templates,omitempty" yaml:"templates,omitempty"` + ReplicasCount int `json:"replicasCount,omitempty" yaml:"replicasCount,omitempty"` + // TODO refactor into map[string]Host + Hosts []*apiChi.Host `json:"replicas,omitempty" yaml:"replicas,omitempty"` + + Runtime ChkShardRuntime `json:"-" yaml:"-"` + + // DefinitionType is DEPRECATED - to be removed soon + DefinitionType string `json:"definitionType,omitempty" yaml:"definitionType,omitempty"` +} + +type ChkShardRuntime struct { + Address ChkShardAddress `json:"-" yaml:"-"` + CHK *ClickHouseKeeperInstallation `json:"-" yaml:"-" testdiff:"ignore"` +} + +func (r ChkShardRuntime) GetAddress() apiChi.IShardAddress { + return &r.Address +} + +func (r *ChkShardRuntime) GetCR() apiChi.ICustomResource { + return r.CHK +} + +func (r *ChkShardRuntime) SetCR(cr apiChi.ICustomResource) { + r.CHK = cr.(*ClickHouseKeeperInstallation) +} + +func (shard *ChkShard) GetName() string { + return shard.Name +} + +func (shard *ChkShard) GetInternalReplication() *types.StringBool { + return shard.InternalReplication +} + +// InheritSettingsFrom inherits settings from specified cluster +func (shard *ChkShard) InheritSettingsFrom(cluster *Cluster) { + shard.Settings = shard.Settings.MergeFrom(cluster.Settings) +} + +// InheritFilesFrom inherits files from specified cluster +func (shard *ChkShard) InheritFilesFrom(cluster *Cluster) { + shard.Files = shard.Files.MergeFrom(cluster.Files) +} + +// InheritTemplatesFrom inherits templates from specified cluster +func (shard *ChkShard) InheritTemplatesFrom(cluster *Cluster) { + shard.Templates = shard.Templates.MergeFrom(cluster.Templates, apiChi.MergeTypeFillEmptyValues) + shard.Templates.HandleDeprecatedFields() +} + +// GetServiceTemplate gets service template +func (shard *ChkShard) GetServiceTemplate() (*apiChi.ServiceTemplate, bool) { + if !shard.Templates.HasShardServiceTemplate() { + return nil, false + } + name := shard.Templates.GetShardServiceTemplate() + return shard.Runtime.CHK.GetServiceTemplate(name) +} + +// HasReplicasCount checks whether shard has replicas count specified +func (shard *ChkShard) HasReplicasCount() bool { + if shard == nil { + return false + } + + return shard.ReplicasCount > 0 +} + +// WalkHosts runs specified function on each host +func (shard *ChkShard) WalkHosts(f func(host *apiChi.Host) error) []error { + if shard == nil { + return nil + } + + res := make([]error, 0) + + for replicaIndex := range shard.Hosts { + host := shard.Hosts[replicaIndex] + res = append(res, f(host)) + } + + return res +} + +// WalkHosts runs specified function on each host +func (shard *ChkShard) WalkHostsAbortOnError(f func(host *apiChi.Host) error) error { + if shard == nil { + return nil + } + + for replicaIndex := range shard.Hosts { + host := shard.Hosts[replicaIndex] + if err := f(host); err != nil { + return err + } + } + + return nil +} + +// FindHost finds host by name or index. +// Expectations: name is expected to be a string, index is expected to be an int. +func (shard *ChkShard) FindHost(needle interface{}) (res *apiChi.Host) { + shard.WalkHosts(func(host *apiChi.Host) error { + switch v := needle.(type) { + case string: + if host.Runtime.Address.HostName == v { + res = host + } + case int: + if host.Runtime.Address.ShardScopeIndex == v { + res = host + } + } + return nil + }) + return +} + +// FirstHost finds first host in the shard +func (shard *ChkShard) FirstHost() *apiChi.Host { + var result *apiChi.Host + shard.WalkHosts(func(host *apiChi.Host) error { + if result == nil { + result = host + } + return nil + }) + return result +} + +// HostsCount returns count of hosts in the shard +func (shard *ChkShard) HostsCount() int { + count := 0 + shard.WalkHosts(func(host *apiChi.Host) error { + count++ + return nil + }) + return count +} + +// GetCHK gets Custom Resource of the shard +func (shard *ChkShard) GetCHK() *ClickHouseKeeperInstallation { + return shard.Runtime.CHK +} + +func (shard *ChkShard) GetAncestor() apiChi.IShard { + return (*ChkShard)(nil) +} + +// GetCluster gets cluster of the shard +func (shard *ChkShard) GetCluster() *Cluster { + return shard.Runtime.CHK.GetSpecT().Configuration.Clusters[shard.Runtime.Address.ClusterIndex] +} + +// HasWeight checks whether shard has applicable weight value specified +func (shard *ChkShard) HasWeight() bool { + if shard == nil { + return false + } + if shard.Weight == nil { + return false + } + return *shard.Weight >= 0 +} + +// GetWeight gets weight +func (shard *ChkShard) GetWeight() int { + if shard.HasWeight() { + return *shard.Weight + } + return 0 +} + +func (shard *ChkShard) GetRuntime() apiChi.IShardRuntime { + if shard == nil { + return (*ChkShardRuntime)(nil) + } + return &shard.Runtime +} + +func (shard *ChkShard) HasSettings() bool { + return shard.GetSettings() != nil +} + +func (shard *ChkShard) GetSettings() *apiChi.Settings { + if shard == nil { + return nil + } + return shard.Settings +} + +func (shard *ChkShard) HasFiles() bool { + return shard.GetFiles() != nil +} + +func (shard *ChkShard) GetFiles() *apiChi.Settings { + if shard == nil { + return nil + } + return shard.Files +} + +func (shard *ChkShard) HasTemplates() bool { + return shard.GetTemplates() != nil +} + +func (shard *ChkShard) GetTemplates() *apiChi.TemplatesList { + if shard == nil { + return nil + } + return shard.Templates +} + +func (shard *ChkShard) IsZero() bool { + return shard == nil +} + +func (shard *ChkShard) IsNonZero() bool { + return shard != nil +} + +// ChiShardAddress defines address of a shard within ClickHouseInstallation +type ChkShardAddress struct { + Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` + CHIName string `json:"chiName,omitempty" yaml:"chiName,omitempty"` + ClusterName string `json:"clusterName,omitempty" yaml:"clusterName,omitempty"` + ClusterIndex int `json:"clusterIndex,omitempty" yaml:"clusterIndex,omitempty"` + ShardName string `json:"shardName,omitempty" yaml:"shardName,omitempty"` + ShardIndex int `json:"shardIndex,omitempty" yaml:"shardIndex,omitempty"` +} + +func (a *ChkShardAddress) GetNamespace() string { + return a.Namespace +} + +func (a *ChkShardAddress) SetNamespace(namespace string) { + a.Namespace = namespace +} + +func (a *ChkShardAddress) GetCRName() string { + return a.CHIName +} + +func (a *ChkShardAddress) SetCRName(name string) { + a.CHIName = name +} + +func (a *ChkShardAddress) GetClusterName() string { + return a.ClusterName +} + +func (a *ChkShardAddress) SetClusterName(name string) { + a.ClusterName = name +} + +func (a *ChkShardAddress) GetClusterIndex() int { + return a.ClusterIndex +} + +func (a *ChkShardAddress) SetClusterIndex(index int) { + a.ClusterIndex = index +} + +func (a *ChkShardAddress) GetShardName() string { + return a.ShardName +} + +func (a *ChkShardAddress) SetShardName(name string) { + a.ShardName = name +} + +func (a *ChkShardAddress) GetShardIndex() int { + return a.ShardIndex +} + +func (a *ChkShardAddress) SetShardIndex(index int) { + a.ShardIndex = index +} diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_spec.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_spec.go new file mode 100644 index 00000000..5d620cc3 --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_spec.go @@ -0,0 +1,130 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + apiChi "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/common/types" +) + +// ChkSpec defines spec section of ClickHouseKeeper resource +type ChkSpec struct { + TaskID *types.Id `json:"taskID,omitempty" yaml:"taskID,omitempty"` + Stop *types.StringBool `json:"stop,omitempty" yaml:"stop,omitempty"` + NamespaceDomainPattern *types.String `json:"namespaceDomainPattern,omitempty" yaml:"namespaceDomainPattern,omitempty"` + Suspend *types.StringBool `json:"suspend,omitempty" yaml:"suspend,omitempty"` + Reconciling *apiChi.ChiReconcile `json:"reconciling,omitempty" yaml:"reconciling,omitempty"` + Reconcile *apiChi.ChiReconcile `json:"reconcile,omitempty" yaml:"reconcile,omitempty"` + Defaults *apiChi.Defaults `json:"defaults,omitempty" yaml:"defaults,omitempty"` + Configuration *Configuration `json:"configuration,omitempty" yaml:"configuration,omitempty"` + Templates *apiChi.Templates `json:"templates,omitempty" yaml:"templates,omitempty"` +} + +// HasTaskID checks whether task id is specified +func (spec *ChkSpec) HasTaskID() bool { + if spec == nil { + return false + } + return spec.TaskID.HasValue() +} + +// GetTaskID gets task id as a string +func (spec *ChkSpec) GetTaskID() *types.Id { + if spec == nil { + return nil + } + return spec.TaskID +} + +func (spec *ChkSpec) GetStop() *types.StringBool { + if spec == nil { + return (*types.StringBool)(nil) + } + return spec.Stop +} + +func (spec *ChkSpec) GetNamespaceDomainPattern() *types.String { + if spec == nil { + return (*types.String)(nil) + } + return spec.NamespaceDomainPattern +} + +func (spec *ChkSpec) GetDefaults() *apiChi.Defaults { + if spec == nil { + return (*apiChi.Defaults)(nil) + } + return spec.Defaults +} + +func (spec *ChkSpec) GetConfiguration() apiChi.IConfiguration { + if spec == nil { + return (*Configuration)(nil) + } + return spec.Configuration +} + +func (spec *ChkSpec) GetTemplates() *apiChi.Templates { + if spec == nil { + return (*apiChi.Templates)(nil) + } + return spec.Templates +} + +// MergeFrom merges from spec +func (spec *ChkSpec) MergeFrom(from *ChkSpec, _type apiChi.MergeType) { + if from == nil { + return + } + + if spec == nil { + spec = &ChkSpec{} + } + + switch _type { + case apiChi.MergeTypeFillEmptyValues: + if !spec.HasTaskID() { + spec.TaskID = spec.TaskID.MergeFrom(from.TaskID) + } + if !spec.Stop.HasValue() { + spec.Stop = spec.Stop.MergeFrom(from.Stop) + } + if !spec.NamespaceDomainPattern.HasValue() { + spec.NamespaceDomainPattern = spec.NamespaceDomainPattern.MergeFrom(from.NamespaceDomainPattern) + } + if !spec.Suspend.HasValue() { + spec.Suspend = spec.Suspend.MergeFrom(from.Suspend) + } + case apiChi.MergeTypeOverrideByNonEmptyValues: + if from.HasTaskID() { + spec.TaskID = spec.TaskID.MergeFrom(from.TaskID) + } + if from.Stop.HasValue() { + // Override by non-empty values only + spec.Stop = from.Stop + } + if from.NamespaceDomainPattern.HasValue() { + spec.NamespaceDomainPattern = spec.NamespaceDomainPattern.MergeFrom(from.NamespaceDomainPattern) + } + if spec.Suspend.HasValue() { + spec.Suspend = spec.Suspend.MergeFrom(from.Suspend) + } + } + + spec.Reconcile = spec.Reconcile.MergeFrom(from.Reconcile, _type) + spec.Defaults = spec.Defaults.MergeFrom(from.Defaults, _type) + spec.Configuration = spec.Configuration.MergeFrom(from.Configuration, _type) + spec.Templates = spec.Templates.MergeFrom(from.Templates, _type) +} diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_status.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_status.go new file mode 100644 index 00000000..486abce3 --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/type_status.go @@ -0,0 +1,899 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "sort" + "sync" + + chi "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/common/types" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/util" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/version" +) + +const ( + maxActions = 10 + maxErrors = 10 + maxTaskIDs = 10 +) + +// Possible CR statuses +const ( + StatusInProgress = "InProgress" + StatusCompleted = "Completed" + StatusAborted = "Aborted" + StatusTerminating = "Terminating" +) + +// Status defines status section of the custom resource. +// +// Note: application level reads and writes to Status fields should be done through synchronized getter/setter functions. +// While all of these fields need to be exported for JSON and YAML serialization/deserialization, we can at least audit +// that application logic sticks to the synchronized getter/setters by auditing whether all explicit Go field-level +// accesses are strictly within _this_ source file OR the generated deep copy source file. +type Status struct { + CHOpVersion string `json:"chop-version,omitempty" yaml:"chop-version,omitempty"` + CHOpCommit string `json:"chop-commit,omitempty" yaml:"chop-commit,omitempty"` + CHOpDate string `json:"chop-date,omitempty" yaml:"chop-date,omitempty"` + CHOpIP string `json:"chop-ip,omitempty" yaml:"chop-ip,omitempty"` + ClustersCount int `json:"clusters,omitempty" yaml:"clusters,omitempty"` + ShardsCount int `json:"shards,omitempty" yaml:"shards,omitempty"` + ReplicasCount int `json:"replicas,omitempty" yaml:"replicas,omitempty"` + HostsCount int `json:"hosts,omitempty" yaml:"hosts,omitempty"` + Status string `json:"status,omitempty" yaml:"status,omitempty"` + TaskID string `json:"taskID,omitempty" yaml:"taskID,omitempty"` + TaskIDsStarted []string `json:"taskIDsStarted,omitempty" yaml:"taskIDsStarted,omitempty"` + TaskIDsCompleted []string `json:"taskIDsCompleted,omitempty" yaml:"taskIDsCompleted,omitempty"` + Action string `json:"action,omitempty" yaml:"action,omitempty"` + Actions []string `json:"actions,omitempty" yaml:"actions,omitempty"` + Error string `json:"error,omitempty" yaml:"error,omitempty"` + Errors []string `json:"errors,omitempty" yaml:"errors,omitempty"` + HostsUpdatedCount int `json:"hostsUpdated,omitempty" yaml:"hostsUpdated,omitempty"` + HostsAddedCount int `json:"hostsAdded,omitempty" yaml:"hostsAdded,omitempty"` + HostsUnchangedCount int `json:"hostsUnchanged,omitempty" yaml:"hostsUnchanged,omitempty"` + HostsFailedCount int `json:"hostsFailed,omitempty" yaml:"hostsFailed,omitempty"` + HostsCompletedCount int `json:"hostsCompleted,omitempty" yaml:"hostsCompleted,omitempty"` + HostsDeletedCount int `json:"hostsDeleted,omitempty" yaml:"hostsDeleted,omitempty"` + HostsDeleteCount int `json:"hostsDelete,omitempty" yaml:"hostsDelete,omitempty"` + Pods []string `json:"pods,omitempty" yaml:"pods,omitempty"` + PodIPs []string `json:"pod-ips,omitempty" yaml:"pod-ips,omitempty"` + FQDNs []string `json:"fqdns,omitempty" yaml:"fqdns,omitempty"` + Endpoint string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"` + Endpoints []string `json:"endpoints,omitempty" yaml:"endpoints,omitempty"` + NormalizedCR *ClickHouseKeeperInstallation `json:"normalized,omitempty" yaml:"normalized,omitempty"` + NormalizedCRCompleted *ClickHouseKeeperInstallation `json:"normalizedCompleted,omitempty" yaml:"normalizedCompleted,omitempty"` + ActionPlan *chi.ActionPlan `json:"actionPlan,omitempty" yaml:"actionPlan,omitempty"` + HostsWithTablesCreated []string `json:"hostsWithTablesCreated,omitempty" yaml:"hostsWithTablesCreated,omitempty"` + HostsWithReplicaCaughtUp []string `json:"hostsWithReplicaCaughtUp,omitempty" yaml:"hostsWithReplicaCaughtUp,omitempty"` + UsedTemplates []*chi.TemplateRef `json:"usedTemplates,omitempty" yaml:"usedTemplates,omitempty"` + + mu sync.RWMutex `json:"-" yaml:"-"` +} + +// FillStatusParams is a struct used to fill status params +type FillStatusParams struct { + CHOpIP string + ClustersCount int + ShardsCount int + HostsCount int + TaskID string + HostsUpdatedCount int + HostsAddedCount int + HostsUnchangedCount int + HostsCompletedCount int + HostsDeleteCount int + HostsDeletedCount int + Pods []string + FQDNs []string + Endpoint string + Endpoints []string + NormalizedCR *ClickHouseKeeperInstallation +} + +// Fill is a synchronized setter for a fairly large number of fields. We take a struct type "params" argument to avoid +// confusion of similarly typed positional arguments, and to avoid defining a lot of separate synchronized setters +// for these fields that are typically all set together at once (during "fills"). +func (s *Status) Fill(params *FillStatusParams) { + doWithWriteLock(s, func(s *Status) { + // We always set these (build-hardcoded) version fields. + s.CHOpVersion = version.Version + s.CHOpCommit = version.GitSHA + s.CHOpDate = version.BuiltAt + + // Now, set fields from the provided input. + s.CHOpIP = params.CHOpIP + s.ClustersCount = params.ClustersCount + s.ShardsCount = params.ShardsCount + s.HostsCount = params.HostsCount + s.TaskID = params.TaskID + s.HostsUpdatedCount = params.HostsUpdatedCount + s.HostsAddedCount = params.HostsAddedCount + s.HostsUnchangedCount = params.HostsUnchangedCount + s.HostsCompletedCount = params.HostsCompletedCount + s.HostsDeleteCount = params.HostsDeleteCount + s.HostsDeletedCount = params.HostsDeletedCount + s.Pods = params.Pods + s.FQDNs = util.NormalizeFQDNs(params.FQDNs) + s.Endpoint = util.NormalizeFQDN(params.Endpoint) + s.Endpoints = util.NormalizeFQDNs(params.Endpoints) + s.NormalizedCR = params.NormalizedCR + }) +} + +// SetError sets status error +func (s *Status) SetError(err string) { + doWithWriteLock(s, func(s *Status) { + s.Error = err + }) +} + +// PushError sets and pushes error into status +func (s *Status) PushError(error string) { + doWithWriteLock(s, func(s *Status) { + s.Errors = append([]string{error}, s.Errors...) + if len(s.Errors) > maxErrors { + s.Errors = s.Errors[:maxErrors] + } + }) +} + +// SetAndPushError sets and pushes error into status +func (s *Status) SetAndPushError(err string) { + doWithWriteLock(s, func(s *Status) { + s.Error = err + s.Errors = append([]string{err}, s.Errors...) + if len(s.Errors) > maxErrors { + s.Errors = s.Errors[:maxErrors] + } + }) +} + +// PushHostReplicaCaughtUp pushes host to the list of hosts with replica caught-up +func (s *Status) PushHostReplicaCaughtUp(host string) { + host = util.NormalizeFQDN(host) + doWithWriteLock(s, func(s *Status) { + if util.InArray(host, s.HostsWithReplicaCaughtUp) { + return + } + s.HostsWithReplicaCaughtUp = append(s.HostsWithReplicaCaughtUp, host) + }) +} + +// PushHostTablesCreated pushes host to the list of hosts with created tables +func (s *Status) PushHostTablesCreated(host string) { + host = util.NormalizeFQDN(host) + doWithWriteLock(s, func(s *Status) { + if util.InArray(host, s.HostsWithTablesCreated) { + return + } + s.HostsWithTablesCreated = append(s.HostsWithTablesCreated, host) + }) +} + +// SyncHostTablesCreated syncs list of hosts with tables created with actual list of hosts +func (s *Status) SyncHostTablesCreated() { + doWithWriteLock(s, func(s *Status) { + if s.FQDNs == nil { + return + } + // Normalize both sides to handle status data written by older operator versions + // that may contain trailing dots. + s.HostsWithTablesCreated = util.IntersectStringArrays( + util.NormalizeFQDNs(s.HostsWithTablesCreated), + util.NormalizeFQDNs(s.FQDNs), + ) + }) +} + +// PushUsedTemplate pushes used templates to the list of used templates +func (s *Status) PushUsedTemplate(templateRefs ...*chi.TemplateRef) { + if len(templateRefs) > 0 { + doWithWriteLock(s, func(s *Status) { + s.UsedTemplates = append(s.UsedTemplates, templateRefs...) + }) + } +} + +// GetUsedTemplatesCount gets used templates count +func (s *Status) GetUsedTemplatesCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return len(s.UsedTemplates) + }) +} + +// SetAction action setter +func (s *Status) SetAction(action string) { + doWithWriteLock(s, func(s *Status) { + s.Action = action + }) +} + +// PushAction pushes action into status +func (s *Status) PushAction(action string) { + doWithWriteLock(s, func(s *Status) { + s.Actions = append([]string{action}, s.Actions...) + trimActionsNoSync(s) + }) +} + +// HasNormalizedCRCompleted is a checker +func (s *Status) HasNormalizedCRCompleted() bool { + return s.GetNormalizedCRCompleted() != nil +} + +// HasNormalizedCR is a checker +func (s *Status) HasNormalizedCR() bool { + return s.GetNormalizedCR() != nil +} + +// SetPodIPs sets pod IPs +func (s *Status) SetPodIPs(podIPs []string) { + doWithWriteLock(s, func(s *Status) { + s.PodIPs = podIPs + }) +} + +// HostDeleted increments deleted hosts counter +func (s *Status) HostDeleted() { + doWithWriteLock(s, func(s *Status) { + s.HostsDeletedCount++ + }) +} + +// HostUpdated increments updated hosts counter +func (s *Status) HostUpdated() { + doWithWriteLock(s, func(s *Status) { + s.HostsUpdatedCount++ + }) +} + +// HostAdded increments added hosts counter +func (s *Status) HostAdded() { + doWithWriteLock(s, func(s *Status) { + s.HostsAddedCount++ + }) +} + +// HostUnchanged increments unchanged hosts counter +func (s *Status) HostUnchanged() { + doWithWriteLock(s, func(s *Status) { + s.HostsUnchangedCount++ + }) +} + +// HostFailed increments failed hosts counter +func (s *Status) HostFailed() { + doWithWriteLock(s, func(s *Status) { + s.HostsFailedCount++ + }) +} + +// HostCompleted increments completed hosts counter +func (s *Status) HostCompleted() { + doWithWriteLock(s, func(s *Status) { + s.HostsCompletedCount++ + }) +} + +// ReconcileStart marks reconcile start +func (s *Status) ReconcileStart(ap chi.IActionPlan) { + doWithWriteLock(s, func(s *Status) { + if s == nil { + return + } + s.Status = StatusInProgress + s.HostsUpdatedCount = 0 + s.HostsAddedCount = 0 + s.HostsUnchangedCount = 0 + s.HostsCompletedCount = 0 + s.HostsDeletedCount = 0 + s.HostsDeleteCount = ap.GetRemovedHostsNum() + s.ActionPlan = ap.(*chi.ActionPlan) + pushTaskIDStartedNoSync(s) + }) +} + +// ReconcileComplete marks reconcile completion +func (s *Status) ReconcileComplete() { + doWithWriteLock(s, func(s *Status) { + if s == nil { + return + } + s.Status = StatusCompleted + s.Action = "" + pushTaskIDCompletedNoSync(s) + }) +} + +// ReconcileAbort marks reconcile abortion +func (s *Status) ReconcileAbort() { + doWithWriteLock(s, func(s *Status) { + if s == nil { + return + } + s.Status = StatusAborted + s.Action = "" + pushTaskIDCompletedNoSync(s) + }) +} + +// DeleteStart marks deletion start +func (s *Status) DeleteStart() { + doWithWriteLock(s, func(s *Status) { + if s == nil { + return + } + s.Status = StatusTerminating + s.HostsUpdatedCount = 0 + s.HostsAddedCount = 0 + s.HostsUnchangedCount = 0 + s.HostsCompletedCount = 0 + s.HostsDeletedCount = 0 + s.HostsDeleteCount = 0 + pushTaskIDStartedNoSync(s) + }) +} + +// SetActionPlan sets action plan +func (s *Status) SetActionPlan(ap chi.IActionPlan) { + doWithWriteLock(s, func(s *Status) { + s.ActionPlan = ap.(*chi.ActionPlan) + }) +} + +func prepareOptions(opts types.CopyStatusOptions) types.CopyStatusOptions { + if opts.FieldGroupInheritable { + opts.Copy.TaskIDsStarted = true + opts.Copy.TaskIDsCompleted = true + opts.Copy.Actions = true + opts.Copy.Errors = true + opts.Copy.UsedTemplates = true + } + + if opts.FieldGroupActions { + opts.Copy.Action = true + opts.Merge.Actions = true + opts.Copy.UsedTemplates = true + } + + if opts.FieldGroupErrors { + opts.Copy.Error = true + opts.Merge.Errors = true + } + + if opts.FieldGroupMain { + opts.Copy.CHOpVersion = true + opts.Copy.CHOpCommit = true + opts.Copy.CHOpDate = true + opts.Copy.CHOpIP = true + opts.Copy.ClustersCount = true + opts.Copy.ShardsCount = true + opts.Copy.ReplicasCount = true + opts.Copy.HostsCount = true + opts.Copy.Status = true + opts.Copy.TaskID = true + opts.Copy.TaskIDsStarted = true + opts.Copy.TaskIDsCompleted = true + opts.Copy.Action = true + opts.Merge.Actions = true + opts.Copy.Error = true + opts.Copy.Errors = true + opts.Copy.HostsUpdatedCount = true + opts.Copy.HostsAddedCount = true + opts.Copy.HostsUnchangedCount = true + opts.Copy.HostsCompletedCount = true + opts.Copy.HostsDeletedCount = true + opts.Copy.HostsDeleteCount = true + opts.Copy.Pods = true + opts.Copy.PodIPs = true + opts.Copy.FQDNs = true + opts.Copy.Endpoint = true + opts.Copy.NormalizedCR = true + opts.Copy.ActionPlan = true + opts.Copy.UsedTemplates = true + } + + if opts.FieldGroupNormalized { + opts.Copy.NormalizedCR = true + opts.Copy.ActionPlan = true + } + + if opts.FieldGroupWholeStatus { + opts.Copy.CHOpVersion = true + opts.Copy.CHOpCommit = true + opts.Copy.CHOpDate = true + opts.Copy.CHOpIP = true + opts.Copy.ClustersCount = true + opts.Copy.ShardsCount = true + opts.Copy.ReplicasCount = true + opts.Copy.HostsCount = true + opts.Copy.Status = true + opts.Copy.TaskID = true + opts.Copy.TaskIDsStarted = true + opts.Copy.TaskIDsCompleted = true + opts.Copy.Action = true + opts.Merge.Actions = true + opts.Copy.Error = true + opts.Copy.Errors = true + opts.Copy.HostsUpdatedCount = true + opts.Copy.HostsAddedCount = true + opts.Copy.HostsUnchangedCount = true + opts.Copy.HostsCompletedCount = true + opts.Copy.HostsDeletedCount = true + opts.Copy.HostsDeleteCount = true + opts.Copy.Pods = true + opts.Copy.PodIPs = true + opts.Copy.FQDNs = true + opts.Copy.Endpoint = true + opts.Copy.NormalizedCR = true + opts.Copy.NormalizedCRCompleted = true + opts.Copy.ActionPlan = true + opts.Copy.UsedTemplates = true + } + + return opts +} + +// CopyFrom copies the state of a given Status f into the receiver Status of the call. +func (s *Status) CopyFrom(f *Status, opts types.CopyStatusOptions) { + doWithWriteLock(s, func(s *Status) { + doWithReadLock(f, func(from *Status) { + if s == nil || from == nil { + return + } + + opts = prepareOptions(opts) + + // Copy fields + if opts.Copy.CHOpVersion { + s.CHOpVersion = from.CHOpVersion + } + if opts.Copy.CHOpCommit { + s.CHOpCommit = from.CHOpCommit + } + if opts.Copy.CHOpDate { + s.CHOpDate = from.CHOpDate + } + if opts.Copy.CHOpIP { + s.CHOpIP = from.CHOpIP + } + if opts.Copy.ClustersCount { + s.ClustersCount = from.ClustersCount + } + if opts.Copy.ShardsCount { + s.ShardsCount = from.ShardsCount + } + if opts.Copy.ReplicasCount { + s.ReplicasCount = from.ReplicasCount + } + if opts.Copy.HostsCount { + s.HostsCount = from.HostsCount + } + if opts.Copy.Status { + s.Status = from.Status + } + if opts.Copy.TaskID { + s.TaskID = from.TaskID + } + if opts.Copy.TaskIDsStarted { + s.TaskIDsStarted = from.TaskIDsStarted + } + if opts.Copy.TaskIDsCompleted { + s.TaskIDsCompleted = from.TaskIDsCompleted + } + if opts.Copy.Action { + s.Action = from.Action + } + if opts.Merge.Actions { + mergeActionsNoSync(s, from) + } + if opts.Copy.Error { + s.Error = from.Error + } + if opts.Copy.Errors { + s.Errors = from.Errors + } + if opts.Merge.Errors { + s.Errors = util.MergeStringArrays(s.Errors, from.Errors) + sort.Sort(sort.Reverse(sort.StringSlice(s.Errors))) + } + if opts.Copy.HostsUpdatedCount { + s.HostsUpdatedCount = from.HostsUpdatedCount + } + if opts.Copy.HostsAddedCount { + s.HostsAddedCount = from.HostsAddedCount + } + if opts.Copy.HostsUnchangedCount { + s.HostsUnchangedCount = from.HostsUnchangedCount + } + if opts.Copy.HostsCompletedCount { + s.HostsCompletedCount = from.HostsCompletedCount + } + if opts.Copy.HostsDeletedCount { + s.HostsDeletedCount = from.HostsDeletedCount + } + if opts.Copy.HostsDeleteCount { + s.HostsDeleteCount = from.HostsDeleteCount + } + if opts.Copy.Pods { + s.Pods = from.Pods + } + if opts.Copy.PodIPs { + s.PodIPs = from.PodIPs + } + if opts.Copy.FQDNs { + s.FQDNs = from.FQDNs + } + if opts.Copy.Endpoint { + s.Endpoint = from.Endpoint + s.Endpoints = from.Endpoints + } + if opts.Copy.NormalizedCR { + s.NormalizedCR = from.NormalizedCR + } + if opts.Copy.NormalizedCRCompleted { + s.NormalizedCRCompleted = from.NormalizedCRCompleted + } + if opts.Copy.ActionPlan { + s.ActionPlan = from.ActionPlan + } + if opts.Copy.HostsWithTablesCreated { + s.HostsWithTablesCreated = nil + if len(from.HostsWithTablesCreated) > 0 { + s.HostsWithTablesCreated = append(s.HostsWithTablesCreated, from.HostsWithTablesCreated...) + } + s.HostsWithReplicaCaughtUp = nil + if len(from.HostsWithReplicaCaughtUp) > 0 { + s.HostsWithReplicaCaughtUp = append(s.HostsWithReplicaCaughtUp, from.HostsWithReplicaCaughtUp...) + } + } + if opts.Copy.UsedTemplates { + if len(from.UsedTemplates) > len(s.UsedTemplates) { + s.UsedTemplates = nil + s.UsedTemplates = append(s.UsedTemplates, from.UsedTemplates...) + } + } + }) + }) +} + +// ClearNormalizedCR clears normalized CR in status +func (s *Status) ClearNormalizedCR() { + doWithWriteLock(s, func(s *Status) { + s.NormalizedCR = nil + }) +} + +// SetNormalizedCompletedFromCurrentNormalized sets completed CR from current CR +func (s *Status) SetNormalizedCompletedFromCurrentNormalized() { + doWithWriteLock(s, func(s *Status) { + s.NormalizedCRCompleted = s.NormalizedCR + }) +} + +// GetCHOpVersion gets operator version +func (s *Status) GetCHOpVersion() string { + return getStringWithReadLock(s, func(s *Status) string { + return s.CHOpVersion + }) +} + +// GetCHOpCommit gets operator build commit +func (s *Status) GetCHOpCommit() string { + return getStringWithReadLock(s, func(s *Status) string { + return s.CHOpCommit + }) +} + +// GetCHOpDate gets operator build date +func (s *Status) GetCHOpDate() string { + return getStringWithReadLock(s, func(s *Status) string { + return s.CHOpDate + }) +} + +// GetCHOpIP gets operator pod's IP +func (s *Status) GetCHOpIP() string { + return getStringWithReadLock(s, func(s *Status) string { + return s.CHOpIP + }) +} + +// GetClustersCount gets clusters count +func (s *Status) GetClustersCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return s.ClustersCount + }) +} + +// GetShardsCount gets shards count +func (s *Status) GetShardsCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return s.ShardsCount + }) +} + +// GetReplicasCount gets replicas count +func (s *Status) GetReplicasCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return s.ReplicasCount + }) +} + +// GetHostsCount gets hosts count +func (s *Status) GetHostsCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return s.HostsCount + }) +} + +// GetStatus gets status +func (s *Status) GetStatus() string { + return getStringWithReadLock(s, func(s *Status) string { + return s.Status + }) +} + +// GetTaskID gets task ipd +func (s *Status) GetTaskID() string { + return getStringWithReadLock(s, func(s *Status) string { + return s.TaskID + }) +} + +// GetTaskIDsStarted gets started task id +func (s *Status) GetTaskIDsStarted() []string { + return getStringArrWithReadLock(s, func(s *Status) []string { + return s.TaskIDsStarted + }) +} + +// GetTaskIDsCompleted gets completed task id +func (s *Status) GetTaskIDsCompleted() []string { + return getStringArrWithReadLock(s, func(s *Status) []string { + return s.TaskIDsCompleted + }) +} + +// GetAction gets last action +func (s *Status) GetAction() string { + return getStringWithReadLock(s, func(s *Status) string { + return s.Action + }) +} + +// GetActions gets all actions +func (s *Status) GetActions() []string { + return getStringArrWithReadLock(s, func(s *Status) []string { + return s.Actions + }) +} + +// GetError gets last error +func (s *Status) GetError() string { + return getStringWithReadLock(s, func(s *Status) string { + return s.Error + }) +} + +// GetErrors gets all errors +func (s *Status) GetErrors() []string { + return getStringArrWithReadLock(s, func(s *Status) []string { + return s.Errors + }) +} + +// GetHostsUpdatedCount gets updated hosts counter +func (s *Status) GetHostsUpdatedCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return s.HostsUpdatedCount + }) +} + +// GetHostsAddedCount gets added hosts counter +func (s *Status) GetHostsAddedCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return s.HostsAddedCount + }) +} + +// GetHostsUnchangedCount gets unchanged hosts counter +func (s *Status) GetHostsUnchangedCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return s.HostsUnchangedCount + }) +} + +// GetHostsFailedCount gets failed hosts counter +func (s *Status) GetHostsFailedCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return s.HostsFailedCount + }) +} + +// GetHostsCompletedCount gets completed hosts counter +func (s *Status) GetHostsCompletedCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return s.HostsCompletedCount + }) +} + +// GetHostsDeletedCount gets deleted hosts counter +func (s *Status) GetHostsDeletedCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return s.HostsDeletedCount + }) +} + +// GetHostsDeleteCount gets hosts to be deleted counter +func (s *Status) GetHostsDeleteCount() int { + return getIntWithReadLock(s, func(s *Status) int { + return s.HostsDeleteCount + }) +} + +// GetPods gets list of pods +func (s *Status) GetPods() []string { + return getStringArrWithReadLock(s, func(s *Status) []string { + return s.Pods + }) +} + +// GetPodIPs gets list of pod ips +func (s *Status) GetPodIPs() []string { + return getStringArrWithReadLock(s, func(s *Status) []string { + return s.PodIPs + }) +} + +// GetFQDNs gets list of all FQDNs of hosts +func (s *Status) GetFQDNs() []string { + return getStringArrWithReadLock(s, func(s *Status) []string { + return s.FQDNs + }) +} + +// GetEndpoint gets API endpoint +func (s *Status) GetEndpoint() string { + return getStringWithReadLock(s, func(s *Status) string { + return s.Endpoint + }) +} + +// GetNormalizedCR gets target CR +func (s *Status) GetNormalizedCR() *ClickHouseKeeperInstallation { + return getCRWithReadLock(s, func(s *Status) *ClickHouseKeeperInstallation { + return s.NormalizedCR + }) +} + +// GetNormalizedCRCompleted gets completed CR +func (s *Status) GetNormalizedCRCompleted() *ClickHouseKeeperInstallation { + return getCRWithReadLock(s, func(s *Status) *ClickHouseKeeperInstallation { + return s.NormalizedCRCompleted + }) +} + +// GetHostsWithTablesCreated gets hosts with created tables +func (s *Status) GetHostsWithTablesCreated() []string { + return getStringArrWithReadLock(s, func(s *Status) []string { + return util.NormalizeFQDNs(s.HostsWithTablesCreated) + }) +} + +// GetHostsWithReplicaCaughtUp gets hosts with replica caught-up +func (s *Status) GetHostsWithReplicaCaughtUp() []string { + return getStringArrWithReadLock(s, func(s *Status) []string { + return util.NormalizeFQDNs(s.HostsWithReplicaCaughtUp) + }) +} + +// Begin helpers + +func doWithWriteLock(s *Status, f func(*Status)) { + if s == nil { + return + } + + s.mu.Lock() + defer s.mu.Unlock() + f(s) +} + +func doWithReadLock(s *Status, f func(*Status)) { + if s == nil { + return + } + + s.mu.RLock() + defer s.mu.RUnlock() + f(s) +} + +func getIntWithReadLock(s *Status, f func(*Status) int) int { + var zeroVal int + if s == nil { + return zeroVal + } + + s.mu.RLock() + defer s.mu.RUnlock() + return f(s) +} + +func getStringWithReadLock(s *Status, f func(*Status) string) string { + var zeroVal string + if s == nil { + return zeroVal + } + + s.mu.RLock() + defer s.mu.RUnlock() + return f(s) +} + +func getCRWithReadLock(s *Status, f func(*Status) *ClickHouseKeeperInstallation) *ClickHouseKeeperInstallation { + var zeroVal *ClickHouseKeeperInstallation + if s == nil { + return zeroVal + } + + s.mu.RLock() + defer s.mu.RUnlock() + return f(s) +} + +func getStringArrWithReadLock(s *Status, f func(*Status) []string) []string { + emptyArr := make([]string, 0, 0) + if s == nil { + return emptyArr + } + + s.mu.RLock() + defer s.mu.RUnlock() + return f(s) +} + +// mergeActionsNoSync merges the actions of from into those of s (without synchronization, because synchronized +// functions call into this). +func mergeActionsNoSync(s *Status, from *Status) { + s.Actions = util.MergeStringArrays(s.Actions, from.Actions) + sort.Sort(sort.Reverse(sort.StringSlice(s.Actions))) + trimActionsNoSync(s) +} + +// trimActionsNoSync trims actions (without synchronization, because synchronized functions call into this). +func trimActionsNoSync(s *Status) { + if len(s.Actions) > maxActions { + s.Actions = s.Actions[:maxActions] + } +} + +// pushTaskIDStartedNoSync pushes task id into status +func pushTaskIDStartedNoSync(s *Status) { + s.TaskIDsStarted = append([]string{s.TaskID}, s.TaskIDsStarted...) + if len(s.TaskIDsStarted) > maxTaskIDs { + s.TaskIDsStarted = s.TaskIDsStarted[:maxTaskIDs] + } +} + +// pushTaskIDCompletedNoSync pushes task id into status w/o sync +func pushTaskIDCompletedNoSync(s *Status) { + s.TaskIDsCompleted = append([]string{s.TaskID}, s.TaskIDsCompleted...) + if len(s.TaskIDsCompleted) > maxTaskIDs { + s.TaskIDsCompleted = s.TaskIDsCompleted[:maxTaskIDs] + } +} diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/types.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/types.go new file mode 100644 index 00000000..38522eac --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/types.go @@ -0,0 +1,74 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "sync" + + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + + apiChi "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/swversion" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ClickHouseKeeperInstallation defines a ClickHouse Keeper ChkCluster +type ClickHouseKeeperInstallation struct { + meta.TypeMeta `json:",inline" yaml:",inline"` + meta.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` + + Spec ChkSpec `json:"spec" yaml:"spec"` + Status *Status `json:"status,omitempty" yaml:"status,omitempty"` + + runtime *ClickHouseKeeperInstallationRuntime `json:"-" yaml:"-"` + statusCreatorMutex sync.Mutex `json:"-" yaml:"-"` + runtimeCreatorMutex sync.Mutex `json:"-" yaml:"-"` +} + +type ClickHouseKeeperInstallationRuntime struct { + attributes *apiChi.ComparableAttributes `json:"-" yaml:"-"` + commonConfigMutex sync.Mutex `json:"-" yaml:"-"` + MinVersion *swversion.SoftWareVersion `json:"-" yaml:"-"` + MaxVersion *swversion.SoftWareVersion `json:"-" yaml:"-"` + ActionPlan apiChi.IActionPlan `json:"-" yaml:"-"` +} + +func newClickHouseKeeperInstallationRuntime() *ClickHouseKeeperInstallationRuntime { + return &ClickHouseKeeperInstallationRuntime{ + attributes: &apiChi.ComparableAttributes{}, + } +} + +func (runtime *ClickHouseKeeperInstallationRuntime) GetAttributes() *apiChi.ComparableAttributes { + return runtime.attributes +} + +func (runtime *ClickHouseKeeperInstallationRuntime) LockCommonConfig() { + runtime.commonConfigMutex.Lock() +} + +func (runtime *ClickHouseKeeperInstallationRuntime) UnlockCommonConfig() { + runtime.commonConfigMutex.Unlock() +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ClickHouseKeeperList defines a list of ClickHouseKeeper resources +type ClickHouseKeeperInstallationList struct { + meta.TypeMeta `json:",inline" yaml:",inline"` + meta.ListMeta `json:"metadata" yaml:"metadata"` + Items []ClickHouseKeeperInstallation `json:"items" yaml:"items"` +} diff --git a/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/zz_generated.deepcopy.go b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/zz_generated.deepcopy.go new file mode 100644 index 00000000..7d2b04c5 --- /dev/null +++ b/pkg/vendored/altinity-clickhouse/clickhouse-keeper.altinity.com/v1/zz_generated.deepcopy.go @@ -0,0 +1,682 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1 + +import ( + clickhousealtinitycomv1 "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/clickhouse.altinity.com/v1" + types "github.com/wandb/operator/pkg/vendored/altinity-clickhouse/common/types" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChkClusterAddress) DeepCopyInto(out *ChkClusterAddress) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChkClusterAddress. +func (in *ChkClusterAddress) DeepCopy() *ChkClusterAddress { + if in == nil { + return nil + } + out := new(ChkClusterAddress) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChkClusterLayout) DeepCopyInto(out *ChkClusterLayout) { + *out = *in + if in.Shards != nil { + in, out := &in.Shards, &out.Shards + *out = make([]*ChkShard, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(ChkShard) + (*in).DeepCopyInto(*out) + } + } + } + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = make([]*ChkReplica, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(ChkReplica) + (*in).DeepCopyInto(*out) + } + } + } + if in.HostsField != nil { + in, out := &in.HostsField, &out.HostsField + *out = new(clickhousealtinitycomv1.HostsField) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChkClusterLayout. +func (in *ChkClusterLayout) DeepCopy() *ChkClusterLayout { + if in == nil { + return nil + } + out := new(ChkClusterLayout) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChkClusterRuntime) DeepCopyInto(out *ChkClusterRuntime) { + *out = *in + out.Address = in.Address + if in.CHK != nil { + in, out := &in.CHK, &out.CHK + *out = new(ClickHouseKeeperInstallation) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChkClusterRuntime. +func (in *ChkClusterRuntime) DeepCopy() *ChkClusterRuntime { + if in == nil { + return nil + } + out := new(ChkClusterRuntime) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChkReplica) DeepCopyInto(out *ChkReplica) { + *out = *in + if in.Settings != nil { + in, out := &in.Settings, &out.Settings + *out = new(clickhousealtinitycomv1.Settings) + (*in).DeepCopyInto(*out) + } + if in.Files != nil { + in, out := &in.Files, &out.Files + *out = new(clickhousealtinitycomv1.Settings) + (*in).DeepCopyInto(*out) + } + if in.Templates != nil { + in, out := &in.Templates, &out.Templates + *out = new(clickhousealtinitycomv1.TemplatesList) + (*in).DeepCopyInto(*out) + } + if in.Hosts != nil { + in, out := &in.Hosts, &out.Hosts + *out = make([]*clickhousealtinitycomv1.Host, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(clickhousealtinitycomv1.Host) + (*in).DeepCopyInto(*out) + } + } + } + in.Runtime.DeepCopyInto(&out.Runtime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChkReplica. +func (in *ChkReplica) DeepCopy() *ChkReplica { + if in == nil { + return nil + } + out := new(ChkReplica) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChkReplicaAddress) DeepCopyInto(out *ChkReplicaAddress) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChkReplicaAddress. +func (in *ChkReplicaAddress) DeepCopy() *ChkReplicaAddress { + if in == nil { + return nil + } + out := new(ChkReplicaAddress) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChkReplicaRuntime) DeepCopyInto(out *ChkReplicaRuntime) { + *out = *in + out.Address = in.Address + if in.CHK != nil { + in, out := &in.CHK, &out.CHK + *out = new(ClickHouseKeeperInstallation) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChkReplicaRuntime. +func (in *ChkReplicaRuntime) DeepCopy() *ChkReplicaRuntime { + if in == nil { + return nil + } + out := new(ChkReplicaRuntime) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChkShard) DeepCopyInto(out *ChkShard) { + *out = *in + if in.Weight != nil { + in, out := &in.Weight, &out.Weight + *out = new(int) + **out = **in + } + if in.InternalReplication != nil { + in, out := &in.InternalReplication, &out.InternalReplication + *out = new(types.StringBool) + **out = **in + } + if in.Settings != nil { + in, out := &in.Settings, &out.Settings + *out = new(clickhousealtinitycomv1.Settings) + (*in).DeepCopyInto(*out) + } + if in.Files != nil { + in, out := &in.Files, &out.Files + *out = new(clickhousealtinitycomv1.Settings) + (*in).DeepCopyInto(*out) + } + if in.Templates != nil { + in, out := &in.Templates, &out.Templates + *out = new(clickhousealtinitycomv1.TemplatesList) + (*in).DeepCopyInto(*out) + } + if in.Hosts != nil { + in, out := &in.Hosts, &out.Hosts + *out = make([]*clickhousealtinitycomv1.Host, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(clickhousealtinitycomv1.Host) + (*in).DeepCopyInto(*out) + } + } + } + in.Runtime.DeepCopyInto(&out.Runtime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChkShard. +func (in *ChkShard) DeepCopy() *ChkShard { + if in == nil { + return nil + } + out := new(ChkShard) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChkShardAddress) DeepCopyInto(out *ChkShardAddress) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChkShardAddress. +func (in *ChkShardAddress) DeepCopy() *ChkShardAddress { + if in == nil { + return nil + } + out := new(ChkShardAddress) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChkShardRuntime) DeepCopyInto(out *ChkShardRuntime) { + *out = *in + out.Address = in.Address + if in.CHK != nil { + in, out := &in.CHK, &out.CHK + *out = new(ClickHouseKeeperInstallation) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChkShardRuntime. +func (in *ChkShardRuntime) DeepCopy() *ChkShardRuntime { + if in == nil { + return nil + } + out := new(ChkShardRuntime) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChkSpec) DeepCopyInto(out *ChkSpec) { + *out = *in + if in.TaskID != nil { + in, out := &in.TaskID, &out.TaskID + *out = new(types.Id) + **out = **in + } + if in.Stop != nil { + in, out := &in.Stop, &out.Stop + *out = new(types.StringBool) + **out = **in + } + if in.NamespaceDomainPattern != nil { + in, out := &in.NamespaceDomainPattern, &out.NamespaceDomainPattern + *out = new(types.String) + **out = **in + } + if in.Suspend != nil { + in, out := &in.Suspend, &out.Suspend + *out = new(types.StringBool) + **out = **in + } + if in.Reconciling != nil { + in, out := &in.Reconciling, &out.Reconciling + *out = new(clickhousealtinitycomv1.ChiReconcile) + (*in).DeepCopyInto(*out) + } + if in.Reconcile != nil { + in, out := &in.Reconcile, &out.Reconcile + *out = new(clickhousealtinitycomv1.ChiReconcile) + (*in).DeepCopyInto(*out) + } + if in.Defaults != nil { + in, out := &in.Defaults, &out.Defaults + *out = new(clickhousealtinitycomv1.Defaults) + (*in).DeepCopyInto(*out) + } + if in.Configuration != nil { + in, out := &in.Configuration, &out.Configuration + *out = new(Configuration) + (*in).DeepCopyInto(*out) + } + if in.Templates != nil { + in, out := &in.Templates, &out.Templates + *out = new(clickhousealtinitycomv1.Templates) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChkSpec. +func (in *ChkSpec) DeepCopy() *ChkSpec { + if in == nil { + return nil + } + out := new(ChkSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClickHouseKeeperInstallation) DeepCopyInto(out *ClickHouseKeeperInstallation) { + // *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(Status) + (*in).DeepCopyInto(*out) + } + if in.runtime != nil { + in, out := &in.runtime, &out.runtime + *out = new(ClickHouseKeeperInstallationRuntime) + (*in).DeepCopyInto(*out) + } + // out.statusCreatorMutex = in.statusCreatorMutex + // out.runtimeCreatorMutex = in.runtimeCreatorMutex + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClickHouseKeeperInstallation. +func (in *ClickHouseKeeperInstallation) DeepCopy() *ClickHouseKeeperInstallation { + if in == nil { + return nil + } + out := new(ClickHouseKeeperInstallation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClickHouseKeeperInstallation) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClickHouseKeeperInstallationList) DeepCopyInto(out *ClickHouseKeeperInstallationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClickHouseKeeperInstallation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClickHouseKeeperInstallationList. +func (in *ClickHouseKeeperInstallationList) DeepCopy() *ClickHouseKeeperInstallationList { + if in == nil { + return nil + } + out := new(ClickHouseKeeperInstallationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClickHouseKeeperInstallationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClickHouseKeeperInstallationRuntime) DeepCopyInto(out *ClickHouseKeeperInstallationRuntime) { + // *out = *in + if in.attributes != nil { + in, out := &in.attributes, &out.attributes + *out = new(clickhousealtinitycomv1.ComparableAttributes) + (*in).DeepCopyInto(*out) + } + // out.commonConfigMutex = in.commonConfigMutex + if in.MinVersion != nil { + in, out := &in.MinVersion, &out.MinVersion + *out = (*in).DeepCopy() + } + if in.MaxVersion != nil { + in, out := &in.MaxVersion, &out.MaxVersion + *out = (*in).DeepCopy() + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClickHouseKeeperInstallationRuntime. +func (in *ClickHouseKeeperInstallationRuntime) DeepCopy() *ClickHouseKeeperInstallationRuntime { + if in == nil { + return nil + } + out := new(ClickHouseKeeperInstallationRuntime) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Cluster) DeepCopyInto(out *Cluster) { + *out = *in + if in.Settings != nil { + in, out := &in.Settings, &out.Settings + *out = new(clickhousealtinitycomv1.Settings) + (*in).DeepCopyInto(*out) + } + if in.Files != nil { + in, out := &in.Files, &out.Files + *out = new(clickhousealtinitycomv1.Settings) + (*in).DeepCopyInto(*out) + } + if in.Templates != nil { + in, out := &in.Templates, &out.Templates + *out = new(clickhousealtinitycomv1.TemplatesList) + (*in).DeepCopyInto(*out) + } + if in.Layout != nil { + in, out := &in.Layout, &out.Layout + *out = new(ChkClusterLayout) + (*in).DeepCopyInto(*out) + } + if in.PDBManaged != nil { + in, out := &in.PDBManaged, &out.PDBManaged + *out = new(types.StringBool) + **out = **in + } + if in.PDBMaxUnavailable != nil { + in, out := &in.PDBMaxUnavailable, &out.PDBMaxUnavailable + *out = new(types.Int32) + **out = **in + } + if in.Reconcile != nil { + in, out := &in.Reconcile, &out.Reconcile + *out = new(clickhousealtinitycomv1.ClusterReconcile) + (*in).DeepCopyInto(*out) + } + in.Runtime.DeepCopyInto(&out.Runtime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cluster. +func (in *Cluster) DeepCopy() *Cluster { + if in == nil { + return nil + } + out := new(Cluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Configuration) DeepCopyInto(out *Configuration) { + *out = *in + if in.Settings != nil { + in, out := &in.Settings, &out.Settings + *out = new(clickhousealtinitycomv1.Settings) + (*in).DeepCopyInto(*out) + } + if in.Files != nil { + in, out := &in.Files, &out.Files + *out = new(clickhousealtinitycomv1.Settings) + (*in).DeepCopyInto(*out) + } + if in.Clusters != nil { + in, out := &in.Clusters, &out.Clusters + *out = make([]*Cluster, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Cluster) + (*in).DeepCopyInto(*out) + } + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. +func (in *Configuration) DeepCopy() *Configuration { + if in == nil { + return nil + } + out := new(Configuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FillStatusParams) DeepCopyInto(out *FillStatusParams) { + *out = *in + if in.Pods != nil { + in, out := &in.Pods, &out.Pods + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.FQDNs != nil { + in, out := &in.FQDNs, &out.FQDNs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Endpoints != nil { + in, out := &in.Endpoints, &out.Endpoints + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.NormalizedCR != nil { + in, out := &in.NormalizedCR, &out.NormalizedCR + *out = new(ClickHouseKeeperInstallation) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FillStatusParams. +func (in *FillStatusParams) DeepCopy() *FillStatusParams { + if in == nil { + return nil + } + out := new(FillStatusParams) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Status) DeepCopyInto(out *Status) { + // *out = *in + out.CHOpVersion = in.CHOpVersion + out.CHOpCommit = in.CHOpCommit + out.CHOpDate = in.CHOpDate + out.CHOpIP = in.CHOpIP + out.ClustersCount = in.ClustersCount + out.ShardsCount = in.ShardsCount + out.ReplicasCount = in.ReplicasCount + out.HostsCount = in.HostsCount + out.Status = in.Status + out.TaskID = in.TaskID + out.Action = in.Action + out.Error = in.Error + out.HostsUpdatedCount = in.HostsUpdatedCount + out.HostsAddedCount = in.HostsAddedCount + out.HostsUnchangedCount = in.HostsUnchangedCount + out.HostsFailedCount = in.HostsFailedCount + out.HostsCompletedCount = in.HostsCompletedCount + out.HostsDeletedCount = in.HostsDeletedCount + out.HostsDeleteCount = in.HostsDeleteCount + out.Endpoint = in.Endpoint + if in.TaskIDsStarted != nil { + in, out := &in.TaskIDsStarted, &out.TaskIDsStarted + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TaskIDsCompleted != nil { + in, out := &in.TaskIDsCompleted, &out.TaskIDsCompleted + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Actions != nil { + in, out := &in.Actions, &out.Actions + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Errors != nil { + in, out := &in.Errors, &out.Errors + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Pods != nil { + in, out := &in.Pods, &out.Pods + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PodIPs != nil { + in, out := &in.PodIPs, &out.PodIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.FQDNs != nil { + in, out := &in.FQDNs, &out.FQDNs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Endpoints != nil { + in, out := &in.Endpoints, &out.Endpoints + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.NormalizedCR != nil { + in, out := &in.NormalizedCR, &out.NormalizedCR + *out = new(ClickHouseKeeperInstallation) + (*in).DeepCopyInto(*out) + } + if in.NormalizedCRCompleted != nil { + in, out := &in.NormalizedCRCompleted, &out.NormalizedCRCompleted + *out = new(ClickHouseKeeperInstallation) + (*in).DeepCopyInto(*out) + } + if in.ActionPlan != nil { + in, out := &in.ActionPlan, &out.ActionPlan + *out = new(clickhousealtinitycomv1.ActionPlan) + (*in).DeepCopyInto(*out) + } + if in.HostsWithTablesCreated != nil { + in, out := &in.HostsWithTablesCreated, &out.HostsWithTablesCreated + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.HostsWithReplicaCaughtUp != nil { + in, out := &in.HostsWithReplicaCaughtUp, &out.HostsWithReplicaCaughtUp + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.UsedTemplates != nil { + in, out := &in.UsedTemplates, &out.UsedTemplates + *out = make([]*clickhousealtinitycomv1.TemplateRef, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(clickhousealtinitycomv1.TemplateRef) + **out = **in + } + } + } + // out.mu = in.mu + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Status. +func (in *Status) DeepCopy() *Status { + if in == nil { + return nil + } + out := new(Status) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/vendored/seaweedfs-operator/README.md b/pkg/vendored/seaweedfs-operator/README.md index e0a84175..d5e9d2c7 100644 --- a/pkg/vendored/seaweedfs-operator/README.md +++ b/pkg/vendored/seaweedfs-operator/README.md @@ -5,8 +5,9 @@ This directory contains vendored API types from the [SeaweedFS Operator](https:/ ## Source - **Repository**: https://github.com/seaweedfs/seaweedfs-operator -- **Version**: v0.1.13 -- **Date Vendored**: 2026-04-13 +- **Operator Version**: 1.0.32 +- **Helm Chart Version**: 0.1.35 +- **Date Vendored**: 2026-07-17 ## Reason for Vendoring diff --git a/pkg/vendored/seaweedfs-operator/crds/seaweed.seaweedfs.com_seaweeds.yaml b/pkg/vendored/seaweedfs-operator/crds/seaweed.seaweedfs.com_seaweeds.yaml index 9b08316a..425796e5 100644 --- a/pkg/vendored/seaweedfs-operator/crds/seaweed.seaweedfs.com_seaweeds.yaml +++ b/pkg/vendored/seaweedfs-operator/crds/seaweed.seaweedfs.com_seaweeds.yaml @@ -473,6 +473,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -480,6 +482,76 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + containerSecurityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object credentialsSecret: properties: name: @@ -519,6 +591,23 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -598,6 +687,12 @@ spec: type: object type: array type: object + initContainers: + x-kubernetes-preserve-unknown-fields: true + labels: + additionalProperties: + type: string + type: object limits: additionalProperties: anyOf: @@ -606,14 +701,127 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + livenessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object + loggingArgs: + items: + type: string + type: array + x-kubernetes-list-type: atomic metricsPort: type: integer nodeSelector: additionalProperties: type: string type: object + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object priorityClassName: type: string + readinessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + successThreshold: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object requests: additionalProperties: anyOf: @@ -637,6 +845,10 @@ spec: type: type: string type: object + serviceAccountName: + type: string + sidecars: + x-kubernetes-preserve-unknown-fields: true statefulSetUpdateStrategy: type: string terminationGracePeriodSeconds: @@ -705,10 +917,12 @@ spec: diskURI: type: string fsType: + default: ext4 type: string kind: type: string readOnly: + default: false type: boolean required: - diskName @@ -1062,6 +1276,13 @@ spec: required: - path type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object iscsi: properties: chapAuthDiscovery: @@ -1075,6 +1296,7 @@ spec: iqn: type: string iscsiInterface: + default: default type: string lun: type: integer @@ -1253,6 +1475,28 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + type: integer + signerName: + type: string + userAnnotations: + additionalProperties: + type: string + type: object + required: + - keyType + - signerName + type: object secret: properties: items: @@ -1317,6 +1561,7 @@ spec: image: type: string keyring: + default: /etc/ceph/keyring type: string monitors: items: @@ -1324,6 +1569,7 @@ spec: type: array x-kubernetes-list-type: atomic pool: + default: rbd type: string readOnly: type: boolean @@ -1335,6 +1581,7 @@ spec: type: object x-kubernetes-map-type: atomic user: + default: admin type: string required: - image @@ -1343,6 +1590,7 @@ spec: scaleIO: properties: fsType: + default: xfs type: string gateway: type: string @@ -1360,6 +1608,7 @@ spec: sslEnabled: type: boolean storageMode: + default: ThinProvisioned type: string storagePool: type: string @@ -1872,6 +2121,167 @@ spec: additionalProperties: type: string type: object + backup: + properties: + dataMirror: + items: + properties: + filerPath: + default: / + type: string + storageName: + maxLength: 50 + minLength: 1 + type: string + required: + - storageName + type: object + maxItems: 32 + type: array + x-kubernetes-list-map-keys: + - storageName + x-kubernetes-list-type: map + image: + type: string + schedule: + items: + properties: + filerPath: + default: / + type: string + keep: + minimum: 0 + type: integer + name: + maxLength: 50 + minLength: 1 + type: string + schedule: + maxLength: 120 + minLength: 1 + type: string + storageName: + maxLength: 50 + minLength: 1 + type: string + suspend: + default: false + type: boolean + required: + - name + - schedule + - storageName + type: object + maxItems: 32 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + storages: + additionalProperties: + properties: + azure: + properties: + accountName: + minLength: 1 + type: string + container: + minLength: 1 + type: string + directory: + default: / + type: string + required: + - accountName + - container + type: object + b2: + properties: + bucket: + minLength: 1 + type: string + directory: + default: / + type: string + region: + type: string + required: + - bucket + type: object + credentialsSecret: + type: string + filesystem: + properties: + existingClaim: + minLength: 1 + type: string + mountPath: + default: /backup + type: string + subPath: + type: string + required: + - existingClaim + type: object + gcs: + properties: + bucket: + minLength: 1 + type: string + directory: + default: / + type: string + required: + - bucket + type: object + s3: + properties: + bucket: + minLength: 1 + type: string + directory: + default: / + type: string + endpoint: + type: string + forcePathStyle: + default: true + type: boolean + region: + type: string + required: + - bucket + type: object + type: + enum: + - s3 + - gcs + - azure + - b2 + - filesystem + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: storage must set the sub-block matching its type + rule: (self.type != 's3' || has(self.s3)) && (self.type != + 'gcs' || has(self.gcs)) && (self.type != 'azure' || has(self.azure)) + && (self.type != 'b2' || has(self.b2)) && (self.type != + 'filesystem' || has(self.filesystem)) + maxProperties: 32 + minProperties: 1 + type: object + required: + - storages + type: object + x-kubernetes-validations: + - message: schedule.storageName must reference a defined storage + rule: '!has(self.schedule) || self.schedule.all(s, s.storageName + in self.storages)' + - message: dataMirror.storageName must reference a defined storage + rule: '!has(self.dataMirror) || self.dataMirror.all(m, m.storageName + in self.storages)' enablePVReclaim: type: boolean filer: @@ -2321,6 +2731,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -2330,6 +2742,76 @@ spec: x-kubernetes-list-type: map config: type: string + containerSecurityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object env: items: properties: @@ -2362,6 +2844,23 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -2401,6 +2900,33 @@ spec: type: string type: array x-kubernetes-list-type: atomic + grpcIngress: + properties: + annotations: + additionalProperties: + type: string + type: object + className: + type: string + enabled: + type: boolean + host: + type: string + path: + default: / + type: string + tls: + items: + properties: + hosts: + items: + type: string + type: array + secretName: + type: string + type: object + type: array + type: object hostNetwork: type: boolean iam: @@ -2454,6 +2980,12 @@ spec: type: object type: array type: object + initContainers: + x-kubernetes-preserve-unknown-fields: true + labels: + additionalProperties: + type: string + type: object limits: additionalProperties: anyOf: @@ -2462,6 +2994,26 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + livenessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object + loggingArgs: + items: + type: string + type: array + x-kubernetes-list-type: atomic maxMB: type: integer metricsPort: @@ -2478,6 +3030,10 @@ spec: items: type: string type: array + annotations: + additionalProperties: + type: string + type: object dataSource: properties: apiGroup: @@ -2496,6 +3052,10 @@ spec: type: boolean existingClaim: type: string + labels: + additionalProperties: + type: string + type: object mountPath: default: /data type: string @@ -2557,8 +3117,101 @@ spec: volumeName: type: string type: object + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object priorityClassName: type: string + readinessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + successThreshold: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object replicas: minimum: 1 type: integer @@ -2631,6 +3284,10 @@ spec: type: type: string type: object + serviceAccountName: + type: string + sidecars: + x-kubernetes-preserve-unknown-fields: true statefulSetUpdateStrategy: type: string terminationGracePeriodSeconds: @@ -2699,10 +3356,12 @@ spec: diskURI: type: string fsType: + default: ext4 type: string kind: type: string readOnly: + default: false type: boolean required: - diskName @@ -3056,6 +3715,13 @@ spec: required: - path type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object iscsi: properties: chapAuthDiscovery: @@ -3069,6 +3735,7 @@ spec: iqn: type: string iscsiInterface: + default: default type: string lun: type: integer @@ -3247,6 +3914,28 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + type: integer + signerName: + type: string + userAnnotations: + additionalProperties: + type: string + type: object + required: + - keyType + - signerName + type: object secret: properties: items: @@ -3311,6 +4000,7 @@ spec: image: type: string keyring: + default: /etc/ceph/keyring type: string monitors: items: @@ -3318,6 +4008,7 @@ spec: type: array x-kubernetes-list-type: atomic pool: + default: rbd type: string readOnly: type: boolean @@ -3329,6 +4020,7 @@ spec: type: object x-kubernetes-map-type: atomic user: + default: admin type: string required: - image @@ -3337,6 +4029,7 @@ spec: scaleIO: properties: fsType: + default: xfs type: string gateway: type: string @@ -3354,6 +4047,7 @@ spec: sslEnabled: type: boolean storageMode: + default: ThinProvisioned type: string storagePool: type: string @@ -3445,6 +4139,15 @@ spec: type: object x-kubernetes-map-type: atomic type: array + labels: + additionalProperties: + type: string + type: object + loggingArgs: + items: + type: string + type: array + x-kubernetes-list-type: atomic master: properties: affinity: @@ -3892,6 +4595,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -3903,6 +4608,76 @@ spec: type: boolean config: type: string + containerSecurityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object defaultReplication: type: string env: @@ -3937,6 +4712,23 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -3997,46 +4789,1851 @@ spec: additionalProperties: type: string type: object - className: - type: string - enabled: - type: boolean - host: - type: string - path: - default: / + className: + type: string + enabled: + type: boolean + host: + type: string + path: + default: / + type: string + tls: + items: + properties: + hosts: + items: + type: string + type: array + secretName: + type: string + type: object + type: array + type: object + initContainers: + x-kubernetes-preserve-unknown-fields: true + labels: + additionalProperties: + type: string + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + livenessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object + loggingArgs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + metricsPort: + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + priorityClassName: + type: string + pulseSeconds: + type: integer + readinessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + successThreshold: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object + replicas: + minimum: 1 + type: integer + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + schedulerName: + type: string + service: + properties: + annotations: + additionalProperties: + type: string + type: object + clusterIP: + type: string + loadBalancerIP: + type: string + type: + type: string + type: object + serviceAccountName: + type: string + sidecars: + x-kubernetes-preserve-unknown-fields: true + statefulSetUpdateStrategy: + type: string + terminationGracePeriodSeconds: + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + type: integer + value: + type: string + type: object + type: array + version: + type: string + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + volumePreallocate: + type: boolean + volumeSizeLimitMB: + type: integer + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + default: ext4 + type: string + kind: + type: string + readOnly: + default: false + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + type: integer + items: + items: + properties: + key: + type: string + mode: + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + default: default + type: string + lun: + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + type: integer + signerName: + type: string + userAnnotations: + additionalProperties: + type: string + type: object + required: + - keyType + - signerName + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + default: /etc/ceph/keyring + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + default: xfs + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + default: ThinProvisioned + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + type: integer + items: + items: + properties: + key: + type: string + mode: + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - replicas + type: object + metricsAddress: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + pvReclaimPolicy: + type: string + s3: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + configSecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + containerSecurityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + domainName: + type: string + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + extraArgs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + hostNetwork: + type: boolean + iam: + default: true + type: boolean + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + ingress: + properties: + annotations: + additionalProperties: + type: string + type: object + className: + type: string + enabled: + type: boolean + host: + type: string + path: + default: / + type: string + tls: + items: + properties: + hosts: + items: + type: string + type: array + secretName: + type: string + type: object + type: array + type: object + initContainers: + x-kubernetes-preserve-unknown-fields: true + labels: + additionalProperties: + type: string + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + livenessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object + loggingArgs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + metricsPort: + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: type: string - tls: + sysctls: items: properties: - hosts: - items: - type: string - type: array - secretName: + name: type: string + value: + type: string + required: + - name + - value type: object type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object type: object - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - metricsPort: + port: type: integer - nodeSelector: - additionalProperties: - type: string - type: object priorityClassName: type: string - pulseSeconds: - type: integer + readinessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + successThreshold: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object replicas: + default: 1 minimum: 1 type: integer requests: @@ -4062,6 +6659,10 @@ spec: type: type: string type: object + serviceAccountName: + type: string + sidecars: + x-kubernetes-preserve-unknown-fields: true statefulSetUpdateStrategy: type: string terminationGracePeriodSeconds: @@ -4105,10 +6706,6 @@ spec: - name type: object type: array - volumePreallocate: - type: boolean - volumeSizeLimitMB: - type: integer volumes: items: properties: @@ -4134,10 +6731,12 @@ spec: diskURI: type: string fsType: + default: ext4 type: string kind: type: string readOnly: + default: false type: boolean required: - diskName @@ -4491,6 +7090,13 @@ spec: required: - path type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object iscsi: properties: chapAuthDiscovery: @@ -4504,6 +7110,7 @@ spec: iqn: type: string iscsiInterface: + default: default type: string lun: type: integer @@ -4682,6 +7289,28 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + type: integer + signerName: + type: string + userAnnotations: + additionalProperties: + type: string + type: object + required: + - keyType + - signerName + type: object secret: properties: items: @@ -4746,6 +7375,7 @@ spec: image: type: string keyring: + default: /etc/ceph/keyring type: string monitors: items: @@ -4753,6 +7383,7 @@ spec: type: array x-kubernetes-list-type: atomic pool: + default: rbd type: string readOnly: type: boolean @@ -4764,6 +7395,7 @@ spec: type: object x-kubernetes-map-type: atomic user: + default: admin type: string required: - image @@ -4772,6 +7404,7 @@ spec: scaleIO: properties: fsType: + default: xfs type: string gateway: type: string @@ -4789,6 +7422,7 @@ spec: sslEnabled: type: boolean storageMode: + default: ThinProvisioned type: string storagePool: type: string @@ -4863,15 +7497,9 @@ spec: required: - replicas type: object - metricsAddress: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - pvReclaimPolicy: + schedulerName: type: string - s3: + sftp: properties: affinity: properties: @@ -5313,11 +7941,15 @@ spec: additionalProperties: type: string type: object + authMethods: + type: string claims: items: properties: name: type: string + request: + type: string required: - name type: object @@ -5325,21 +7957,76 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map - configSecret: + containerSecurityContext: properties: - key: - type: string - name: - default: "" + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: type: string - optional: + readOnlyRootFilesystem: type: boolean - required: - - key + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object type: object - x-kubernetes-map-type: atomic - domainName: - type: string env: items: properties: @@ -5372,6 +8059,23 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -5411,11 +8115,15 @@ spec: type: string type: array x-kubernetes-list-type: atomic + hostKeysSecret: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic hostNetwork: type: boolean - iam: - default: true - type: boolean imagePullPolicy: type: string imagePullSecrets: @@ -5454,6 +8162,12 @@ spec: type: object type: array type: object + initContainers: + x-kubernetes-preserve-unknown-fields: true + labels: + additionalProperties: + type: string + type: object limits: additionalProperties: anyOf: @@ -5462,16 +8176,134 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + livenessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object + loggingArgs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxAuthTries: + minimum: 1 + type: integer metricsPort: type: integer nodeSelector: additionalProperties: type: string type: object + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object port: + maximum: 65535 + minimum: 1 type: integer priorityClassName: type: string + readinessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + successThreshold: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object replicas: default: 1 minimum: 1 @@ -5499,6 +8331,10 @@ spec: type: type: string type: object + serviceAccountName: + type: string + sidecars: + x-kubernetes-preserve-unknown-fields: true statefulSetUpdateStrategy: type: string terminationGracePeriodSeconds: @@ -5518,6 +8354,19 @@ spec: type: string type: object type: array + userStoreSecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic version: type: string volumeMounts: @@ -5567,10 +8416,12 @@ spec: diskURI: type: string fsType: + default: ext4 type: string kind: type: string readOnly: + default: false type: boolean required: - diskName @@ -5924,6 +8775,13 @@ spec: required: - path type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object iscsi: properties: chapAuthDiscovery: @@ -5937,6 +8795,7 @@ spec: iqn: type: string iscsiInterface: + default: default type: string lun: type: integer @@ -6115,6 +8974,28 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + type: integer + signerName: + type: string + userAnnotations: + additionalProperties: + type: string + type: object + required: + - keyType + - signerName + type: object secret: properties: items: @@ -6179,6 +9060,7 @@ spec: image: type: string keyring: + default: /etc/ceph/keyring type: string monitors: items: @@ -6186,6 +9068,7 @@ spec: type: array x-kubernetes-list-type: atomic pool: + default: rbd type: string readOnly: type: boolean @@ -6197,6 +9080,7 @@ spec: type: object x-kubernetes-map-type: atomic user: + default: admin type: string required: - image @@ -6205,6 +9089,7 @@ spec: scaleIO: properties: fsType: + default: xfs type: string gateway: type: string @@ -6222,6 +9107,7 @@ spec: sslEnabled: type: boolean storageMode: + default: ThinProvisioned type: string storagePool: type: string @@ -6296,8 +9182,6 @@ spec: required: - replicas type: object - schedulerName: - type: string statefulSetUpdateStrategy: type: string tls: @@ -6785,6 +9669,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -6794,6 +9680,76 @@ spec: x-kubernetes-list-type: map compactionMBps: type: integer + containerSecurityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object dataCenter: type: string env: @@ -6828,6 +9784,23 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -6873,6 +9846,32 @@ spec: type: boolean hostNetwork: type: boolean + hostPath: + items: + properties: + maxVolumeCount: + minimum: 0 + type: integer + path: + minLength: 1 + type: string + type: + default: DirectoryOrCreate + enum: + - "" + - DirectoryOrCreate + - Directory + - FileOrCreate + - File + - Socket + - CharDevice + - BlockDevice + type: string + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic idleTimeout: type: integer imagePullPolicy: @@ -6913,6 +9912,18 @@ spec: type: object type: array type: object + initContainers: + x-kubernetes-preserve-unknown-fields: true + kind: + default: StatefulSet + enum: + - StatefulSet + - DaemonSet + type: string + labels: + additionalProperties: + type: string + type: object limits: additionalProperties: anyOf: @@ -6921,6 +9932,26 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + livenessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object + loggingArgs: + items: + type: string + type: array + x-kubernetes-list-type: atomic maxVolumeCounts: type: integer metricsPort: @@ -6931,10 +9962,103 @@ spec: additionalProperties: type: string type: object + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object priorityClassName: type: string rack: type: string + readinessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + successThreshold: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object replicas: minimum: 0 type: integer @@ -6961,10 +10085,48 @@ spec: type: type: string type: object + serviceAccountName: + type: string + sidecars: + x-kubernetes-preserve-unknown-fields: true statefulSetUpdateStrategy: type: string + storageAnnotations: + additionalProperties: + type: string + type: object storageClassName: type: string + storageLabels: + additionalProperties: + type: string + type: object + storageSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic terminationGracePeriodSeconds: type: integer tolerations: @@ -7031,10 +10193,12 @@ spec: diskURI: type: string fsType: + default: ext4 type: string kind: type: string readOnly: + default: false type: boolean required: - diskName @@ -7388,6 +10552,13 @@ spec: required: - path type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object iscsi: properties: chapAuthDiscovery: @@ -7401,6 +10572,7 @@ spec: iqn: type: string iscsiInterface: + default: default type: string lun: type: integer @@ -7579,6 +10751,28 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + type: integer + signerName: + type: string + userAnnotations: + additionalProperties: + type: string + type: object + required: + - keyType + - signerName + type: object secret: properties: items: @@ -7643,6 +10837,7 @@ spec: image: type: string keyring: + default: /etc/ceph/keyring type: string monitors: items: @@ -7650,6 +10845,7 @@ spec: type: array x-kubernetes-list-type: atomic pool: + default: rbd type: string readOnly: type: boolean @@ -7661,6 +10857,7 @@ spec: type: object x-kubernetes-map-type: atomic user: + default: admin type: string required: - image @@ -7669,6 +10866,7 @@ spec: scaleIO: properties: fsType: + default: xfs type: string gateway: type: string @@ -7686,6 +10884,7 @@ spec: sslEnabled: type: boolean storageMode: + default: ThinProvisioned type: string storagePool: type: string @@ -8210,6 +11409,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -8219,6 +11420,76 @@ spec: x-kubernetes-list-type: map compactionMBps: type: integer + containerSecurityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object dataCenter: type: string env: @@ -8253,6 +11524,23 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -8311,6 +11599,12 @@ spec: type: object x-kubernetes-map-type: atomic type: array + initContainers: + x-kubernetes-preserve-unknown-fields: true + labels: + additionalProperties: + type: string + type: object limits: additionalProperties: anyOf: @@ -8319,6 +11613,26 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + livenessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object + loggingArgs: + items: + type: string + type: array + x-kubernetes-list-type: atomic maxVolumeCounts: type: integer metricsPort: @@ -8329,10 +11643,103 @@ spec: additionalProperties: type: string type: object + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object priorityClassName: type: string rack: type: string + readinessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + successThreshold: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object replicas: minimum: 0 type: integer @@ -8359,10 +11766,48 @@ spec: type: type: string type: object + serviceAccountName: + type: string + sidecars: + x-kubernetes-preserve-unknown-fields: true statefulSetUpdateStrategy: type: string + storageAnnotations: + additionalProperties: + type: string + type: object storageClassName: type: string + storageLabels: + additionalProperties: + type: string + type: object + storageSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic terminationGracePeriodSeconds: type: integer tolerations: @@ -8429,10 +11874,12 @@ spec: diskURI: type: string fsType: + default: ext4 type: string kind: type: string readOnly: + default: false type: boolean required: - diskName @@ -8786,6 +12233,13 @@ spec: required: - path type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object iscsi: properties: chapAuthDiscovery: @@ -8799,6 +12253,7 @@ spec: iqn: type: string iscsiInterface: + default: default type: string lun: type: integer @@ -8977,6 +12432,28 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + type: integer + signerName: + type: string + userAnnotations: + additionalProperties: + type: string + type: object + required: + - keyType + - signerName + type: object secret: properties: items: @@ -9041,6 +12518,7 @@ spec: image: type: string keyring: + default: /etc/ceph/keyring type: string monitors: items: @@ -9048,6 +12526,7 @@ spec: type: array x-kubernetes-list-type: atomic pool: + default: rbd type: string readOnly: type: boolean @@ -9059,6 +12538,7 @@ spec: type: object x-kubernetes-map-type: atomic user: + default: admin type: string required: - image @@ -9067,6 +12547,7 @@ spec: scaleIO: properties: fsType: + default: xfs type: string gateway: type: string @@ -9084,6 +12565,7 @@ spec: sslEnabled: type: boolean storageMode: + default: ThinProvisioned type: string storagePool: type: string @@ -9608,6 +13090,8 @@ spec: properties: name: type: string + request: + type: string required: - name type: object @@ -9615,6 +13099,76 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + containerSecurityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object env: items: properties: @@ -9647,6 +13201,23 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -9699,9 +13270,15 @@ spec: type: object x-kubernetes-map-type: atomic type: array + initContainers: + x-kubernetes-preserve-unknown-fields: true jobType: default: all type: string + labels: + additionalProperties: + type: string + type: object limits: additionalProperties: anyOf: @@ -9710,6 +13287,26 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + livenessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object + loggingArgs: + items: + type: string + type: array + x-kubernetes-list-type: atomic maxDetect: minimum: 1 type: integer @@ -9730,6 +13327,10 @@ spec: items: type: string type: array + annotations: + additionalProperties: + type: string + type: object dataSource: properties: apiGroup: @@ -9748,6 +13349,10 @@ spec: type: boolean existingClaim: type: string + labels: + additionalProperties: + type: string + type: object mountPath: default: /data type: string @@ -9809,8 +13414,101 @@ spec: volumeName: type: string type: object + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object priorityClassName: type: string + readinessProbe: + properties: + failureThreshold: + minimum: 1 + type: integer + initialDelaySeconds: + minimum: 0 + type: integer + periodSeconds: + minimum: 1 + type: integer + successThreshold: + minimum: 1 + type: integer + timeoutSeconds: + minimum: 1 + type: integer + type: object replicas: default: 1 minimum: 1 @@ -9825,6 +13523,10 @@ spec: type: object schedulerName: type: string + serviceAccountName: + type: string + sidecars: + x-kubernetes-preserve-unknown-fields: true statefulSetUpdateStrategy: type: string terminationGracePeriodSeconds: @@ -9893,10 +13595,12 @@ spec: diskURI: type: string fsType: + default: ext4 type: string kind: type: string readOnly: + default: false type: boolean required: - diskName @@ -10250,6 +13954,13 @@ spec: required: - path type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object iscsi: properties: chapAuthDiscovery: @@ -10263,6 +13974,7 @@ spec: iqn: type: string iscsiInterface: + default: default type: string lun: type: integer @@ -10441,6 +14153,28 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + type: integer + signerName: + type: string + userAnnotations: + additionalProperties: + type: string + type: object + required: + - keyType + - signerName + type: object secret: properties: items: @@ -10505,6 +14239,7 @@ spec: image: type: string keyring: + default: /etc/ceph/keyring type: string monitors: items: @@ -10512,6 +14247,7 @@ spec: type: array x-kubernetes-list-type: atomic pool: + default: rbd type: string readOnly: type: boolean @@ -10523,6 +14259,7 @@ spec: type: object x-kubernetes-map-type: atomic user: + default: admin type: string required: - image @@ -10531,6 +14268,7 @@ spec: scaleIO: properties: fsType: + default: xfs type: string gateway: type: string @@ -10548,6 +14286,7 @@ spec: sslEnabled: type: boolean storageMode: + default: ThinProvisioned type: string storagePool: type: string @@ -10634,6 +14373,22 @@ spec: minimum: 0 type: integer type: object + backupMirrors: + items: + properties: + deploymentName: + type: string + ready: + type: boolean + storageName: + type: string + required: + - storageName + type: object + type: array + x-kubernetes-list-map-keys: + - storageName + x-kubernetes-list-type: map conditions: items: properties: @@ -10701,6 +14456,15 @@ spec: minimum: 0 type: integer type: object + sftp: + properties: + readyReplicas: + minimum: 0 + type: integer + replicas: + minimum: 0 + type: integer + type: object volume: properties: readyReplicas: diff --git a/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/groupversion_info.go b/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/groupversion_info.go index ce077e8e..45227715 100644 --- a/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/groupversion_info.go +++ b/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/groupversion_info.go @@ -12,6 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package v1 contains API Schema definitions for the seaweed v1 API group. package v1 import ( diff --git a/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/types.go b/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/types.go index f8222941..fcea5928 100644 --- a/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/types.go +++ b/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/types.go @@ -242,6 +242,26 @@ type WorkerSpec struct { MaxExecute *int32 `json:"maxExecute,omitempty"` } +// ProbeOverride tunes the timing fields of an operator-managed readiness probe. +// Nil fields retain the SeaweedFS operator defaults; the probe handler itself +// remains managed by the SeaweedFS operator. +type ProbeOverride struct { + InitialDelaySeconds *int32 `json:"initialDelaySeconds,omitempty"` + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"` + PeriodSeconds *int32 `json:"periodSeconds,omitempty"` + SuccessThreshold *int32 `json:"successThreshold,omitempty"` + FailureThreshold *int32 `json:"failureThreshold,omitempty"` +} + +// LivenessProbeOverride is ProbeOverride without SuccessThreshold, which +// Kubernetes requires to remain 1 for liveness probes. +type LivenessProbeOverride struct { + InitialDelaySeconds *int32 `json:"initialDelaySeconds,omitempty"` + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"` + PeriodSeconds *int32 `json:"periodSeconds,omitempty"` + FailureThreshold *int32 `json:"failureThreshold,omitempty"` +} + // ComponentSpec is the base spec of each component type ComponentSpec struct { Version *string `json:"version,omitempty"` @@ -260,6 +280,8 @@ type ComponentSpec struct { Volumes []corev1.Volume `json:"volumes,omitempty"` VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` ExtraArgs []string `json:"extraArgs,omitempty"` + ReadinessProbe *ProbeOverride `json:"readinessProbe,omitempty"` + LivenessProbe *LivenessProbeOverride `json:"livenessProbe,omitempty"` } // ServiceSpec is a subset of the original k8s spec diff --git a/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/zz_generated.deepcopy.go b/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/zz_generated.deepcopy.go index b9d323a9..2342eb7b 100644 --- a/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/zz_generated.deepcopy.go +++ b/pkg/vendored/seaweedfs-operator/seaweed.seaweedfs.com/v1/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ //go:build !ignore_autogenerated /* - +Copyright 2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ package v1 import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" + runtime "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -153,6 +153,16 @@ func (in *ComponentSpec) DeepCopyInto(out *ComponentSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ReadinessProbe != nil { + in, out := &in.ReadinessProbe, &out.ReadinessProbe + *out = new(ProbeOverride) + (*in).DeepCopyInto(*out) + } + if in.LivenessProbe != nil { + in, out := &in.LivenessProbe, &out.LivenessProbe + *out = new(LivenessProbeOverride) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentSpec. @@ -316,6 +326,41 @@ func (in *IngressTLS) DeepCopy() *IngressTLS { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LivenessProbeOverride) DeepCopyInto(out *LivenessProbeOverride) { + *out = *in + if in.InitialDelaySeconds != nil { + in, out := &in.InitialDelaySeconds, &out.InitialDelaySeconds + *out = new(int32) + **out = **in + } + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int32) + **out = **in + } + if in.PeriodSeconds != nil { + in, out := &in.PeriodSeconds, &out.PeriodSeconds + *out = new(int32) + **out = **in + } + if in.FailureThreshold != nil { + in, out := &in.FailureThreshold, &out.FailureThreshold + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LivenessProbeOverride. +func (in *LivenessProbeOverride) DeepCopy() *LivenessProbeOverride { + if in == nil { + return nil + } + out := new(LivenessProbeOverride) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MasterSpec) DeepCopyInto(out *MasterSpec) { *out = *in @@ -439,6 +484,46 @@ func (in *PersistenceSpec) DeepCopy() *PersistenceSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProbeOverride) DeepCopyInto(out *ProbeOverride) { + *out = *in + if in.InitialDelaySeconds != nil { + in, out := &in.InitialDelaySeconds, &out.InitialDelaySeconds + *out = new(int32) + **out = **in + } + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int32) + **out = **in + } + if in.PeriodSeconds != nil { + in, out := &in.PeriodSeconds, &out.PeriodSeconds + *out = new(int32) + **out = **in + } + if in.SuccessThreshold != nil { + in, out := &in.SuccessThreshold, &out.SuccessThreshold + *out = new(int32) + **out = **in + } + if in.FailureThreshold != nil { + in, out := &in.FailureThreshold, &out.FailureThreshold + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProbeOverride. +func (in *ProbeOverride) DeepCopy() *ProbeOverride { + if in == nil { + return nil + } + out := new(ProbeOverride) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *S3Config) DeepCopyInto(out *S3Config) { *out = *in diff --git a/pkg/wandb/manifest/load_manifest_from_files_test.go b/pkg/wandb/manifest/load_manifest_from_files_test.go index 1dea3480..18b2b4c1 100644 --- a/pkg/wandb/manifest/load_manifest_from_files_test.go +++ b/pkg/wandb/manifest/load_manifest_from_files_test.go @@ -31,7 +31,7 @@ var _ = Describe("LoadManifestFromFiles", func() { repository := "file://" + filepath.ToSlash(manifestRoot) // Load the manifest using the public API which will internally call loadManifestFromFiles - m, err := manifest.LoadManifestFromFile(ctx, repository, "0.78.0-pre") + m, err := manifest.LoadManifestFromFile(ctx, repository, "0.83.0-clickhouse-keeper.2") Expect(err).NotTo(HaveOccurred()) Expect(m).NotTo(BeNil()) @@ -55,8 +55,8 @@ var _ = Describe("LoadManifestFromFiles", func() { // Test sizing configurations that come from sizing.yaml // Check that Kafka sizing was loaded Expect(m.Kafka.Sizing).NotTo(BeEmpty()) - Expect(m.Kafka.Sizing["default"].Replicas).To(Equal(int32(1))) - Expect(m.Kafka.Sizing["micro"].Replicas).To(Equal(int32(3))) + Expect(m.Kafka.Sizing["default"].Replicas).To(Equal(int32(2))) + Expect(m.Kafka.Sizing["micro"].Replicas).To(Equal(int32(2))) // Check that Bucket sizing was loaded Expect(m.Bucket["default"].Sizing["default"].Replicas).To(Equal(int32(1))) @@ -68,7 +68,7 @@ var _ = Describe("LoadManifestFromFiles", func() { // Check that Redis sizing was loaded Expect(m.Redis["default"].Sizing["default"].Replicas).To(Equal(int32(1))) - Expect(m.Redis["default"].Sizing["micro"].Replicas).To(Equal(int32(3))) + Expect(m.Redis["default"].Sizing["micro"].Replicas).To(Equal(int32(2))) // Check that Clickhouse sizing was loaded Expect(m.Clickhouse["default"].Sizing["default"].Shards).To(Equal(int32(1))) @@ -86,12 +86,12 @@ var _ = Describe("LoadManifestFromFiles", func() { repository := "file://" + filepath.ToSlash(manifestRoot) // This will internally call loadManifestFromFiles with multiple files - m, err := manifest.LoadManifestFromFile(ctx, repository, "0.78.0-pre") + m, err := manifest.LoadManifestFromFile(ctx, repository, "0.83.0-clickhouse-keeper.2") Expect(err).NotTo(HaveOccurred()) // Confirm we have merged data from both files Expect(m.RequiredOperatorVersion).To(Equal("^2.0.0")) // From manifest.yaml - Expect(m.Kafka.Sizing["default"].Replicas).To(Equal(int32(1))) // From sizing.yaml + Expect(m.Kafka.Sizing["default"].Replicas).To(Equal(int32(2))) // From sizing.yaml }) }) }) diff --git a/pkg/wandb/manifest/manifest.go b/pkg/wandb/manifest/manifest.go index fe666607..1cf06bd7 100644 --- a/pkg/wandb/manifest/manifest.go +++ b/pkg/wandb/manifest/manifest.go @@ -49,6 +49,7 @@ type Manifest struct { CommonVolumeMounts map[string][]VolumeMount `yaml:"commonVolumeMounts,omitempty"` Bucket map[string]InfraConfig `yaml:"bucket"` Clickhouse map[string]InfraConfig `yaml:"clickhouse"` + ClickhouseKeeper map[string]InfraConfig `yaml:"clickhouseKeeper"` Kafka KafkaConfig `yaml:"kafka"` Mysql map[string]InfraConfig `yaml:"mysql"` Redis map[string]InfraConfig `yaml:"redis"` @@ -145,10 +146,13 @@ type AppKafkaSection struct { // Application describes one entry in the applications list. type Application struct { - Name string `yaml:"name"` - Image ImageRef `yaml:"image"` - Args []string `yaml:"args,omitempty"` - Command []string `yaml:"command,omitempty"` + Name string `yaml:"name"` + // LegacyKey is the v1 operator-wandb helm values key for this application + // when it differs from the name (e.g. nginx-proxy was `nginx`). + LegacyKey string `yaml:"legacyKey,omitempty"` + Image ImageRef `yaml:"image"` + Args []string `yaml:"args,omitempty"` + Command []string `yaml:"command,omitempty"` // CommonEnvs is a list of keys referencing top-level commonEnvvars groups // to be included for this application (e.g., ["gorillaMysql", "gorillaBucket"]). CommonEnvs []string `yaml:"commonEnvs,omitempty"` @@ -182,9 +186,12 @@ type AppIngressSpec struct { type SizingConfig struct { Replicas int32 `yaml:"replicas,omitempty"` Shards int32 `yaml:"shards,omitempty"` + Copies int32 `yaml:"copies,omitempty"` VolumeSize string `yaml:"volumeSize,omitempty"` Resources *corev1.ResourceRequirements `yaml:"resources,omitempty"` Autoscaling *AutoscalingConfig `yaml:"autoscaling,omitempty"` + // MetadataVolumeSize sizes the disk backing an object store's index/metadata + MetadataVolumeSize string `yaml:"metadataVolumeSize,omitempty"` } type KafkaSizingConfig struct { @@ -405,6 +412,12 @@ func mergeSimple(dst, src *Manifest) { } mergeInfraConfigs(dst.Clickhouse, src.Clickhouse) } + if src.ClickhouseKeeper != nil { + if dst.ClickhouseKeeper == nil { + dst.ClickhouseKeeper = make(map[string]InfraConfig) + } + mergeInfraConfigs(dst.ClickhouseKeeper, src.ClickhouseKeeper) + } // Kafka sizing if src.Kafka.Sizing != nil { @@ -768,7 +781,7 @@ func (m *Manifest) FeaturesEnabled(topicFeatures []string) bool { return false } -func (m *Manifest) ResolveServiceURL(src EnvSource) (string, bool) { +func (m *Manifest) ResolveServiceURL(src EnvSource, namespace string) (string, bool) { if src.Name == "" { return "", false } @@ -787,7 +800,19 @@ func (m *Manifest) ResolveServiceURL(src EnvSource) (string, bool) { if src.Proto != "" { protoPrefix = fmt.Sprintf("%s://", src.Proto) } - return fmt.Sprintf("%s%s:%d%s", protoPrefix, src.Name, port, src.Path), true + // Emit the fully-qualified service host (..svc.cluster.local) + // rather than the bare service name. A bare single-label host only resolves + // via the consuming pod's DNS search domain, and — critically — is not + // matched by NO_PROXY suffix rules (.svc/.svc.cluster.local), so when a proxy + // is configured these internal service-to-service calls hairpin through it. + // The FQDN resolves unambiguously and is covered by the standard cluster-DNS + // NO_PROXY suffixes. Matches the FQDN convention the managed-infra reconcilers + // already use for datastore hosts. + host := src.Name + if namespace != "" { + host = fmt.Sprintf("%s.%s.svc.cluster.local", src.Name, namespace) + } + return fmt.Sprintf("%s%s:%d%s", protoPrefix, host, port, src.Path), true } func (a *Application) ResolveServicePortFromManifest(requestedPort string) (int32, bool) { diff --git a/pkg/wandb/manifest/manifest_decode_test.go b/pkg/wandb/manifest/manifest_decode_test.go index 59bb42d4..7d902df5 100644 --- a/pkg/wandb/manifest/manifest_decode_test.go +++ b/pkg/wandb/manifest/manifest_decode_test.go @@ -17,28 +17,8 @@ var _ = Describe("Server manifest YAML decode", func() { return "file://" + filepath.ToSlash(manifestRoot) } - It("loads a single root manifest file", func() { - m, err := manifest.LoadManifestFromFile(context.Background(), manifestRepository(), "0.78.0-single-file") - Expect(err).NotTo(HaveOccurred()) - - Expect(m.Features).NotTo(BeNil()) - Expect(m.Features["filestreamQueue"]).To(BeFalse()) - Expect(m.Kafka.Topics).To(HaveLen(4)) - Expect(m.Applications).NotTo(BeEmpty()) - }) - - It("loads a single manifest file from a version directory", func() { - m, err := manifest.LoadManifestFromFile(context.Background(), manifestRepository(), "0.78.0") - Expect(err).NotTo(HaveOccurred()) - - Expect(m.Features).NotTo(BeNil()) - Expect(m.Features["proxy"]).To(BeFalse()) - Expect(m.Kafka.Topics).To(HaveLen(4)) - Expect(m.Migrations).To(HaveKey("gorilla")) - }) - It("merges multiple manifest files from a version directory", func() { - m, err := manifest.LoadManifestFromFile(context.Background(), manifestRepository(), "0.78.0-pre") + m, err := manifest.LoadManifestFromFile(context.Background(), manifestRepository(), "0.83.0-clickhouse-keeper.2") Expect(err).NotTo(HaveOccurred()) // Features (match current testing manifest values) @@ -49,7 +29,7 @@ var _ = Describe("Server manifest YAML decode", func() { Expect(m.Kafka.Topics).To(HaveLen(4)) Expect(m.Kafka.Topics[0].Name).To(Equal("filestream")) Expect(m.Kafka.Topics[0].Topic).To(Equal("filestream")) - Expect(m.Kafka.Topics[0].PartitionCount).To(Equal(48)) + Expect(m.Kafka.Topics[0].PartitionCount).To(Equal(96)) Expect(m.Kafka.Topics[0].Features).To(ContainElement("filestreamQueue")) // Applications basic presence @@ -67,7 +47,7 @@ var _ = Describe("Server manifest YAML decode", func() { Expect(m.Migrations["gorilla"].Args).To(ContainElement("migrate")) // Sizing comes from the split sizing.yaml file. - Expect(m.Kafka.Sizing["default"].Replicas).To(Equal(int32(1))) + Expect(m.Kafka.Sizing["default"].Replicas).To(Equal(int32(2))) Expect(m.Bucket["default"].Sizing["default"].Replicas).To(Equal(int32(1))) }) }) diff --git a/pkg/wandb/spec/charts/charts.go b/pkg/wandb/spec/charts/charts.go index 286c454e..82d5795c 100644 --- a/pkg/wandb/spec/charts/charts.go +++ b/pkg/wandb/spec/charts/charts.go @@ -33,6 +33,7 @@ type ValidatableRelease interface { func Get(maybeRelease interface{}) spec.Chart { releases := []ValidatableRelease{ new(LocalRelease), + new(OCIRelease), new(RepoRelease), } diff --git a/pkg/wandb/spec/charts/oci.go b/pkg/wandb/spec/charts/oci.go new file mode 100644 index 00000000..ab7c2fc8 --- /dev/null +++ b/pkg/wandb/spec/charts/oci.go @@ -0,0 +1,227 @@ +package charts + +import ( + "bytes" + "context" + "fmt" + "net/url" + "strings" + + "github.com/go-playground/validator/v10" + v1 "github.com/wandb/operator/api/v1" + "github.com/wandb/operator/pkg/helm" + "github.com/wandb/operator/pkg/wandb/spec" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/registry" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + orasregistry "oras.land/oras-go/v2/registry" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" +) + +// OCIRelease pulls a Helm chart from an OCI-based registry. +// The URL must use the oci:// scheme (e.g., oci://ghcr.io/wandb/helm-charts/operator-wandb). +type OCIRelease struct { + URL string `validate:"required,ociurl" json:"url"` + Version string `validate:"ociversion" json:"version,omitempty"` + + CredentialSecret *CredentialSecret `json:"credentialSecret,omitempty"` + Password string `json:"password"` + Username string `json:"username"` + + PlainHTTP bool `json:"plainHTTP,omitempty"` + Debug bool `json:"debug"` +} + +func validateOCIURL(fl validator.FieldLevel) bool { + return registry.IsOCI(fl.Field().String()) +} + +func validateOCIVersion(fl validator.FieldLevel) bool { + release, ok := fl.Parent().Interface().(OCIRelease) + if !ok { + return false + } + if release.Version != "" { + return true + } + + parsedRef, err := orasregistry.ParseReference(strings.TrimPrefix( + release.URL, + fmt.Sprintf("%s://", registry.OCIScheme), + )) + if err != nil { + return false + } + + return parsedRef.Reference != "" +} + +func (c OCIRelease) Validate() error { + v := validator.New() + if err := v.RegisterValidation("ociurl", validateOCIURL); err != nil { + return fmt.Errorf("register OCI URL validation: %w", err) + } + if err := v.RegisterValidation("ociversion", validateOCIVersion); err != nil { + return fmt.Errorf("register OCI version validation: %w", err) + } + return v.Struct(c) +} + +func (r OCIRelease) Chart() (*chart.Chart, error) { + return r.pullChart() +} + +func (r OCIRelease) pullReference(registryClient *registry.Client) (string, error) { + parsedURL, err := url.Parse(r.URL) + if err != nil { + return "", fmt.Errorf("invalid OCI URL %q: %w", r.URL, err) + } + + normalizedRef, err := registryClient.ValidateReference(r.URL, r.Version, parsedURL) + if err != nil { + return "", fmt.Errorf("failed to validate OCI reference %s: %w", r.URL, err) + } + + return normalizedRef.String(), nil +} + +func (r OCIRelease) pullChart() (*chart.Chart, error) { + log := ctrllog.Log.WithName("chart-oci") + + opts := []registry.ClientOption{ + registry.ClientOptEnableCache(true), + } + if r.Debug { + opts = append(opts, registry.ClientOptDebug(true)) + } + if (r.Username == "") != (r.Password == "") { + return nil, fmt.Errorf("both username and password must be set together for OCI basic auth") + } + if r.Username != "" && r.Password != "" { + opts = append(opts, registry.ClientOptBasicAuth(r.Username, r.Password)) + } + if r.PlainHTTP { + opts = append(opts, registry.ClientOptPlainHTTP()) + } + + registryClient, err := registry.NewClient(opts...) + if err != nil { + log.Error(err, "Failed to create registry client") + return nil, fmt.Errorf("failed to create registry client: %w", err) + } + + ref, err := r.pullReference(registryClient) + if err != nil { + log.Error(err, "Failed to normalize OCI reference", "url", r.URL, "version", r.Version) + return nil, err + } + + if r.Debug { + log.Info("Pulling OCI chart", "ref", ref) + } + + result, err := registryClient.Pull(ref) + if err != nil { + log.Error(err, "Failed to pull chart", "ref", ref) + return nil, fmt.Errorf("failed to pull chart from %s: %w", r.URL, err) + } + + if result.Chart == nil { + return nil, fmt.Errorf("registry returned empty chart for %s", r.URL) + } + + if r.Debug { + log.Info("Chart pulled successfully", + "name", result.Chart.Meta.Name, + "version", result.Chart.Meta.Version, + "size", result.Chart.Size) + } + + // Load the chart directly from the pulled bytes + chrt, err := loader.LoadArchive(bytes.NewReader(result.Chart.Data)) + if err != nil { + log.Error(err, "Failed to load chart archive") + return nil, fmt.Errorf("failed to load chart archive: %w", err) + } + + return chrt, nil +} + +func (r *OCIRelease) getActionableChart(wandb *v1.WeightsAndBiases) (*helm.ActionableChart, error) { + namespace := wandb.GetNamespace() + releaseName := wandb.GetName() + return helm.NewActionableChart(releaseName, namespace) +} + +func (r OCIRelease) Apply( + ctx context.Context, + c client.Client, + wandb *v1.WeightsAndBiases, + scheme *runtime.Scheme, + config spec.Values, +) error { + log := ctrllog.Log.WithName("chart-oci") + if r.CredentialSecret != nil { + if r.CredentialSecret.UsernameKey == "" { + r.CredentialSecret.UsernameKey = CredentialUsernameKey + } + if r.CredentialSecret.PasswordKey == "" { + r.CredentialSecret.PasswordKey = CredentialPasswordKey + } + log.Info("Retrieving credentials from secret", + "name", r.CredentialSecret.Name, + "usernameKey", r.CredentialSecret.UsernameKey, + "passwordKey", r.CredentialSecret.PasswordKey) + + secret := &corev1.Secret{} + err := c.Get(ctx, client.ObjectKey{Name: r.CredentialSecret.Name, Namespace: wandb.Namespace}, secret) + if err != nil { + log.Error(err, "Failed to get credentials from secret") + return err + } + usernameBytes, ok := secret.Data[r.CredentialSecret.UsernameKey] + if !ok || len(usernameBytes) == 0 { + return fmt.Errorf("credential secret %s/%s missing key %q", + wandb.Namespace, r.CredentialSecret.Name, r.CredentialSecret.UsernameKey) + } + passwordBytes, ok := secret.Data[r.CredentialSecret.PasswordKey] + if !ok || len(passwordBytes) == 0 { + return fmt.Errorf("credential secret %s/%s missing key %q", + wandb.Namespace, r.CredentialSecret.Name, r.CredentialSecret.PasswordKey) + } + r.Username = string(usernameBytes) + r.Password = string(passwordBytes) + } + + chrt, err := r.pullChart() + if err != nil { + return err + } + + actionableChart, err := r.getActionableChart(wandb) + if err != nil { + return err + } + + _, err = actionableChart.Apply(chrt, config) + return err +} + +func (r OCIRelease) Prune( + ctx context.Context, + c client.Client, + wandb *v1.WeightsAndBiases, + scheme *runtime.Scheme, + _ spec.Values, +) error { + actionableChart, err := r.getActionableChart(wandb) + if err != nil { + return err + } + + _, err = actionableChart.Uninstall() + return err +} diff --git a/pkg/wandb/spec/charts/oci_test.go b/pkg/wandb/spec/charts/oci_test.go new file mode 100644 index 00000000..683d1c72 --- /dev/null +++ b/pkg/wandb/spec/charts/oci_test.go @@ -0,0 +1,349 @@ +package charts + +import ( + "context" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "github.com/wandb/operator/api/v1" + "github.com/wandb/operator/pkg/wandb/spec" + "helm.sh/helm/v3/pkg/registry" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var _ = Describe("OCIRelease", func() { + var ociRelease *OCIRelease + + BeforeEach(func() { + ociRelease = &OCIRelease{ + URL: "oci://ghcr.io/wandb/helm-charts/operator-wandb", + Version: "1.0.0", + Debug: false, + } + }) + + Describe("Validate", func() { + Context("with valid OCI URL", func() { + It("should validate successfully", func() { + err := ociRelease.Validate() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("with various valid OCI URLs", func() { + DescribeTable("should validate successfully", + func(url string) { + ociRelease.URL = url + err := ociRelease.Validate() + Expect(err).NotTo(HaveOccurred()) + }, + Entry("ghcr.io", "oci://ghcr.io/wandb/charts/wandb"), + Entry("docker.io", "oci://docker.io/library/nginx"), + Entry("custom registry with port", "oci://registry.example.com:5000/charts/wandb"), + Entry("tagged reference without version", "oci://ghcr.io/wandb/charts/wandb:1.0.0"), + Entry("digest reference without version", "oci://ghcr.io/wandb/charts/wandb@sha256:c6841b3a895f1444a6738b5d04564a57e860ce42f8519c3be807fb6d9bee7888"), + ) + }) + + Context("with missing version", func() { + It("should fail validation for unqualified repository URLs", func() { + ociRelease.Version = "" + err := ociRelease.Validate() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Version")) + }) + + It("should allow tagged OCI URLs", func() { + ociRelease.URL = "oci://ghcr.io/wandb/charts/wandb:1.0.0" + ociRelease.Version = "" + err := ociRelease.Validate() + Expect(err).NotTo(HaveOccurred()) + }) + + It("should allow digest OCI URLs", func() { + ociRelease.URL = "oci://ghcr.io/wandb/charts/wandb@sha256:c6841b3a895f1444a6738b5d04564a57e860ce42f8519c3be807fb6d9bee7888" + ociRelease.Version = "" + err := ociRelease.Validate() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("with missing URL", func() { + It("should fail validation", func() { + ociRelease.URL = "" + err := ociRelease.Validate() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("URL")) + }) + }) + + Context("with non-OCI URL", func() { + DescribeTable("should fail validation", + func(url string) { + ociRelease.URL = url + err := ociRelease.Validate() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("URL")) + }, + Entry("https URL", "https://charts.example.com"), + Entry("http URL", "http://charts.example.com"), + Entry("plain hostname", "ghcr.io/wandb/charts/wandb"), + Entry("empty scheme", "://ghcr.io/wandb/charts/wandb"), + ) + }) + }) + + Describe("Chart dispatcher", func() { + It("should return OCIRelease for OCI URL", func() { + input := map[string]interface{}{ + "url": "oci://ghcr.io/wandb/charts/wandb", + "version": "1.0.0", + } + result := Get(input) + Expect(result).NotTo(BeNil()) + Expect(result).To(BeAssignableToTypeOf(&OCIRelease{})) + }) + + It("should return RepoRelease for HTTPS URL", func() { + input := map[string]interface{}{ + "url": "https://charts.example.com", + "name": "wandb", + } + result := Get(input) + Expect(result).NotTo(BeNil()) + Expect(result).To(BeAssignableToTypeOf(&RepoRelease{})) + }) + + It("should return LocalRelease for path", func() { + input := map[string]interface{}{ + "path": "/opt/charts/wandb.tgz", + } + result := Get(input) + Expect(result).NotTo(BeNil()) + Expect(result).To(BeAssignableToTypeOf(&LocalRelease{})) + }) + + It("should not match OCI URL as RepoRelease even with name field", func() { + input := map[string]interface{}{ + "url": "oci://ghcr.io/wandb/charts/wandb", + "name": "wandb", + "version": "1.0.0", + } + release := new(RepoRelease) + err := Is(release, input) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("pullChart", func() { + It("should return error for unreachable registry", func() { + ociRelease.URL = "oci://localhost:1/nonexistent/chart" + ociRelease.PlainHTTP = true + _, err := ociRelease.pullChart() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to pull chart")) + }) + }) + + Describe("pullReference", func() { + var registryClient *registry.Client + + BeforeEach(func() { + var err error + registryClient, err = registry.NewClient() + Expect(err).NotTo(HaveOccurred()) + }) + + It("should preserve an explicit tag from the URL", func() { + ociRelease.URL = "oci://ghcr.io/wandb/charts/wandb:1.2.3" + ociRelease.Version = "" + + ref, err := ociRelease.pullReference(registryClient) + Expect(err).NotTo(HaveOccurred()) + Expect(ref).To(Equal("oci://ghcr.io/wandb/charts/wandb:1.2.3")) + }) + + It("should preserve an explicit digest from the URL", func() { + ociRelease.URL = "oci://ghcr.io/wandb/charts/wandb@sha256:c6841b3a895f1444a6738b5d04564a57e860ce42f8519c3be807fb6d9bee7888" + ociRelease.Version = "" + + ref, err := ociRelease.pullReference(registryClient) + Expect(err).NotTo(HaveOccurred()) + Expect(ref).To(Equal("oci://ghcr.io/wandb/charts/wandb@sha256:c6841b3a895f1444a6738b5d04564a57e860ce42f8519c3be807fb6d9bee7888")) + }) + + It("should combine a repository URL with the version field", func() { + ref, err := ociRelease.pullReference(registryClient) + Expect(err).NotTo(HaveOccurred()) + Expect(ref).To(Equal("oci://ghcr.io/wandb/helm-charts/operator-wandb:1.0.0")) + }) + + It("should reject mismatched tagged URLs and version fields", func() { + ociRelease.URL = "oci://ghcr.io/wandb/charts/wandb:1.2.3" + ociRelease.Version = "2.0.0" + + _, err := ociRelease.pullReference(registryClient) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("chart reference and version mismatch")) + }) + }) + + Describe("Apply", func() { + It("should return error when pullChart fails", func() { + ociRelease.URL = "oci://localhost:1/nonexistent/chart" + ociRelease.PlainHTTP = true + err := ociRelease.Apply(context.TODO(), nil, &v1.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + }, nil, nil) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("Prune", func() { + It("should return error when actionable chart creation fails", func() { + // Empty name will fail release name validation + err := ociRelease.Prune(context.TODO(), nil, &v1.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{ + Name: "", + Namespace: "default", + }, + }, nil, nil) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("CredentialSecret", func() { + var ( + fakeClient client.Client + scheme *runtime.Scheme + wandb *v1.WeightsAndBiases + config spec.Values + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(v1.AddToScheme(scheme)).To(Succeed()) + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + fakeClient = fake.NewClientBuilder().WithScheme(scheme).Build() + + wandb = &v1.WeightsAndBiases{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-wandb", + Namespace: "test-namespace", + }, + } + + config = spec.Values{} + + GinkgoT().Setenv("HELM_CACHE_HOME", filepath.Join(os.TempDir(), "oci-test-cache")) + GinkgoT().Setenv("HELM_CONFIG_HOME", filepath.Join(os.TempDir(), "oci-test-config")) + GinkgoT().Setenv("HELM_DATA_HOME", filepath.Join(os.TempDir(), "oci-test-data")) + }) + + Context("with valid credential secret", func() { + BeforeEach(func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-oci-credentials", + Namespace: "test-namespace", + }, + Data: map[string][]byte{ + "HELM_USERNAME": []byte("secret-user"), + "HELM_PASSWORD": []byte("secret-pass"), + }, + } + Expect(fakeClient.Create(context.TODO(), secret)).To(Succeed()) + + ociRelease.URL = "oci://localhost:1/nonexistent/chart" + ociRelease.PlainHTTP = true + ociRelease.CredentialSecret = &CredentialSecret{ + Name: "test-oci-credentials", + UsernameKey: "HELM_USERNAME", + PasswordKey: "HELM_PASSWORD", + } + ociRelease.Username = "" + ociRelease.Password = "" + }) + + It("should retrieve credentials from secret before pull", func() { + err := ociRelease.Apply(context.TODO(), fakeClient, wandb, scheme, config) + // Should fail at pull step, not credential retrieval + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring("Failed to get credentials from secret")) + }) + }) + + Context("with default credential keys", func() { + BeforeEach(func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-oci-defaults", + Namespace: "test-namespace", + }, + Data: map[string][]byte{ + "HELM_USERNAME": []byte("default-user"), + "HELM_PASSWORD": []byte("default-pass"), + }, + } + Expect(fakeClient.Create(context.TODO(), secret)).To(Succeed()) + + ociRelease.URL = "oci://localhost:1/nonexistent/chart" + ociRelease.PlainHTTP = true + ociRelease.CredentialSecret = &CredentialSecret{ + Name: "test-oci-defaults", + } + }) + + It("should use default credential keys", func() { + err := ociRelease.Apply(context.TODO(), fakeClient, wandb, scheme, config) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring("Failed to get credentials from secret")) + }) + }) + + Context("with missing credential secret", func() { + BeforeEach(func() { + ociRelease.URL = "oci://localhost:1/nonexistent/chart" + ociRelease.PlainHTTP = true + ociRelease.CredentialSecret = &CredentialSecret{ + Name: "non-existent-secret", + UsernameKey: "HELM_USERNAME", + PasswordKey: "HELM_PASSWORD", + } + }) + + It("should fail when secret does not exist", func() { + err := ociRelease.Apply(context.TODO(), fakeClient, wandb, scheme, config) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("non-existent-secret")) + }) + }) + + Context("without credential secret", func() { + BeforeEach(func() { + ociRelease.URL = "oci://localhost:1/nonexistent/chart" + ociRelease.PlainHTTP = true + ociRelease.CredentialSecret = nil + ociRelease.Username = "direct-user" + ociRelease.Password = "direct-pass" + }) + + It("should use direct credentials", func() { + err := ociRelease.Apply(context.TODO(), fakeClient, wandb, scheme, config) + // Should fail at pull step, not credential setup + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring("Failed to get credentials from secret")) + }) + }) + }) +}) diff --git a/pkg/wandb/spec/charts/repo.go b/pkg/wandb/spec/charts/repo.go index 083a5610..2229b247 100644 --- a/pkg/wandb/spec/charts/repo.go +++ b/pkg/wandb/spec/charts/repo.go @@ -17,6 +17,7 @@ import ( "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/downloader" "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/registry" "helm.sh/helm/v3/pkg/repo" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -27,7 +28,7 @@ const CredentialUsernameKey = "HELM_USERNAME" const CredentialPasswordKey = "HELM_PASSWORD" type RepoRelease struct { - URL string `validate:"required,url" json:"url"` + URL string `validate:"required,url,nonociurl" json:"url"` Name string `validate:"required" json:"name"` // If version is not set, download latest. @@ -77,8 +78,16 @@ func (r RepoRelease) Chart() (*chart.Chart, error) { return local.Chart() } +func validateNonOCIURL(fl validator.FieldLevel) bool { + return !registry.IsOCI(fl.Field().String()) +} + func (c RepoRelease) Validate() error { - return validator.New().Struct(c) + v := validator.New() + if err := v.RegisterValidation("nonociurl", validateNonOCIURL); err != nil { + return fmt.Errorf("register non-OCI URL validation: %w", err) + } + return v.Struct(c) } func (r RepoRelease) ToLocalRelease() (*LocalRelease, error) { diff --git a/tilt-settings.sample.star b/tilt-settings.sample.star index 3bdd3871..4aab4811 100644 --- a/tilt-settings.sample.star +++ b/tilt-settings.sample.star @@ -41,6 +41,14 @@ SETTINGS = { "logFormat": "pretty", + # Optional composable Tilt infra settings. External infra installs the + # local test-infra chart for the selected service. useCustomCA generates + # test CA material through the normal W&B CR and user ConfigMap inputs. + "useExternalMysql": False, + "useExternalRedis": False, + "useExternalObjectStore": False, + "useCustomCA": False, + # CRC/OpenShift Local uses the crc-admin context. Tilt auto-enables # openshiftSCC on CRC; set it explicitly for other OpenShift clusters. # "allowedContexts": ["crc-admin"],