feat(backend): Unify labels for pods

.github/CODEOWNERS

@@ -1 +1 @@
-* @wandb/delivery-tooling-team
+* @wandb/on-prem-team

.github/dependabot.yml

@@ -14,9 +14,3 @@ updates:
     reviewers:
       - wandb/delivery-tooling-team
 
-  - package-ecosystem: github-actions
-    directory: /.github/workflows
-    schedule:
-      interval: daily
-    reviewers:
-      - wandb/delivery-tooling-team

.github/renovate.json5

@@ -0,0 +1,4 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": ["github>wandb/renovate-config"]
+}

.github/workflows/chart-validation.yaml

@@ -0,0 +1,71 @@
+name: Chart Validation
+
+on:
+  push:
+    branches: [v2, main]
+  pull_request:
+    branches: [v2, main]
+
+jobs:
+  chart-validation:
+    name: Chart Validation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Install Helm
+        uses: azure/setup-helm@bf6a7d304bc2fdb57e0331155b7ebf2c504acf0a # v4
+        with:
+          version: v3.19.0
+
+      - name: Install chart-testing
+        uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1
+        with:
+          version: v3.14.0
+
+      - name: Resolve chart dependencies
+        run: |
+          helm repo add ci-wandb https://charts.wandb.ai/
+          helm repo add ci-moco https://cybozu-go.github.io/moco/
+          helm repo add ci-ot-container-kit https://ot-container-kit.github.io/helm-charts
+          helm repo add ci-seaweedfs https://seaweedfs.github.io/seaweedfs-operator/
+          helm repo add ci-prometheus-community https://prometheus-community.github.io/helm-charts
+          helm repo add ci-altinity https://helm.altinity.com
+          helm repo add ci-victoria-metrics https://victoriametrics.github.io/helm-charts/
+          helm repo add ci-grafana https://grafana.github.io/helm-charts
+          helm dependency build deploy/operator
+          git diff --exit-code deploy/operator/Chart.lock
+
+      - name: Run chart-testing
+        run: ct lint --all --config deploy/ct.yaml
+
+      - name: Validate values schema
+        run: |
+          set -euo pipefail
+          helm lint --strict deploy/operator
+          for profile in deploy/operator/profiles/*.yaml; do
+            extra_args=()
+            if [[ "${profile}" == *telemetry-forward.yaml ]]; then
+              extra_args+=(--set-string telemetry.forwarding.otlp.endpoint=https://example.invalid:4317)
+            fi
+            helm lint --strict deploy/operator --values "${profile}" "${extra_args[@]}"
+          done
+
+      - name: Render representative configurations
+        run: |
+          set -euo pipefail
+          helm template wandb-operator deploy/operator \
+            --namespace wandb-operators \
+            --include-crds >/dev/null
+          for profile in deploy/operator/profiles/*.yaml; do
+            extra_args=()
+            if [[ "${profile}" == *telemetry-forward.yaml ]]; then
+              extra_args+=(--set-string telemetry.forwarding.otlp.endpoint=https://example.invalid:4317)
+            fi
+            helm template wandb-operator deploy/operator \
+              --namespace wandb-operators \
+              --include-crds \
+              --values "${profile}" \
+              "${extra_args[@]}" >/dev/null
+          done

.github/workflows/docker-build-scan.yml

@@ -2,22 +2,22 @@ name: Docker Build and Security Scan
 
 on:
   push:
-    branches: [main]
+    branches: [v2, main]
   pull_request:
-    branches: [main]
+    branches: [v2, main]
 
 jobs:
   build-and-scan:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
 
       - name: Build Docker image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
         with:
           context: .
           push: false
@@ -27,7 +27,7 @@ jobs:
           cache-to: type=gha,mode=max
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@c07df6fec6fa692e6fd1200d50aaa1fdd66f03c8 # master
         with:
           image-ref: wandb/operator:${{ github.sha }}
           format: "table"

.github/workflows/internal-chart-publish.yaml

@@ -5,75 +5,88 @@ on:
 
 jobs:
   release:
-    name: Release
+    name: Publish prerelease chart
     runs-on: ubuntu-latest
     permissions:
-      contents: 'read'
-      id-token: 'write'
+      contents: read
+      id-token: write
+    env:
+      CHART_REPOSITORY: us-docker.pkg.dev/wandb-production/public/wandb/charts
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
-          persist-credentials: false
           fetch-depth: 0
+          persist-credentials: false
+
+      - name: Validate prerelease chart version
+        id: chart
+        shell: bash
+        run: |
+          set -euo pipefail
+          version="$(awk '$1 == "version:" { print $2; exit }' deploy/operator/Chart.yaml | tr -d '\"')"
+          if [[ ! "${version}" =~ ^2\.[0-9]+\.[0-9]+-[0-9A-Za-z][0-9A-Za-z.-]*$ ]]; then
+            echo "Internal chart publishing requires a v2 prerelease version; got ${version}" >&2
+            exit 1
+          fi
+          echo "version=${version}" >> "${GITHUB_OUTPUT}"
+
+      - name: Install Helm
+        uses: azure/setup-helm@bf6a7d304bc2fdb57e0331155b7ebf2c504acf0a # v4
+        with:
+          version: v3.19.0
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+      - name: Install chart-testing
+        uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1
+        with:
+          version: v3.14.0
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+      - name: Lint charts
+        run: ct lint --all --config deploy/ct.yaml
 
       - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v3.0.1
+        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1
 
       - id: auth
         name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v3
+        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
         with:
           create_credentials_file: 'true'
           token_format: access_token
           project_id: wandb-production
           workload_identity_provider: ${{ secrets.CI_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.CI_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }}
 
-      - name: Authorize Docker to use Google Container Registry
-        run: |
-          gcloud auth configure-docker us-docker.pkg.dev
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: 1.25
+      - name: Authorize Docker for Artifact Registry
+        run: gcloud auth configure-docker us-docker.pkg.dev --quiet
 
-      - name: Install Helm
-        uses: azure/setup-helm@v4
-        with:
-          version: v3.19.0
-
-      - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.6.1
-        with:
-          version: v3.14.0
-
-      - name: Run chart-testing (list-changed)
-        id: list-changed
+      - name: Reject existing chart version
+        env:
+          VERSION: ${{ steps.chart.outputs.version }}
+        shell: bash
         run: |
-          changed=$(ct list-changed --config deploy/ct.yaml || true)
-          if [[ -n "$changed" ]]; then
-            echo "changed=true" >> $GITHUB_OUTPUT
+          set -euo pipefail
+          artifact="${CHART_REPOSITORY}/operator:${VERSION}"
+          set +e
+          output="$(gcloud artifacts docker images describe "${artifact}" --format='value(image_summary.digest)' 2>&1)"
+          status=$?
+          set -e
+          if [[ ${status} -eq 0 ]]; then
+            echo "Refusing to overwrite existing chart ${artifact}" >&2
+            exit 1
+          fi
+          if ! grep -Eqi 'NOT_FOUND|not found' <<< "${output}"; then
+            echo "Could not safely determine whether ${artifact} exists:" >&2
+            echo "${output}" >&2
+            exit 1
           fi
 
-      - name: Run chart-testing  (lint)
-        run: ct lint --config deploy/ct.yaml
-
-      - name: Install Ginkgo
-        run: go install github.com/onsi/ginkgo/v2/ginkgo@latest
-
-      - name: Build and Push to GAR
-        if: steps.list-changed.outputs.changed == 'true'
+      - name: Package and publish prerelease chart
+        env:
+          VERSION: ${{ steps.chart.outputs.version }}
         run: |
-          VERSION=$(grep "^version:" deploy/operator/Chart.yaml | awk '{print $2}')
+          set -euo pipefail
           helm dependency build deploy/operator
-          helm package deploy/operator
-          helm push operator-${VERSION}.tgz oci://$REPOSITORY
-        env:
-          REPOSITORY: us-docker.pkg.dev/wandb-production/public/wandb/charts
+          mkdir -p dist
+          helm package deploy/operator --destination dist
+          helm push "dist/operator-${VERSION}.tgz" "oci://${CHART_REPOSITORY}"

.github/workflows/internal-image-publish.yaml

@@ -5,57 +5,50 @@ on:
     inputs:
       image_tag:
         type: string
-        description: 'Tags for Images in GAR'
+        description: 'Development tag in the form dev-<name>-<7-to-40-character-sha>'
         required: true
 
 jobs:
   release:
-    name: Release
+    name: Publish development image
     runs-on: ubuntu-latest
     permissions:
-      contents: 'read'
-      id-token: 'write'
+      contents: read
+      id-token: write
     steps:
+      - name: Validate development tag
+        env:
+          VERSION: ${{ inputs.image_tag }}
+        run: |
+          if [[ ! "${VERSION}" =~ ^dev-[a-z0-9][a-z0-9._-]*-[0-9a-f]{7,40}$ ]]; then
+            echo "Development tags must use dev-<name>-<7-to-40-character-sha>" >&2
+            exit 1
+          fi
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
-          persist-credentials: false
           fetch-depth: 0
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+          persist-credentials: false
 
       - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v3.0.1
+        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1
 
       - id: auth
         name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v3
+        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
         with:
           create_credentials_file: 'true'
           token_format: access_token
           project_id: wandb-production
           workload_identity_provider: ${{ secrets.CI_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.CI_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }}
 
-      - name: Authorize Docker to use Google Container Registry
-        run: |
-          gcloud auth configure-docker us-docker.pkg.dev
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: 1.25
-
-      - name: Install Ginkgo
-        run: go install github.com/onsi/ginkgo/v2/ginkgo@latest
+      - name: Authorize Docker for Artifact Registry
+        run: gcloud auth configure-docker us-docker.pkg.dev --quiet
 
-      - name: Build and Push to GAR
-        run: |
-          export IMG=$IMAGE_TAG_BASE:$VERSION
-          make docker-build docker-push
+      - name: Build and publish development image
         env:
-          IMAGE_TAG_BASE: us-docker.pkg.dev/wandb-production/public/wandb/operator
-          VERSION: ${{ github.event.inputs.image_tag }}
+          IMAGE_REPOSITORY: us-docker.pkg.dev/wandb-production/public/wandb/operator
+          VERSION: ${{ inputs.image_tag }}
+        run: make docker-build docker-push IMG="${IMAGE_REPOSITORY}:${VERSION}"

.github/workflows/pr-title.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # https://github.com/amannn/action-semantic-pull-request/releases
-      - uses: amannn/action-semantic-pull-request@v4.2.0
+      - uses: amannn/action-semantic-pull-request@0eb081bc9c35210408951834a444794406eff6f8 # v4.2.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with: