diff --git a/.github/workflows/internal-chart-publish.yaml b/.github/workflows/internal-chart-publish.yaml index 1d05842e..0962199d 100644 --- a/.github/workflows/internal-chart-publish.yaml +++ b/.github/workflows/internal-chart-publish.yaml @@ -5,7 +5,7 @@ on: jobs: release: - name: Publish prerelease chart + name: Publish development chart runs-on: ubuntu-latest permissions: contents: read @@ -19,14 +19,14 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Validate prerelease chart version + - name: Validate development chart version id: chart shell: bash run: | set -euo pipefail version="$(awk '$1 == "version:" { print $2; exit }' deploy/operator/Chart.yaml | tr -d '\"')" - if [[ ! "${version}" =~ ^2\.[0-9]+\.[0-9]+-[0-9A-Za-z][0-9A-Za-z.-]*$ ]]; then - echo "Internal chart publishing requires a v2 prerelease version; got ${version}" >&2 + if [[ ! "${version}" =~ ^2\.[0-9]+\.[0-9]+-dev\.[0-9A-Za-z]+([.-][0-9A-Za-z]+)*$ ]]; then + echo "Internal chart publishing requires 2.x.y-dev.; got ${version}" >&2 exit 1 fi echo "version=${version}" >> "${GITHUB_OUTPUT}" @@ -81,7 +81,7 @@ jobs: exit 1 fi - - name: Package and publish prerelease chart + - name: Package and publish development chart env: VERSION: ${{ steps.chart.outputs.version }} run: | diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9b6e9f72..2b52d897 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -38,8 +38,12 @@ jobs: tag="${GITHUB_REF_NAME}" - if [[ ! "${tag}" =~ ^v2\.([0-9]+)\.([0-9]+)(\-.*)?$ ]]; then - echo "Expected a stable v2 tag in the form v2.x.y; got ${tag}" >&2 + if [[ ! "${tag}" =~ ^v2\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$ ]]; then + echo "Expected v2.x.y or v2.x.y-; got ${tag}" >&2 + exit 1 + fi + if [[ "${tag}" == *-dev || "${tag}" == *-dev.* ]]; then + echo "Development versions are published by Internal Chart Publish, not the tagged release workflow" >&2 exit 1 fi @@ -59,6 +63,10 @@ jobs: fi version="${tag#v}" + prerelease=false + if [[ "${version}" == *-* ]]; then + prerelease=true + fi chart_version="$(awk '$1 == "version:" { print $2; exit }' deploy/operator/Chart.yaml | tr -d '\"')" app_version="$(awk '$1 == "appVersion:" { print $2; exit }' deploy/operator/Chart.yaml | tr -d '\"')" image_tag="$(awk ' @@ -78,6 +86,7 @@ jobs: echo "tag=${tag}" >> "${GITHUB_OUTPUT}" echo "tagged_commit=${tagged_commit}" >> "${GITHUB_OUTPUT}" echo "version=${version}" >> "${GITHUB_OUTPUT}" + echo "prerelease=${prerelease}" >> "${GITHUB_OUTPUT}" - name: Install Helm uses: azure/setup-helm@bf6a7d304bc2fdb57e0331155b7ebf2c504acf0a # v4 @@ -157,12 +166,19 @@ jobs: - name: Create GitHub release env: GH_TOKEN: ${{ github.token }} + PRERELEASE: ${{ steps.release.outputs.prerelease }} TAG: ${{ steps.release.outputs.tag }} TAGGED_COMMIT: ${{ steps.release.outputs.tagged_commit }} run: | + set -euo pipefail + release_args=() + if [[ "${PRERELEASE}" == "true" ]]; then + release_args+=(--prerelease) + fi gh release create "${TAG}" \ --repo "${GITHUB_REPOSITORY}" \ --target "${TAGGED_COMMIT}" \ --verify-tag \ --generate-notes \ - --title "${TAG}" + --title "${TAG}" \ + "${release_args[@]}" diff --git a/README.md b/README.md index eedb9385..e7782efc 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,14 @@ The operator and its component dependencies are distributed as a Helm chart ```bash helm install wandb-operator \ oci://us-docker.pkg.dev/wandb-production/public/wandb/charts/operator \ + --version \ --namespace wandb-operators --create-namespace ``` +Operator v2 is published through this OCI path. The `wandb/operator` chart in +the legacy `charts.wandb.ai` Helm repository is Operator v1; do not use that +repository to install or upgrade Operator v2. + Then apply a `WeightsAndBiases` resource describing your deployment: ```yaml @@ -83,6 +88,7 @@ for the available chart options and which component operators are enabled. - [Configuration API](docs/config-api.md) - [Infrastructure Connection Settings](docs/infra-connection-settings.md) +- [Migrating from Operator v1 to v2](docs/migrating-v1-to-v2.md) - [Monitoring and Telemetry Guide](docs/monitoring.md) - [Deploying on OpenShift](docs/openshift.md) diff --git a/docs/migrating-v1-to-v2.md b/docs/migrating-v1-to-v2.md new file mode 100644 index 00000000..1de956cf --- /dev/null +++ b/docs/migrating-v1-to-v2.md @@ -0,0 +1,172 @@ +# Migrating from Operator v1 to v2 + +Use this runbook for an existing W&B deployment managed by Operator v1. Treat +the migration as a controlled cutover: preserve stateful dependencies, run one +operator controller at a time, and keep a tested rollback path until v2 is +healthy. + +Operator v2 does not uninstall the v1 Helm releases or modify migration +metadata inside MySQL or ClickHouse. Those actions can destroy data or make +rollback impossible and must remain explicit operator decisions. + +## Before the change window + +1. Record the v1 operator and W&B Helm release names, namespaces, chart + versions, values, and current replica counts. +2. Back up MySQL, object storage, ClickHouse, and Redis according to the + provider's restore procedure, and verify that the backups can be read. +3. Use `spec.retentionPolicy.onDelete: detach` in the v2 resource while + validating the migration. Do not delete the v1 resource or uninstall a + state-owning Helm release as a migration step. +4. Inspect the converted `apps.wandb.com/v2` `WeightsAndBiases` resource before + applying it: + + - Preserve the public `spec.wandb.hostname`. + - Verify every external infrastructure selector names an existing Secret + and key. + - Verify Redis has a non-empty host and port (or a complete URL), not only a + password. + - Verify object-store endpoint, bucket, region, TLS, and path-style settings. + - Remove `status` and other server-populated metadata from a live-resource + export before applying it as desired state. + +5. Determine whether the v1 W&B Helm release owns Redis or another dependency + that v2 will continue using: + + ```bash + helm get manifest -n + kubectl get statefulset,service,pvc -n + ``` + + If the release owns Redis, keep that release installed until Redis has been + migrated to an external service or rehomed outside the release's ownership. + Pointing v2 at the v1 Redis Service does not make it safe to uninstall the + v1 release. + +## Enter single-controller mode + +Do not let Operator v1 and v2 reconcile the same W&B deployment concurrently. +The v1 controller can restore legacy values or Secrets while v2 is trying to +apply the converted configuration. + +1. Identify the v1 controller Deployment from its Helm release and save its + replica count. +2. Scale that Deployment to zero. +3. Confirm no v1 controller pods remain before applying the v2 resource. + +For example: + +```bash +kubectl -n scale \ + deployment/ --replicas=0 +kubectl -n get pods +``` + +Do not uninstall the v1 operator yet. Leaving the release installed makes the +controller rollback reversible. + +## Install Operator v2 and apply the resource + +Pin a reviewed version from the Operator v2 OCI repository: + +```bash +helm upgrade --install wandb-operator \ + oci://us-docker.pkg.dev/wandb-production/public/wandb/charts/operator \ + --version \ + --namespace \ + --create-namespace + +kubectl apply -f +``` + +The `wandb/operator` chart in `charts.wandb.ai` is Operator v1 and is not an +Operator v2 upgrade source. + +The v2 chart installs and upgrades its CRDs with server-side apply. If the CRD +installer Job fails, inspect that Job and resolve the ownership or RBAC error; +do not delete existing CRDs as a recovery shortcut. + +## Wait for the cutover gates + +Applications are gated on infrastructure readiness, MySQL initialization, and +the W&B migration Jobs. Do not switch traffic or remove v1 workloads until all +of these checks pass: + +```bash +kubectl -n get wandb -o yaml +kubectl -n get jobs \ + -l app.kubernetes.io/instance=,app.kubernetes.io/component=migration +kubectl -n get applications +kubectl -n get deployments +``` + +Confirm: + +- Every configured infrastructure status is ready. +- `status.wandb.migration.ready` is `true`, its version matches + `spec.wandb.version`, and every migration Job succeeded. +- Every v2 application Deployment is fully rolled out with available replicas. +- The public route targets the intended v2 Services and no longer returns + transient 5xx responses. + +If a migration Job fails, collect its status and logs: + +```bash +kubectl -n describe job/ +kubectl -n logs job/ -c migrate +``` + +Do not infer migration success from a similarly named Deployment. Do not +automatically clear `partially_applied_version` or edit migration tables: that +requires a migration-specific recovery decision and a verified backup. + +## Smoke test before cleanup + +Run the tests through the customer-facing hostname: + +1. Sign in and load the main UI. +2. Create an SDK run using an existing, valid entity. +3. Upload and download an artifact through the SDK. +4. Download the artifact in a browser using the presigned URL. +5. Check the browser request for the expected external object-store endpoint, + trusted TLS, and successful CORS headers. +6. Exercise Runs and Weave views used by the deployment. + +An SDK upload alone is not sufficient for S3-compatible storage. Browser +downloads can still fail when the external endpoint or CORS policy is wrong. + +## Retire v1 in the safe order + +1. Keep the v1 controller at zero replicas. +2. Verify the v2 route and workloads remain healthy for the agreed observation + period. +3. Inventory remaining v1 Deployments and scale any that are still serving + traffic to zero. Operator v2 removes the known legacy `*-bc` Deployments + only after all desired v2 Deployments are ready; verify rather than assume + that all v1 workloads are covered. +4. Uninstall the v1 operator release only after the rollback window no longer + requires it. +5. Uninstall the v1 W&B release only after confirming it owns no Redis, + database, object-store, ClickHouse, PVC, Secret, or other resource still + used by v2. + +Never uninstall the v1 W&B release while v2 still references its Helm-owned +Redis. The uninstall deletes the Redis workload even if the v2 resource calls +it "external." + +## Rollback + +Rollback must also use one controller at a time: + +1. Stop traffic to the v2 workloads or restore the previous route. +2. Scale the v2 operator controller to zero. +3. Restore the saved v1 configuration and workload replica counts. +4. Scale the v1 operator controller back to its saved replica count. +5. Verify database compatibility before rolling the W&B application version + back; a completed forward migration may not be reversible by changing only + the image tag. +6. Re-run the UI, SDK, artifact, and browser-download smoke tests. + +Do not delete the v2 resource or its dependencies during rollback. Keep them +detached until the incident is understood and the retained data is no longer +needed. diff --git a/docs/releasing.md b/docs/releasing.md index 1c7d7f17..d0539aae 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,10 +1,11 @@ # Releasing Operator v2 -Production v2 releases are prepared through a reviewed pull request and -published from an annotated `v2.x.y` tag. A single workflow publishes the -operator image and Helm chart from the same commit. +Stable and preview v2 releases are prepared through a reviewed pull request and +published from an annotated `v2.x.y` or `v2.x.y-` tag. A single +workflow publishes the operator image and Helm chart from the same commit, then +creates the matching GitHub Release. -## Production release +## Tagged release 1. Open a release pull request against `main`. 2. Update `wandb.version` in `deploy/operator/values.yaml` to the intended W&B @@ -31,7 +32,8 @@ operator image and Helm chart from the same commit. test "$(git rev-parse HEAD)" = "$(git rev-parse origin/main)" ``` -7. Create and push an annotated release tag at that commit: +7. Create and push an annotated release tag at that commit. Use a SemVer + prerelease suffix such as `-beta.2` or `-rc.1` for previews: ```bash version=v2.0.0 @@ -41,14 +43,21 @@ operator image and Helm chart from the same commit. 8. Monitor the `Release v2` workflow. It publishes the versioned GAR image first, then the matching OCI Helm chart, and creates the GitHub Release only - after both artifacts succeed. + after both artifacts succeed. A tag with a prerelease suffix creates a + GitHub prerelease. 9. Record the source commit, image digest, chart digest, and GitHub Release URL in the release record. -Production versions are immutable. Never move, delete, reuse, or overwrite a -`v2.x.y` tag or its `2.x.y` image/chart tags. If a release is incorrect or only -partially publishes, fix it with a new patch version. The production workflow -does not publish a `latest` tag. +Tagged versions are immutable. Never move, delete, reuse, or overwrite a +`v2.x.y` or `v2.x.y-` tag or its matching image/chart tags. If a +release is incorrect or only partially publishes, fix it with a new version. +The release workflow does not publish a `latest` tag. + +Every version offered to users must use the tagged release workflow so the OCI +chart version, operator image tag, and GitHub Release stay aligned. Operator v2 +charts are published at +`oci://us-docker.pkg.dev/wandb-production/public/wandb/charts/operator`; the +legacy `charts.wandb.ai` repository is not the Operator v2 release channel. ## Development artifacts @@ -56,6 +65,9 @@ The `Internal Image Publish` workflow accepts only tags in the form `dev--<7-to-40-character-sha>`, for example `dev-bucket-proxy-1106901`. It cannot publish production-style tags. -The `Internal Chart Publish` workflow accepts the prerelease version already -declared in `deploy/operator/Chart.yaml`, such as `2.0.0-rc.1`. It rejects -stable versions and refuses to overwrite an existing prerelease chart tag. +The `Internal Chart Publish` workflow is for engineering-only chart validation. +It accepts only a development version already declared in +`deploy/operator/Chart.yaml`, such as `2.0.0-dev.1106901`. It rejects stable +versions and release prereleases such as `-beta.2` or `-rc.1`, and refuses to +overwrite an existing development chart tag. Do not offer these chart-only +builds to users; publish a tagged release instead.