From 4bfdc63af093bb50a75d3359959355512c78799d Mon Sep 17 00:00:00 2001 From: Jvst Me Date: Tue, 19 Aug 2025 15:00:17 +0200 Subject: [PATCH 1/5] CI refactoring - Split GitHub Actions workflows into reusable workflows to avoid duplication. - Split some jobs into two stages: building an artifact and uploading it. This allows to upload artifacts only after all were successfully built. Also allows to test building artifacts in PRs from forks, where credentials for uploading are not available. --- .github/workflows/build-artifacts.yml | 256 ++++++++++++++ .github/workflows/build-docs.yml | 37 +++ .github/workflows/build.yml | 312 +++--------------- .github/workflows/docs.yaml | 33 +- .github/workflows/release.yml | 304 +++-------------- .../workflows/upload-post-pypi-artifacts.yml | 55 +++ .../workflows/upload-pre-pypi-artifacts.yml | 57 ++++ .pre-commit-config.yaml | 2 +- src/dstack/version.py | 4 +- 9 files changed, 508 insertions(+), 552 deletions(-) create mode 100644 .github/workflows/build-artifacts.yml create mode 100644 .github/workflows/build-docs.yml create mode 100644 .github/workflows/upload-post-pypi-artifacts.yml create mode 100644 .github/workflows/upload-pre-pypi-artifacts.yml diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 0000000000..ecb686bd66 --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,256 @@ +name: Build Artifacts + +on: + workflow_call: + inputs: + version: + type: string + required: true + staging: + type: boolean + required: true + go-integration-tests: + type: boolean + required: true + +jobs: + code-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + - run: uv tool install pre-commit + - run: pre-commit run -a --show-diff-on-failure + + frontend-build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + - name: Restore cached build + id: cache-build + uses: actions/cache@v4 + with: + path: frontend/build + key: frontend-build-${{ hashFiles('frontend/**') }} + restore-keys: | + frontend-build- + - name: Set up Node + if: steps.cache-build.outputs.cache-hit != 'true' + uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Install packages + if: steps.cache-build.outputs.cache-hit != 'true' + run: npm ci + - name: Build dist + if: steps.cache-build.outputs.cache-hit != 'true' + run: npm run build + - name: Upload dist + uses: actions/upload-artifact@v4 + with: + name: frontend-build + path: frontend/build + retention-days: 1 + + python-test: + needs: [code-lint, frontend-build] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: astral-sh/setup-uv@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: uv sync --all-extras + - name: Download frontend build + uses: actions/download-artifact@v4 + with: + name: frontend-build + path: src/dstack/_internal/server/statics + - name: Run pytest on POSIX + if: matrix.os != 'windows-latest' + # Skip Postgres tests on macos since macos runner doesn't have Docker. + run: | + RUNPOSTGRES="" + if [ "${{ matrix.os }}" != "macos-latest" ]; then + RUNPOSTGRES="--runpostgres" + fi + uv run pytest -n auto src/tests --runui $RUNPOSTGRES + - name: Run pytest on Windows + if: matrix.os == 'windows-latest' + run: | + uv run pytest -n auto src/tests --runui --runpostgres + + runner-test: + defaults: + run: + working-directory: runner + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: runner/go.mod + cache-dependency-path: runner/go.sum + - name: Check if go.mod and go.sum are up-to-date + run: go mod tidy -diff + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: v1.62.0 # Should match .pre-commit-config.yaml + args: --timeout=20m + working-directory: runner + - name: Test + # Only run slow tests if requested by workflow call inputs. + run: | + SHORT="-short" + if [[ "${{ inputs.go-integration-tests }}" == "true" ]]; then + SHORT="" + fi + # Skip failing integration tests on macOS for release builds. + # TODO: https://github.com/dstackai/dstack/issues/3005 + if [[ "${{ inputs.staging }}" == "false" && "${{ startsWith(matrix.os, 'macos') }}" == "true" ]]; then + SHORT="-short" + fi + go version + go fmt $(go list ./... | grep -v /vendor/) + go vet $(go list ./... | grep -v /vendor/) + go test $SHORT -race $(go list ./... | grep -v /vendor/) + + runner-compile: + needs: [runner-test] + defaults: + run: + working-directory: runner + env: + REPO_NAME: github.com/dstackai/dstack + strategy: + matrix: + include: + - { runs-on: "ubuntu-24.04", goos: "linux", goarch: "amd64" } + - { runs-on: "ubuntu-24.04-arm", goos: "linux", goarch: "arm64" } + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: runner/go.mod + cache-dependency-path: runner/go.sum + - name: build + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + run: | + CGO_ENABLED=0 go build -ldflags "-X 'main.Version=${{ inputs.version }}' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner + CGO_ENABLED=1 go build -ldflags "-X 'main.Version=${{ inputs.version }}'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim + - uses: actions/upload-artifact@v4 + with: + name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} + path: | + runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} + runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }} + retention-days: 1 + + gateway-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + - name: Build package + working-directory: gateway + run: | + echo "__version__ = \"${{ inputs.version }}\"" > src/dstack/gateway/version.py + # TODO: depend on a specific dstackai/dstack commit for staging builds? + if [[ "${{ inputs.staging }}" == "false" ]]; then + sed \ + -i.old \ + "s|@ git+https://github.com/dstackai/dstack.git@master|== ${{ inputs.version }}|" \ + pyproject.toml + diff pyproject.toml pyproject.toml.old > /dev/null && echo "Could not set version" && exit 1 + fi + uv build + - uses: actions/upload-artifact@v4 + with: + name: dstack-gateway + path: gateway/dist/dstack_gateway-${{ inputs.version }}-py3-none-any.whl + retention-days: 1 + + python-build: + needs: [code-lint, frontend-build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + - name: Download frontend build + uses: actions/download-artifact@v4 + with: + name: frontend-build + path: src/dstack/_internal/server/statics + - name: Build dstack Python package + # TODO: set __version__ to inputs.version regardless of inputs.staging, + # so that staging builds are also tied to a specific runner and gateway version. + # May require changing how dstack handles __version__. + run: | + if [[ "${{ inputs.staging }}" == "true" ]]; then + VERSION=0.0.0 + IS_RELEASE=False + else + VERSION=${{ inputs.version }} + IS_RELEASE=True + fi + BASE_IMAGE=$(cat src/dstack/version.py | grep "base_image = ") + BASE_IMAGE_UBUNTU_VERSION=$(cat src/dstack/version.py | grep "base_image_ubuntu_version = ") + echo "__version__ = \"$VERSION\"" > src/dstack/version.py + echo "__is_release__ = $IS_RELEASE" >> src/dstack/version.py + echo $BASE_IMAGE >> src/dstack/version.py + echo $BASE_IMAGE_UBUNTU_VERSION >> src/dstack/version.py + cp README.md src + uv build + - uses: actions/upload-artifact@v4 + with: + name: python-build + path: dist + retention-days: 1 + + generate-json-schema: + needs: [code-lint] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + - name: Install dstack + run: uv sync + - name: Generate json schema + run: | + mkdir /tmp/json-schemas + uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > /tmp/json-schemas/configuration.json + uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json())" > /tmp/json-schemas/profiles.json + - uses: actions/upload-artifact@v4 + with: + name: json-schemas + path: /tmp/json-schemas + retention-days: 1 diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml new file mode 100644 index 0000000000..572d9c6d3e --- /dev/null +++ b/.github/workflows/build-docs.yml @@ -0,0 +1,37 @@ +name: Build Docs + +on: + workflow_call: + inputs: + release_tag: + type: string + required: false + +jobs: + build-docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + - name: Install dstack + run: | + uv pip install examples/plugins/example_plugin_server + if [ -n "${{ inputs.release_tag }}" ]; then + uv pip install "dstack[server]==${{ inputs.release_tag }}" + else + uv pip install -e '.[server]' + fi + - name: Build + run: | + uv pip install pillow cairosvg + sudo apt-get update && sudo apt-get install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev + uv pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade + uv pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git + uv run mkdocs build -s + - uses: actions/upload-artifact@v4 + with: + name: site + path: site + retention-days: 1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8a97c67c7d..dcd92e696f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build +name: Test Build on: push: @@ -12,290 +12,58 @@ on: - "master" workflow_dispatch: inputs: - intergation-tests: + go-integration-tests: type: boolean required: true default: false + description: Go integration tests env: BUILD_INCREMENT: 150 - PIP_DISABLE_PIP_VERSION_CHECK: on - PIP_DEFAULT_TIMEOUT: 10 - PIP_PROGRESS_BAR: off jobs: - code-lint: + compute-version: runs-on: ubuntu-latest + outputs: + version: ${{ steps.set-version.outputs.version }} steps: - - uses: actions/checkout@v4 - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - run: uv tool install pre-commit - - run: pre-commit run -a --show-diff-on-failure - - frontend-build: - runs-on: ubuntu-latest - defaults: - run: - working-directory: frontend - steps: - - uses: actions/checkout@v4 - - name: Restore cached build - id: cache-build - uses: actions/cache@v4 - with: - path: frontend/build - key: frontend-build-${{ hashFiles('frontend/**') }} - restore-keys: | - frontend-build- - - name: Set up Node - if: steps.cache-build.outputs.cache-hit != 'true' - uses: actions/setup-node@v4 - with: - node-version: 18 - - name: Install packages - if: steps.cache-build.outputs.cache-hit != 'true' - run: npm ci - - name: Build dist - if: steps.cache-build.outputs.cache-hit != 'true' - run: npm run build - - name: Upload dist - uses: actions/upload-artifact@v4 - with: - name: frontend-build - path: frontend/build - - python-test: - needs: [code-lint, frontend-build] - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: astral-sh/setup-uv@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: uv sync --all-extras - - name: Download frontend build - uses: actions/download-artifact@v4 - with: - name: frontend-build - path: src/dstack/_internal/server/statics - - name: Run pytest on POSIX - if: matrix.os != 'windows-latest' - # Skip Postgres tests on macos since macos runner doesn't have Docker. - run: | - RUNPOSTGRES="" - if [ "${{ matrix.os }}" != "macos-latest" ]; then - RUNPOSTGRES="--runpostgres" - fi - uv run pytest -n auto src/tests --runui $RUNPOSTGRES - - name: Run pytest on Windows - if: matrix.os == 'windows-latest' - run: | - uv run pytest -n auto src/tests --runui --runpostgres - - update-get-dstack: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - needs: [python-test] - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - steps: - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - run: | - VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) - echo $VERSION | aws s3 cp - s3://get-dstack/stgn-cli/latest-version --acl public-read - - runner-test: - defaults: - run: - working-directory: runner - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: runner/go.mod - cache-dependency-path: runner/go.sum - - name: Check if go.mod and go.sum are up-to-date - run: go mod tidy -diff - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6 - with: - version: v1.62.0 - args: --timeout=20m - working-directory: runner - - name: Test - # Do not run slow integration tests automatically. - # Slow tests can be run manually via workflow_dispatch when required. - run: | - SHORT="-short" - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - if [[ "${{ github.event.inputs.intergation-tests }}" == "true" ]]; then - SHORT="" - fi - fi - go version - go fmt $(go list ./... | grep -v /vendor/) - go vet $(go list ./... | grep -v /vendor/) - go test $SHORT -race $(go list ./... | grep -v /vendor/) - - runner-compile: - needs: [runner-test] - defaults: - run: - working-directory: runner - env: - REPO_NAME: github.com/dstackai/dstack - strategy: - matrix: - include: - - { runs-on: "ubuntu-24.04", goos: "linux", goarch: "amd64" } - - { runs-on: "ubuntu-24.04-arm", goos: "linux", goarch: "arm64" } - runs-on: ${{ matrix.runs-on }} - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: runner/go.mod - cache-dependency-path: runner/go.sum - - name: build - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - run: | - VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) - CGO_ENABLED=0 go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner - CGO_ENABLED=1 go build -ldflags "-X 'main.Version=$VERSION'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim - echo $VERSION - - uses: actions/upload-artifact@v4 - with: - name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} - path: | - runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} - runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }} - retention-days: 1 - - runner-upload: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - needs: [runner-compile] - runs-on: ubuntu-latest - steps: - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Download Runner - uses: actions/download-artifact@v4 - with: - pattern: dstack-runner-* - merge-multiple: true - path: runner - - name: Upload to S3 - working-directory: runner - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - id: set-version run: | - VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) - aws s3 cp . "s3://dstack-runner-downloads-stgn/$VERSION/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read - aws s3 cp . "s3://dstack-runner-downloads-stgn/latest/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read + echo "version=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))" >> $GITHUB_OUTPUT - generate-json-schema: + build-artifacts: + needs: [compute-version] + uses: ./.github/workflows/build-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + staging: true + # TODO: run integration tests on every 'push' event + # https://github.com/dstackai/dstack/issues/3005 + go-integration-tests: ${{ github.event_name == 'workflow_dispatch' && inputs.go-integration-tests }} + + upload-pre-pypi-artifacts: + needs: [compute-version, build-artifacts] + # Skip for PRs from forks, where AWS S3 credentials are not available if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - needs: [python-test] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Install dstack - run: uv sync - - name: Generate json schema - run: | - uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > configuration.json - uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json())" > profiles.json - - name: Upload json schema to S3 - run: | - VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) - aws s3 cp configuration.json "s3://dstack-runner-downloads-stgn/$VERSION/schemas/configuration.json" --acl public-read - aws s3 cp configuration.json "s3://dstack-runner-downloads-stgn/latest/schemas/configuration.json" --acl public-read - aws s3 cp profiles.json "s3://dstack-runner-downloads-stgn/$VERSION/schemas/profiles.json" --acl public-read - aws s3 cp profiles.json "s3://dstack-runner-downloads-stgn/latest/schemas/profiles.json" --acl public-read - - gateway-build: + uses: ./.github/workflows/upload-pre-pypi-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + staging: true + secrets: inherit + + upload-post-pypi-artifacts: + needs: [compute-version, build-artifacts] + # Skip for PRs from forks, where AWS S3 credentials are not available if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - defaults: - run: - working-directory: gateway - steps: - - uses: actions/checkout@v4 - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Compute version - run: echo VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) > $GITHUB_ENV - - name: Build package - run: | - echo "__version__ = \"${{ env.VERSION }}\"" > src/dstack/gateway/version.py - uv build - - name: Upload to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - WHEEL=dstack_gateway-${{ env.VERSION }}-py3-none-any.whl - aws s3 cp dist/$WHEEL "s3://dstack-gateway-downloads/stgn/$WHEEL" - echo "${{ env.VERSION }}" | aws s3 cp - "s3://dstack-gateway-downloads/stgn/latest-version" + uses: ./.github/workflows/upload-post-pypi-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + is-latest-version: true + staging: true + secrets: inherit - docs-build: - # Skip for PRs from forks since mkdocs-material-insiders is not available in forks + build-docs: + # Skip for PRs from forks, where mkdocs-material-insiders is not available if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install dstack - run: | - uv pip install examples/plugins/example_plugin_server - uv pip install -e '.[server]' - # Move these deps into an extra and install that way - - name: Build - run: | - uv pip install pillow cairosvg - sudo apt-get update && sudo apt-get install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev - uv pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade - uv pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git - uv run mkdocs build -s + uses: ./.github/workflows/build-docs.yml + secrets: inherit diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index e1f43877bf..c9c44cc103 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,34 +1,25 @@ -name: Deploy Docs +name: Build & Deploy Docs on: workflow_dispatch: inputs: - release_tag: + release-tag: description: "dstack version" jobs: - docs-deploy: + build-docs: + uses: ./.github/workflows/build-docs.yml + with: + release-tag: ${{ inputs.release-tag }} + secrets: inherit + + deploy-docs: + needs: [build-docs] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 + - uses: actions/download-artifact@v4 with: - python-version: 3.11 - - name: Install dstack - run: | - uv pip install examples/plugins/example_plugin_server - if [ -n "${{ inputs.release_tag }}" ]; then - uv pip install "dstack[server]==${{ inputs.release_tag }}" - else - uv pip install -e '.[server]' - fi - - name: Build - run: | - uv pip install pillow cairosvg - sudo apt-get update && sudo apt-get install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev - uv pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade - uv pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git - uv run mkdocs build -s + name: site - name: Deploy uses: JamesIves/github-pages-deploy-action@v4.6.4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1fe8b12b58..08cbf90e08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,256 +8,77 @@ on: - "[0-9]+.[0-9]+.[0-9]+" - "[0-9]+.[0-9]+.[0-9]+.post[0-9]+" -env: - BUILD_INCREMENT: 150 - PIP_DISABLE_PIP_VERSION_CHECK: on - PIP_DEFAULT_TIMEOUT: 10 - PIP_PROGRESS_BAR: off - jobs: - code-lint: + compute-version: runs-on: ubuntu-latest + outputs: + version: ${{ steps.set-version.outputs.version }} + latest: ${{ steps.set-latest.outputs.latest }} steps: - - uses: actions/checkout@v4 - - name: Set up uv + - name: Set up Python uses: astral-sh/setup-uv@v5 with: python-version: 3.11 - - run: uv tool install pre-commit - - run: pre-commit run -a --show-diff-on-failure - - frontend-build: - runs-on: ubuntu-latest - defaults: - run: - working-directory: frontend - steps: - - uses: actions/checkout@v4 - - name: Set up Node - uses: actions/setup-node@v4 - with: - node-version: 18 - - name: Install packages - run: npm ci - - name: Build dist - run: npm run build - - name: Upload dist - uses: actions/upload-artifact@v4 - with: - name: frontend-build - path: frontend/build - - python-test: - needs: [code-lint, frontend-build] - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: astral-sh/setup-uv@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: uv sync --all-extras - - name: Download frontend build - uses: actions/download-artifact@v4 - with: - name: frontend-build - path: src/dstack/_internal/server/statics - - name: Run pytest on POSIX - if: matrix.os != 'windows-latest' - # Skip Postgres tests on macos since macos runner doesn't have Docker. + # settings to prevent warnings when running uv without a repo checkout + enable-cache: false + ignore-empty-workdir: true + - id: set-version run: | - RUNPOSTGRES="" - if [ "${{ matrix.os }}" != "macos-latest" ]; then - RUNPOSTGRES="--runpostgres" - fi - uv run pytest -n auto src/tests --runui $RUNPOSTGRES - - name: Run pytest on Windows - if: matrix.os == 'windows-latest' + echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + - id: set-latest run: | - uv run pytest -n auto src/tests --runui --runpostgres + uv pip install packaging + VERSION=${{ steps.set-version.outputs.version }} + LATEST=$(python -c "from packaging import version as pkg_version; print('' if pkg_version.parse('$VERSION').is_prerelease else '1', end='')") + echo "latest=$LATEST" >> "$GITHUB_OUTPUT" - runner-test: - defaults: - run: - working-directory: runner - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: runner/go.mod - cache-dependency-path: runner/go.sum - - name: Check if go.mod and go.sum are up-to-date - run: go mod tidy -diff - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6 - with: - version: v1.62.0 - args: --timeout=20m - working-directory: runner - - name: Test - run: | - go version - go fmt $(go list ./... | grep -v /vendor/) - go vet $(go list ./... | grep -v /vendor/) - go test -race $(go list ./... | grep -v /vendor/) + build-artifacts: + needs: [compute-version] + uses: ./.github/workflows/build-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + staging: true + go-integration-tests: true - runner-compile: - needs: [runner-test] - defaults: - run: - working-directory: runner - env: - REPO_NAME: github.com/dstackai/dstack - strategy: - matrix: - include: - - { runs-on: "ubuntu-24.04", goos: "linux", goarch: "amd64" } - - { runs-on: "ubuntu-24.04-arm", goos: "linux", goarch: "arm64" } - runs-on: ${{ matrix.runs-on }} - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: runner/go.mod - cache-dependency-path: runner/go.sum - - name: build - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - run: | - VERSION=${GITHUB_REF#refs/tags/} - CGO_ENABLED=0 go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner - CGO_ENABLED=1 go build -ldflags "-X 'main.Version=$VERSION'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim - - uses: actions/upload-artifact@v4 - with: - name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} - path: | - runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} - runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }} - - gateway-build: - runs-on: ubuntu-latest - defaults: - run: - working-directory: gateway - steps: - - uses: actions/checkout@v4 - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Store version - run: echo VERSION=${GITHUB_REF#refs/tags/} > $GITHUB_ENV - - name: Build package - run: | - echo "__version__ = \"${{ env.VERSION }}\"" > src/dstack/gateway/version.py - sed \ - -i.old \ - "s|@ git+https://github.com/dstackai/dstack.git@master|== ${{ env.VERSION }}|" \ - pyproject.toml - diff pyproject.toml pyproject.toml.old > /dev/null && echo "Could not set version" && exit 1 - uv build - - name: Upload to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - WHEEL=dstack_gateway-${{ env.VERSION }}-py3-none-any.whl - aws s3 cp dist/$WHEEL "s3://dstack-gateway-downloads/release/$WHEEL" - echo "${{ env.VERSION }}" | aws s3 cp - "s3://dstack-gateway-downloads/release/latest-version" - - runner-upload: - needs: [runner-compile, gateway-build, python-test] - runs-on: ubuntu-latest - steps: - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Download Runner - uses: actions/download-artifact@v4 - with: - pattern: dstack-runner-* - merge-multiple: true - path: runner - - name: Upload to S3 - working-directory: runner - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - VERSION=${GITHUB_REF#refs/tags/} - aws s3 cp . "s3://dstack-runner-downloads/$VERSION/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read - aws s3 cp . "s3://dstack-runner-downloads/latest/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read + upload-pre-pypi-artifacts: + needs: [compute-version, build-artifacts] + uses: ./.github/workflows/upload-pre-pypi-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + staging: false + secrets: inherit pypi-upload: - needs: [python-test, runner-upload] + needs: [compute-version, upload-pre-pypi-artifacts] runs-on: ubuntu-latest - outputs: - LATEST: ${{ steps.set_latest.outputs.LATEST }} - name: Set latest variable steps: - - uses: actions/checkout@v4 - name: Set up uv uses: astral-sh/setup-uv@v5 with: python-version: 3.11 - - name: Download frontend build + # settings to prevent warnings when running uv without a repo checkout + enable-cache: false + ignore-empty-workdir: true + - name: Download Python package uses: actions/download-artifact@v4 with: - name: frontend-build - path: src/dstack/_internal/server/statics - - name: Set output - id: set_latest + name: python-build + path: dist + - name: Upload Python package to PyPI run: | - uv pip install packaging - VERSION=${GITHUB_REF#refs/tags/} - LATEST=$(python -c "from packaging import version as pkg_version; print('' if pkg_version.parse('$VERSION').is_prerelease else '1', end='')") - echo "LATEST=$LATEST" >> "$GITHUB_OUTPUT" - - name: Upload pip package - run: | - VERSION=${GITHUB_REF#refs/tags/} - BASE_IMAGE=$(cat src/dstack/version.py | grep "base_image = ") - BASE_IMAGE_UBUNTU_VERSION=$(cat src/dstack/version.py | grep "base_image_ubuntu_version = ") - echo "__version__ = \"$VERSION\"" > src/dstack/version.py - echo "__is_release__ = True" >> src/dstack/version.py - echo $BASE_IMAGE >> src/dstack/version.py - echo $BASE_IMAGE_UBUNTU_VERSION >> src/dstack/version.py - cp README.md src - uv build uv publish --username ${{ secrets.PYPI_USERNAME }} --password ${{ secrets.PYPI_PASSWORD }} - update-get-dstack-tag: - needs: [pypi-upload] - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - steps: - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - run: | - VERSION=${GITHUB_REF#refs/tags/} - echo $VERSION | aws s3 cp - s3://get-dstack/cli/latest-version --acl public-read + upload-post-pypi-artifacts: + needs: [compute-version, pypi-upload] + uses: ./.github/workflows/upload-post-pypi-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + is-latest-version: ${{ needs.compute-version.outputs.latest == '1' }} + staging: false + secrets: inherit server-docker-upload: - needs: [pypi-upload] + needs: [compute-version, pypi-upload] defaults: run: working-directory: docker/server @@ -276,12 +97,12 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Build and upload to DockerHub run: | - VERSION=${GITHUB_REF#refs/tags/} + VERSION=${{ needs.compute-version.outputs.version }} docker buildx build --platform linux/arm64/v8 --build-arg VERSION=$VERSION --push --provenance=false --tag dstackai/dstack:$VERSION-arm64 -f release/Dockerfile . docker buildx build --platform linux/amd64 --build-arg VERSION=$VERSION --push --provenance=false --tag dstackai/dstack:$VERSION-amd64 -f release/Dockerfile . docker manifest create dstackai/dstack:$VERSION --amend dstackai/dstack:$VERSION-arm64 --amend dstackai/dstack:$VERSION-amd64 docker manifest push dstackai/dstack:$VERSION - if [ -n "${{ needs.pypi-upload.outputs.LATEST }}" ]; then + if [ -n "${{ needs.compute-version.outputs.latest }}" ]; then docker manifest create dstackai/dstack:latest --amend dstackai/dstack:$VERSION-arm64 --amend dstackai/dstack:$VERSION-amd64 docker manifest push dstackai/dstack:latest fi @@ -292,32 +113,3 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} repository: dstackai/dstack readme-filepath: ./docker/server/README.md - - generate-json-schema: - needs: [pypi-upload] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Install dstack - run: uv sync - - name: Generate json schema - run: | - uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > configuration.json - uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json())" > profiles.json - - name: Upload json schema to S3 - run: | - VERSION=${GITHUB_REF#refs/tags/} - aws s3 cp configuration.json "s3://dstack-runner-downloads/$VERSION/schemas/configuration.json" --acl public-read - aws s3 cp profiles.json "s3://dstack-runner-downloads/$VERSION/schemas/profiles.json" --acl public-read - if [ -n "${{ needs.pypi-upload.outputs.LATEST }}" ]; then - aws s3 cp configuration.json "s3://dstack-runner-downloads/latest/schemas/configuration.json" --acl public-read - aws s3 cp profiles.json "s3://dstack-runner-downloads/latest/schemas/profiles.json" --acl public-read - fi diff --git a/.github/workflows/upload-post-pypi-artifacts.yml b/.github/workflows/upload-post-pypi-artifacts.yml new file mode 100644 index 0000000000..4d9899deda --- /dev/null +++ b/.github/workflows/upload-post-pypi-artifacts.yml @@ -0,0 +1,55 @@ +name: Upload Post-PyPI Artifacts + +on: + workflow_call: + inputs: + version: + type: string + required: true + is-latest-version: + type: boolean + required: true + staging: + type: boolean + required: true + +jobs: + upload-post-pypi-artifacts: + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + # settings to prevent warnings when running uv without a repo checkout + enable-cache: false + ignore-empty-workdir: true + - name: Install AWS + run: uv tool install awscli + - name: Download JSON schemas + uses: actions/download-artifact@v4 + with: + name: json-schemas + path: json-schemas + - name: Upload JSON schemas to S3 + working-directory: json-schemas + run: | + BUCKET=dstack-runner-downloads + if [ "${{ inputs.staging }}" = "true" ]; then + BUCKET=dstack-runner-downloads-stgn + fi + aws s3 cp configuration.json "s3://$BUCKET/${{ inputs.version }}/schemas/configuration.json" --acl public-read + aws s3 cp profiles.json "s3://$BUCKET/${{ inputs.version }}/schemas/profiles.json" --acl public-read + if [ "${{ inputs.is-latest-version }}" = "true" ]; then + aws s3 cp configuration.json "s3://$BUCKET/latest/schemas/configuration.json" --acl public-read + aws s3 cp profiles.json "s3://$BUCKET/latest/schemas/profiles.json" --acl public-read + fi + - name: Set latest version in S3 + run: | + CHANNEL=cli + if [ "${{ inputs.staging }}" = "true" ]; then + CHANNEL=stgn-cli + fi + echo ${{ inputs.version }} | aws s3 cp - s3://get-dstack/$CHANNEL/latest-version --acl public-read diff --git a/.github/workflows/upload-pre-pypi-artifacts.yml b/.github/workflows/upload-pre-pypi-artifacts.yml new file mode 100644 index 0000000000..542f445e2e --- /dev/null +++ b/.github/workflows/upload-pre-pypi-artifacts.yml @@ -0,0 +1,57 @@ +name: Upload Pre-PyPI Artifacts + +on: + workflow_call: + inputs: + version: + type: string + required: true + staging: + type: boolean + required: true + +jobs: + upload-pre-pypi-artifacts: + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + # settings to prevent warnings when running uv without a repo checkout + enable-cache: false + ignore-empty-workdir: true + - name: Install AWS + run: uv tool install awscli + - name: Download dstack-gateway + uses: actions/download-artifact@v4 + with: + name: dstack-gateway + path: gateway + - name: Upload dstack-gateway to S3 + working-directory: gateway + run: | + CHANNEL=release + if [ "${{ inputs.staging }}" = "true" ]; then + CHANNEL=stgn + fi + WHEEL=dstack_gateway-${{ inputs.version }}-py3-none-any.whl + aws s3 cp $WHEEL "s3://dstack-gateway-downloads/$CHANNEL/$WHEEL" + echo "${{ inputs.version }}" | aws s3 cp - "s3://dstack-gateway-downloads/$CHANNEL/latest-version" + - name: Download dstack-runner + uses: actions/download-artifact@v4 + with: + pattern: dstack-runner-* + merge-multiple: true + path: runner + - name: Upload dstack-runner to S3 + working-directory: runner + run: | + BUCKET=dstack-runner-downloads + if [ "${{ inputs.staging }}" = "true" ]; then + BUCKET=dstack-runner-downloads-stgn + fi + aws s3 cp . "s3://$BUCKET/${{ inputs.version }}/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read + aws s3 cp . "s3://$BUCKET/latest/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9af26f3354..22f7dd3cb8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: ['--fix'] - id: ruff-format - repo: https://github.com/golangci/golangci-lint - rev: v1.62.0 # Should match .github/workflows/build.yml + rev: v1.62.0 # Should match .github/workflows/build-artifacts.yml hooks: - id: golangci-lint-full language_version: 1.23.8 # Should match runner/go.mod diff --git a/src/dstack/version.py b/src/dstack/version.py index cc3efc230a..b6ed6c9133 100644 --- a/src/dstack/version.py +++ b/src/dstack/version.py @@ -1,7 +1,7 @@ # WARNING: # This file is overwritten when building the dstack package in CI. -# If you are adding, removing, or renaming variables, -# remember to update and test the release.yml workflow. +# If you are making any changes, +# remember to update and test the build-artifacts.yml workflow. __version__ = "0.0.0" __is_release__ = False From 6415c6aca00472b4e1c30e7095684b033315a3f8 Mon Sep 17 00:00:00 2001 From: Jvst Me Date: Wed, 20 Aug 2025 12:30:57 +0200 Subject: [PATCH 2/5] Fix release artifacts build parameter --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 08cbf90e08..078bf451d2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: uses: ./.github/workflows/build-artifacts.yml with: version: ${{ needs.compute-version.outputs.version }} - staging: true + staging: false go-integration-tests: true upload-pre-pypi-artifacts: From 7c88a0d75389b391a261a12c603af7ac51390468 Mon Sep 17 00:00:00 2001 From: Jvst Me Date: Wed, 20 Aug 2025 14:45:09 +0200 Subject: [PATCH 3/5] Fix build-docs parameter name --- .github/workflows/build-docs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 572d9c6d3e..9789440550 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -3,7 +3,7 @@ name: Build Docs on: workflow_call: inputs: - release_tag: + release-tag: type: string required: false @@ -18,8 +18,8 @@ jobs: - name: Install dstack run: | uv pip install examples/plugins/example_plugin_server - if [ -n "${{ inputs.release_tag }}" ]; then - uv pip install "dstack[server]==${{ inputs.release_tag }}" + if [ -n "${{ inputs.release-tag }}" ]; then + uv pip install "dstack[server]==${{ inputs.release-tag }}" else uv pip install -e '.[server]' fi From 3ea844d969f9536c878b1a48dd84a8a16dc5106b Mon Sep 17 00:00:00 2001 From: Jvst Me Date: Wed, 20 Aug 2025 14:53:46 +0200 Subject: [PATCH 4/5] Fix site directory in docs.yaml --- .github/workflows/docs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index c9c44cc103..a3ad89154e 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -20,6 +20,7 @@ jobs: - uses: actions/download-artifact@v4 with: name: site + path: site - name: Deploy uses: JamesIves/github-pages-deploy-action@v4.6.4 with: From d434c063a7194f670e33ae1007b0f818fbca2c3f Mon Sep 17 00:00:00 2001 From: Jvst Me Date: Wed, 20 Aug 2025 15:02:10 +0200 Subject: [PATCH 5/5] Checkout repo before deploying docs --- .github/workflows/docs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a3ad89154e..eedfa5ee4d 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -17,6 +17,7 @@ jobs: needs: [build-docs] runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: name: site