diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 0000000000..ecb686bd66 --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,256 @@ +name: Build Artifacts + +on: + workflow_call: + inputs: + version: + type: string + required: true + staging: + type: boolean + required: true + go-integration-tests: + type: boolean + required: true + +jobs: + code-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + - run: uv tool install pre-commit + - run: pre-commit run -a --show-diff-on-failure + + frontend-build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + - name: Restore cached build + id: cache-build + uses: actions/cache@v4 + with: + path: frontend/build + key: frontend-build-${{ hashFiles('frontend/**') }} + restore-keys: | + frontend-build- + - name: Set up Node + if: steps.cache-build.outputs.cache-hit != 'true' + uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Install packages + if: steps.cache-build.outputs.cache-hit != 'true' + run: npm ci + - name: Build dist + if: steps.cache-build.outputs.cache-hit != 'true' + run: npm run build + - name: Upload dist + uses: actions/upload-artifact@v4 + with: + name: frontend-build + path: frontend/build + retention-days: 1 + + python-test: + needs: [code-lint, frontend-build] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: astral-sh/setup-uv@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: uv sync --all-extras + - name: Download frontend build + uses: actions/download-artifact@v4 + with: + name: frontend-build + path: src/dstack/_internal/server/statics + - name: Run pytest on POSIX + if: matrix.os != 'windows-latest' + # Skip Postgres tests on macos since macos runner doesn't have Docker. + run: | + RUNPOSTGRES="" + if [ "${{ matrix.os }}" != "macos-latest" ]; then + RUNPOSTGRES="--runpostgres" + fi + uv run pytest -n auto src/tests --runui $RUNPOSTGRES + - name: Run pytest on Windows + if: matrix.os == 'windows-latest' + run: | + uv run pytest -n auto src/tests --runui --runpostgres + + runner-test: + defaults: + run: + working-directory: runner + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: runner/go.mod + cache-dependency-path: runner/go.sum + - name: Check if go.mod and go.sum are up-to-date + run: go mod tidy -diff + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: v1.62.0 # Should match .pre-commit-config.yaml + args: --timeout=20m + working-directory: runner + - name: Test + # Only run slow tests if requested by workflow call inputs. + run: | + SHORT="-short" + if [[ "${{ inputs.go-integration-tests }}" == "true" ]]; then + SHORT="" + fi + # Skip failing integration tests on macOS for release builds. + # TODO: https://github.com/dstackai/dstack/issues/3005 + if [[ "${{ inputs.staging }}" == "false" && "${{ startsWith(matrix.os, 'macos') }}" == "true" ]]; then + SHORT="-short" + fi + go version + go fmt $(go list ./... | grep -v /vendor/) + go vet $(go list ./... | grep -v /vendor/) + go test $SHORT -race $(go list ./... | grep -v /vendor/) + + runner-compile: + needs: [runner-test] + defaults: + run: + working-directory: runner + env: + REPO_NAME: github.com/dstackai/dstack + strategy: + matrix: + include: + - { runs-on: "ubuntu-24.04", goos: "linux", goarch: "amd64" } + - { runs-on: "ubuntu-24.04-arm", goos: "linux", goarch: "arm64" } + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: runner/go.mod + cache-dependency-path: runner/go.sum + - name: build + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + run: | + CGO_ENABLED=0 go build -ldflags "-X 'main.Version=${{ inputs.version }}' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner + CGO_ENABLED=1 go build -ldflags "-X 'main.Version=${{ inputs.version }}'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim + - uses: actions/upload-artifact@v4 + with: + name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} + path: | + runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} + runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }} + retention-days: 1 + + gateway-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + - name: Build package + working-directory: gateway + run: | + echo "__version__ = \"${{ inputs.version }}\"" > src/dstack/gateway/version.py + # TODO: depend on a specific dstackai/dstack commit for staging builds? + if [[ "${{ inputs.staging }}" == "false" ]]; then + sed \ + -i.old \ + "s|@ git+https://github.com/dstackai/dstack.git@master|== ${{ inputs.version }}|" \ + pyproject.toml + diff pyproject.toml pyproject.toml.old > /dev/null && echo "Could not set version" && exit 1 + fi + uv build + - uses: actions/upload-artifact@v4 + with: + name: dstack-gateway + path: gateway/dist/dstack_gateway-${{ inputs.version }}-py3-none-any.whl + retention-days: 1 + + python-build: + needs: [code-lint, frontend-build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + - name: Download frontend build + uses: actions/download-artifact@v4 + with: + name: frontend-build + path: src/dstack/_internal/server/statics + - name: Build dstack Python package + # TODO: set __version__ to inputs.version regardless of inputs.staging, + # so that staging builds are also tied to a specific runner and gateway version. + # May require changing how dstack handles __version__. + run: | + if [[ "${{ inputs.staging }}" == "true" ]]; then + VERSION=0.0.0 + IS_RELEASE=False + else + VERSION=${{ inputs.version }} + IS_RELEASE=True + fi + BASE_IMAGE=$(cat src/dstack/version.py | grep "base_image = ") + BASE_IMAGE_UBUNTU_VERSION=$(cat src/dstack/version.py | grep "base_image_ubuntu_version = ") + echo "__version__ = \"$VERSION\"" > src/dstack/version.py + echo "__is_release__ = $IS_RELEASE" >> src/dstack/version.py + echo $BASE_IMAGE >> src/dstack/version.py + echo $BASE_IMAGE_UBUNTU_VERSION >> src/dstack/version.py + cp README.md src + uv build + - uses: actions/upload-artifact@v4 + with: + name: python-build + path: dist + retention-days: 1 + + generate-json-schema: + needs: [code-lint] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + - name: Install dstack + run: uv sync + - name: Generate json schema + run: | + mkdir /tmp/json-schemas + uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > /tmp/json-schemas/configuration.json + uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json())" > /tmp/json-schemas/profiles.json + - uses: actions/upload-artifact@v4 + with: + name: json-schemas + path: /tmp/json-schemas + retention-days: 1 diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml new file mode 100644 index 0000000000..9789440550 --- /dev/null +++ b/.github/workflows/build-docs.yml @@ -0,0 +1,37 @@ +name: Build Docs + +on: + workflow_call: + inputs: + release-tag: + type: string + required: false + +jobs: + build-docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + - name: Install dstack + run: | + uv pip install examples/plugins/example_plugin_server + if [ -n "${{ inputs.release-tag }}" ]; then + uv pip install "dstack[server]==${{ inputs.release-tag }}" + else + uv pip install -e '.[server]' + fi + - name: Build + run: | + uv pip install pillow cairosvg + sudo apt-get update && sudo apt-get install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev + uv pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade + uv pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git + uv run mkdocs build -s + - uses: actions/upload-artifact@v4 + with: + name: site + path: site + retention-days: 1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8a97c67c7d..dcd92e696f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build +name: Test Build on: push: @@ -12,290 +12,58 @@ on: - "master" workflow_dispatch: inputs: - intergation-tests: + go-integration-tests: type: boolean required: true default: false + description: Go integration tests env: BUILD_INCREMENT: 150 - PIP_DISABLE_PIP_VERSION_CHECK: on - PIP_DEFAULT_TIMEOUT: 10 - PIP_PROGRESS_BAR: off jobs: - code-lint: + compute-version: runs-on: ubuntu-latest + outputs: + version: ${{ steps.set-version.outputs.version }} steps: - - uses: actions/checkout@v4 - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - run: uv tool install pre-commit - - run: pre-commit run -a --show-diff-on-failure - - frontend-build: - runs-on: ubuntu-latest - defaults: - run: - working-directory: frontend - steps: - - uses: actions/checkout@v4 - - name: Restore cached build - id: cache-build - uses: actions/cache@v4 - with: - path: frontend/build - key: frontend-build-${{ hashFiles('frontend/**') }} - restore-keys: | - frontend-build- - - name: Set up Node - if: steps.cache-build.outputs.cache-hit != 'true' - uses: actions/setup-node@v4 - with: - node-version: 18 - - name: Install packages - if: steps.cache-build.outputs.cache-hit != 'true' - run: npm ci - - name: Build dist - if: steps.cache-build.outputs.cache-hit != 'true' - run: npm run build - - name: Upload dist - uses: actions/upload-artifact@v4 - with: - name: frontend-build - path: frontend/build - - python-test: - needs: [code-lint, frontend-build] - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: astral-sh/setup-uv@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: uv sync --all-extras - - name: Download frontend build - uses: actions/download-artifact@v4 - with: - name: frontend-build - path: src/dstack/_internal/server/statics - - name: Run pytest on POSIX - if: matrix.os != 'windows-latest' - # Skip Postgres tests on macos since macos runner doesn't have Docker. - run: | - RUNPOSTGRES="" - if [ "${{ matrix.os }}" != "macos-latest" ]; then - RUNPOSTGRES="--runpostgres" - fi - uv run pytest -n auto src/tests --runui $RUNPOSTGRES - - name: Run pytest on Windows - if: matrix.os == 'windows-latest' - run: | - uv run pytest -n auto src/tests --runui --runpostgres - - update-get-dstack: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - needs: [python-test] - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - steps: - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - run: | - VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) - echo $VERSION | aws s3 cp - s3://get-dstack/stgn-cli/latest-version --acl public-read - - runner-test: - defaults: - run: - working-directory: runner - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: runner/go.mod - cache-dependency-path: runner/go.sum - - name: Check if go.mod and go.sum are up-to-date - run: go mod tidy -diff - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6 - with: - version: v1.62.0 - args: --timeout=20m - working-directory: runner - - name: Test - # Do not run slow integration tests automatically. - # Slow tests can be run manually via workflow_dispatch when required. - run: | - SHORT="-short" - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - if [[ "${{ github.event.inputs.intergation-tests }}" == "true" ]]; then - SHORT="" - fi - fi - go version - go fmt $(go list ./... | grep -v /vendor/) - go vet $(go list ./... | grep -v /vendor/) - go test $SHORT -race $(go list ./... | grep -v /vendor/) - - runner-compile: - needs: [runner-test] - defaults: - run: - working-directory: runner - env: - REPO_NAME: github.com/dstackai/dstack - strategy: - matrix: - include: - - { runs-on: "ubuntu-24.04", goos: "linux", goarch: "amd64" } - - { runs-on: "ubuntu-24.04-arm", goos: "linux", goarch: "arm64" } - runs-on: ${{ matrix.runs-on }} - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: runner/go.mod - cache-dependency-path: runner/go.sum - - name: build - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - run: | - VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) - CGO_ENABLED=0 go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner - CGO_ENABLED=1 go build -ldflags "-X 'main.Version=$VERSION'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim - echo $VERSION - - uses: actions/upload-artifact@v4 - with: - name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} - path: | - runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} - runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }} - retention-days: 1 - - runner-upload: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - needs: [runner-compile] - runs-on: ubuntu-latest - steps: - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Download Runner - uses: actions/download-artifact@v4 - with: - pattern: dstack-runner-* - merge-multiple: true - path: runner - - name: Upload to S3 - working-directory: runner - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - id: set-version run: | - VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) - aws s3 cp . "s3://dstack-runner-downloads-stgn/$VERSION/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read - aws s3 cp . "s3://dstack-runner-downloads-stgn/latest/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read + echo "version=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }}))" >> $GITHUB_OUTPUT - generate-json-schema: + build-artifacts: + needs: [compute-version] + uses: ./.github/workflows/build-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + staging: true + # TODO: run integration tests on every 'push' event + # https://github.com/dstackai/dstack/issues/3005 + go-integration-tests: ${{ github.event_name == 'workflow_dispatch' && inputs.go-integration-tests }} + + upload-pre-pypi-artifacts: + needs: [compute-version, build-artifacts] + # Skip for PRs from forks, where AWS S3 credentials are not available if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - needs: [python-test] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Install dstack - run: uv sync - - name: Generate json schema - run: | - uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > configuration.json - uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json())" > profiles.json - - name: Upload json schema to S3 - run: | - VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) - aws s3 cp configuration.json "s3://dstack-runner-downloads-stgn/$VERSION/schemas/configuration.json" --acl public-read - aws s3 cp configuration.json "s3://dstack-runner-downloads-stgn/latest/schemas/configuration.json" --acl public-read - aws s3 cp profiles.json "s3://dstack-runner-downloads-stgn/$VERSION/schemas/profiles.json" --acl public-read - aws s3 cp profiles.json "s3://dstack-runner-downloads-stgn/latest/schemas/profiles.json" --acl public-read - - gateway-build: + uses: ./.github/workflows/upload-pre-pypi-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + staging: true + secrets: inherit + + upload-post-pypi-artifacts: + needs: [compute-version, build-artifacts] + # Skip for PRs from forks, where AWS S3 credentials are not available if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - defaults: - run: - working-directory: gateway - steps: - - uses: actions/checkout@v4 - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Compute version - run: echo VERSION=$((${{ github.run_number }} + ${{ env.BUILD_INCREMENT }})) > $GITHUB_ENV - - name: Build package - run: | - echo "__version__ = \"${{ env.VERSION }}\"" > src/dstack/gateway/version.py - uv build - - name: Upload to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - WHEEL=dstack_gateway-${{ env.VERSION }}-py3-none-any.whl - aws s3 cp dist/$WHEEL "s3://dstack-gateway-downloads/stgn/$WHEEL" - echo "${{ env.VERSION }}" | aws s3 cp - "s3://dstack-gateway-downloads/stgn/latest-version" + uses: ./.github/workflows/upload-post-pypi-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + is-latest-version: true + staging: true + secrets: inherit - docs-build: - # Skip for PRs from forks since mkdocs-material-insiders is not available in forks + build-docs: + # Skip for PRs from forks, where mkdocs-material-insiders is not available if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install dstack - run: | - uv pip install examples/plugins/example_plugin_server - uv pip install -e '.[server]' - # Move these deps into an extra and install that way - - name: Build - run: | - uv pip install pillow cairosvg - sudo apt-get update && sudo apt-get install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev - uv pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade - uv pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git - uv run mkdocs build -s + uses: ./.github/workflows/build-docs.yml + secrets: inherit diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index e1f43877bf..eedfa5ee4d 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,34 +1,27 @@ -name: Deploy Docs +name: Build & Deploy Docs on: workflow_dispatch: inputs: - release_tag: + release-tag: description: "dstack version" jobs: - docs-deploy: + build-docs: + uses: ./.github/workflows/build-docs.yml + with: + release-tag: ${{ inputs.release-tag }} + secrets: inherit + + deploy-docs: + needs: [build-docs] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 + - uses: actions/download-artifact@v4 with: - python-version: 3.11 - - name: Install dstack - run: | - uv pip install examples/plugins/example_plugin_server - if [ -n "${{ inputs.release_tag }}" ]; then - uv pip install "dstack[server]==${{ inputs.release_tag }}" - else - uv pip install -e '.[server]' - fi - - name: Build - run: | - uv pip install pillow cairosvg - sudo apt-get update && sudo apt-get install -y libcairo2-dev libfreetype6-dev libffi-dev libjpeg-dev libpng-dev libz-dev - uv pip install mkdocs-material "mkdocs-material[imaging]" mkdocs-material-extensions mkdocs-redirects mkdocs-gen-files "mkdocstrings[python]" mkdocs-render-swagger-plugin --upgrade - uv pip install git+https://${{ secrets.GH_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git - uv run mkdocs build -s + name: site + path: site - name: Deploy uses: JamesIves/github-pages-deploy-action@v4.6.4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1fe8b12b58..078bf451d2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,256 +8,77 @@ on: - "[0-9]+.[0-9]+.[0-9]+" - "[0-9]+.[0-9]+.[0-9]+.post[0-9]+" -env: - BUILD_INCREMENT: 150 - PIP_DISABLE_PIP_VERSION_CHECK: on - PIP_DEFAULT_TIMEOUT: 10 - PIP_PROGRESS_BAR: off - jobs: - code-lint: + compute-version: runs-on: ubuntu-latest + outputs: + version: ${{ steps.set-version.outputs.version }} + latest: ${{ steps.set-latest.outputs.latest }} steps: - - uses: actions/checkout@v4 - - name: Set up uv + - name: Set up Python uses: astral-sh/setup-uv@v5 with: python-version: 3.11 - - run: uv tool install pre-commit - - run: pre-commit run -a --show-diff-on-failure - - frontend-build: - runs-on: ubuntu-latest - defaults: - run: - working-directory: frontend - steps: - - uses: actions/checkout@v4 - - name: Set up Node - uses: actions/setup-node@v4 - with: - node-version: 18 - - name: Install packages - run: npm ci - - name: Build dist - run: npm run build - - name: Upload dist - uses: actions/upload-artifact@v4 - with: - name: frontend-build - path: frontend/build - - python-test: - needs: [code-lint, frontend-build] - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: astral-sh/setup-uv@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: uv sync --all-extras - - name: Download frontend build - uses: actions/download-artifact@v4 - with: - name: frontend-build - path: src/dstack/_internal/server/statics - - name: Run pytest on POSIX - if: matrix.os != 'windows-latest' - # Skip Postgres tests on macos since macos runner doesn't have Docker. + # settings to prevent warnings when running uv without a repo checkout + enable-cache: false + ignore-empty-workdir: true + - id: set-version run: | - RUNPOSTGRES="" - if [ "${{ matrix.os }}" != "macos-latest" ]; then - RUNPOSTGRES="--runpostgres" - fi - uv run pytest -n auto src/tests --runui $RUNPOSTGRES - - name: Run pytest on Windows - if: matrix.os == 'windows-latest' + echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + - id: set-latest run: | - uv run pytest -n auto src/tests --runui --runpostgres + uv pip install packaging + VERSION=${{ steps.set-version.outputs.version }} + LATEST=$(python -c "from packaging import version as pkg_version; print('' if pkg_version.parse('$VERSION').is_prerelease else '1', end='')") + echo "latest=$LATEST" >> "$GITHUB_OUTPUT" - runner-test: - defaults: - run: - working-directory: runner - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: runner/go.mod - cache-dependency-path: runner/go.sum - - name: Check if go.mod and go.sum are up-to-date - run: go mod tidy -diff - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6 - with: - version: v1.62.0 - args: --timeout=20m - working-directory: runner - - name: Test - run: | - go version - go fmt $(go list ./... | grep -v /vendor/) - go vet $(go list ./... | grep -v /vendor/) - go test -race $(go list ./... | grep -v /vendor/) + build-artifacts: + needs: [compute-version] + uses: ./.github/workflows/build-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + staging: false + go-integration-tests: true - runner-compile: - needs: [runner-test] - defaults: - run: - working-directory: runner - env: - REPO_NAME: github.com/dstackai/dstack - strategy: - matrix: - include: - - { runs-on: "ubuntu-24.04", goos: "linux", goarch: "amd64" } - - { runs-on: "ubuntu-24.04-arm", goos: "linux", goarch: "arm64" } - runs-on: ${{ matrix.runs-on }} - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: runner/go.mod - cache-dependency-path: runner/go.sum - - name: build - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - run: | - VERSION=${GITHUB_REF#refs/tags/} - CGO_ENABLED=0 go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner - CGO_ENABLED=1 go build -ldflags "-X 'main.Version=$VERSION'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim - - uses: actions/upload-artifact@v4 - with: - name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} - path: | - runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }} - runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }} - - gateway-build: - runs-on: ubuntu-latest - defaults: - run: - working-directory: gateway - steps: - - uses: actions/checkout@v4 - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Store version - run: echo VERSION=${GITHUB_REF#refs/tags/} > $GITHUB_ENV - - name: Build package - run: | - echo "__version__ = \"${{ env.VERSION }}\"" > src/dstack/gateway/version.py - sed \ - -i.old \ - "s|@ git+https://github.com/dstackai/dstack.git@master|== ${{ env.VERSION }}|" \ - pyproject.toml - diff pyproject.toml pyproject.toml.old > /dev/null && echo "Could not set version" && exit 1 - uv build - - name: Upload to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - WHEEL=dstack_gateway-${{ env.VERSION }}-py3-none-any.whl - aws s3 cp dist/$WHEEL "s3://dstack-gateway-downloads/release/$WHEEL" - echo "${{ env.VERSION }}" | aws s3 cp - "s3://dstack-gateway-downloads/release/latest-version" - - runner-upload: - needs: [runner-compile, gateway-build, python-test] - runs-on: ubuntu-latest - steps: - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Download Runner - uses: actions/download-artifact@v4 - with: - pattern: dstack-runner-* - merge-multiple: true - path: runner - - name: Upload to S3 - working-directory: runner - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - VERSION=${GITHUB_REF#refs/tags/} - aws s3 cp . "s3://dstack-runner-downloads/$VERSION/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read - aws s3 cp . "s3://dstack-runner-downloads/latest/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read + upload-pre-pypi-artifacts: + needs: [compute-version, build-artifacts] + uses: ./.github/workflows/upload-pre-pypi-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + staging: false + secrets: inherit pypi-upload: - needs: [python-test, runner-upload] + needs: [compute-version, upload-pre-pypi-artifacts] runs-on: ubuntu-latest - outputs: - LATEST: ${{ steps.set_latest.outputs.LATEST }} - name: Set latest variable steps: - - uses: actions/checkout@v4 - name: Set up uv uses: astral-sh/setup-uv@v5 with: python-version: 3.11 - - name: Download frontend build + # settings to prevent warnings when running uv without a repo checkout + enable-cache: false + ignore-empty-workdir: true + - name: Download Python package uses: actions/download-artifact@v4 with: - name: frontend-build - path: src/dstack/_internal/server/statics - - name: Set output - id: set_latest + name: python-build + path: dist + - name: Upload Python package to PyPI run: | - uv pip install packaging - VERSION=${GITHUB_REF#refs/tags/} - LATEST=$(python -c "from packaging import version as pkg_version; print('' if pkg_version.parse('$VERSION').is_prerelease else '1', end='')") - echo "LATEST=$LATEST" >> "$GITHUB_OUTPUT" - - name: Upload pip package - run: | - VERSION=${GITHUB_REF#refs/tags/} - BASE_IMAGE=$(cat src/dstack/version.py | grep "base_image = ") - BASE_IMAGE_UBUNTU_VERSION=$(cat src/dstack/version.py | grep "base_image_ubuntu_version = ") - echo "__version__ = \"$VERSION\"" > src/dstack/version.py - echo "__is_release__ = True" >> src/dstack/version.py - echo $BASE_IMAGE >> src/dstack/version.py - echo $BASE_IMAGE_UBUNTU_VERSION >> src/dstack/version.py - cp README.md src - uv build uv publish --username ${{ secrets.PYPI_USERNAME }} --password ${{ secrets.PYPI_PASSWORD }} - update-get-dstack-tag: - needs: [pypi-upload] - runs-on: ubuntu-latest - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - steps: - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - run: | - VERSION=${GITHUB_REF#refs/tags/} - echo $VERSION | aws s3 cp - s3://get-dstack/cli/latest-version --acl public-read + upload-post-pypi-artifacts: + needs: [compute-version, pypi-upload] + uses: ./.github/workflows/upload-post-pypi-artifacts.yml + with: + version: ${{ needs.compute-version.outputs.version }} + is-latest-version: ${{ needs.compute-version.outputs.latest == '1' }} + staging: false + secrets: inherit server-docker-upload: - needs: [pypi-upload] + needs: [compute-version, pypi-upload] defaults: run: working-directory: docker/server @@ -276,12 +97,12 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Build and upload to DockerHub run: | - VERSION=${GITHUB_REF#refs/tags/} + VERSION=${{ needs.compute-version.outputs.version }} docker buildx build --platform linux/arm64/v8 --build-arg VERSION=$VERSION --push --provenance=false --tag dstackai/dstack:$VERSION-arm64 -f release/Dockerfile . docker buildx build --platform linux/amd64 --build-arg VERSION=$VERSION --push --provenance=false --tag dstackai/dstack:$VERSION-amd64 -f release/Dockerfile . docker manifest create dstackai/dstack:$VERSION --amend dstackai/dstack:$VERSION-arm64 --amend dstackai/dstack:$VERSION-amd64 docker manifest push dstackai/dstack:$VERSION - if [ -n "${{ needs.pypi-upload.outputs.LATEST }}" ]; then + if [ -n "${{ needs.compute-version.outputs.latest }}" ]; then docker manifest create dstackai/dstack:latest --amend dstackai/dstack:$VERSION-arm64 --amend dstackai/dstack:$VERSION-amd64 docker manifest push dstackai/dstack:latest fi @@ -292,32 +113,3 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} repository: dstackai/dstack readme-filepath: ./docker/server/README.md - - generate-json-schema: - needs: [pypi-upload] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Install AWS - run: uv tool install awscli - - name: Install dstack - run: uv sync - - name: Generate json schema - run: | - uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > configuration.json - uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json())" > profiles.json - - name: Upload json schema to S3 - run: | - VERSION=${GITHUB_REF#refs/tags/} - aws s3 cp configuration.json "s3://dstack-runner-downloads/$VERSION/schemas/configuration.json" --acl public-read - aws s3 cp profiles.json "s3://dstack-runner-downloads/$VERSION/schemas/profiles.json" --acl public-read - if [ -n "${{ needs.pypi-upload.outputs.LATEST }}" ]; then - aws s3 cp configuration.json "s3://dstack-runner-downloads/latest/schemas/configuration.json" --acl public-read - aws s3 cp profiles.json "s3://dstack-runner-downloads/latest/schemas/profiles.json" --acl public-read - fi diff --git a/.github/workflows/upload-post-pypi-artifacts.yml b/.github/workflows/upload-post-pypi-artifacts.yml new file mode 100644 index 0000000000..4d9899deda --- /dev/null +++ b/.github/workflows/upload-post-pypi-artifacts.yml @@ -0,0 +1,55 @@ +name: Upload Post-PyPI Artifacts + +on: + workflow_call: + inputs: + version: + type: string + required: true + is-latest-version: + type: boolean + required: true + staging: + type: boolean + required: true + +jobs: + upload-post-pypi-artifacts: + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + # settings to prevent warnings when running uv without a repo checkout + enable-cache: false + ignore-empty-workdir: true + - name: Install AWS + run: uv tool install awscli + - name: Download JSON schemas + uses: actions/download-artifact@v4 + with: + name: json-schemas + path: json-schemas + - name: Upload JSON schemas to S3 + working-directory: json-schemas + run: | + BUCKET=dstack-runner-downloads + if [ "${{ inputs.staging }}" = "true" ]; then + BUCKET=dstack-runner-downloads-stgn + fi + aws s3 cp configuration.json "s3://$BUCKET/${{ inputs.version }}/schemas/configuration.json" --acl public-read + aws s3 cp profiles.json "s3://$BUCKET/${{ inputs.version }}/schemas/profiles.json" --acl public-read + if [ "${{ inputs.is-latest-version }}" = "true" ]; then + aws s3 cp configuration.json "s3://$BUCKET/latest/schemas/configuration.json" --acl public-read + aws s3 cp profiles.json "s3://$BUCKET/latest/schemas/profiles.json" --acl public-read + fi + - name: Set latest version in S3 + run: | + CHANNEL=cli + if [ "${{ inputs.staging }}" = "true" ]; then + CHANNEL=stgn-cli + fi + echo ${{ inputs.version }} | aws s3 cp - s3://get-dstack/$CHANNEL/latest-version --acl public-read diff --git a/.github/workflows/upload-pre-pypi-artifacts.yml b/.github/workflows/upload-pre-pypi-artifacts.yml new file mode 100644 index 0000000000..542f445e2e --- /dev/null +++ b/.github/workflows/upload-pre-pypi-artifacts.yml @@ -0,0 +1,57 @@ +name: Upload Pre-PyPI Artifacts + +on: + workflow_call: + inputs: + version: + type: string + required: true + staging: + type: boolean + required: true + +jobs: + upload-pre-pypi-artifacts: + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - uses: astral-sh/setup-uv@v5 + with: + python-version: 3.11 + # settings to prevent warnings when running uv without a repo checkout + enable-cache: false + ignore-empty-workdir: true + - name: Install AWS + run: uv tool install awscli + - name: Download dstack-gateway + uses: actions/download-artifact@v4 + with: + name: dstack-gateway + path: gateway + - name: Upload dstack-gateway to S3 + working-directory: gateway + run: | + CHANNEL=release + if [ "${{ inputs.staging }}" = "true" ]; then + CHANNEL=stgn + fi + WHEEL=dstack_gateway-${{ inputs.version }}-py3-none-any.whl + aws s3 cp $WHEEL "s3://dstack-gateway-downloads/$CHANNEL/$WHEEL" + echo "${{ inputs.version }}" | aws s3 cp - "s3://dstack-gateway-downloads/$CHANNEL/latest-version" + - name: Download dstack-runner + uses: actions/download-artifact@v4 + with: + pattern: dstack-runner-* + merge-multiple: true + path: runner + - name: Upload dstack-runner to S3 + working-directory: runner + run: | + BUCKET=dstack-runner-downloads + if [ "${{ inputs.staging }}" = "true" ]; then + BUCKET=dstack-runner-downloads-stgn + fi + aws s3 cp . "s3://$BUCKET/${{ inputs.version }}/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read + aws s3 cp . "s3://$BUCKET/latest/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9af26f3354..22f7dd3cb8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: ['--fix'] - id: ruff-format - repo: https://github.com/golangci/golangci-lint - rev: v1.62.0 # Should match .github/workflows/build.yml + rev: v1.62.0 # Should match .github/workflows/build-artifacts.yml hooks: - id: golangci-lint-full language_version: 1.23.8 # Should match runner/go.mod diff --git a/src/dstack/version.py b/src/dstack/version.py index cc3efc230a..b6ed6c9133 100644 --- a/src/dstack/version.py +++ b/src/dstack/version.py @@ -1,7 +1,7 @@ # WARNING: # This file is overwritten when building the dstack package in CI. -# If you are adding, removing, or renaming variables, -# remember to update and test the release.yml workflow. +# If you are making any changes, +# remember to update and test the build-artifacts.yml workflow. __version__ = "0.0.0" __is_release__ = False