Skip to content

Commit e32b8e5

Browse files
jvstmejvstme
committed
CI refactoring WIP
1 parent 3e54439 commit e32b8e5

File tree

5 files changed

+368
-457
lines changed

5 files changed

+368
-457
lines changed

‎.github/workflows/build-artifacts.yml‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
name: Build Artifacts
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
version:
7+
type: string
8+
required: true
9+
staging:
10+
type: boolean
11+
required: true
12+
default: true
13+
go-integration-tests:
14+
type: boolean
15+
required: true
16+
default: true
17+
18+
env:
19+
BUILD_INCREMENT: 150
20+
PIP_DISABLE_PIP_VERSION_CHECK: on
21+
PIP_DEFAULT_TIMEOUT: 10
22+
PIP_PROGRESS_BAR: off
23+
24+
jobs:
25+
code-lint:
26+
runs-on: ubuntu-latest
27+
steps:
28+
- uses: actions/checkout@v4
29+
- name: Set up uv
30+
uses: astral-sh/setup-uv@v5
31+
with:
32+
python-version: 3.11
33+
- run: uv tool install pre-commit
34+
- run: pre-commit run -a --show-diff-on-failure
35+
36+
frontend-build:
37+
runs-on: ubuntu-latest
38+
defaults:
39+
run:
40+
working-directory: frontend
41+
steps:
42+
- uses: actions/checkout@v4
43+
- name: Restore cached build
44+
id: cache-build
45+
uses: actions/cache@v4
46+
with:
47+
path: frontend/build
48+
key: frontend-build-${{ hashFiles('frontend/**') }}
49+
restore-keys: |
50+
frontend-build-
51+
- name: Set up Node
52+
if: steps.cache-build.outputs.cache-hit != 'true'
53+
uses: actions/setup-node@v4
54+
with:
55+
node-version: 18
56+
- name: Install packages
57+
if: steps.cache-build.outputs.cache-hit != 'true'
58+
run: npm ci
59+
- name: Build dist
60+
if: steps.cache-build.outputs.cache-hit != 'true'
61+
run: npm run build
62+
- name: Upload dist
63+
uses: actions/upload-artifact@v4
64+
with:
65+
name: frontend-build
66+
path: frontend/build
67+
68+
python-test:
69+
needs: [code-lint, frontend-build]
70+
runs-on: ${{ matrix.os }}
71+
strategy:
72+
matrix:
73+
os: [macos-latest, ubuntu-latest, windows-latest]
74+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
75+
steps:
76+
- uses: actions/checkout@v4
77+
- name: Set up Python ${{ matrix.python-version }}
78+
uses: astral-sh/setup-uv@v5
79+
with:
80+
python-version: ${{ matrix.python-version }}
81+
- name: Install dependencies
82+
run: uv sync --all-extras
83+
- name: Download frontend build
84+
uses: actions/download-artifact@v4
85+
with:
86+
name: frontend-build
87+
path: src/dstack/_internal/server/statics
88+
# - name: Run pytest on POSIX
89+
# if: matrix.os != 'windows-latest'
90+
# # Skip Postgres tests on macos since macos runner doesn't have Docker.
91+
# run: |
92+
# RUNPOSTGRES=""
93+
# if [ "${{ matrix.os }}" != "macos-latest" ]; then
94+
# RUNPOSTGRES="--runpostgres"
95+
# fi
96+
# uv run pytest -n auto src/tests --runui $RUNPOSTGRES
97+
# - name: Run pytest on Windows
98+
# if: matrix.os == 'windows-latest'
99+
# run: |
100+
# uv run pytest -n auto src/tests --runui --runpostgres
101+
102+
runner-test:
103+
defaults:
104+
run:
105+
working-directory: runner
106+
runs-on: ${{ matrix.os }}
107+
strategy:
108+
matrix:
109+
os: [ubuntu-latest, macos-latest]
110+
steps:
111+
- uses: actions/checkout@v4
112+
- name: Set up Go
113+
uses: actions/setup-go@v5
114+
with:
115+
go-version-file: runner/go.mod
116+
cache-dependency-path: runner/go.sum
117+
- name: Check if go.mod and go.sum are up-to-date
118+
run: go mod tidy -diff
119+
- name: Run golangci-lint
120+
uses: golangci/golangci-lint-action@v6
121+
with:
122+
version: v1.62.0 # Should match .pre-commit-config.yaml
123+
args: --timeout=20m
124+
working-directory: runner
125+
- name: Test
126+
# Only run slow tests if requested by workflow call inputs.
127+
run: |
128+
SHORT="-short"
129+
if [[ "${{ github.event_name }}" == "workflow_call" ]]; then
130+
if [[ "${{ github.event.inputs.go-integration-tests }}" == "true" ]]; then
131+
SHORT=""
132+
fi
133+
fi
134+
go version
135+
go fmt $(go list ./... | grep -v /vendor/)
136+
go vet $(go list ./... | grep -v /vendor/)
137+
go test $SHORT -race $(go list ./... | grep -v /vendor/)
138+
139+
runner-compile:
140+
needs: [runner-test]
141+
defaults:
142+
run:
143+
working-directory: runner
144+
env:
145+
REPO_NAME: github.com/dstackai/dstack
146+
strategy:
147+
matrix:
148+
include:
149+
- { runs-on: "ubuntu-24.04", goos: "linux", goarch: "amd64" }
150+
- { runs-on: "ubuntu-24.04-arm", goos: "linux", goarch: "arm64" }
151+
runs-on: ${{ matrix.runs-on }}
152+
steps:
153+
- uses: actions/checkout@v4
154+
- name: Set up Go
155+
uses: actions/setup-go@v5
156+
with:
157+
go-version-file: runner/go.mod
158+
cache-dependency-path: runner/go.sum
159+
- name: build
160+
env:
161+
GOOS: ${{ matrix.goos }}
162+
GOARCH: ${{ matrix.goarch }}
163+
run: |
164+
CGO_ENABLED=0 go build -ldflags "-X 'main.Version=${{ inputs.version }}' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner
165+
CGO_ENABLED=1 go build -ldflags "-X 'main.Version=${{ inputs.version }}'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim
166+
- uses: actions/upload-artifact@v4
167+
with:
168+
name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
169+
path: |
170+
runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
171+
runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }}
172+
retention-days: 1
173+
174+
gateway-build:
175+
runs-on: ubuntu-latest
176+
steps:
177+
- uses: actions/checkout@v4
178+
- name: Set up uv
179+
uses: astral-sh/setup-uv@v5
180+
with:
181+
python-version: 3.11
182+
- name: Build package
183+
working-directory: gateway
184+
run: |
185+
echo "__version__ = \"${{ inputs.version }}\"" > src/dstack/gateway/version.py
186+
# TODO: depend on a specific dstackai/dstack commit for staging builds?
187+
if [[ "${{ inputs.staging }}" == "false" ]]; then
188+
sed \
189+
-i.old \
190+
"s|@ git+https://github.com/dstackai/dstack.git@master|== ${{ inputs.version }}|" \
191+
pyproject.toml
192+
diff pyproject.toml pyproject.toml.old > /dev/null && echo "Could not set version" && exit 1
193+
fi
194+
uv build
195+
- uses: actions/upload-artifact@v4
196+
with:
197+
name: dstack-gateway
198+
path: gateway/dist/dstack_gateway-${{ inputs.version }}-py3-none-any.whl
199+
retention-days: 1
200+
201+
python-build:
202+
needs: [code-lint, frontend-build]
203+
runs-on: ubuntu-latest
204+
steps:
205+
- uses: actions/checkout@v4
206+
- name: Set up uv
207+
uses: astral-sh/setup-uv@v5
208+
with:
209+
python-version: 3.11
210+
- name: Download frontend build
211+
uses: actions/download-artifact@v4
212+
with:
213+
name: frontend-build
214+
path: src/dstack/_internal/server/statics
215+
- name: Build dstack Python package
216+
# TODO: set __version__ to inputs.version regardless of inputs.staging,
217+
# so that staging builds are also tied to a specific runner and gateway version.
218+
run: |
219+
if [[ "${{ inputs.staging }}" == "true" ]]; then
220+
VERSION=0.0.0
221+
IS_RELEASE=False
222+
else
223+
VERSION=${{ inputs.version }}
224+
IS_RELEASE=True
225+
fi
226+
BASE_IMAGE=$(cat src/dstack/version.py | grep "base_image = ")
227+
BASE_IMAGE_UBUNTU_VERSION=$(cat src/dstack/version.py | grep "base_image_ubuntu_version = ")
228+
echo "__version__ = \"$VERSION\"" > src/dstack/version.py
229+
echo "__is_release__ = $IS_RELEASE" >> src/dstack/version.py
230+
echo $BASE_IMAGE >> src/dstack/version.py
231+
echo $BASE_IMAGE_UBUNTU_VERSION >> src/dstack/version.py
232+
cp README.md src
233+
uv build
234+
- uses: actions/upload-artifact@v4
235+
with:
236+
name: python-build
237+
path: dist
238+
239+
generate-json-schema:
240+
needs: [code-lint]
241+
runs-on: ubuntu-latest
242+
steps:
243+
- uses: actions/checkout@v4
244+
- uses: astral-sh/setup-uv@v5
245+
with:
246+
python-version: 3.11
247+
- name: Install dstack
248+
run: uv sync
249+
- name: Generate json schema
250+
run: |
251+
mkdir /tmp/json-schemas
252+
uv run python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > /tmp/json-schemas/configuration.json
253+
uv run python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json())" > /tmp/json-schemas/profiles.json
254+
- uses: actions/upload-artifact@v4
255+
with:
256+
name: json-schemas
257+
path: /tmp/json-schemas

0 commit comments

Comments
 (0)