diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ab94de10..8886f58c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -41,15 +41,30 @@ jobs: uses: ./.github/workflows/ci.yml secrets: inherit - integration-tests: + collect-integration-tests: needs: [validate-release, run-ci] runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.collect.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - id: collect + run: | + echo "matrix=$(find tests/integration_tests -name 'test_*.py' -not -path '*/__pycache__/*' | sort | jq -R -s -c 'split("\n") | map(select(length > 0))')" >> $GITHUB_OUTPUT + + integration-tests: + needs: [validate-release, run-ci, collect-integration-tests] + runs-on: ubuntu-latest timeout-minutes: 30 environment: name: integration-tests permissions: contents: read id-token: write + strategy: + fail-fast: false + matrix: + test: ${{ fromJson(needs.collect-integration-tests.outputs.matrix) }} steps: - uses: actions/checkout@v4 - id: auth @@ -59,8 +74,9 @@ jobs: service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} - uses: ./.github/actions/setup-python-uv - run: uv sync --extra gcp - - name: Run ${{ matrix.package }} integration tests + - name: Run ${{ matrix.test }} env: + TEST_FILE: ${{ matrix.test }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} BFL_API_KEY: ${{ secrets.BFL_API_KEY }} @@ -75,17 +91,21 @@ jobs: MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }} DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} run: | - uv run pytest tests/integration_tests -m integration -v --dist=worksteal -n auto \ - --ignore=tests/integration_tests/images/test_stream_edit.py \ + uv run pytest "$TEST_FILE" \ + -m integration -v \ + --reruns 2 --reruns-delay 10 \ + "--deselect=tests/integration_tests/text/test_generate.py::test_vertex_generate[anthropic-claude-haiku-4-5-us-east5]" \ + "--deselect=tests/integration_tests/text/test_stream_generate.py::test_vertex_stream_generate[anthropic-claude-haiku-4-5-us-east5]" \ "--deselect=tests/integration_tests/audio/test_speak.py::test_speak[google-gemini-2.5-flash-tts]" \ "--deselect=tests/integration_tests/audio/test_speak.py::test_speak[google-gemini-2.5-pro-tts]" build: - needs: [validate-release, run-ci, integration-tests] + needs: [validate-release, run-ci, collect-integration-tests, integration-tests] if: | always() && needs.validate-release.result == 'success' && needs.run-ci.result == 'success' && + needs.collect-integration-tests.result == 'success' && (needs.integration-tests.result == 'success' || needs.integration-tests.result == 'skipped') runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index 113f5e82..b07321b8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help sync lint lint-fix format typecheck test integration-test security ci clean +.PHONY: help sync lint lint-fix format typecheck test integration-test integration-retest security ci clean # Default target help: @@ -8,8 +8,8 @@ help: @echo " make format - Apply Ruff formatting" @echo " make typecheck - Run mypy type checking" @echo " make test - Run all tests (core + packages) with coverage" - @echo " make integration-test [capability] - Run integration tests (all or specific)" - @echo " (e.g., make integration-test image-intelligence)" + @echo " make integration-test - Run all integration tests" + @echo " make integration-retest - Rerun only last-failed integration tests" @echo " make security - Run Bandit security scan" @echo " make ci - Run full CI/CD pipeline" @echo " make clean - Clean cache directories" @@ -40,11 +40,15 @@ typecheck: test: uv run pytest tests/unit_tests --cov=celeste --cov-report=term-missing --cov-fail-under=80 -v -# Integration testing (requires API keys) +# Integration testing (requires API keys and/or ADC credentials) # Runs tests from tests/integration_tests/ integration-test: uv run pytest tests/integration_tests/ -m integration -v --dist=worksteal -n auto +# Rerun only last-failed integration tests (without xdist — --lf is incompatible with -n) +integration-retest: + uv run pytest tests/integration_tests/ -m integration -v --lf + # Catch capability names as no-op targets %: @: diff --git a/pyproject.toml b/pyproject.toml index 7287df23..3dcc1c22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "celeste-ai" -version = "0.9.7" +version = "0.10.0" description = "Open source, type-safe primitives for multi-modal AI. All capabilities, all providers, one interface" authors = [{name = "Kamilbenkirane", email = "kamil@withceleste.ai"}] readme = "README.md" @@ -48,6 +48,7 @@ dev = [ "mypy>=1.13.0", "bandit[toml]>=1.7.5", "pre-commit>=3.5.0", + "pytest-rerunfailures>=16.1", ] [build-system]