[MNT] Dockerized tests for CI runs using localhost #2179
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Tests | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| - develop | |
| tags: | |
| - "v*.*.*" | |
| pull_request: | |
| branches: | |
| - main | |
| - develop | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test: | |
| name: (${{ matrix.os }},Py${{ matrix.python-version }},sk${{ matrix.scikit-learn }}${{ matrix.pandas-version != '' && format(',pd:{0}', matrix.pandas-version) || '' }},sk-only:${{ matrix.sklearn-only }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] | |
| scikit-learn: ["1.3.*", "1.4.*", "1.5.*", "1.6.*", "1.7.*"] | |
| os: [ubuntu-latest] | |
| sklearn-only: ["true"] | |
| exclude: | |
| # incompatible version combinations | |
| - python-version: "3.13" | |
| scikit-learn: "1.3.*" | |
| - python-version: "3.13" | |
| scikit-learn: "1.4.*" | |
| - python-version: "3.14" | |
| scikit-learn: "1.3.*" | |
| - python-version: "3.14" | |
| scikit-learn: "1.4.*" | |
| include: | |
| # Full test run on ubuntu, 3.14 | |
| - os: ubuntu-latest | |
| python-version: "3.14" | |
| scikit-learn: "1.7.*" | |
| sklearn-only: "false" | |
| # Full test run on Windows | |
| - os: windows-latest | |
| python-version: "3.12" | |
| scikit-learn: "1.5.*" | |
| sklearn-only: "false" | |
| # Coverage run | |
| - os: ubuntu-latest | |
| python-version: "3.12" | |
| scikit-learn: "1.5.*" | |
| sklearn-only: "false" | |
| code-cov: true | |
| # Pandas 2 run | |
| - os: ubuntu-latest | |
| python-version: "3.12" | |
| scikit-learn: "1.5.*" | |
| sklearn-only: "false" | |
| pandas-version: "2.*" | |
| code-cov: false | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 2 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install test dependencies, scikit-learn, and optional pandas | |
| shell: bash | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e .[test] scikit-learn==${{ matrix.scikit-learn }} | |
| if [ "${{ matrix.pandas-version }}" != "" ]; then | |
| echo "Installing specific pandas version: ${{ matrix.pandas-version }}" | |
| pip install "pandas==${{ matrix.pandas-version }}" | |
| fi | |
| - name: Store repository status | |
| id: status-before | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| git_status=$(git status --porcelain -b) | |
| echo "BEFORE=$git_status" >> $GITHUB_ENV | |
| echo "Repository status before tests: $git_status" | |
| - name: Clone Services | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| git clone --depth 1 https://github.com/openml/services.git | |
| cd services | |
| git config user.email "ci@openml.org" | |
| git config user.name "CI" | |
| git fetch origin pull/13/head:pr-13 | |
| git merge pr-13 --no-edit | |
| git fetch origin pull/15/head:pr-15 | |
| git merge pr-15 --no-edit | |
| sed -i 's/localhost:8000/172.28.0.2:8000/g' config/database/update.sh | |
| # Verify the change | |
| grep "172.28.0.2" config/database/update.sh || echo "Patch failed!" | |
| - name: Start Docker Services | |
| if: matrix.os == 'ubuntu-latest' | |
| working-directory: ./services | |
| run: | | |
| sudo systemctl stop mysql.service | |
| docker compose --profile rest-api --profile minio --profile evaluation-engine up -d --build | |
| echo "1. Waiting for Database population..." | |
| docker wait openml-test-database-setup | |
| echo "2. Waiting for Elasticsearch (this is the slow part)..." | |
| # Wait up to 5 minutes for ES to go green | |
| timeout 300s bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} openml-elasticsearch)" == "healthy" ]; do sleep 5; done' | |
| echo "3. Waiting for PHP API..." | |
| # Wait up to 5 minutes for PHP to accept connections | |
| timeout 300s bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} openml-php-rest-api)" == "healthy" ]; do sleep 5; done' | |
| echo "4. Docker Stack is Healthy!" | |
| docker ps | |
| - name: Error | |
| working-directory: ./services | |
| run: | | |
| echo "---------------------------------------------------" | |
| echo "1. PROBING: Can we reach the API at all?" | |
| timeout 60s bash -c 'until curl -sSf http://localhost:8000/api/v1/xml/data/1 > /dev/null; do sleep 5; done' || echo "WARNING: Main API is slow/down" | |
| echo "---------------------------------------------------" | |
| echo "2. PROBING: Waiting for Task 119 Splits (The Failure Point)..." | |
| # We wait 60s. If it works, great. If not, we want the logs. | |
| timeout 60s bash -c 'until curl -sSf http://localhost:8000/api_splits/get/119/Task_119_splits.arff > /dev/null; do | |
| echo " ... file not ready yet" | |
| sleep 5 | |
| done' || echo "FAILURE: Task 119 splits were NOT generated." | |
| echo "---------------------------------------------------" | |
| echo "3.DUMPING EVALUATION ENGINE LOGS (STDOUT)" | |
| docker logs openml-evaluation-engine | |
| echo "---------------------------------------------------" | |
| echo "4.DUMPING INTERNAL CRON LOGS (The Hidden Logs)" | |
| # The engine runs via cron, so the real errors are often in this file, NOT in docker logs | |
| docker exec openml-evaluation-engine cat /cron.log || echo "Could not read /cron.log" | |
| echo "---------------------------------------------------" | |
| echo "5.DUMPING PHP API LOGS (Why did it throw 412?)" | |
| docker logs openml-php-rest-api | grep "412" -B 5 -A 5 || echo "No 412 errors found in logs?" | |
| echo "---------------------------------------------------" | |
| echo "6.CHECKING NETWORK (Can the container see Nginx?)" | |
| # This checks if the container can actually resolve 'localhost' to the host machine | |
| docker exec openml-evaluation-engine curl -v http://localhost:8000/api/v1/xml/data/1 || echo "Container cannot connect to localhost:8000" | |
| # Force fail so you see the red X and check logs | |
| exit 1 | |
| - name: Verify API and Splits | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| echo "Checking Data API..." | |
| timeout 60s bash -c 'until [ "$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/api/v1/xml/data/1)" == "200" ]; do sleep 5; done' | |
| echo "Checking Task Splits (The 412 Killer)..." | |
| # If this fails, the evaluation engine is broken | |
| timeout 120s bash -c 'until [ "$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/api_splits/get/1/Task_1_splits.arff)" == "200" ]; do | |
| echo "Splits not ready yet. Waiting..." | |
| sleep 5 | |
| done' | |
| echo "System is fully operational." | |
| - name: Show installed dependencies | |
| run: python -m pip list | |
| - name: Run tests on Ubuntu Test | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| if [ "${{ matrix.code-cov }}" = "true" ]; then | |
| codecov="--cov=openml --long --cov-report=xml" | |
| fi | |
| if [ "${{ matrix.sklearn-only }}" = "true" ]; then | |
| marks="sklearn and not production" | |
| else | |
| marks="not production" | |
| fi | |
| pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" | |
| - name: Run tests on Ubuntu Production | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| if [ "${{ matrix.code-cov }}" = "true" ]; then | |
| codecov="--cov=openml --long --cov-report=xml" | |
| fi | |
| if [ "${{ matrix.sklearn-only }}" = "true" ]; then | |
| marks="sklearn and production" | |
| else | |
| marks="production" | |
| fi | |
| pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" | |
| - name: Run tests on Windows | |
| if: matrix.os == 'windows-latest' | |
| run: | # we need a separate step because of the bash-specific if-statement in the previous one. | |
| pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not uses_test_server" | |
| - name: Upload coverage | |
| if: matrix.code-cov && always() | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: coverage.xml | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: true | |
| verbose: true | |
| - name: Cleanup Docker setup | |
| if: matrix.os == 'ubuntu-latest' && always() | |
| run: | | |
| sudo rm -rf services | |
| - name: Check for files left behind by test | |
| if: matrix.os != 'windows-latest' && always() | |
| run: | | |
| before="${{ env.BEFORE }}" | |
| after="$(git status --porcelain -b)" | |
| if [[ "$before" != "$after" ]]; then | |
| echo "git status from before: $before" | |
| echo "git status from after: $after" | |
| echo "Not all generated files have been deleted!" | |
| exit 1 | |
| fi | |
| dummy_windows_py_sk024: | |
| name: (windows-latest, Py, sk0.24.*, sk-only:false) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Dummy step | |
| run: | | |
| echo "This is a temporary dummy job." | |
| echo "Always succeeds." | |
| dummy_windows_py_sk023: | |
| name: (ubuntu-latest, Py3.8, sk0.23.1, sk-only:false) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Dummy step | |
| run: | | |
| echo "This is a temporary dummy job." | |
| echo "Always succeeds." | |
| dummy_docker: | |
| name: docker | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Dummy step | |
| run: | | |
| echo "This is a temporary dummy docker job." | |
| echo "Always succeeds." |