From cc7ebe37e62d7b3820394a08a946b5a465225ac7 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 14 Apr 2026 02:00:39 +0000 Subject: [PATCH 01/11] wip --- .github/workflows/experiment.yaml | 71 +++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/experiment.yaml diff --git a/.github/workflows/experiment.yaml b/.github/workflows/experiment.yaml new file mode 100644 index 000000000000..25ebb10c0640 --- /dev/null +++ b/.github/workflows/experiment.yaml @@ -0,0 +1,71 @@ +name: Next-Gen CI Prototype + +on: + pull_request: + branches: [ main ] + +# 1. CONCURRENCY: Stop burning money on abandoned commits +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + # 2. DISCOVERY: Find exactly what changed and output JSON + discover: + runs-on: ubuntu-latest + outputs: + packages: ${{ steps.set-matrix.outputs.packages }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Needed to diff against main + + - name: Calculate Changed Packages + id: set-matrix + run: | + # Find all modified directories inside packages/ + CHANGED=$(git diff --name-only origin/main...HEAD | grep "^packages/" | cut -d/ -f2 | sort -u || true) + + if [ -z "$CHANGED" ]; then + echo "packages=[]" >> $GITHUB_OUTPUT + exit 0 + fi + + # Convert bash list to JSON array using jq + JSON_ARRAY=$(jq -R -s -c 'split("\n")[:-1]' <<< "$CHANGED") + echo "Discovered packages: $JSON_ARRAY" + echo "packages=$JSON_ARRAY" >> $GITHUB_OUTPUT + + # 3. EXECUTION: Native Fan-out Matrix + unit-test: + needs: discover + if: ${{ needs.discover.outputs.packages != '[]' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false # Don't kill the whole matrix if one package fails + matrix: + package: ${{ fromJSON(needs.discover.outputs.packages) }} + # Risk-Tiering: Smoke test on presubmit to save money + python: ["3.11"] + + steps: + - uses: actions/checkout@v4 + + # 4. THE ENGINE SWAP: Rust-based uv instead of setup-python + pip + - name: Install uv and Python + uses: astral-sh/setup-uv@v5 + with: + python-version: ${{ matrix.python }} + enable-cache: true + cache-dependency-glob: "packages/${{ matrix.package }}/setup.py" + + - name: Execute Tests (High-Density) + # uvx downloads and runs nox in milliseconds without a global pip install + run: | + cd packages/${{ matrix.package }} + + # Force nox to use uv as its backend for lightning-fast venv creation + export VIRTUALENV_CREATOR=uv + + echo "Running targeted tests for ${{ matrix.package }} on Python ${{ matrix.python }}" + uvx nox -s unit-${{ matrix.python }} \ No newline at end of file From f2a1f6072076a1ccece1a066ab32fca0c0a94a73 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 14 Apr 2026 02:02:41 +0000 Subject: [PATCH 02/11] test: dummy changes to verify CI fan-out --- .github/workflows/experiment.yaml | 9 +++++---- packages/google-cloud-kms/tests/unit/test_dummy.py | 1 + packages/google-cloud-storage/tests/unit/test_dummy.py | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 packages/google-cloud-kms/tests/unit/test_dummy.py create mode 100644 packages/google-cloud-storage/tests/unit/test_dummy.py diff --git a/.github/workflows/experiment.yaml b/.github/workflows/experiment.yaml index 25ebb10c0640..100ed6980fc6 100644 --- a/.github/workflows/experiment.yaml +++ b/.github/workflows/experiment.yaml @@ -60,12 +60,13 @@ jobs: cache-dependency-glob: "packages/${{ matrix.package }}/setup.py" - name: Execute Tests (High-Density) - # uvx downloads and runs nox in milliseconds without a global pip install run: | cd packages/${{ matrix.package }} - # Force nox to use uv as its backend for lightning-fast venv creation - export VIRTUALENV_CREATOR=uv + # Force Nox to natively use uv instead of the legacy virtualenv module + export NOX_DEFAULT_VENV_BACKEND=uv echo "Running targeted tests for ${{ matrix.package }} on Python ${{ matrix.python }}" - uvx nox -s unit-${{ matrix.python }} \ No newline at end of file + + # Use uvx to run nox (with the uv plugin injected to guarantee compatibility) + uvx --with 'nox[uv]' nox -s unit-${{ matrix.python }} \ No newline at end of file diff --git a/packages/google-cloud-kms/tests/unit/test_dummy.py b/packages/google-cloud-kms/tests/unit/test_dummy.py new file mode 100644 index 000000000000..2127e58f9386 --- /dev/null +++ b/packages/google-cloud-kms/tests/unit/test_dummy.py @@ -0,0 +1 @@ +# CI Prototype Verification diff --git a/packages/google-cloud-storage/tests/unit/test_dummy.py b/packages/google-cloud-storage/tests/unit/test_dummy.py new file mode 100644 index 000000000000..2127e58f9386 --- /dev/null +++ b/packages/google-cloud-storage/tests/unit/test_dummy.py @@ -0,0 +1 @@ +# CI Prototype Verification From 1c5332c10aee894fc279e656124045bae2ee27c2 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 14 Apr 2026 02:16:32 +0000 Subject: [PATCH 03/11] test: inject dummy unit tests across 15 packages to verify concurrent execution --- .../google-analytics-admin/tests/unit/test_scale_validation.py | 3 +++ .../google-cloud-automl/tests/unit/test_scale_validation.py | 3 +++ .../tests/unit/test_scale_validation.py | 3 +++ .../tests/unit/test_scale_validation.py | 3 +++ .../google-cloud-domains/tests/unit/test_scale_validation.py | 3 +++ .../tests/unit/test_scale_validation.py | 3 +++ .../tests/unit/test_scale_validation.py | 3 +++ packages/google-cloud-ndb/tests/unit/test_scale_validation.py | 3 +++ .../tests/unit/test_scale_validation.py | 3 +++ .../tests/unit/test_scale_validation.py | 3 +++ .../tests/unit/test_scale_validation.py | 3 +++ .../google-cloud-os-login/tests/unit/test_scale_validation.py | 3 +++ .../google-cloud-pubsub/tests/unit/test_scale_validation.py | 3 +++ .../tests/unit/test_scale_validation.py | 3 +++ .../tests/unit/test_scale_validation.py | 3 +++ 15 files changed, 45 insertions(+) create mode 100644 packages/google-analytics-admin/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-automl/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-cloudsecuritycompliance/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-discoveryengine/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-domains/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-edgenetwork/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-hypercomputecluster/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-ndb/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-network-management/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-network-security/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-org-policy/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-os-login/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-pubsub/tests/unit/test_scale_validation.py create mode 100644 packages/google-cloud-securesourcemanager/tests/unit/test_scale_validation.py create mode 100644 packages/googleapis-common-protos/tests/unit/test_scale_validation.py diff --git a/packages/google-analytics-admin/tests/unit/test_scale_validation.py b/packages/google-analytics-admin/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-analytics-admin/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-automl/tests/unit/test_scale_validation.py b/packages/google-cloud-automl/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-automl/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-cloudsecuritycompliance/tests/unit/test_scale_validation.py b/packages/google-cloud-cloudsecuritycompliance/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-cloudsecuritycompliance/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-discoveryengine/tests/unit/test_scale_validation.py b/packages/google-cloud-discoveryengine/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-discoveryengine/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-domains/tests/unit/test_scale_validation.py b/packages/google-cloud-domains/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-domains/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-edgenetwork/tests/unit/test_scale_validation.py b/packages/google-cloud-edgenetwork/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-edgenetwork/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-hypercomputecluster/tests/unit/test_scale_validation.py b/packages/google-cloud-hypercomputecluster/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-hypercomputecluster/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-ndb/tests/unit/test_scale_validation.py b/packages/google-cloud-ndb/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-ndb/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-network-management/tests/unit/test_scale_validation.py b/packages/google-cloud-network-management/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-network-management/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-network-security/tests/unit/test_scale_validation.py b/packages/google-cloud-network-security/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-network-security/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-org-policy/tests/unit/test_scale_validation.py b/packages/google-cloud-org-policy/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-org-policy/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-os-login/tests/unit/test_scale_validation.py b/packages/google-cloud-os-login/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-os-login/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-pubsub/tests/unit/test_scale_validation.py b/packages/google-cloud-pubsub/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-pubsub/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/google-cloud-securesourcemanager/tests/unit/test_scale_validation.py b/packages/google-cloud-securesourcemanager/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/google-cloud-securesourcemanager/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True diff --git a/packages/googleapis-common-protos/tests/unit/test_scale_validation.py b/packages/googleapis-common-protos/tests/unit/test_scale_validation.py new file mode 100644 index 000000000000..64bb4375d915 --- /dev/null +++ b/packages/googleapis-common-protos/tests/unit/test_scale_validation.py @@ -0,0 +1,3 @@ +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True From 075935c6a84e4e48fc02627f0c55868f65f0bae9 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 14 Apr 2026 02:37:04 +0000 Subject: [PATCH 04/11] update discover --- .github/workflows/experiment.yaml | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/.github/workflows/experiment.yaml b/.github/workflows/experiment.yaml index 100ed6980fc6..5925a3d915f9 100644 --- a/.github/workflows/experiment.yaml +++ b/.github/workflows/experiment.yaml @@ -10,31 +10,24 @@ concurrency: cancel-in-progress: true jobs: - # 2. DISCOVERY: Find exactly what changed and output JSON discover: runs-on: ubuntu-latest outputs: - packages: ${{ steps.set-matrix.outputs.packages }} + packages: ${{ steps.changes.outputs.all_changed_files }} steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 # Needed to diff against main - - - name: Calculate Changed Packages - id: set-matrix - run: | - # Find all modified directories inside packages/ - CHANGED=$(git diff --name-only origin/main...HEAD | grep "^packages/" | cut -d/ -f2 | sort -u || true) + fetch-depth: 0 - if [ -z "$CHANGED" ]; then - echo "packages=[]" >> $GITHUB_OUTPUT - exit 0 - fi - - # Convert bash list to JSON array using jq - JSON_ARRAY=$(jq -R -s -c 'split("\n")[:-1]' <<< "$CHANGED") - echo "Discovered packages: $JSON_ARRAY" - echo "packages=$JSON_ARRAY" >> $GITHUB_OUTPUT + - name: Detect Changed Packages + id: changes + uses: tj-actions/changed-files@v44 + with: + files: packages/** # Only watch the packages directory + dir_names: true # Return folder names, not files + dir_names_max_depth: 2 # Strip it down to just "google-cloud-storage" + json: true # Output a perfect JSON array for the matrix + escape_json: false # 3. EXECUTION: Native Fan-out Matrix unit-test: From f82dc1164037d417950484796e8624aa7a364534 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 14 Apr 2026 02:43:24 +0000 Subject: [PATCH 05/11] update infra --- .github/workflows/experiment.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/experiment.yaml b/.github/workflows/experiment.yaml index 5925a3d915f9..98220e78ddf8 100644 --- a/.github/workflows/experiment.yaml +++ b/.github/workflows/experiment.yaml @@ -24,9 +24,9 @@ jobs: uses: tj-actions/changed-files@v44 with: files: packages/** # Only watch the packages directory - dir_names: true # Return folder names, not files - dir_names_max_depth: 2 # Strip it down to just "google-cloud-storage" - json: true # Output a perfect JSON array for the matrix + dir_names: true # Return folder names, not files + dir_names_max_depth: 2 # Output format: "packages/google-cloud-storage" + json: true # Output a perfect JSON array for the matrix escape_json: false # 3. EXECUTION: Native Fan-out Matrix @@ -50,11 +50,13 @@ jobs: with: python-version: ${{ matrix.python }} enable-cache: true - cache-dependency-glob: "packages/${{ matrix.package }}/setup.py" + # Rely on the full path provided by tj-actions + cache-dependency-glob: "${{ matrix.package }}/setup.py" - name: Execute Tests (High-Density) run: | - cd packages/${{ matrix.package }} + # Step into the exact directory output by the matrix + cd ${{ matrix.package }} # Force Nox to natively use uv instead of the legacy virtualenv module export NOX_DEFAULT_VENV_BACKEND=uv From e371c1500f7250a076aba0cf5f2ca84232059c79 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 14 Apr 2026 03:05:32 +0000 Subject: [PATCH 06/11] add all jobs --- .github/workflows/experiment.yaml | 151 +++++++++++++++++++++++++----- 1 file changed, 127 insertions(+), 24 deletions(-) diff --git a/.github/workflows/experiment.yaml b/.github/workflows/experiment.yaml index 98220e78ddf8..1a5c4a2840b0 100644 --- a/.github/workflows/experiment.yaml +++ b/.github/workflows/experiment.yaml @@ -1,15 +1,21 @@ -name: Next-Gen CI Prototype +name: Next-Gen CI Pipeline on: pull_request: - branches: [ main ] + branches: [ main, preview ] + # Native Merge Queue support for O(1) batching + merge_group: + types: [checks_requested] -# 1. CONCURRENCY: Stop burning money on abandoned commits +# Stop burning money on abandoned iterative commits concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: + # ========================================== + # 1. DISCOVERY ENGINE (The Router) + # ========================================== discover: runs-on: ubuntu-latest outputs: @@ -23,45 +29,142 @@ jobs: id: changes uses: tj-actions/changed-files@v44 with: - files: packages/** # Only watch the packages directory - dir_names: true # Return folder names, not files - dir_names_max_depth: 2 # Output format: "packages/google-cloud-storage" - json: true # Output a perfect JSON array for the matrix + files: packages/** + dir_names: true + dir_names_max_depth: 2 + json: true escape_json: false - # 3. EXECUTION: Native Fan-out Matrix - unit-test: + # ========================================== + # 2. STATIC ANALYSIS (Grouped for Speed) + # ========================================== + static-checks: needs: discover if: ${{ needs.discover.outputs.packages != '[]' }} runs-on: ubuntu-latest strategy: - fail-fast: false # Don't kill the whole matrix if one package fails + fail-fast: false matrix: package: ${{ fromJSON(needs.discover.outputs.packages) }} - # Risk-Tiering: Smoke test on presubmit to save money - python: ["3.11"] - steps: - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.14" + enable-cache: true + cache-dependency-glob: "${{ matrix.package }}/setup.py" + + - name: Run Lint and MyPy + run: | + cd ${{ matrix.package }} + export NOX_DEFAULT_VENV_BACKEND=uv + # Chaining sessions executes them in a single fast VM + uvx --with 'nox[uv]' nox -s lint mypy lint_setup_py - # 4. THE ENGINE SWAP: Rust-based uv instead of setup-python + pip - - name: Install uv and Python - uses: astral-sh/setup-uv@v5 + # ========================================== + # 3. DOCUMENTATION BUILD + # ========================================== + docs-build: + needs: discover + if: ${{ needs.discover.outputs.packages != '[]' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.discover.outputs.packages) }} + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.10" + enable-cache: true + cache-dependency-glob: "${{ matrix.package }}/setup.py" + + - name: Build Docs and DocFX + run: | + cd ${{ matrix.package }} + export NOX_DEFAULT_VENV_BACKEND=uv + uvx --with 'nox[uv]' nox -s docs docfx + + # ========================================== + # 4. UNIT TESTS (The 2D Multiplier Matrix) + # ========================================== + unit-tests: + needs: discover + if: ${{ needs.discover.outputs.packages != '[]' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.discover.outputs.packages) }} + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 with: python-version: ${{ matrix.python }} enable-cache: true - # Rely on the full path provided by tj-actions cache-dependency-glob: "${{ matrix.package }}/setup.py" - - name: Execute Tests (High-Density) + - name: Execute Unit Tests run: | - # Step into the exact directory output by the matrix cd ${{ matrix.package }} - - # Force Nox to natively use uv instead of the legacy virtualenv module export NOX_DEFAULT_VENV_BACKEND=uv + uvx --with 'nox[uv]' nox -s unit-${{ matrix.python }} + + # ========================================== + # 5. SYSTEM TESTS + # ========================================== + system-tests: + needs: discover + if: ${{ needs.discover.outputs.packages != '[]' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.discover.outputs.packages) }} + python: ["3.11"] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: ${{ matrix.python }} + enable-cache: true + cache-dependency-glob: "${{ matrix.package }}/setup.py" + + - name: Execute System Tests + env: + RUN_SYSTEM_TESTS: "true" + run: | + cd ${{ matrix.package }} + export NOX_DEFAULT_VENV_BACKEND=uv + uvx --with 'nox[uv]' nox -s system + + # ========================================== + # 6. THE GATEKEEPER (Status Check Rollup) + # ========================================== + presubmit-passed: + # Always runs so GitHub can definitively mark the PR as passed/failed + if: always() + needs: + - discover + - static-checks + - docs-build + - unit-tests + - system-tests + runs-on: ubuntu-latest + steps: + - name: Evaluate Pipeline Status + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "::error::One or more required CI jobs failed or were cancelled." + exit 1 + fi - echo "Running targeted tests for ${{ matrix.package }} on Python ${{ matrix.python }}" + # If the router output was empty, it means no Python code changed. + if [[ "${{ needs.discover.outputs.packages }}" == "[]" ]]; then + echo "No Python packages changed. Safely bypassing execution." + exit 0 + fi - # Use uvx to run nox (with the uv plugin injected to guarantee compatibility) - uvx --with 'nox[uv]' nox -s unit-${{ matrix.python }} \ No newline at end of file + echo "All dynamically generated CI jobs completed successfully." \ No newline at end of file From 9f464d45a6aca5da4cb7ec09ff5004fd9ecc669c Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 14 Apr 2026 22:20:40 +0000 Subject: [PATCH 07/11] attempt retries --- .github/workflows/experiment.yaml | 42 ++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/.github/workflows/experiment.yaml b/.github/workflows/experiment.yaml index 1a5c4a2840b0..491e45bc486c 100644 --- a/.github/workflows/experiment.yaml +++ b/.github/workflows/experiment.yaml @@ -3,7 +3,7 @@ name: Next-Gen CI Pipeline on: pull_request: branches: [ main, preview ] - # Native Merge Queue support for O(1) batching + # Native Merge Queue support for exhaustive batching merge_group: types: [checks_requested] @@ -20,6 +20,8 @@ jobs: runs-on: ubuntu-latest outputs: packages: ${{ steps.changes.outputs.all_changed_files }} + # Expose the dynamic Python matrix to downstream jobs + python_versions: ${{ steps.set-python.outputs.matrix }} steps: - uses: actions/checkout@v4 with: @@ -35,6 +37,17 @@ jobs: json: true escape_json: false + - name: Determine Python Matrix (Risk-Tiering) + id: set-python + run: | + if [[ "${{ github.event_name }}" == "merge_group" ]]; then + echo "Merge Queue detected. Deploying exhaustive matrix." + echo 'matrix=["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]' >> $GITHUB_OUTPUT + else + echo "Pull Request detected. Deploying Min/Max Boundary matrix." + echo 'matrix=["3.9", "3.14"]' >> $GITHUB_OUTPUT + fi + # ========================================== # 2. STATIC ANALYSIS (Grouped for Speed) # ========================================== @@ -58,7 +71,6 @@ jobs: run: | cd ${{ matrix.package }} export NOX_DEFAULT_VENV_BACKEND=uv - # Chaining sessions executes them in a single fast VM uvx --with 'nox[uv]' nox -s lint mypy lint_setup_py # ========================================== @@ -87,7 +99,7 @@ jobs: uvx --with 'nox[uv]' nox -s docs docfx # ========================================== - # 4. UNIT TESTS (The 2D Multiplier Matrix) + # 4. UNIT TESTS (Dynamic 2D Matrix + Retries) # ========================================== unit-tests: needs: discover @@ -97,7 +109,8 @@ jobs: fail-fast: false matrix: package: ${{ fromJSON(needs.discover.outputs.packages) }} - python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + # Reads the array generated by the Discovery job + python: ${{ fromJSON(needs.discover.outputs.python_versions) }} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 @@ -106,11 +119,24 @@ jobs: enable-cache: true cache-dependency-glob: "${{ matrix.package }}/setup.py" - - name: Execute Unit Tests + - name: Execute Unit Tests (With Shock Absorbers) run: | cd ${{ matrix.package }} export NOX_DEFAULT_VENV_BACKEND=uv - uvx --with 'nox[uv]' nox -s unit-${{ matrix.python }} + + # 3-Attempt retry loop to mask legacy flaky tests + for i in 1 2 3; do + echo "Attempt $i of 3 for Python ${{ matrix.python }}..." + if uvx --with 'nox[uv]' nox -s unit-${{ matrix.python }}; then + echo "Tests passed successfully!" + exit 0 + fi + echo "Tests failed. Waiting 5 seconds before retrying..." + sleep 5 + done + + echo "::error::Tests failed after 3 attempts. This is a hard failure." + exit 1 # ========================================== # 5. SYSTEM TESTS @@ -144,7 +170,6 @@ jobs: # 6. THE GATEKEEPER (Status Check Rollup) # ========================================== presubmit-passed: - # Always runs so GitHub can definitively mark the PR as passed/failed if: always() needs: - discover @@ -161,10 +186,9 @@ jobs: exit 1 fi - # If the router output was empty, it means no Python code changed. if [[ "${{ needs.discover.outputs.packages }}" == "[]" ]]; then echo "No Python packages changed. Safely bypassing execution." exit 0 fi - echo "All dynamically generated CI jobs completed successfully." \ No newline at end of file + echo "All dynamically generated CI jobs completed successfully." From c7a7a71c1f59484d086c30398a2c8449dcc29183 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 14 Apr 2026 22:29:17 +0000 Subject: [PATCH 08/11] authenticate for system tests --- .github/workflows/experiment.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/experiment.yaml b/.github/workflows/experiment.yaml index 491e45bc486c..fca69607271c 100644 --- a/.github/workflows/experiment.yaml +++ b/.github/workflows/experiment.yaml @@ -138,6 +138,9 @@ jobs: echo "::error::Tests failed after 3 attempts. This is a hard failure." exit 1 + # ========================================== + # 5. SYSTEM TESTS + # ========================================== # ========================================== # 5. SYSTEM TESTS # ========================================== @@ -152,6 +155,13 @@ jobs: python: ["3.11"] steps: - uses: actions/checkout@v4 + - id: 'auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v2' + with: + workload_identity_provider: ${{ secrets.WIF_PROVIDER }} + service_account: ${{ secrets.WIF_SERVICE_ACCOUNT }} + - uses: astral-sh/setup-uv@v5 with: python-version: ${{ matrix.python }} @@ -161,10 +171,14 @@ jobs: - name: Execute System Tests env: RUN_SYSTEM_TESTS: "true" + # ADDED: Inject the required environment variables for the tests + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} # Update to match your legacy YAML's secret name run: | cd ${{ matrix.package }} export NOX_DEFAULT_VENV_BACKEND=uv - uvx --with 'nox[uv]' nox -s system + + # FIXED: Append the exact matrix version to stop Nox from running all versions sequentially + uvx --with 'nox[uv]' nox -s system-${{ matrix.python }} # ========================================== # 6. THE GATEKEEPER (Status Check Rollup) From 25c17e92cd569c58d32990c94f10c5adbce0e283 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 14 Apr 2026 22:37:44 +0000 Subject: [PATCH 09/11] add py version to system tests --- .github/workflows/experiment.yaml | 14 -------------- test.py | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 14 deletions(-) create mode 100755 test.py diff --git a/.github/workflows/experiment.yaml b/.github/workflows/experiment.yaml index fca69607271c..f1c77af19ba6 100644 --- a/.github/workflows/experiment.yaml +++ b/.github/workflows/experiment.yaml @@ -138,9 +138,6 @@ jobs: echo "::error::Tests failed after 3 attempts. This is a hard failure." exit 1 - # ========================================== - # 5. SYSTEM TESTS - # ========================================== # ========================================== # 5. SYSTEM TESTS # ========================================== @@ -155,13 +152,6 @@ jobs: python: ["3.11"] steps: - uses: actions/checkout@v4 - - id: 'auth' - name: 'Authenticate to Google Cloud' - uses: 'google-github-actions/auth@v2' - with: - workload_identity_provider: ${{ secrets.WIF_PROVIDER }} - service_account: ${{ secrets.WIF_SERVICE_ACCOUNT }} - - uses: astral-sh/setup-uv@v5 with: python-version: ${{ matrix.python }} @@ -171,13 +161,9 @@ jobs: - name: Execute System Tests env: RUN_SYSTEM_TESTS: "true" - # ADDED: Inject the required environment variables for the tests - PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} # Update to match your legacy YAML's secret name run: | cd ${{ matrix.package }} export NOX_DEFAULT_VENV_BACKEND=uv - - # FIXED: Append the exact matrix version to stop Nox from running all versions sequentially uvx --with 'nox[uv]' nox -s system-${{ matrix.python }} # ========================================== diff --git a/test.py b/test.py new file mode 100755 index 000000000000..44b41cb2e540 --- /dev/null +++ b/test.py @@ -0,0 +1,20 @@ +# Ensure you are on the sandbox branch +git checkout experiment-testing-infra + +# Loop through the first 15 packages and inject an actual test file +for dir in $(find packages -mindepth 1 -maxdepth 1 -type d | head -n 15); do + # Ensure the target directory exists + mkdir -p "${dir}/tests/unit" + + # Inject the passing test + cat << 'EOF' > "${dir}/tests/unit/test_scale_validation.py" +def test_fan_out_execution(): + """Verify L5 Fan-Out architecture executes successfully.""" + assert True +EOF +done + +# Commit and push the massive execution test +git add packages/ +git commit -m "test: inject dummy unit tests across 15 packages to verify concurrent execution" +git push origin ci-architecture-prototype \ No newline at end of file From 9aa84fd88285e508585af2182e0290cad3149804 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 14 Apr 2026 23:20:05 +0000 Subject: [PATCH 10/11] update system test execution --- .kokoro/system-single.sh | 15 ++++++++++++++- .kokoro/system.sh | 29 +++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/.kokoro/system-single.sh b/.kokoro/system-single.sh index 0ec5ae7ebf1b..9db33548d852 100755 --- a/.kokoro/system-single.sh +++ b/.kokoro/system-single.sh @@ -32,4 +32,17 @@ NOX_FILE_ARG="" [[ -z "${NOX_FILE}" ]] || NOX_FILE_ARG="-f ${NOX_FILE}" -python3 -m nox ${NOX_SESSION_ARG} $NOX_FILE_ARG +python3 -m pip install uv +for attempt in 1 2 3; do + echo "Execution attempt $attempt of 3..." + if uvx --with 'nox[uv]' nox ${NOX_SESSION_ARG} $NOX_FILE_ARG; then + echo "Tests passed successfully!" + exit 0 + fi + + echo "Tests failed. Backing off for 15 seconds..." + sleep 15 +done + +echo "Tests failed after 3 attempts. Hard failure." +exit 1 diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 67215245bcf1..df7af86f8a60 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -117,6 +117,8 @@ system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh" packages_with_system_tests_pattern=$(printf "|*%s*" "${packages_with_system_tests[@]}") packages_with_system_tests_pattern="${packages_with_system_tests_pattern:1}" # Remove the leading pipe +declare -A pids + # Run system tests for each package with directory packages/*/tests/system for path in `find 'packages' \ \( -type d -wholename 'packages/*/tests/system' \) -o \ @@ -141,11 +143,30 @@ for path in `find 'packages' \ package_modified=$(git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- ${files_to_check} | wc -l) set -e - if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then - # Call the function - its internal exports won't affect the next loop - run_package_test "$package_name" || RETVAL=$? +if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then + echo ">>> Dispatching ${package_name} in the background <<<" + + # Execute inside an isolated subshell ( ) to prevent GCP credential collisions + ( + run_package_test "$package_name" + ) & + + # Capture the PID of the subshell + pids["$package_name"]=$! else echo "No changes in ${package_name} and not a continuous build, skipping." fi done -exit ${RETVAL} + +echo "============================================================" +echo "Waiting for all concurrent system tests to complete..." +echo "============================================================" + +for package in "${!pids[@]}"; do + wait ${pids[$package]} || { + echo "ERROR: System tests failed for $package" + RETVAL=1 + } +done + +exit ${RETVAL} \ No newline at end of file From 2bad24a226170559125bfd66a1cf9bed743ec4cc Mon Sep 17 00:00:00 2001 From: ohmayr Date: Tue, 14 Apr 2026 23:54:13 +0000 Subject: [PATCH 11/11] update kokoro scripts for system tests --- .kokoro/system-single.sh | 38 ++++++++------------------- .kokoro/system.sh | 56 +++++++++++----------------------------- 2 files changed, 26 insertions(+), 68 deletions(-) diff --git a/.kokoro/system-single.sh b/.kokoro/system-single.sh index 9db33548d852..6d8b87cd5ec5 100755 --- a/.kokoro/system-single.sh +++ b/.kokoro/system-single.sh @@ -1,48 +1,32 @@ #!/bin/bash # Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# `-e` enables the script to automatically fail when a command fails -# `-o pipefail` sets the exit code to non-zero if any command fails, -# or zero if all commands in the pipeline exit successfully. +# Licensed under the Apache License, Version 2.0 (the "License"); ... set -eo pipefail pwd -# If NOX_SESSION is set, it only runs the specified session, -# otherwise run all the sessions. NOX_SESSION_ARG="" - -# IF NOX_FILE is set, it runs the specific nox file, -# otherwise it runs noxfile.py in the package directory. NOX_FILE_ARG="" [[ -z "${NOX_SESSION}" ]] || NOX_SESSION_ARG="-s ${NOX_SESSION}" - [[ -z "${NOX_FILE}" ]] || NOX_FILE_ARG="-f ${NOX_FILE}" -python3 -m pip install uv +# 3-Attempt retry loop to absorb GCP quota limits and network blips for attempt in 1 2 3; do + echo "============================================" echo "Execution attempt $attempt of 3..." - if uvx --with 'nox[uv]' nox ${NOX_SESSION_ARG} $NOX_FILE_ARG; then + echo "============================================" + + if uvx --with 'nox[uv]' nox ${NOX_SESSION_ARG} ${NOX_FILE_ARG}; then echo "Tests passed successfully!" exit 0 fi - echo "Tests failed. Backing off for 15 seconds..." - sleep 15 + if [[ $attempt -lt 3 ]]; then + echo "Tests failed. Backing off for 15 seconds to absorb quota limits..." + sleep 15 + fi done echo "Tests failed after 3 attempts. Hard failure." -exit 1 +exit 1 \ No newline at end of file diff --git a/.kokoro/system.sh b/.kokoro/system.sh index df7af86f8a60..140cc25d26f4 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -1,45 +1,26 @@ #!/bin/bash # Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# `-e` enables the script to automatically fail when a command fails -# `-o pipefail` sets the exit code to non-zero if any command fails, -# or zero if all commands in the pipeline exit successfully. +# Licensed under the Apache License, Version 2.0 (the "License"); ... set -eo pipefail -# Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 - -# Setup firestore account credentials export FIRESTORE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/firebase-credentials.json - export PROJECT_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/..) cd "$PROJECT_ROOT" - -# This is needed in order for `git diff` to succeed git config --global --add safe.directory $(realpath .) -RETVAL=0 +# HOISTED: Install uv exactly once globally before we fan out +echo "Installing uv globally..." +python3 -m pip install uv +RETVAL=0 pwd run_package_test() { local package_name=$1 local package_path="packages/${package_name}" - # Declare local overrides to prevent bleeding into the next loop iteration local PROJECT_ID local GOOGLE_APPLICATION_CREDENTIALS local NOX_FILE @@ -52,7 +33,6 @@ run_package_test() { case "${package_name}" in "google-auth") - # Copy files needed for google-auth system tests mkdir -p "${package_path}/system_tests/data" cp "${KOKORO_GFILE_DIR}/google-auth-service-account.json" "${package_path}/system_tests/data/service_account.json" cp "${KOKORO_GFILE_DIR}/google-auth-authorized-user.json" "${package_path}/system_tests/data/authorized_user.json" @@ -72,14 +52,13 @@ run_package_test() { ;; esac - # Export variables for the duration of this function's sub-processes export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}" + + # NEW: Subshell-isolated GCP auth. Never modify the global gcloud config! + export CLOUDSDK_CORE_PROJECT="${PROJECT_ID}" + export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${GOOGLE_APPLICATION_CREDENTIALS}" - gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" - gcloud config set project "$PROJECT_ID" - - # Run the actual test pushd "${package_path}" > /dev/null set +e "${system_test_script}" @@ -110,35 +89,27 @@ packages_with_system_tests=( "sqlalchemy-spanner" ) -# A file for running system tests system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh" - -# Join array elements with | for the pattern match packages_with_system_tests_pattern=$(printf "|*%s*" "${packages_with_system_tests[@]}") -packages_with_system_tests_pattern="${packages_with_system_tests_pattern:1}" # Remove the leading pipe +packages_with_system_tests_pattern="${packages_with_system_tests_pattern:1}" declare -A pids -# Run system tests for each package with directory packages/*/tests/system for path in `find 'packages' \ \( -type d -wholename 'packages/*/tests/system' \) -o \ \( -type d -wholename 'packages/*/system_tests' \) -o \ \( -type f -wholename 'packages/*/tests/system.py' \)`; do - # Extract the package name and define the relative package path - # 1. Remove the 'packages/' prefix - # 2. Remove everything after the first '/' package_name=${path#packages/} package_name=${package_name%%/*} package_path="packages/${package_name}" - # Determine if we should skip based on git diff files_to_check="${package_path}/CHANGELOG.md" if [[ $package_name == @($packages_with_system_tests_pattern) ]]; then files_to_check="${package_path}" fi - echo "checking changes with 'git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- ${files_to_check}'" + echo "checking changes with 'git diff ${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT} -- ${files_to_check}'" set +e package_modified=$(git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- ${files_to_check} | wc -l) set -e @@ -159,14 +130,17 @@ if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"conti done echo "============================================================" -echo "Waiting for all concurrent system tests to complete..." +echo "All affected packages dispatched. Waiting for completion..." echo "============================================================" for package in "${!pids[@]}"; do wait ${pids[$package]} || { + echo "============================================================" echo "ERROR: System tests failed for $package" + echo "============================================================" RETVAL=1 } done +echo "All concurrent tests completed." exit ${RETVAL} \ No newline at end of file