diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index 9811ae4fed..141cd02205 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -214,6 +214,7 @@ jobs: run: dotnet restore CodeIndex.sln --locked-mode - name: Audit NuGet package vulnerabilities + if: matrix.os == 'ubuntu-latest' && matrix.test-framework == 'net8.0' shell: pwsh run: | $auditOutput = dotnet list src/CodeIndex/CodeIndex.csproj package --vulnerable --include-transitive 2>&1 @@ -230,6 +231,7 @@ jobs: } - name: Verify formatting + if: matrix.os == 'ubuntu-latest' && matrix.test-framework == 'net8.0' run: dotnet format whitespace CodeIndex.sln --verify-no-changes --no-restore --verbosity minimal - name: Verify Release solution build @@ -261,11 +263,11 @@ jobs: "--results-directory", "./TestResults" ) - $collectCoverage = "${{ matrix.os }}" -ne "windows-latest" -or "${{ matrix.test-framework }}" -ne "net9.0" + $collectCoverage = "${{ matrix.os }}" -eq "ubuntu-latest" -and "${{ matrix.test-framework }}" -eq "net8.0" if ($collectCoverage) { $testArgs += @("--collect", "XPlat Code Coverage") } else { - Write-Host "Skipping XPlat Code Coverage for windows-latest/net9.0 to keep the slowest matrix lane under the test session timeout; see #3358." + Write-Host "Skipping XPlat Code Coverage outside ubuntu-latest/net8.0 so platform/framework matrix lanes run only the test suite." } New-Item -ItemType Directory -Force -Path ./TestResults | Out-Null @@ -325,7 +327,7 @@ jobs: TestResults/**/*.hangdump - name: Upload coverage reports - if: always() && !(matrix.os == 'windows-latest' && matrix.test-framework == 'net9.0') + if: always() && matrix.os == 'ubuntu-latest' && matrix.test-framework == 'net8.0' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: Coverage-${{ matrix.os }}-${{ matrix.test-framework }} diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 6e455cd153..8afe307e62 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -38,7 +38,7 @@ Development contracts: | Shell completions | Generated shell completion scripts include a comment with the `cdidx` version that produced them. When command or flag schema changes, update completion tests and keep the README guidance that installed completions should be regenerated after upgrades. | | Target frameworks | The production CLI and NuGet tool packaging target `net8.0`. The test project multi-targets `net8.0;net9.0`, and CI runs the test suite on both frameworks across Linux, Windows, and macOS. Use a .NET SDK that can restore and run both target frameworks when validating the full CI-equivalent test matrix. | | SDK selection | `global.json` pins the repository SDK to `9.0.301` with `rollForward` disabled. CI installs both `8.0.413` and `9.0.301` explicitly: `8.0.413` provides the `net8.0` runtime lane, while `9.0.301` is the selected SDK for restore, build, test, publish, and changelog validation. When rolling SDKs, update `global.json`, every `actions/setup-dotnet` version list, the Docker build image, and this guide together. | -| Test diagnostics | CI uses `tests/CodeIndex.Tests/CodeIndex.Tests.runsettings` plus VSTest blame crash/hang collection and a bounded one-time retry to distinguish repeatable failures from pass-on-retry flakes. For test suite structure, shared helpers, state-isolation rules, timeout diagnostics, and test-writing conventions, see [TESTING_GUIDE.md](TESTING_GUIDE.md). | +| Test diagnostics | CI uses `tests/CodeIndex.Tests/CodeIndex.Tests.runsettings` plus VSTest blame crash/hang collection and a bounded one-time retry to distinguish repeatable failures from pass-on-retry flakes. The Build and Test workflow collects XPlat Code Coverage only on `ubuntu-latest` / `net8.0`; the remaining OS/framework lanes run the full suite without coverage collector overhead. For test suite structure, shared helpers, state-isolation rules, timeout diagnostics, and test-writing conventions, see [TESTING_GUIDE.md](TESTING_GUIDE.md). | | Mutation testing | The weekly `Mutation testing` workflow runs Stryker.NET against `src/CodeIndex/Database/DbWriter.cs` using `stryker-config.json`. Keep this scope focused on transaction, savepoint, rollback, and batch-write behavior unless the runtime budget is intentionally expanded. The workflow installs `dotnet-stryker` version `4.14.0` explicitly, and the mutation score gates are high 75, low 70, and break 65 so changes that weaken rollback or savepoint coverage fail outside the regular PR test path. | ## CI / Artifact Distribution @@ -2229,7 +2229,7 @@ net9 CI lane に合わせる場合は `FRAMEWORK=net9.0 make test` を使いま | shell completion | 生成された shell completion script には、生成元の `cdidx` version comment が含まれます。command や flag の schema を変えた場合は completion test を更新し、upgrade 後に installed completion を再生成する README guidance も保ってください。 | | target framework | 製品版 CLI と NuGet tool packaging は `net8.0` を対象にしています。test project は `net8.0;net9.0` の multi-target で、CI は Linux、Windows、macOS の各 lane で両方の framework に対して test suite を実行します。CI 相当の full matrix を検証する場合は、両方の target framework を restore / 実行できる .NET SDK を使ってください。 | | SDK selection | `global.json` は repository SDK を `9.0.301` に固定し、`rollForward` を無効化します。CI は `8.0.413` と `9.0.301` を明示的に install します。`8.0.413` は `net8.0` runtime lane を提供し、`9.0.301` は restore、build、test、publish、changelog 検証で選択される SDK です。SDK を更新する場合は、`global.json`、すべての `actions/setup-dotnet` version list、Docker build image、この guide を同じ変更で更新してください。 | -| test diagnostics | CI は `tests/CodeIndex.Tests/CodeIndex.Tests.runsettings` と VSTest の crash/hang blame collection、上限付きの 1 回だけの retry を使い、再現性のある失敗と retry で通る flake を区別します。test suite の構成、共有 helper、state-isolation rule、timeout diagnostics、test-writing convention については [TESTING_GUIDE.md#テストガイド](TESTING_GUIDE.md#テストガイド) を参照してください。 | +| test diagnostics | CI は `tests/CodeIndex.Tests/CodeIndex.Tests.runsettings` と VSTest の crash/hang blame collection、上限付きの 1 回だけの retry を使い、再現性のある失敗と retry で通る flake を区別します。Build and Test workflow は XPlat Code Coverage を `ubuntu-latest` / `net8.0` でのみ収集し、それ以外の OS/framework lane は coverage collector overhead なしで full suite を実行します。test suite の構成、共有 helper、state-isolation rule、timeout diagnostics、test-writing convention については [TESTING_GUIDE.md#テストガイド](TESTING_GUIDE.md#テストガイド) を参照してください。 | | mutation testing | weekly の `Mutation testing` workflow は `stryker-config.json` を使い、`src/CodeIndex/Database/DbWriter.cs` に対して Stryker.NET を実行します。runtime budget を意図的に広げる場合を除き、transaction、savepoint、rollback、batch-write behavior に scope を集中させてください。workflow は `dotnet-stryker` version `4.14.0` を明示的に install し、mutation score gate は high 75、low 70、break 65 で、rollback や savepoint coverage を弱める変更は通常の PR test path の外で失敗します。 | ## CI / アーティファクト配布 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index fef09c5011..8f42579533 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -26,7 +26,7 @@ Use the full suite by default. Use targeted filters only while iterating locally - These test-only packages are separate from the production dependency rule in `src/CodeIndex`, which still allows only `Microsoft.Data.Sqlite` at runtime. - `FsCheck.Xunit` is reserved for property-based tests that assert universal invariants (never-throws contracts, idempotence, "output is parseable by downstream consumer") across randomly generated inputs. Use it to complement, not replace, the example-based `[Fact]` / `[Theory]` tests — pick FsCheck when the property is a universally quantified claim, and an example test when a specific concrete case is the contract. - Test parallelism: enabled by default across independent test classes. Tests that touch process-global state such as SQLite pool resets, environment variables, or current-directory overrides must use an explicit non-parallel collection, and tests that swap `Console.Out` / `Console.Error` must lock on `TestConsoleLock.Gate`. -- CI runs the test project through `tests/CodeIndex.Tests/CodeIndex.Tests.runsettings`, enables VSTest blame crash and hang collection, applies a 45-minute session timeout plus 60-second xUnit long-running diagnostics, and reruns the suite once after an initial failure. If the retry passes, CI uploads `TestResults/flaky-retry.txt` with the TRX and blame artifacts so the run is treated as suspect instead of silently trusted. +- CI runs the test project through `tests/CodeIndex.Tests/CodeIndex.Tests.runsettings`, enables VSTest blame crash and hang collection, applies a 45-minute session timeout plus 60-second xUnit long-running diagnostics, and reruns the suite once after an initial failure. If the retry passes, CI uploads `TestResults/flaky-retry.txt` with the TRX and blame artifacts so the run is treated as suspect instead of silently trusted. XPlat Code Coverage collection is limited to the `ubuntu-latest` / `net8.0` lane so every OS/framework lane still exercises the full suite without paying collector overhead. ## Test Layout @@ -234,7 +234,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - これらの test-only package は `src/CodeIndex` の本番依存ルールとは別であり、runtime 側は引き続き `Microsoft.Data.Sqlite` のみを許容する。 - `FsCheck.Xunit` はランダム生成入力に対する普遍的不変条件(never-throws、idempotence、"出力が downstream consumer で parse 可能" 等)を表明する property-based テスト専用です。例ベースの `[Fact]` / `[Theory]` を置き換えるのではなく補完するもので、普遍量化された主張なら FsCheck、特定の具体ケースが契約なら例ベースという形で使い分けてください。 - テスト並列実行: 独立したテストクラス間ではデフォルトで有効です。SQLite pool の解放、環境変数の変更、カレントディレクトリの上書きのような process-global 状態を触るテストは、明示的な non-parallel collection に入れてください。`Console.Out` / `Console.Error` を差し替えるテストは `TestConsoleLock.Gate` で lock してください。 -- CI は `tests/CodeIndex.Tests/CodeIndex.Tests.runsettings` 経由でテストプロジェクトを実行し、VSTest の blame crash / hang 収集、45分のセッションタイムアウト、60秒の xUnit long-running 診断を有効にします。初回失敗時は suite を1回だけ再実行し、再実行で成功した場合は TRX / blame artifact と一緒に `TestResults/flaky-retry.txt` を upload して、その実行を疑わしい flaky run として扱います。 +- CI は `tests/CodeIndex.Tests/CodeIndex.Tests.runsettings` 経由でテストプロジェクトを実行し、VSTest の blame crash / hang 収集、45分のセッションタイムアウト、60秒の xUnit long-running 診断を有効にします。初回失敗時は suite を1回だけ再実行し、再実行で成功した場合は TRX / blame artifact と一緒に `TestResults/flaky-retry.txt` を upload して、その実行を疑わしい flaky run として扱います。XPlat Code Coverage の収集は `ubuntu-latest` / `net8.0` lane に限定し、すべての OS/framework lane で full suite を実行しつつ collector overhead を避けます。 ## テスト構成 diff --git a/changelog.d/unreleased/+ci-build-test-runtime.internal.md b/changelog.d/unreleased/+ci-build-test-runtime.internal.md new file mode 100644 index 0000000000..ae131ab7c3 --- /dev/null +++ b/changelog.d/unreleased/+ci-build-test-runtime.internal.md @@ -0,0 +1,16 @@ +--- +category: internal +affected: + - .github/workflows/dotnet.yml + - tests/CodeIndex.Tests/CiWorkflowTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Build and Test CI now avoids repeated slow validation work** — vulnerability auditing, formatting validation, and XPlat Code Coverage collection now run on the representative `ubuntu-latest` / `net8.0` lane while the full OS/framework matrix continues to run the test suite. + +## 日本語 + +- **Build and Test CI が重い検証処理の重複を避けるようになりました** — 脆弱性監査、format 検証、XPlat Code Coverage 収集を代表 `ubuntu-latest` / `net8.0` lane に寄せ、OS/framework の full matrix では引き続き test suite を実行します。 diff --git a/tests/CodeIndex.Tests/CiWorkflowTests.cs b/tests/CodeIndex.Tests/CiWorkflowTests.cs index c96541d23f..8973b5137e 100644 --- a/tests/CodeIndex.Tests/CiWorkflowTests.cs +++ b/tests/CodeIndex.Tests/CiWorkflowTests.cs @@ -9,9 +9,17 @@ public class CiWorkflowTests public void DotnetWorkflow_RunsTestsWithRunsettingsBlameRetryAndArtifacts() { var workflow = File.ReadAllText(Path.Combine(GetRepositoryRoot(), ".github", "workflows", "dotnet.yml")); + var normalizedWorkflow = workflow.ReplaceLineEndings("\n"); Assert.Contains("--settings\", \"tests/CodeIndex.Tests/CodeIndex.Tests.runsettings", workflow); - Assert.Contains("Skipping XPlat Code Coverage for windows-latest/net9.0", workflow); + Assert.Contains( + "- name: Audit NuGet package vulnerabilities\n if: matrix.os == 'ubuntu-latest' && matrix.test-framework == 'net8.0'", + normalizedWorkflow); + Assert.Contains( + "- name: Verify formatting\n if: matrix.os == 'ubuntu-latest' && matrix.test-framework == 'net8.0'", + normalizedWorkflow); + Assert.Contains("\"${{ matrix.os }}\" -eq \"ubuntu-latest\" -and \"${{ matrix.test-framework }}\" -eq \"net8.0\"", workflow); + Assert.Contains("Skipping XPlat Code Coverage outside ubuntu-latest/net8.0", workflow); Assert.Contains("--blame-crash", workflow); Assert.Contains("--blame-hang", workflow); Assert.Contains("--blame-hang-timeout\", \"5m", workflow); @@ -24,7 +32,8 @@ public void DotnetWorkflow_RunsTestsWithRunsettingsBlameRetryAndArtifacts() Assert.Contains("TestResults/**/*Sequence*.xml", workflow); Assert.Contains("TestResults/**/*.dmp", workflow); Assert.Contains("TestResults/**/*.dump", workflow); - Assert.Contains("always() && !(matrix.os == 'windows-latest' && matrix.test-framework == 'net9.0')", workflow); + Assert.Contains("always() && matrix.os == 'ubuntu-latest' && matrix.test-framework == 'net8.0'", workflow); + Assert.DoesNotContain("always() && !(matrix.os == 'windows-latest' && matrix.test-framework == 'net9.0')", workflow); } [Fact]