Skip to content

grouped-tests: QA group defaults to Julia v1 only #104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ChrisRackauckas merged 2 commits into from
Jun 24, 2026
Merged

grouped-tests: QA group defaults to Julia v1 only #104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b9b6076
grouped-tests: QA group defaults to Julia v1 only
ChrisRackauckas Jun 23, 2026
aeee328
test: update matrix assertions for QA→v1 default
ChrisRackauckas Jun 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 19 additions & 6 deletions scripts/compute_affected_sublibraries.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@
# versions = ["lts", "1", "pre"]
#
# [QA]
# versions = ["lts", "1"]
# versions = ["lts", "1"] # NOTE: ignored — QA always runs once on "1".
# # Aqua/JET/ExplicitImports are version-agnostic, so
# # the QA group is centrally clamped to ["1"] (see
# # QA_VERSIONS); no need to trim this per repo.
#
# [GPU]
# versions = ["1"]
Expand All @@ -39,7 +42,7 @@
#
# If no test/test_groups.toml exists, the default is:
# Core on ["lts", "1", "pre"]
# QA on ["lts", "1"]
# QA on ["1"]
#
# A group that needs test-only deps beyond the sublibrary's [targets].test list should
# carry an isolated environment at test/<group>/Project.toml that runtests.jl activates
Expand Down Expand Up @@ -76,7 +79,7 @@ using TOML

const DEFAULT_TEST_GROUPS = Dict(
"Core" => ["lts", "1", "pre"],
"QA" => ["lts", "1"],
"QA" => ["1"],
)

function build_dependency_graph(lib_dir::String)
Expand Down Expand Up @@ -241,6 +244,13 @@ const EXCLUDES = Set(

const DOWNSTREAM_VERSION = "1"

# QA (Aqua/JET/ExplicitImports) is version-agnostic, so running it per declared
# version just doubles the job count with no added signal — and the lts half is
# where the <1.11 [sources]-backport QA failures bite. Central policy: the QA
# group always runs once, on the latest stable, regardless of the per-repo
# [QA] versions. Repos therefore needn't trim [QA] versions in test_groups.toml.
const QA_VERSIONS = ["1"]

function build_matrix(
direct::Set{String}, transitive::Set{String}, lib_dir::String
)
Expand All @@ -255,7 +265,8 @@ function build_matrix(
is_downstream && config.local_only && continue
ci_group = group_name == "Core" ? pkg : "$(pkg)_$(group_name)"
# Downstream (transitive) deps only run on latest stable.
versions = is_downstream ? [DOWNSTREAM_VERSION] : config.versions
versions = group_name == "QA" ? QA_VERSIONS :
is_downstream ? [DOWNSTREAM_VERSION] : config.versions
for ver in versions
(ci_group, ver) in EXCLUDES && continue
push!(
Expand Down Expand Up @@ -313,7 +324,8 @@ function build_projects_matrix(
config = groups[group_name]
is_downstream && config.local_only && continue
ci_group = group_name == "Core" ? pkg : "$(pkg)_$(group_name)"
versions = is_downstream ? [DOWNSTREAM_VERSION] : config.versions
versions = group_name == "QA" ? QA_VERSIONS :
is_downstream ? [DOWNSTREAM_VERSION] : config.versions
for ver in versions
(ci_group, ver) in EXCLUDES && continue
push!(
Expand Down Expand Up @@ -370,7 +382,8 @@ function build_root_matrix(repo_root::String)
for group_name in sort!(collect(keys(groups)))
config = groups[group_name]
runners = isempty(config.os) ? Any[config.runner] : Any[o for o in config.os]
for ver in config.versions
vers = group_name == "QA" ? QA_VERSIONS : config.versions
for ver in vers
for runner in runners
push!(
entries,
Expand Down
15 changes: 8 additions & 7 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,10 @@ end
@testset "projects-matrix: default groups + downstream→v1" begin
direct, trans = compute_affected(["lib/A/src/A.jl"], graph, rev)
m = build_projects_matrix(direct, trans, lib)
# A is directly changed: default Core on lts,1,pre + QA on lts,1
# A is directly changed: default Core on lts,1,pre + QA on 1 (QA defaults to v1 only)
a = filter(e -> e.project == "lib/A", m)
@test Set((e.group, e.version) for e in a) ==
Set([("Core", "lts"), ("Core", "1"), ("Core", "pre"), ("QA", "lts"), ("QA", "1")])
Set([("Core", "lts"), ("Core", "1"), ("Core", "pre"), ("QA", "1")])
# B and C are downstream: version "1" only
for p in ("lib/B", "lib/C")
ds = filter(e -> e.project == p, m)
Expand Down Expand Up @@ -210,7 +210,8 @@ end
m = build_root_matrix(d)
cells = Set((e.group, e.version) for e in m)
@test ("AD", "1") ∉ cells && ("AD", "pre") ∉ cells && ("AD", "lts") in cells
@test ("QA", "pre") ∉ cells && ("QA", "lts") in cells && ("QA", "1") in cells
# QA is centrally clamped to v1 only, regardless of the per-group `versions`.
@test ("QA", "pre") ∉ cells && ("QA", "lts") ∉ cells && ("QA", "1") in cells
# continue_on_error rides only on the Downstream group.
@test all(e -> e.continue_on_error, filter(e -> e.group == "Downstream", m))
@test all(e -> !e.continue_on_error, filter(e -> e.group != "Downstream", m))
Expand Down Expand Up @@ -252,8 +253,8 @@ end

@testset "root matrix faithfully reproduces OrdinaryDiffEq's embedded matrix" begin
# ODE's root CI.yml is 17 groups × [lts,1,pre] minus excludes (AD->lts only,
# QA->lts/1, ODEInterfaceRegression->lts only). Per-group `versions`
# expresses the same 46 cells, which is the migration this enables.
# ODEInterfaceRegression->lts only). QA is centrally clamped to v1 only (see
# QA_VERSIONS), so it intentionally diverges from ODE's old QA-on-lts/1: 45 cells.
base = [
"InterfaceI", "InterfaceII", "InterfaceIII", "InterfaceIV", "InterfaceV",
"Integrators_I", "Integrators_II", "AlgConvergence_I", "AlgConvergence_II",
Expand All @@ -273,11 +274,11 @@ end
cells = Set((e.group, e.version) for e in build_root_matrix(d))
groups17 = vcat(base, ["AD", "QA", "ODEInterfaceRegression"])
expected = Set((g, v) for g in groups17 for v in ["lts", "1", "pre"])
for ex in [("AD", "1"), ("AD", "pre"), ("QA", "pre"), ("ODEInterfaceRegression", "1"), ("ODEInterfaceRegression", "pre")]
for ex in [("AD", "1"), ("AD", "pre"), ("QA", "pre"), ("QA", "lts"), ("ODEInterfaceRegression", "1"), ("ODEInterfaceRegression", "pre")]
delete!(expected, ex)
end
@test cells == expected
@test length(cells) == 46
@test length(cells) == 45
end

@testset "--root-matrix CLI (no lib/ required) + JSON shape" begin
Expand Down
Loading