diff --git a/.conformance-catalog-ref b/.conformance-catalog-ref new file mode 100644 index 0000000..efa9db0 --- /dev/null +++ b/.conformance-catalog-ref @@ -0,0 +1 @@ +b4c758a7dac698d7fcacd32dafcd4bb2f5dbddaf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33f3148..781d2f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,10 +28,24 @@ jobs: # surface to SHA-pin. Keeps the catalog outside the workspace so # release.yml's `git add -A` cannot stage it as an embedded gitlink. - name: Check out shared conformance catalog (outside workspace) + # Conformance catalog pinned by SHA (was: clone of the latest default + # branch). The single source of truth for the ref is the tracked + # `.conformance-catalog-ref` file at the repo root — bump it when + # adopting new catalog cases, together with the SDK-side conformance + # coverage, so a catalog change can never break CI on its own. The + # Checkout step above must precede this read. Source: + # github.com/AuthPlane/conformance. run: | - git -c advice.detachedHead=false clone --depth=1 \ - https://github.com/AuthPlane/conformance.git \ - "${{ runner.temp }}/conformance" + CONFORMANCE_CATALOG_REF="$(cat "$GITHUB_WORKSPACE/.conformance-catalog-ref")" + # Guard the pin: a non-SHA value would silently un-pin CI to whatever + # ref resolves at fetch time. + grep -Eq '^[0-9a-f]{40}$' <<<"$CONFORMANCE_CATALOG_REF" \ + || { echo "::error::.conformance-catalog-ref must be a 40-hex commit SHA"; exit 1; } + git init -q "${{ runner.temp }}/conformance" + git -C "${{ runner.temp }}/conformance" \ + fetch --depth=1 https://github.com/AuthPlane/conformance.git "$CONFORMANCE_CATALOG_REF" \ + || { echo "::error::Pinned conformance catalog ref $CONFORMANCE_CATALOG_REF is unreachable"; exit 1; } + git -C "${{ runner.temp }}/conformance" checkout -q FETCH_HEAD - name: Setup Java uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 diff --git a/.github/workflows/conformance-catalog-drift.yml b/.github/workflows/conformance-catalog-drift.yml new file mode 100644 index 0000000..bc843a0 --- /dev/null +++ b/.github/workflows/conformance-catalog-drift.yml @@ -0,0 +1,121 @@ +name: Conformance catalog drift + +# Early-warning for conformance-catalog drift. +# +# PR/release CI pins the catalog to the SHA in `.conformance-catalog-ref`, so a +# new catalog case can never break CI on its own. The trade-off is that new +# cases go unnoticed until someone bumps the ref. This job closes that gap: on a +# weekly schedule it clones the catalog's *default* branch (latest, unpinned), +# points the harness at it, and runs the alignment assertion +# (ConformanceCatalogTest#catalogCasesAndConformanceMappingsAgree), which fails +# when a catalog case id has no matching @ConformanceCase mapping in the suite +# (or vice versa). PR and release CI run that same assertion against the *pinned* +# catalog; the only difference here is which catalog it points at. This job has no +# `pull_request` trigger, so failing the scheduled run cannot block PRs; it deliberately FAILS +# on drift so the run turns red and the `::warning::` plus job-summary line are +# not buried in an otherwise-green run — prompting a coordinated ref bump plus +# SDK-side coverage. + +on: + schedule: + # Mondays 07:00 UTC + - cron: "0 7 * * 1" + workflow_dispatch: + +# Least-privilege default; this workflow only reads the repo. +permissions: + contents: read + +jobs: + drift: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # Clone the catalog's default branch (latest) — deliberately unpinned, + # unlike ci.yml/release.yml which pin to `.conformance-catalog-ref`. This + # is what lets the job detect cases added since the pinned ref. + - name: Check out latest shared conformance catalog (outside workspace) + run: | + git clone --depth=1 https://github.com/AuthPlane/conformance.git \ + "${{ runner.temp }}/conformance" \ + || { echo "::error::Could not clone the conformance catalog default branch"; exit 1; } + + - name: Setup Java + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 + with: + distribution: temurin + java-version: "21" + cache: maven + + # Runs the alignment assertion against the latest catalog. That assertion + # (ConformanceCatalogTest#catalogCasesAndConformanceMappingsAgree) fails + # when a catalog case id has no matching @ConformanceCase mapping in the + # suite — i.e. a case added since the pinned ref that the SDK does not yet + # cover. PR/release CI runs the same assertion against the catalog pinned + # in .conformance-catalog-ref; pointing CONFORMANCE_CATALOG_PATH at the + # unpinned tip is this job's entire contribution. The step is deliberately + # allowed to fail the job (no `continue-on-error`) so scheduled drift turns + # the run red instead of hiding in a green run; the workflow has no + # `pull_request` trigger, so this never blocks PR CI. + - name: Run catalog-alignment check against latest catalog + id: align + env: + CONFORMANCE_CATALOG_PATH: ${{ runner.temp }}/conformance/oauth-sdk-conformance-catalog.yaml + run: mvn -B -ntp -pl core test -Dtest=ConformanceCatalogTest + + # Runs even when the alignment step fails the job, so the `::warning::` + # and job summary are always written on drift. A `failure` outcome alone + # does not mean drift — the step also fails on a compile error, a Maven + # resolution failure, or a failure of the other test in the class. Real + # drift is identified by the `Conformance-catalog drift:` marker the + # assertion writes into the surefire report; anything else that failed the + # step is reported as an infrastructure problem, as are skipped/cancelled + # outcomes (the alignment step never ran because an earlier step failed). + - name: Report drift + if: always() + run: | + pinned="$(cat "$GITHUB_WORKSPACE/.conformance-catalog-ref")" + grep -Eq '^[0-9a-f]{40}$' <<<"$pinned" \ + || { echo "::error::.conformance-catalog-ref must be a 40-hex commit SHA"; exit 1; } + case "${{ steps.align.outcome }}" in + success) + echo "No conformance-catalog drift detected against the latest catalog." >> "$GITHUB_STEP_SUMMARY" + echo "Pinned ref: \`$pinned\`" >> "$GITHUB_STEP_SUMMARY" + ;; + failure) + if grep -rqF 'Conformance-catalog drift:' core/target/surefire-reports 2>/dev/null; then + echo "::warning::Conformance-catalog drift: the catalog-alignment check fails against the latest catalog. New or changed cases exist since the pinned ref ($pinned). Review github.com/AuthPlane/conformance, add SDK-side coverage, then bump .conformance-catalog-ref." + { + echo "### ⚠️ Conformance-catalog drift detected" + echo "" + echo "The catalog-alignment check fails against the **latest** catalog default branch." + echo "New or changed cases exist since the pinned ref \`$pinned\`." + echo "" + echo "**Next steps:** review [AuthPlane/conformance](https://github.com/AuthPlane/conformance), add SDK-side coverage for any new cases, then bump \`.conformance-catalog-ref\` in the same change." + } >> "$GITHUB_STEP_SUMMARY" + else + echo "::warning::The catalog-alignment step failed without the drift marker (no 'Conformance-catalog drift:' in the surefire report). This is a build or harness problem — a compile error, a Maven resolution failure, or the other test in the class — not catalog drift." + { + echo "### ⚠️ Conformance drift check failed for another reason" + echo "" + echo "The alignment step failed, but the surefire report carries no \`Conformance-catalog drift:\` marker." + echo "That points at a build or harness problem (compile error, dependency resolution, or the other test in the class), **not** catalog drift." + echo "" + echo "Read the step log above before touching \`.conformance-catalog-ref\`." + } >> "$GITHUB_STEP_SUMMARY" + fi + ;; + *) + echo "::warning::Conformance drift check did not run: the alignment step was '${{ steps.align.outcome }}' (the catalog clone or Setup Java likely failed). This is an infrastructure problem, not catalog drift." + { + echo "### ⚠️ Conformance drift check could not run" + echo "" + echo "The alignment step was \`${{ steps.align.outcome }}\`, so drift was not evaluated." + echo "This is an infrastructure issue (e.g. the catalog clone or Setup Java failed), **not** catalog drift." + echo "" + echo "Re-run the workflow; if it keeps failing, investigate the failing setup step above." + } >> "$GITHUB_STEP_SUMMARY" + ;; + esac diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 89ac4d0..0986909 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -95,10 +95,24 @@ jobs: # surface to SHA-pin. Keeps the catalog outside the workspace so the # `git add -A` below cannot stage it as an embedded gitlink. - name: Check out shared conformance catalog (outside workspace) + # Conformance catalog pinned by SHA (was: clone of the latest default + # branch). The single source of truth for the ref is the tracked + # `.conformance-catalog-ref` file at the repo root — bump it when + # adopting new catalog cases, together with the SDK-side conformance + # coverage, so a catalog change can never break CI on its own. The + # checkout step above must precede this read. Source: + # github.com/AuthPlane/conformance. run: | - git -c advice.detachedHead=false clone --depth=1 \ - https://github.com/AuthPlane/conformance.git \ - "${{ runner.temp }}/conformance" + CONFORMANCE_CATALOG_REF="$(cat "$GITHUB_WORKSPACE/.conformance-catalog-ref")" + # Guard the pin: a non-SHA value would silently un-pin CI to whatever + # ref resolves at fetch time. + grep -Eq '^[0-9a-f]{40}$' <<<"$CONFORMANCE_CATALOG_REF" \ + || { echo "::error::.conformance-catalog-ref must be a 40-hex commit SHA"; exit 1; } + git init -q "${{ runner.temp }}/conformance" + git -C "${{ runner.temp }}/conformance" \ + fetch --depth=1 https://github.com/AuthPlane/conformance.git "$CONFORMANCE_CATALOG_REF" \ + || { echo "::error::Pinned conformance catalog ref $CONFORMANCE_CATALOG_REF is unreachable"; exit 1; } + git -C "${{ runner.temp }}/conformance" checkout -q FETCH_HEAD - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 09af7b7..af7740f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,12 +46,14 @@ The RFC conformance tests load the shared [Authplane Conformance Catalog](https: # ├── java-sdk/ ← this repo # └── conformance/ ← catalog repo git clone https://github.com/AuthPlane/conformance.git ../conformance +# Check out the same ref CI pins, so local runs match CI: +git -C ../conformance checkout "$(cat .conformance-catalog-ref)" # Option B — clone it anywhere and point CONFORMANCE_CATALOG_PATH at the file: export CONFORMANCE_CATALOG_PATH=/path/to/conformance/oauth-sdk-conformance-catalog.yaml ``` -See [`core/src/conformance/README.md`](core/src/conformance/README.md) for details. CI clones the catalog automatically. +CI pins the catalog to the SHA in [`.conformance-catalog-ref`](.conformance-catalog-ref) (a single source of truth read by `ci.yml` and `release.yml`); a weekly `conformance-catalog-drift.yml` job fails when the latest catalog adds cases the SDK does not yet cover (it runs only on a schedule, so it never blocks PRs). Bumping the pinned ref must accompany the matching SDK-side coverage. See [`core/src/conformance/README.md`](core/src/conformance/README.md) for details. ## Local Verification diff --git a/core/src/conformance/README.md b/core/src/conformance/README.md index 226b84a..111f7e3 100644 --- a/core/src/conformance/README.md +++ b/core/src/conformance/README.md @@ -20,8 +20,16 @@ Clone the catalog next to the SDK (recommended for local work): ```bash # From the parent directory of java-sdk/ git clone git@github.com:AuthPlane/conformance.git conformance +# Check out the same ref CI pins, so local runs match CI: +git -C conformance checkout "$(cat java-sdk/.conformance-catalog-ref)" ``` +CI pins the catalog to the SHA in `java-sdk/.conformance-catalog-ref` — a single +source of truth read by `ci.yml` and `release.yml`. Bump that file (together with +the SDK-side coverage for any new cases) to adopt a newer catalog; the weekly +`conformance-catalog-drift.yml` job fails when the latest catalog drifts ahead of +the pinned ref. It runs only on a schedule, so it never blocks PR CI. + Expected layout: ``` diff --git a/core/src/conformance/java/ai/authplane/sdk/core/conformance/ConformanceCatalogTest.java b/core/src/conformance/java/ai/authplane/sdk/core/conformance/ConformanceCatalogTest.java index 1bf1869..761d52a 100644 --- a/core/src/conformance/java/ai/authplane/sdk/core/conformance/ConformanceCatalogTest.java +++ b/core/src/conformance/java/ai/authplane/sdk/core/conformance/ConformanceCatalogTest.java @@ -2,12 +2,22 @@ import static org.assertj.core.api.Assertions.assertThat; +import java.lang.reflect.Method; +import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; import org.junit.jupiter.api.Test; class ConformanceCatalogTest { + private static final String NL = System.lineSeparator(); + @Test void load_readsCatalogMetadataAndCaseIds() { Path projectRoot = Path.of(System.getProperty("user.dir")).toAbsolutePath().normalize(); @@ -20,4 +30,157 @@ void load_readsCatalogMetadataAndCaseIds() { assertThat(catalog.caseIds()).isNotEmpty(); assertThat(catalog.caseIds()).contains("rfc6749-client-credentials-success-response"); } + + /** + * Alignment check between the resolved catalog and the {@link ConformanceCase} mappings in the + * suite, asserted in both directions: + * + * + * + *

This runs unconditionally, against whichever catalog {@link ConformanceCatalogPaths} + * resolves: the SHA pinned in {@code .conformance-catalog-ref} in PR and release CI, and the + * catalog's unpinned tip in the scheduled drift job (which points {@code + * CONFORMANCE_CATALOG_PATH} at its own clone). Asserting it at PR time is what makes a bump of + * {@code .conformance-catalog-ref} safe: a bump that adds cases without SDK-side coverage turns + * the PR red instead of merging green and publishing a report full of silent {@code not_run} + * entries. + */ + @Test + void catalogCasesAndConformanceMappingsAgree() throws Exception { + Path projectRoot = Path.of(System.getProperty("user.dir")).toAbsolutePath().normalize(); + ConformanceCatalog catalog = + ConformanceCatalog.load(ConformanceCatalogPaths.resolve(projectRoot)); + + SuiteScan scan = scanConformanceSuites(); + + // Preconditions on the scan itself. Without these, a scan that read nothing (or read only + // part of the suite) is indistinguishable from the catalog having drifted: every case id + // would be reported as uncovered. + assertThat(scan.loadFailures()) + .withFailMessage( + "Conformance-suite scan incomplete: %d classpath entr(ies) could not be" + + " read, so any @ConformanceCase they declare is missing from this" + + " check. Fix the scan before trusting its result:%n - %s", + scan.loadFailures().size(), String.join(NL + " - ", scan.loadFailures())) + .isEmpty(); + assertThat(scan.caseIds()) + .withFailMessage( + "Conformance-suite scan found no @ConformanceSuite classes on the" + + " classpath. The scan failed — this is not catalog drift.") + .isNotEmpty(); + + Set catalogIds = new TreeSet<>(catalog.caseIds()); + + List uncovered = + catalogIds.stream().filter(id -> !scan.caseIds().contains(id)).sorted().toList(); + assertThat(uncovered) + .withFailMessage( + "Conformance-catalog drift: %d catalog case(s) have no @ConformanceCase" + + " mapping in the suite. Add SDK-side coverage for each, then bump" + + " .conformance-catalog-ref:%n - %s", + uncovered.size(), String.join(NL + " - ", uncovered)) + .isEmpty(); + + List unknown = + scan.caseIds().stream().filter(id -> !catalogIds.contains(id)).sorted().toList(); + assertThat(unknown) + .withFailMessage( + "Conformance-catalog drift: %d conformance test(s) register unknown case" + + " id(s) (not in catalog). Their results are dropped from the report," + + " which is built by iterating the catalog. Correct the id or drop the" + + " @ConformanceCase annotation:%n - %s", + unknown.size(), String.join(NL + " - ", unknown)) + .isEmpty(); + } + + /** Case ids declared across the suite, plus whatever the scan could not read. */ + private record SuiteScan(TreeSet caseIds, List loadFailures) {} + + /** + * Collects every {@link ConformanceCase} case id declared by a {@link ConformanceSuite} test + * class in this package or below it. Scans the compiled conformance classes on the classpath so + * newly added suite classes are discovered automatically (no hand-maintained registry). + * + *

Anything the scan cannot read is recorded rather than skipped: a class that fails to load, + * or a classpath root the scan cannot walk, silently loses its mappings and would be reported + * as catalog drift. + */ + private static SuiteScan scanConformanceSuites() throws Exception { + TreeSet ids = new TreeSet<>(); + List loadFailures = new ArrayList<>(); + String packageName = ConformanceCatalogTest.class.getPackageName(); + String packagePath = packageName.replace('.', '/'); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + Enumeration roots = classLoader.getResources(packagePath); + while (roots.hasMoreElements()) { + URL root = roots.nextElement(); + if (!"file".equals(root.getProtocol())) { + // Conformance classes compile to a directory on the test classpath. A packaged jar + // (or any other protocol) is not expected here, and silently skipping it would + // under-report the mappings. + loadFailures.add(root + " (classpath root is not a directory this scan can walk)"); + continue; + } + Path dir = Path.of(root.toURI()); + if (!Files.isDirectory(dir)) { + continue; + } + for (String className : classNamesIn(dir, packageName)) { + Class clazz; + try { + clazz = Class.forName(className, false, classLoader); + } catch (Throwable t) { + // NoClassDefFoundError / ExceptionInInitializerError included: a suite class + // that cannot be loaded loses every mapping it declares. + loadFailures.add(className + " -> " + t); + continue; + } + if (!clazz.isAnnotationPresent(ConformanceSuite.class)) { + continue; + } + for (Method method : clazz.getDeclaredMethods()) { + ConformanceCase mapping = method.getAnnotation(ConformanceCase.class); + if (mapping != null) { + ids.add(mapping.value()); + } + } + } + } + return new SuiteScan(ids, loadFailures); + } + + /** + * Binary names of every {@code .class} file under {@code root}, walked recursively so a suite + * class placed in a subpackage is still discovered. The package is derived from the file's path + * relative to {@code root}. + */ + private static List classNamesIn(Path root, String packageName) throws Exception { + List names = new ArrayList<>(); + try (var entries = Files.walk(root)) { + for (Path entry : (Iterable) entries::iterator) { + if (!Files.isRegularFile(entry)) { + continue; + } + String fileName = entry.getFileName().toString(); + if (!fileName.endsWith(".class")) { + continue; + } + Path relative = root.relativize(entry); + StringBuilder binaryName = new StringBuilder(packageName); + for (int i = 0; i < relative.getNameCount() - 1; i++) { + binaryName.append('.').append(relative.getName(i)); + } + binaryName.append('.').append(fileName, 0, fileName.length() - ".class".length()); + names.add(binaryName.toString()); + } + } + return names; + } } diff --git a/core/src/conformance/java/ai/authplane/sdk/core/conformance/Rfc7662ConformanceTest.java b/core/src/conformance/java/ai/authplane/sdk/core/conformance/Rfc7662ConformanceTest.java index 820ea20..3a0eafd 100644 --- a/core/src/conformance/java/ai/authplane/sdk/core/conformance/Rfc7662ConformanceTest.java +++ b/core/src/conformance/java/ai/authplane/sdk/core/conformance/Rfc7662ConformanceTest.java @@ -210,8 +210,12 @@ void rfc7662_introspection_standard_fields_must_round_trip() { assertThat(response.raw()).containsEntry("jti", "unique-id-001"); } + // No @ConformanceCase: the catalog has no case for the introspection response exposing + // cnf.jkt. Its only cnf.jkt case, rfc9449-dpop-bound-token-must-contain-cnf-jkt, covers the + // verifier rejecting a DPoP-bound token that lacks the claim (mapped in + // Rfc9449ConformanceTest) — a different requirement. Kept as SDK-side coverage of RFC 9449 + // §6.2 Figure 11, reported under uncatalogued tests. @Test - @ConformanceCase("rfc9449-introspection-response-must-expose-cnf-jkt") void rfc9449_introspection_response_must_expose_cnf_jkt() { // RFC 9449 §6.2 Figure 11: cnf.jkt is a top-level member of the introspection response. // The SDK introspection result type must expose that thumbprint without forcing callers