From 0cce635a2d2da700ce85346689fe8e4f03ae4115 Mon Sep 17 00:00:00 2001 From: Gerry Campion Date: Thu, 25 Jun 2026 20:55:08 -0400 Subject: [PATCH 1/8] action to update published rule results --- .../workflows/update-published-results.yml | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 .github/workflows/update-published-results.yml diff --git a/.github/workflows/update-published-results.yml b/.github/workflows/update-published-results.yml new file mode 100644 index 000000000..eba2baa3d --- /dev/null +++ b/.github/workflows/update-published-results.yml @@ -0,0 +1,197 @@ +# ============================================================================== +# update-published-results.yml +# +# Runs the CORE engine against every Published rule's test cases, writes the +# actual results.csv files back into the repo, then opens a PR so the changes +# can be reviewed and merged. +# +# Triggers: +# - Manual (workflow_dispatch) — optionally target a specific engine branch +# and/or a subset of rules +# - Weekly schedule (Monday 00:00 UTC) +# ============================================================================== +name: Update Published Results + +on: + # TODO: remove on push before merging + push: + branches: + - automatic-pr + workflow_dispatch: + inputs: + engine_ref: + description: "Branch/tag/SHA of cdisc-rules-engine to run" + required: false + default: "main" + type: string + core_ids: + description: >- + Space-separated rule IDs to update (e.g. CORE-000001 CORE-000002). + Leave blank to update all Published rules. + required: false + default: "" + type: string + schedule: + - cron: "0 0 * * 1" + +env: + engine_ref: ${{ inputs.engine_ref || 'main' }} + +permissions: + contents: write + pull-requests: write + +jobs: + update-results: + runs-on: ubuntu-latest + + steps: + # ----------------------------------------------------------------------- + # 1. Checkout this repo (cdisc-open-rules) — full history for the PR step + # ----------------------------------------------------------------------- + - name: Checkout cdisc-open-rules + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # ----------------------------------------------------------------------- + # 2. Checkout the engine into ./engine/ + # ----------------------------------------------------------------------- + - name: Checkout cdisc-rules-engine + uses: actions/checkout@v4 + with: + repository: cdisc-org/cdisc-rules-engine + ref: ${{ env.engine_ref }} + path: engine + + # ----------------------------------------------------------------------- + # 3. Python + dependencies + # ----------------------------------------------------------------------- + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install engine dependencies + run: | + python -m venv venv + venv/bin/pip install --upgrade pip + cd engine + ../venv/bin/pip install . --group dev + + - name: Install script dependencies + run: venv/bin/pip install tabulate jmespath pyyaml + + # ----------------------------------------------------------------------- + # 4. Run the engine for every Published rule, writing actual results.csv + # directly into each case's results/ directory (overwriting the old baseline) + # ----------------------------------------------------------------------- + - name: Run engine and write results + id: run_engine + continue-on-error: true + run: | + chmod +x .github/scripts/run_validation.sh + + CORE_IDS_ARG="" + if [ -n "${{ inputs.core_ids }}" ]; then + CORE_IDS_ARG="--core-ids ${{ inputs.core_ids }}" + fi + + CORE_IDS_ARG="--core-ids CORE-000007" + + ENGINE_DIR_OVERRIDE="$(pwd)/engine" \ + venv/bin/python engine/scripts/validate_published_rules.py \ + --rules-root "$(pwd)" \ + --engine-dir "$(pwd)/engine" \ + --python-cmd "$(pwd)/venv/bin/python" \ + --output-dir "$(pwd)" \ + $CORE_IDS_ARG + + # ----------------------------------------------------------------------- + # 5. Write summary to Job Summary + # ----------------------------------------------------------------------- + - name: Write summary to workflow summary + if: always() + run: | + [ -f summary_table.md ] && cat summary_table.md >> $GITHUB_STEP_SUMMARY || true + + # ----------------------------------------------------------------------- + # 6. Upload reports as artifacts (regardless of outcome) + # ----------------------------------------------------------------------- + - name: Upload validation artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: update-results-${{ github.run_id }} + path: | + Published/**/results/results.csv + summary_table.md + detail_report.md + if-no-files-found: warn + + # ----------------------------------------------------------------------- + # 7. Determine if any results.csv files changed + # ----------------------------------------------------------------------- + - name: Check for changed results + id: check_changes + run: | + git add Published/**/results/results.csv 2>/dev/null || true + if git diff --cached --quiet; then + echo "changed=false" >> $GITHUB_OUTPUT + echo "No results.csv files changed — no PR needed." + else + echo "changed=true" >> $GITHUB_OUTPUT + CHANGED_COUNT=$(git diff --cached --name-only | grep 'results\.csv' | wc -l | tr -d ' ') + echo "changed_count=$CHANGED_COUNT" >> $GITHUB_OUTPUT + echo "$CHANGED_COUNT results.csv file(s) changed." + fi + git reset HEAD + + # ----------------------------------------------------------------------- + # 8. Create PR with updated results + # ----------------------------------------------------------------------- + - name: Create Pull Request + id: create_pr + if: steps.check_changes.outputs.changed == 'true' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + add-paths: Published/**/results/results.csv + commit-message: "ci: update Published results.csv baselines (engine@${{ env.engine_ref }})" + branch: update-results-${{ env.engine_ref }} + delete-branch: true + title: "Update Published results.csv baselines (engine@${{ env.engine_ref }})" + draft: ${{ env.engine_ref != 'main' }} + reviewers: ${{ vars.REVIEWERS }} + body: | + This PR updates the `results.csv` baseline files for Published rules + using the latest engine output. + + | | | + |---|---| + | **Engine ref** | [`${{ env.engine_ref }}`](https://github.com/cdisc-org/cdisc-rules-engine/tree/${{ env.engine_ref }}) | + | **Rules updated** | ${{ steps.check_changes.outputs.changed_count }} result file(s) changed | + | **Triggered by** | ${{ github.event_name == 'schedule' && 'Scheduled run (weekly)' || format('Manual run by @{0}', github.actor) }} | + | **Workflow run** | [#${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | + + ### What to review + - Each changed `results.csv` is the raw engine output for that test case. + - The summary table is available in the [workflow run artifacts](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). + - If results changed unexpectedly, check the `detail_report.md` artifact + or rerun the workflow with a specific `engine_ref` to bisect. + + # ----------------------------------------------------------------------- + # 9. Annotate summary with PR link + # ----------------------------------------------------------------------- + - name: Annotate summary with PR link + if: steps.create_pr.outcome == 'success' + run: | + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Pull Request" >> $GITHUB_STEP_SUMMARY + echo "[📝 PR #${{ steps.create_pr.outputs.pull-request-number }}](${{ steps.create_pr.outputs.pull-request-url }})" >> $GITHUB_STEP_SUMMARY + + - name: No changes message + if: steps.check_changes.outputs.changed == 'false' + run: | + echo "" >> $GITHUB_STEP_SUMMARY + echo "✅ All results.csv files are already up to date — no PR created." >> $GITHUB_STEP_SUMMARY From 62e4c19b7e40dc0c55f64d7265ae5007afda9686 Mon Sep 17 00:00:00 2001 From: Gerry Campion Date: Thu, 25 Jun 2026 22:49:30 -0400 Subject: [PATCH 2/8] fix --- .../workflows/update-published-results.yml | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/update-published-results.yml b/.github/workflows/update-published-results.yml index eba2baa3d..224ccc3ae 100644 --- a/.github/workflows/update-published-results.yml +++ b/.github/workflows/update-published-results.yml @@ -108,7 +108,20 @@ jobs: $CORE_IDS_ARG # ----------------------------------------------------------------------- - # 5. Write summary to Job Summary + # 5. Promote actual.csv → results.csv for all cases that ran + # run_validation.sh restores the original baseline after diffing; + # actual.csv holds the true engine output we want to commit. + # ----------------------------------------------------------------------- + - name: Promote actual results to baseline + run: | + find Published -path "*/results/actual.csv" | while IFS= read -r actual; do + dest="$(dirname "$actual")/results.csv" + cp "$actual" "$dest" + echo "Updated: $dest" + done + + # ----------------------------------------------------------------------- + # 6. Write summary to Job Summary # ----------------------------------------------------------------------- - name: Write summary to workflow summary if: always() @@ -116,7 +129,7 @@ jobs: [ -f summary_table.md ] && cat summary_table.md >> $GITHUB_STEP_SUMMARY || true # ----------------------------------------------------------------------- - # 6. Upload reports as artifacts (regardless of outcome) + # 7. Upload reports as artifacts (regardless of outcome) # ----------------------------------------------------------------------- - name: Upload validation artifacts if: always() @@ -130,7 +143,7 @@ jobs: if-no-files-found: warn # ----------------------------------------------------------------------- - # 7. Determine if any results.csv files changed + # 8. Determine if any results.csv files changed # ----------------------------------------------------------------------- - name: Check for changed results id: check_changes @@ -148,7 +161,7 @@ jobs: git reset HEAD # ----------------------------------------------------------------------- - # 8. Create PR with updated results + # 9. Create PR with updated results # ----------------------------------------------------------------------- - name: Create Pull Request id: create_pr @@ -181,7 +194,7 @@ jobs: or rerun the workflow with a specific `engine_ref` to bisect. # ----------------------------------------------------------------------- - # 9. Annotate summary with PR link + # 10. Annotate summary with PR link # ----------------------------------------------------------------------- - name: Annotate summary with PR link if: steps.create_pr.outcome == 'success' From 2e96be7fa910df8ef70d228b32b368d810e1a170 Mon Sep 17 00:00:00 2001 From: Gerry Campion Date: Sat, 27 Jun 2026 00:42:16 -0400 Subject: [PATCH 3/8] force lf for csv --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..2c695418b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Explicitly force LF for files +*.csv text eol=lf From cf8c49827d7a530e03c4aad6ba9f52c096002c3a Mon Sep 17 00:00:00 2001 From: Gerry Campion Date: Sat, 27 Jun 2026 01:07:37 -0400 Subject: [PATCH 4/8] make it draft --- .github/workflows/update-published-results.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-published-results.yml b/.github/workflows/update-published-results.yml index 224ccc3ae..393abef8f 100644 --- a/.github/workflows/update-published-results.yml +++ b/.github/workflows/update-published-results.yml @@ -174,7 +174,7 @@ jobs: branch: update-results-${{ env.engine_ref }} delete-branch: true title: "Update Published results.csv baselines (engine@${{ env.engine_ref }})" - draft: ${{ env.engine_ref != 'main' }} + draft: ${{ env.engine_ref != 'main' || github.ref != format('refs/heads/{0}', github.event.repository.default_branch) }} reviewers: ${{ vars.REVIEWERS }} body: | This PR updates the `results.csv` baseline files for Published rules From cc058c1ce11310d0683ea7b9515304cfe1ab3d9f Mon Sep 17 00:00:00 2001 From: Gerry Campion Date: Sat, 27 Jun 2026 01:12:20 -0400 Subject: [PATCH 5/8] run full test --- .github/workflows/update-published-results.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/update-published-results.yml b/.github/workflows/update-published-results.yml index 393abef8f..65d46397d 100644 --- a/.github/workflows/update-published-results.yml +++ b/.github/workflows/update-published-results.yml @@ -97,8 +97,6 @@ jobs: CORE_IDS_ARG="--core-ids ${{ inputs.core_ids }}" fi - CORE_IDS_ARG="--core-ids CORE-000007" - ENGINE_DIR_OVERRIDE="$(pwd)/engine" \ venv/bin/python engine/scripts/validate_published_rules.py \ --rules-root "$(pwd)" \ From eb657a1b764927a2d95b7257af4264e83559136e Mon Sep 17 00:00:00 2001 From: Gerry Campion Date: Sat, 27 Jun 2026 12:31:05 -0400 Subject: [PATCH 6/8] move pr link --- .github/workflows/update-published-results.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/update-published-results.yml b/.github/workflows/update-published-results.yml index 65d46397d..4483df7e5 100644 --- a/.github/workflows/update-published-results.yml +++ b/.github/workflows/update-published-results.yml @@ -13,10 +13,6 @@ name: Update Published Results on: - # TODO: remove on push before merging - push: - branches: - - automatic-pr workflow_dispatch: inputs: engine_ref: @@ -181,6 +177,7 @@ jobs: | | | |---|---| | **Engine ref** | [`${{ env.engine_ref }}`](https://github.com/cdisc-org/cdisc-rules-engine/tree/${{ env.engine_ref }}) | + | **Open Rules ref** | [`${{ github.sha }}`](https://github.com/${{ github.repository }}/tree/${{ github.sha }}) | | **Rules updated** | ${{ steps.check_changes.outputs.changed_count }} result file(s) changed | | **Triggered by** | ${{ github.event_name == 'schedule' && 'Scheduled run (weekly)' || format('Manual run by @{0}', github.actor) }} | | **Workflow run** | [#${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | @@ -192,17 +189,16 @@ jobs: or rerun the workflow with a specific `engine_ref` to bisect. # ----------------------------------------------------------------------- - # 10. Annotate summary with PR link + # 10. Annotate summary with PR link (prepended before the summary table) # ----------------------------------------------------------------------- - name: Annotate summary with PR link if: steps.create_pr.outcome == 'success' run: | - echo "" >> $GITHUB_STEP_SUMMARY - echo "## Pull Request" >> $GITHUB_STEP_SUMMARY - echo "[📝 PR #${{ steps.create_pr.outputs.pull-request-number }}](${{ steps.create_pr.outputs.pull-request-url }})" >> $GITHUB_STEP_SUMMARY + PR_HEADER="## Pull Request\n[📝 PR #${{ steps.create_pr.outputs.pull-request-number }}](${{ steps.create_pr.outputs.pull-request-url }})\n\n" + EXISTING=$(cat "$GITHUB_STEP_SUMMARY" 2>/dev/null || true) + printf "%b%s" "$PR_HEADER" "$EXISTING" > "$GITHUB_STEP_SUMMARY" - name: No changes message if: steps.check_changes.outputs.changed == 'false' run: | - echo "" >> $GITHUB_STEP_SUMMARY echo "✅ All results.csv files are already up to date — no PR created." >> $GITHUB_STEP_SUMMARY From 7ffad2a8411ff79b0a153329ce4d3206e4f0a23a Mon Sep 17 00:00:00 2001 From: Gerry Campion Date: Sat, 27 Jun 2026 18:14:36 -0400 Subject: [PATCH 7/8] pin to specific version --- .github/workflows/update-published-results.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-published-results.yml b/.github/workflows/update-published-results.yml index 4483df7e5..c29cfe664 100644 --- a/.github/workflows/update-published-results.yml +++ b/.github/workflows/update-published-results.yml @@ -160,7 +160,7 @@ jobs: - name: Create Pull Request id: create_pr if: steps.check_changes.outputs.changed == 'true' - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v8.1.1 with: token: ${{ secrets.GITHUB_TOKEN }} add-paths: Published/**/results/results.csv From c66fe23bbd3bd25af04a6e042bc311a541d58cd9 Mon Sep 17 00:00:00 2001 From: Gerry Campion Date: Sat, 27 Jun 2026 18:18:14 -0400 Subject: [PATCH 8/8] pin to full hash --- .github/workflows/update-published-results.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-published-results.yml b/.github/workflows/update-published-results.yml index c29cfe664..a0e667661 100644 --- a/.github/workflows/update-published-results.yml +++ b/.github/workflows/update-published-results.yml @@ -160,7 +160,7 @@ jobs: - name: Create Pull Request id: create_pr if: steps.check_changes.outputs.changed == 'true' - uses: peter-evans/create-pull-request@v8.1.1 + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 #v8.1.1 with: token: ${{ secrets.GITHUB_TOKEN }} add-paths: Published/**/results/results.csv