diff --git a/.github/scripts/convert_results.py b/.github/scripts/convert_results.py deleted file mode 100644 index 117e2251b..000000000 --- a/.github/scripts/convert_results.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -convert_results.py — converts a CORE engine JSON results file to results.csv. - -Non-USDM output (Issue_Details has dataset/row/variables/values): - CSV columns: Dataset, Record, Variable, Value - One row per variable/value pair per issue. - -USDM output (Issue_Details has path/attributes/values): - CSV columns: Path, Attribute, Value - One row per attribute/value pair per issue. - -Usage: - python convert_results.py - -Exit codes: - 0 — success - 1 — error -""" -import csv -import json -import sys - - -def detect_standard(data: dict) -> str: - return data.get("Conformance_Details", {}).get("Standard", "").upper() - - -def convert_nonusdm(issue_details: list) -> tuple[list[str], list[tuple]]: - header = ["Dataset", "Record", "Variable", "Value"] - rows = [] - for issue in issue_details: - dataset = issue.get("dataset", "").removesuffix(".csv") - record = str(issue.get("row", "")) - variables = issue.get("variables") or [] - values = issue.get("values") or [] - for variable, value in zip(variables, values): - rows.append((dataset, record, variable, str(value))) - return header, rows - - -def convert_usdm(issue_details: list) -> tuple[list[str], list[tuple]]: - header = ["path", "attribute", "value"] - rows = [] - for issue in issue_details: - path = issue.get("path") or "" - attributes = issue.get("attributes") or [] - values = issue.get("values") or [] - # attributes/values may be a plain string on error-type issues - if isinstance(attributes, str): - attributes = [attributes] - if isinstance(values, str): - values = [values] - for attribute, value in zip(attributes, values): - rows.append((path, attribute, str(value))) - return header, rows - - -def convert(json_path: str, csv_path: str) -> None: - with open(json_path) as f: - data = json.load(f) - - standard = detect_standard(data) - issue_details = data.get("Issue_Details", []) - - if standard == "USDM": - header, rows = convert_usdm(issue_details) - else: - header, rows = convert_nonusdm(issue_details) - - with open(csv_path, "w", newline="") as f: - writer = csv.writer(f) - writer.writerow(header) - writer.writerows(rows) - - print(f"Wrote {len(rows)} rows to {csv_path}") - - -def main(): - if len(sys.argv) != 3: - print(f"Usage: {sys.argv[0]} ", file=sys.stderr) - sys.exit(1) - - json_path, csv_path = sys.argv[1], sys.argv[2] - - try: - convert(json_path, csv_path) - except Exception as e: - print(f"ERROR: {e}", file=sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/.github/scripts/diff_results.py b/.github/scripts/diff_results.py index 762c092f1..debbf52fb 100644 --- a/.github/scripts/diff_results.py +++ b/.github/scripts/diff_results.py @@ -1,14 +1,15 @@ """ -diff_results.py — compares a committed results.csv against a freshly generated one. +diff_results.py — compares an actual results.csv against an expected one. Usage: - python .github/scripts/diff_results.py + python .github/scripts/diff_results.py Exit codes: 0 — results match 1 — results differ (failure) 2 — error """ + import csv import sys from itertools import zip_longest @@ -22,24 +23,26 @@ def load(path: str) -> list[tuple]: return sorted(rows) -def diff(committed_path: str, generated_path: str) -> list[str]: +def diff(expected_path: str, actual_path: str) -> list[str]: diffs = [] - c_rows = load(committed_path) - g_rows = load(generated_path) + c_rows = load(expected_path) + g_rows = load(actual_path) if len(c_rows) != len(g_rows): diffs.append( - f"Row count changed: {len(c_rows)} committed -> {len(g_rows)} generated" + f"Row count changed: {len(c_rows)} expected -> {len(g_rows)} actual" ) _ABSENT = object() - for i, (c_row, g_row) in enumerate(zip_longest(c_rows, g_rows, fillvalue=_ABSENT), start=1): + for i, (c_row, g_row) in enumerate( + zip_longest(c_rows, g_rows, fillvalue=_ABSENT), start=1 + ): if c_row is _ABSENT: - diffs.append(f" Row {i}: present in generated only -> {g_row}") + diffs.append(f" Row {i}: present in actual only -> {g_row}") elif g_row is _ABSENT: - diffs.append(f" Row {i}: present in committed only -> {c_row}") + diffs.append(f" Row {i}: present in expected only -> {c_row}") elif c_row != g_row: - diffs.append(f" Row {i}: committed={c_row} -> generated={g_row}") + diffs.append(f" Row {i}: expected={c_row} -> actual={g_row}") return diffs @@ -47,15 +50,15 @@ def diff(committed_path: str, generated_path: str) -> list[str]: def main(): if len(sys.argv) != 4: print( - f"Usage: {sys.argv[0]} ", + f"Usage: {sys.argv[0]} ", file=sys.stderr, ) sys.exit(2) - committed_path, generated_path, case_label = sys.argv[1], sys.argv[2], sys.argv[3] + expected_path, actual_path, case_label = sys.argv[1], sys.argv[2], sys.argv[3] try: - diffs = diff(committed_path, generated_path) + diffs = diff(expected_path, actual_path) except Exception as e: print(f"ERROR comparing results for {case_label}: {e}", file=sys.stderr) sys.exit(2) diff --git a/.github/scripts/run_validation.sh b/.github/scripts/run_validation.sh index a9644b403..74df066be 100644 --- a/.github/scripts/run_validation.sh +++ b/.github/scripts/run_validation.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash # run_validation.sh — iterates all positive/ and negative/ test cases for a rule, # runs the CORE engine against each, converts JSON output to results.csv, -# diffs against any committed results.csv, and writes a markdown report. +# diffs against any expected results.csv, and writes two outputs: +# - $REPO_ROOT/validation_report.md (detailed markdown, legacy/fallback) +# - $REPO_ROOT/case_results.jsonl (one JSON line per test case for the summary table) # # Usage: # bash .github/scripts/run_validation.sh @@ -18,9 +20,11 @@ REPO_ROOT="${3:?repo_root required}" RULE_ID=$(basename "$RULE_REL_PATH") RULE_DIR="$REPO_ROOT/$RULE_REL_PATH" -ENGINE_DIR="$REPO_ROOT/engine" +# Allow caller to override engine location (e.g. when called from rules-engine repo) +ENGINE_DIR="${ENGINE_DIR_OVERRIDE:-$REPO_ROOT/engine}" SCRIPTS_DIR="$REPO_ROOT/.github/scripts" REPORT_FILE="$REPO_ROOT/validation_report.md" +JSONL_FILE="$REPO_ROOT/case_results.jsonl" # --------------------------------------------------------------------------- # Locate the rule YAML @@ -48,6 +52,37 @@ TOTAL_CASES=0 PASSED_CASES=0 FAILED_CASES=0 +# --------------------------------------------------------------------------- +# Helper: append one JSON line to case_results.jsonl +# Passes all values via env vars to avoid shell-quoting issues with paths. +# Args: exec(true|false) expected actual match(true|false) diff_path stderr_path +# --------------------------------------------------------------------------- +emit_result() { + R_RULE="$RULE_ID" \ + R_TYPE="$TEST_TYPE" \ + R_NUM="$CASE_ID" \ + R_EXEC="$1" \ + R_EXPECTED="$2" \ + R_ACTUAL="$3" \ + R_MATCH="$4" \ + R_DIFF="$5" \ + R_STDERR="$6" \ + python -c " +import json, os +e = os.environ +print(json.dumps({ + 'rule': e['R_RULE'], + 'type': e['R_TYPE'], + 'num': e['R_NUM'], + 'exec': e['R_EXEC'] == 'true', + 'expected': e['R_EXPECTED'], + 'actual': e['R_ACTUAL'], + 'match': e['R_MATCH'] == 'true', + 'diff': e['R_DIFF'], + 'stderr': e['R_STDERR'], +}))" >> "$JSONL_FILE" +} + # --------------------------------------------------------------------------- # Iterate test types and cases # --------------------------------------------------------------------------- @@ -55,20 +90,22 @@ for TEST_TYPE in positive negative; do TYPE_DIR="$RULE_DIR/$TEST_TYPE" [ -d "$TYPE_DIR" ] || continue - echo "" >> "$REPORT_FILE" - echo "## $TEST_TYPE" >> "$REPORT_FILE" - echo "" >> "$REPORT_FILE" + { + echo "" + echo "## $TEST_TYPE" + echo "" + } >> "$REPORT_FILE" - for CASE_DIR in $(find "$TYPE_DIR" -mindepth 1 -maxdepth 1 -type d | sort); do + while IFS= read -r -d '' CASE_DIR; do CASE_ID=$(basename "$CASE_DIR") DATA_DIR="$CASE_DIR/data" RESULTS_DIR="$CASE_DIR/results" CASE_LABEL="$TEST_TYPE/$CASE_ID" - TOTAL_CASES=$((TOTAL_CASES + 1)) echo "" echo "--- Processing $RULE_ID / $CASE_LABEL ---" + # -- Skip cases that are structurally incomplete (no jsonl entry emitted) if [ ! -d "$DATA_DIR" ]; then echo "::warning::Missing data/ directory for $CASE_LABEL — skipping" echo "### \`$CASE_LABEL\` — ⚠️ Skipped (no data/ directory)" >> "$REPORT_FILE" @@ -86,41 +123,43 @@ for TEST_TYPE in positive negative; do continue fi echo " .env: $ENV_FILE" + + TOTAL_CASES=$((TOTAL_CASES + 1)) + + # -- Check for missing expected baseline (engine still runs) + MISSING_BASELINE=false if [ ! -f "$RESULTS_DIR/results.csv" ]; then - echo " ERROR: no committed results.csv found for $CASE_LABEL" - { - echo "### \`$CASE_LABEL\` — ❌ Missing results.csv" - echo "" - echo "No \`results.csv\` was found for this test case. Run the rule locally before opening a PR and commit the generated \`results.csv\`." - echo "" - } >> "$REPORT_FILE" - FAILED_CASES=$((FAILED_CASES + 1)) - OVERALL_SUCCESS=false - continue + echo " WARNING: no expected results.csv found for $CASE_LABEL — engine will still run" + MISSING_BASELINE=true + fi + + # Back up expected results.csv before the engine run (only if it exists) + EXPECTED_RESULTS="" + if [ "$MISSING_BASELINE" = false ]; then + cp "$RESULTS_DIR/results.csv" "$RESULTS_DIR/results.expected.csv" + EXPECTED_RESULTS="$RESULTS_DIR/results.expected.csv" fi ENGINE_ARGS=( "-lr" "$RULE_YML" "-d" "$DATA_DIR" "-dep" "$ENV_FILE" - "-of" "JSON" + "-of" "CSV" "-o" "$RESULTS_DIR/results" "-p" "disabled" ) echo " Command: python core.py validate ${ENGINE_ARGS[*]}" - # Back up committed results.csv before the engine run - cp "$RESULTS_DIR/results.csv" "$RESULTS_DIR/results.csv.committed" - COMMITTED_RESULTS="$RESULTS_DIR/results.csv.committed" - # Run the engine ENGINE_LOG="/tmp/engine_${TEST_TYPE}_${CASE_ID}.txt" ENGINE_EXIT=0 (cd "$ENGINE_DIR" && $PYTHON_CMD core.py validate "${ENGINE_ARGS[@]}") \ 2>&1 | tee "$ENGINE_LOG" || ENGINE_EXIT=${PIPESTATUS[0]} - if [ $ENGINE_EXIT -ne 0 ] || [ ! -f "$RESULTS_DIR/results.json" ]; then + ACTUAL_CSV="$RESULTS_DIR/results.csv" + + if [ $ENGINE_EXIT -ne 0 ] || [ ! -f "$ACTUAL_CSV" ]; then echo " ERROR: engine failed or produced no output (exit $ENGINE_EXIT)" { echo "### \`$CASE_LABEL\` — ❌ Engine error" @@ -133,55 +172,51 @@ for TEST_TYPE in positive negative; do echo "" echo "" } >> "$REPORT_FILE" + emit_result "false" "" "" "false" "" "$ENGINE_LOG" FAILED_CASES=$((FAILED_CASES + 1)) OVERALL_SUCCESS=false - mv "$COMMITTED_RESULTS" "$RESULTS_DIR/results.csv" + [ "$MISSING_BASELINE" = false ] && mv "$EXPECTED_RESULTS" "$RESULTS_DIR/results.csv" continue fi - # Convert engine JSON output to a temporary CSV for comparison - GENERATED_CSV="/tmp/results_generated_${TEST_TYPE}_${CASE_ID}.csv" - CONVERT_EXIT=0 - $PYTHON_CMD "$SCRIPTS_DIR/convert_results.py" \ - "$RESULTS_DIR/results.json" "$GENERATED_CSV" \ - 2>&1 | tee -a "$ENGINE_LOG" || CONVERT_EXIT=$? + ACTUAL_COUNT=$(( $(wc -l < "$ACTUAL_CSV") - 1 )) - if [ $CONVERT_EXIT -ne 0 ]; then - echo " ERROR: failed to convert results.json to results.csv" + # -- Missing baseline: report actual count, no diff + if [ "$MISSING_BASELINE" = true ]; then + echo " FAILED — no expected results.csv baseline exists" { - echo "### \`$CASE_LABEL\` — ❌ Conversion error" + echo "### \`$CASE_LABEL\` — ❌ Missing expected results.csv" echo "" - echo "
Conversion output" - echo "" - echo '```' - cat "$ENGINE_LOG" - echo '```' - echo "
" + echo "No expected \`results.csv\` was found for this test case." echo "" } >> "$REPORT_FILE" + emit_result "true" "" "$ACTUAL_COUNT" "false" "" "" FAILED_CASES=$((FAILED_CASES + 1)) OVERALL_SUCCESS=false - mv "$COMMITTED_RESULTS" "$RESULTS_DIR/results.csv" continue fi + # -- Diff + EXPECTED_COUNT=$(( $(wc -l < "$EXPECTED_RESULTS") - 1 )) + DIFF_LOG="/tmp/diff_${TEST_TYPE}_${CASE_ID}.txt" DIFF_EXIT=0 $PYTHON_CMD "$SCRIPTS_DIR/diff_results.py" \ - "$COMMITTED_RESULTS" "$GENERATED_CSV" "$CASE_LABEL" \ + "$EXPECTED_RESULTS" "$ACTUAL_CSV" "$CASE_LABEL" \ > "$DIFF_LOG" 2>&1 || DIFF_EXIT=$? if [ $DIFF_EXIT -eq 0 ]; then - echo " PASSED — results match committed baseline" + echo " PASSED — actual results match expected baseline" { - echo "### \`$CASE_LABEL\` — ✅ Results match committed baseline" + echo "### \`$CASE_LABEL\` — ✅ Actual results match expected baseline" echo "" } >> "$REPORT_FILE" + emit_result "true" "$EXPECTED_COUNT" "$ACTUAL_COUNT" "true" "" "" PASSED_CASES=$((PASSED_CASES + 1)) else - echo " FAILED — committed results do not match engine output" + echo " FAILED — expected results do not match actual engine output" { - echo "### \`$CASE_LABEL\` — ❌ Results do not match engine output" + echo "### \`$CASE_LABEL\` — ❌ Expected results do not match actual engine output" echo "" echo "
Diff details" echo "" @@ -191,24 +226,15 @@ for TEST_TYPE in positive negative; do echo "
" echo "" } >> "$REPORT_FILE" + emit_result "true" "$EXPECTED_COUNT" "$ACTUAL_COUNT" "false" "$DIFF_LOG" "" FAILED_CASES=$((FAILED_CASES + 1)) OVERALL_SUCCESS=false fi - mv "$COMMITTED_RESULTS" "$RESULTS_DIR/results.csv" - if [ -s "$ENGINE_LOG" ]; then - { - echo "
Engine output for \`$CASE_LABEL\`" - echo "" - echo '```' - cat "$ENGINE_LOG" - echo '```' - echo "
" - echo "" - } >> "$REPORT_FILE" - fi - done # cases -done # test types + mv "$EXPECTED_RESULTS" "$RESULTS_DIR/results.csv" + + done < <(find "$TYPE_DIR" -mindepth 1 -maxdepth 1 -type d -print0 | sort -z) +done # --------------------------------------------------------------------------- # Summary @@ -224,4 +250,4 @@ done # test types if [ "$OVERALL_SUCCESS" = false ]; then exit 1 -fi \ No newline at end of file +fi diff --git a/README.md b/README.md index 339278d96..52fbd4df9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # cdisc-open-rules + Contains rules and test data that can be processed by CORE **BEFORE CONTRIBUTING, MAKE SURE YOU HAVE GONE THROUGH THE CDISC VOLUNTEERING ONBOARDING PROCESS** \ @@ -6,60 +7,64 @@ https://www.cdisc.org/volunteer **SUPPLEMENTARY GUIDE** \ Instructions below will guide you step-by-step through the: + - First-time Local Setup Steps - Rule Authoring and Test Data Creation Process # First-time Local Setup Steps -***IMPORTANT NOTE*** \ +**_IMPORTANT NOTE_** \ _You may need your IT support team to install some of the following software for you. In particular, the setup script requires python3.12 to run properly. If you don't have it installed, the script will attempt to install it for you, but this is likely to be blocked by your company settings. If so, you will need to contact IT._ **Follow steps 1 - 9 carefully.** -1) Create a free GitHub account: https://github.com/signup -2) Install Git, following the instructions here: https://git-scm.com/install +1. Create a free GitHub account: https://github.com/signup +2. Install Git, following the instructions here: https://git-scm.com/install - When prompted, ensure you check the **"Add to PATH"** option (or select **"Git from the command line and also from 3rd-party software"**) - Keep all other default settings throughout the installer - You DO NOT need to actually run Git as a program, so close any pop-ups that appear after the installation -3) Install VSCode (***not*** VSCodeUser), following the instructions here: https://code.visualstudio.com/download -4) Open VSCode and a terminal within it: +3. Install VSCode (**_not_** VSCodeUser), following the instructions here: https://code.visualstudio.com/download +4. Open VSCode and a terminal within it: - Top Menu → Terminal → New Terminal (check the three dots in the top menu if you don't see 'Terminal') -5) Open VSCode and a terminal within it: +5. Open VSCode and a terminal within it: - Top Menu → Terminal → New Terminal (check the three dots in the top menu if you don't see 'Terminal') - Configure your Git identity by running the following two commands in the terminal, replacing the placeholder values with your own: - `git config --global user.name "Your Name"` + `git config --global user.name "Your Name"` - `git config --global user.email "you@example.com"` + `git config --global user.email "you@example.com"` Use the same email address you used when creating your GitHub account in step 1. You only need to do this once per machine. -6) Create a new empty directory on your machine for storing the repository and subsequent rule authoring and editing. Navigate to it in the terminal using `cd` commands. Avoid OneDrive if possible. + +6. Create a new empty directory on your machine for storing the repository and subsequent rule authoring and editing. Navigate to it in the terminal using `cd` commands. Avoid OneDrive if possible. - There is sometimes an AI 'helper' box popup in the terminal - make sure you are typing commands into the command line itself, not the box - If any of the folder names you are navigating through have spaces (eg 'My Folder'), you will need to wrap the path in quotes,\ eg: `cd "C:\Users\sam\Documents\Rules Folder"` -7) Clone this repo into that directory by running the following command (**DO NOT RUN MORE THAN ONCE**): \ - `git clone --recurse-submodules https://github.com/cdisc-org/cdisc-open-rules.git` +7. Clone this repo into that directory by running the following command (**DO NOT RUN MORE THAN ONCE**): \ + `git clone --recurse-submodules https://github.com/cdisc-org/cdisc-open-rules.git` + > **NOTE:** If you encounter `The term 'git' is not recognized as the name of a cmdlet, function, script file, or operable program.`, Git's installation directory has not been added to your system PATH. See [this StackOverflow answer](https://stackoverflow.com/questions/4492979/error-git-is-not-recognized-as-an-internal-or-external-command) for instructions on manually adding Git to your PATH. _***IMPORTANT NOTE***\ Unless something goes badly wrong and you need to fully delete the entire directory, you should never need to run this command again._ -8) In VSCode, select "Open Folder" and select the repository folder you just cloned - it should be called `cdisc-open-rules` +8. In VSCode, select "Open Folder" and select the repository folder you just cloned - it should be called `cdisc-open-rules` - VSCode may show a prompt asking if you trust the authors of the files in this folder — click **Yes, I trust the authors** - You should also see a prompt to install the workspace recommended extensions in the bottom right corner — click **Install All**. If you miss this prompt, don't worry — step 10 will cover it -9) This should re-open a new terminal in the repository folder. If this doesn't happen, open a new terminal in VSCode and navigate to the repository folder again. +9. This should re-open a new terminal in the repository folder. If this doesn't happen, open a new terminal in VSCode and navigate to the repository folder again. -10) You will need to setup the python environment, which will take a little bit of time. - - Assuming you are in the cdisc-open-rules folder in the VSCode terminal, run one of the following depending on your operating system (ignore messages and warnings): - - WINDOWS: `.\setup\windows_setup.bat` - - MAC: `./setup/bash_setup.sh` - - Windows might prompt you asking if you want to install python - the answer is yes! +10. You will need to setup the python environment, which will take a little bit of time. - _***IMPORTANT NOTE***\ - If you start the setup script and stop it midway through, you may get some strange errors when you try to run rules in the future. If you have any doubts, rerun the setup script, and make sure it completes._ +- Assuming you are in the cdisc-open-rules folder in the VSCode terminal, run one of the following depending on your operating system (ignore messages and warnings): + - WINDOWS: `.\setup\windows_setup.bat` + - MAC: `./setup/bash_setup.sh` +- Windows might prompt you asking if you want to install python - the answer is yes! + +_***IMPORTANT NOTE***\ + If you start the setup script and stop it midway through, you may get some strange errors when you try to run rules in the future. If you have any doubts, rerun the setup script, and make sure it completes._ -11) Set up the rule authoring auto-completion and real-time schema validation: +11. Set up the rule authoring auto-completion and real-time schema validation: - When you opened the repository folder in step 7, VSCode should have shown a prompt to install the workspace recommended extensions — click **Install All** if you haven't already - If you missed the prompt, go to Extensions on the left sidebar, search for `@recommended` in the Extensions search bar and install them from there - That's it! Schema validation and CSV highlighting will be active automatically once the extensions are installed. If you don't see this behaviour after a few seconds, try restarting VSCode @@ -75,31 +80,31 @@ _In the following section, the exact process to follow with relevant Git command **Create a Local Branch.** -1) Make sure you are on the main branch and that both the main branch and the engine submodule are up to date. To do this, run the following three commands: \ - `git checkout main` \ - `git pull origin main` \ - `git submodule update --recursive` +1. Make sure you are on the main branch and that both the main branch and the engine submodule are up to date. To do this, run the following three commands: \ + `git checkout main` \ + `git pull origin main` \ + `git submodule update --recursive` -2) Create a new branch to work on your changes, named as such: `//` (eg `richard/CORE-000001/edit`): \ - `git branch ` +2. Create a new branch to work on your changes, named as such: `//` (eg `richard/CORE-000001/edit`): \ + `git branch ` **_*IMPORTANT NOTE*_**\ _Whenever you create a local branch to work on a rule, ensure that you are on the main branch. If you create a new local branch, when you are already on a local branch, the new branch will branch off the local branch and not from main. If you would then want to merge changes from your new local branch, it will merge with the first local branch and not with the main branch. Therefore, ensure to be on the main branch first prior to creating a local branch (git checkout main). Once the local branch exists, you can checkout out to it from any branch._ -3) Switch to your new branch: \ - `git checkout ` +3. Switch to your new branch: \ + `git checkout ` **Set up Rule Folder.** **_*IMPORTANT NOTE*_**\ _Step 4 is only applicable in case you want to create a rule for which the folder does not exist yet in the GitHub repository. It is therefore important to first check if a folder is already present. If no folder is present, you can automatically generate the required folder structure for a new rule including a blank YAML template and template files for the test data._ -4) Initialize your new rule folder structure: +4. Initialize your new rule folder structure: - In the base directory of the project, activate the virtual environment by running: - WINDOWS: `venv\Scripts\activate` - MAC: `source venv/bin/activate` - Then run the following command in your terminal: \ - `python new-rule.py` + `python new-rule.py` - It will prompt you a few times. - If the `NEW-RULE` folder already exists, it will check you definitely want to make a new one. (NOTE: If the empty folder is a leftover from a previous branch, which is likely, you SHOULD run the script and overwrite the folder to make a new one, as this will set up the template properly for you). - You will also be prompted to enter the number of positive and negative test cases you want to create. Don't worry if you realise you need more later — you can easily add more manually. @@ -140,19 +145,20 @@ Unpublished/ **Edit the Rule.** -5) Open `Unpublished/NEW-RULE/rule.yml` and edit the rule definition as desired. +5. Open `Unpublished/NEW-RULE/rule.yml` and edit the rule definition as desired. - Ensure that you save any changes (File → Save, or Ctrl/Cmd + S) **Create Test Data.** > **NOTE:** The workspace recommended extensions installed in step 10 include 'Excel Viewer' and 'Rainbow CSV'. Excel Viewer displays CSVs as a formatted table — to use it, open any CSV file and click the table icon in the top right corner of the editor, or right-click (Windows/Linux) / two-finger click (Mac) the file and select **Open With... → Excel Viewer**. Rainbow CSV color-codes each column directly in the raw CSV view to make it easier to read. -6) Each test case's `data/` folder must contain the following files: +6. Each test case's `data/` folder must contain the following files: **`.env`** Specifies the standard and version to validate against. `PRODUCT` and `VERSION` are required; `SUBSTANDARD`, `USE_CASE`, `CT`, and `DEFINE_XML` are optional depending on the rule. See [`.env.example`](./engine/.env.example) for example env environment variables. + ``` PRODUCT=TIG VERSION=1-0 @@ -190,13 +196,13 @@ Unpublished/ ... ``` - | Column | Description | - |--------|-------------| - | `dataset` | Filename of the dataset this variable belongs to (must match a `Filename` in `_datasets.csv`) | - | `variable` | Variable name (e.g. `USUBJID`) | - | `label` | Variable Label | - | `type` | Data type — `Char` or `Num` | - | `length` | Maximum field length | + | Column | Description | + | ---------- | --------------------------------------------------------------------------------------------- | + | `dataset` | Filename of the dataset this variable belongs to (must match a `Filename` in `_datasets.csv`) | + | `variable` | Variable name (e.g. `USUBJID`) | + | `label` | Variable Label | + | `type` | Data type — `Char` or `Num` | + | `length` | Maximum field length | **Dataset CSV files** @@ -211,7 +217,7 @@ Unpublished/ - Column headers must exactly match the `variable` values in `_variables.csv` for that dataset. - For **positive** cases, ensure the data satisfies all rule conditions so no errors are raised. - - For **negative** cases, include data that deliberately triggers the rule. When raising your PR, describe the errors you expect to see in the PR description or as a comment so reviewers can verify the generated `results.json` against your intent. There is no automated validation check for CSV test data; human review of the results is required. + - For **negative** cases, include data that deliberately triggers the rule. When raising your PR, describe the errors you expect to see in the PR description or as a comment so reviewers can verify the actual `results.json` against your expected results. There is no automated validation check for CSV test data; human review of the results is required. **`results/`** @@ -219,7 +225,7 @@ Unpublished/ **Perform Local Testing.** -7) When you want to run the rule against test data locally, make sure you are in the cdisc-open-rules folder and run one of the following: \ +7. When you want to run the rule against test data locally, make sure you are in the cdisc-open-rules folder and run one of the following: \ WINDOWS: `.\run\windows_run.bat` \ MAC: `./run/bash_run.sh` - If you haven't run the setup script before, don't worry; it will run automatically when you execute this command. @@ -227,58 +233,58 @@ Unpublished/ **Verify Results.** -8) Check your run results in the `results/` folder of each test case. +8. Check your run results in the `results/` folder of each test case. - There will be a `results.json` file with the engine output and a `results.csv` summarizing the issues found. - For positive cases, verify that no errors are reported. - For negative cases, verify that the errors reported match the violations you introduced in the test data. Include a summary of the expected errors in your PR description so reviewers can confirm the output is correct. -9) If you are unhappy with the results of your changes, continue to edit and run the rule until you are satisfied. +9. If you are unhappy with the results of your changes, continue to edit and run the rule until you are satisfied. **Request Review via PR.** -> **NOTE:** Once you are satisfied with your results, delete the `results.json` from each test case's results/ folder, but leave the results.csv in place. The CSV is used to verify your results match current engine output results when your PR is reviewed — it should always reflect your latest local run. +> **NOTE:** Once you are satisfied with your results, delete the `results.json` from each test case's results/ folder, but leave the results.csv in place. The CSV is used to verify your results match current engine output results when your PR is reviewed — it should always reflect your latest local run. -10) Create a PR to add your changes to the repository. To do this, run the following commands: \ - `git add .` \ - `git commit -m "your custom message"` \ - `git push origin ` +10. Create a PR to add your changes to the repository. To do this, run the following commands: \ + `git add .` \ + `git commit -m "your custom message"` \ + `git push origin ` - The first time you commit, you may have to log in to GitHub -11) Go to the online repository and create a pull request (PR) from your newly pushed branch +11. Go to the online repository and create a pull request (PR) from your newly pushed branch -12) On the PR page, make sure the information at the top is correct. It should be: \ - `base: main ← compare: ` +12. On the PR page, make sure the information at the top is correct. It should be: \ + `base: main ← compare: ` -13) Name your PR using the format ` ` and add a brief description of your changes. If you are making a new rule, use ` create`. Include a description of the errors expected in any negative test cases so reviewers can audit the generated `results.csv`. +13. Name your PR using the format ` ` and add a brief description of your changes. If you are making a new rule, use ` create`. Include a description of the errors expected in any negative test cases so reviewers can audit the actual `results.csv`. -14) On the PR, add reviewers (both the 'Rules Team' and 'Engineers Team' are required) by clicking the cog in the top right corner, and add yourself as an assignee +14. On the PR, add reviewers (both the 'Rules Team' and 'Engineers Team' are required) by clicking the cog in the top right corner, and add yourself as an assignee -15) You're done! - - The CI pipeline will automatically run the rule against all test cases and post a validation report as a PR comment. It will diff the output against your committed `results.csv`, and post a validation report as a PR comment.. +15. You're done! + - The CI pipeline will automatically run the rule against all test cases and post a validation report as a PR comment. It will diff the output against your expected `results.csv`, and post a validation report as a PR comment.. - Keep an eye on the PR to make sure the automated checks pass, as well as to respond to any comments from reviewers. - If you need to make further changes, simply checkout your branch (`git checkout `), make your changes, and commit and push them — the PR will automatically update and re-run validation. -16) GitHub will automatically validate your changes when a PR is opened. If you did not include a results.csv, the check will fail — run the rule locally and push the generated results.csv to resolve it. If a difference between your results.csv and the engine output is detected, the check will also fail — re-run the rule locally, verify the results look correct, and push the updated results.csv. If the check continues to fail after updating, flag it for the Engineers Team in the PR comments. +16. GitHub will automatically validate your changes when a PR is opened. If you did not include a results.csv, the check will fail — run the rule locally and push the actual results.csv to resolve it. If a difference between your results.csv and the engine output is detected, the check will also fail — re-run the rule locally, verify the results look correct, and push the updated results.csv. If the check continues to fail after updating, flag it for the Engineers Team in the PR comments. - **Rule Schema Validation** will run and post a comment on the PR showing whether your `rule.yaml` is valid. If the schema check fails, the comment will show the specific validation error — for example, the image below shows a failure caused by an empty `check all:` condition in the rule. The comment will automatically update when you push new code and the action re-runs. + **Rule Schema Validation** will run and post a comment on the PR showing whether your `rule.yaml` is valid. If the schema check fails, the comment will show the specific validation error — for example, the image below shows a failure caused by an empty `check all:` condition in the rule. The comment will automatically update when you push new code and the action re-runs. - ![Schema validation failure example](docs/files/schema.png) +![Schema validation failure example](docs/files/schema.png) - **Test Data Validation** will also run and post a comment showing the results of running the rule against your test data. If the check fails, the comment will indicate why — for example, the image below shows a failure caused by a missing `.env` file in the test data. +**Test Data Validation** will also run and post a comment showing the results of running the rule against your test data. If the check fails, the comment will indicate why — for example, the image below shows a failure caused by a missing `.env` file in the test data. - ![Test data validation failure example](docs/files/validation.png) +![Test data validation failure example](docs/files/validation.png) Once your rule and test data pass these checks, the PR can be merged. **Approval - Merge PR** -17) Once your PR is approved, merge your changes to the source code. If you created a new rule in your PR, a new CORE-id will be assigned to it. +17. Once your PR is approved, merge your changes to the source code. If you created a new rule in your PR, a new CORE-id will be assigned to it. **Let's do another rule!** -18) If you want to start editing another rule, don't forget to run the below commands on VSCode terminal again: \ - `git checkout main` \ - `git pull origin main` +18. If you want to start editing another rule, don't forget to run the below commands on VSCode terminal again: \ + `git checkout main` \ + `git pull origin main` For further detail on any of these steps or git in general, see [supplementary guide](https://docs.google.com/document/d/15ydgj4AqtEnFtlXL-J4DLJV32q_Q71gKn0ucu4tYYQw/edit?pli=1&tab=t.0) @@ -289,7 +295,7 @@ If you're stuck or confused, please reach out to Richard (richard@verisian.com) However, here are some quick fixes for common issues you might experience: \
-> ***I accidentally made my changes on the main branch but haven't committed them yet*** +> **_I accidentally made my changes on the main branch but haven't committed them yet_** If the branch you want to move your changes to already exists, run: \ `git checkout main` \ @@ -300,7 +306,7 @@ If you want to move the changes to a new branch, you can run this useful one-lin `git switch -c ` \
-> ***I accidentally made my changes on the main branch and committed them*** +> **_I accidentally made my changes on the main branch and committed them_** In this case, you won't be able to move your changes to an already existing branch easily. If you desperately need to do this, reach out to us. \ Otherwise, create a new branch from main which includes your changes and then reset main: \ @@ -309,17 +315,17 @@ Otherwise, create a new branch from main which includes your changes and then re `git reset --hard HEAD~1` \ `git checkout ` -***IMPORTANT NOTE*** - if you've committed more than once on main, you'll need to replace `HEAD~1` with `HEAD~n` where `n` is the number of commits you've made \ +**_IMPORTANT NOTE_** - if you've committed more than once on main, you'll need to replace `HEAD~1` with `HEAD~n` where `n` is the number of commits you've made \
-> ***I've made some changes that I want to push to the repo and other changes that I don't want to keep*** +> **_I've made some changes that I want to push to the repo and other changes that I don't want to keep_** In the source control sidebar panel (the icon is three dots connected by lines), you will see all of the changes you've made. \ You can right-click on any of these and select 'Discard Changes'. \ This will completely remove your changes, so make sure you don't want them before doing this! \
-> ***I want to work on multiple rules at once!*** +> **_I want to work on multiple rules at once!_** You can! You can create multiple branches for different rules and they will all be isolated from each other. \ Just make sure to use `checkout` commands or the console to switch to the relevant branch before you make changes. \ diff --git a/docs/test_data.md b/docs/test_data.md index dc2b396e6..7d610b804 100644 --- a/docs/test_data.md +++ b/docs/test_data.md @@ -12,8 +12,8 @@ - Volunteer contributors could also use unit testing for debugging rule logic. - Unit testing includes testing rule logic on both positive and negative test data. - - Positive test data: Test data is in compliance with the conformance rule and will not result in output. All test data passes. - - Negative test data: Test data is not in compliance with the conformance rule and will result in output. Not all test data passes. + - Positive test data: Test data is in compliance with the conformance rule and will not result in output. All test data passes. + - Negative test data: Test data is not in compliance with the conformance rule and will result in output. Not all test data passes. - Unit testing will be conducted per CDISC Open Rule. If more than one conformance rule (e.g. conformance rules from 2 different standards like SDTMIG and SENDIG use exactly the same rule logic and scope) is included in 1 CDISC Open Rule, then unit testing should only be done once. The standard used to create test data can be chosen by the volunteer contributor. - Test data should be kept relatively small, just enough data points to assert correctness and full functionality of the rule. - Existing mock studies are available to the volunteer contributors as a valuable resource. @@ -44,7 +44,7 @@ negative/ - Positive and negative test data are kept in separate numbered test case folders. - Multiple datasets can be included in a single test case by providing multiple dataset CSV files. -- The `results/` folder will contain the `results.csv` --this can be generated using the local run script, outlined in the readme. The local script also generates a `results.json` which can we used for testing but should be deleted when sending a rule to QC for publication. +- The `results/` folder will contain the `results.csv` --this can be generated using the local run script, outlined in the readme. The local script also generates a `results.json` which can we used for testing but should be deleted when sending a rule to QC for publication. #### SDTMIG & SENDIG @@ -61,7 +61,7 @@ CT=sdtmct-2024-03-29 ``` - Based on the rule being tested and its IG scope, set the applicable `PRODUCT` and `VERSION`. Example: CG0100 is applicable for SDTMIG 3.3 so `PRODUCT=sdtmig` and `VERSION=3-3`. -- If a rule is applicable for multiple versions of an IG, then only 1 IG should be tested. If there is a clear distinction in rule logic between IG versions--multiple versions for each version should be created. +- If a rule is applicable for multiple versions of an IG, then only 1 IG should be tested. If there is a clear distinction in rule logic between IG versions--multiple versions for each version should be created. - If a rule needs to use CDISC CT, add the `CT` key with the appropriate CT package. **`_datasets.csv`** @@ -93,13 +93,13 @@ dm,USUBJID,Unique Subject Identifier,Char,50 dm,AGE,Age,Num,8 ``` -| Column | Description | -|--------|-------------| -| `dataset` | Filename of the dataset this variable belongs to (must match a `Filename` in `_datasets.csv`) | -| `variable` | Variable name (e.g. `USUBJID`) | -| `label` | Variable label | -| `type` | Data type — `Char` or `Num` | -| `length` | Maximum field length | +| Column | Description | +| ---------- | --------------------------------------------------------------------------------------------- | +| `dataset` | Filename of the dataset this variable belongs to (must match a `Filename` in `_datasets.csv`) | +| `variable` | Variable name (e.g. `USUBJID`) | +| `label` | Variable label | +| `type` | Data type — `Char` or `Num` | +| `length` | Maximum field length | **Dataset CSV files** @@ -123,7 +123,7 @@ under construction - In case a CDISC Open Rule is using metadata captured in a define.xml to execute rule logic, then a test define.xml needs to be created (negative and positive) and uploaded for unit testing in the `data/` directory of each test. - To create this test define.xml, the templates created for the Metadata Submission Guidelines for SDTM and ADaM can be used and adapted accordingly. -- Reference the define.xml in the `.env` file using `DEFINE_XML=define.xml`. This should be equal to the name of the define file contained in the directory. +- Reference the define.xml in the `.env` file using `DEFINE_XML=define.xml`. This should be equal to the name of the define file contained in the directory. #### TIG @@ -151,6 +151,7 @@ under construction **Best Practices** Creating solid, qualitative test data is a skill on its own and needs to be done with care. Below best practices will help you during this process. + - Test data should test **all** functionalities of the rule logic. - Test data should test both **condition** (if applicable) and **rule**. - If more than 1 domain is in scope, test data should be created for more than 1 domain. @@ -164,6 +165,7 @@ Creating solid, qualitative test data is a skill on its own and needs to be done - If the rule being tested references CDISC CT, then the correct name and version should be added as `CT` in the `.env` file. Volunteer contributors should use the information available to create test data, such that: + - Dataset CSV files can be copied from sample data and adapted. - Unused variables can be removed from `_variables.csv` and the dataset CSV. - Variable metadata can be modified in `_variables.csv` in accordance with the test purpose of the associated rule logic. @@ -184,7 +186,7 @@ negative/01/ negative/02/ (only if applicable) ... For negative test cases, there is no automated cell-level validation check for CSV test data — human review of the results is required. When raising your PR: - Describe the errors you expect to see in the PR description or as a comment. -- Reviewers will verify the generated `results.csv` against your stated intent. +- Reviewers will verify the actual `results.csv` against your stated expected results. - The CI pipeline diffs the `results.csv` you commit locally against engine output during review. If a difference is detected, the check will fail — re-run locally, verify the results look correct, and push the updated `results.csv`. A `results.csv` summarizing issues found is generated alongside `results.json` after each local run. **Leave `results/` empty when first creating a test case.** After running locally and confirming results, commit the `results.csv` but delete `results.json` before opening your PR. @@ -193,30 +195,30 @@ A `results.csv` summarizing issues found is generated alongside `results.json` a This section contains links to the different Test Data Templates, Test Data Examples, and Sample Data. Together with the instructions given above, this should give volunteer contributors sufficient information to create consistent, qualitative test data. - #### Template #### +#### Template - - The `_datasets.csv` template contains a list of datasets that can be used for unit testing. - - The `_variables.csv` template includes Identifier, Events, Interventions, Findings, Timing, and Associated Persons variables from SDTM v2.0. - - Domain-specific variable sets are drawn from SDTMIG v3.4, as well as AC, APRELSUB, DI, and TX from SDTM v2.0. +- The `_datasets.csv` template contains a list of datasets that can be used for unit testing. +- The `_variables.csv` template includes Identifier, Events, Interventions, Findings, Timing, and Associated Persons variables from SDTM v2.0. +- Domain-specific variable sets are drawn from SDTMIG v3.4, as well as AC, APRELSUB, DI, and TX from SDTM v2.0. - [unit-test-sdtmig-sendig-template.xlsx](files/unit-test-sdtmig-sendig-template.xlsx ":ignore") +[unit-test-sdtmig-sendig-template.xlsx](files/unit-test-sdtmig-sendig-template.xlsx ":ignore") - #### Examples #### +#### Examples - Also, here is a mock Excel workbook for positive and negative testing against which contains: +Also, here is a mock Excel workbook for positive and negative testing against which contains: - - dm.xpt and ae.xpt. - - Both with variable metadata adjusted, unused columns removed, data rows added. +- dm.xpt and ae.xpt. +- Both with variable metadata adjusted, unused columns removed, data rows added. - [unit-test-ruleid-sdtmigexample-positive.xlsx](files/unit-test-sdtmigexample-positive.xlsx ":ignore") - [unit-test-ruleid-sdtmigexample-negative.xlsx](files/unit-test-sdtmigexample-negative.xlsx ":ignore") +[unit-test-ruleid-sdtmigexample-positive.xlsx](files/unit-test-sdtmigexample-positive.xlsx ":ignore") +[unit-test-ruleid-sdtmigexample-negative.xlsx](files/unit-test-sdtmigexample-negative.xlsx ":ignore") - #### Sample Data #### +#### Sample Data - CDISC has 2 sets of mock study in SDTM format. They have been converted for use as test data. +CDISC has 2 sets of mock study in SDTM format. They have been converted for use as test data. - - [CDISCTestData-sdtm-xpt-xlsx.zip](files/CDISCTestData-sdtm-xpt-xlsx.zip ":ignore") A set of test data files transformed from the CDISCTestData Github repo, sourced from /SDTM/XPT. Per Read Me, this mock study implements "SDTM IG Version 3.2. - - [sdtm-msg-2-0-m5-datasets-xlsx.zip](files/sdtm-msg-2-0-m5-datasets-xlsx.zip ":ignore") A set of test data files transformed from the example submission bundled in the SDTM MSG v2.0, sourced from /m5/datasets/cdiscpilot01/tabulations/sdtm, as well as the split subdirectory. Per documentation, this example submission implements "SDTM v1.7/SDTMIG v3.3, and SDTM Terminology 2020-03-27. +- [CDISCTestData-sdtm-xpt-xlsx.zip](files/CDISCTestData-sdtm-xpt-xlsx.zip ":ignore") A set of test data files transformed from the CDISCTestData Github repo, sourced from /SDTM/XPT. Per Read Me, this mock study implements "SDTM IG Version 3.2. +- [sdtm-msg-2-0-m5-datasets-xlsx.zip](files/sdtm-msg-2-0-m5-datasets-xlsx.zip ":ignore") A set of test data files transformed from the example submission bundled in the SDTM MSG v2.0, sourced from /m5/datasets/cdiscpilot01/tabulations/sdtm, as well as the split subdirectory. Per documentation, this example submission implements "SDTM v1.7/SDTMIG v3.3, and SDTM Terminology 2020-03-27. ## Storage diff --git a/test.py b/test.py index 33990c229..52f8e3090 100644 --- a/test.py +++ b/test.py @@ -12,7 +12,6 @@ from typing import Optional, Dict, List ENGINE_DIR = Path("engine") -CONVERT_SCRIPT = Path(".github/scripts/convert_results.py") LOG_LEVELS = ["info", "debug", "error", "critical", "disabled", "warn"] @@ -51,11 +50,13 @@ def get_test_cases(rule_path: Path) -> Dict[str, List[dict]]: continue for case_dir in sorted(type_dir.iterdir()): if case_dir.is_dir() and (case_dir / "data").is_dir(): - cases[test_type].append({ - "case_id": case_dir.name, - "data_dir": case_dir / "data", - "results_dir": case_dir / "results", - }) + cases[test_type].append( + { + "case_id": case_dir.name, + "data_dir": case_dir / "data", + "results_dir": case_dir / "results", + } + ) return cases @@ -69,15 +70,15 @@ def find_env_file(data_dir: Path) -> Optional[Path]: def next_results_path(results_dir: Path) -> Path: """ Creates results/ if needed. Returns the next available -o path for the engine - (without extension — engine appends .json automatically). - - No results.json yet -> results_dir/results - - results.json exists -> results_dir/results(1), results_dir/results(2), ... + (without extension — engine appends .csv automatically). + - No results.csv yet -> results_dir/results + - results.csv exists -> results_dir/results(1), results_dir/results(2), ... """ results_dir.mkdir(parents=True, exist_ok=True) - if not (results_dir / "results.json").exists(): + if not (results_dir / "results.csv").exists(): return results_dir / "results" n = 1 - while (results_dir / f"results({n}).json").exists(): + while (results_dir / f"results({n}).csv").exists(): n += 1 return results_dir / f"results({n})" @@ -86,6 +87,7 @@ def next_results_path(results_dir: Path) -> Path: # Engine invocation # --------------------------------------------------------------------------- + def run_engine( rule_yml: Path, data_dir: Path, @@ -98,14 +100,23 @@ def run_engine( return False, f"No .env file found in {data_dir}" cmd = [ - sys.executable, "core.py", "validate", - "-lr", str(rule_yml.resolve()), - "-d", str(data_dir.resolve()), - "-dep", str(env_file.resolve()), - "-of", "JSON", - "-o", str(output_path.resolve()), - "-p", "disabled", - "-l", log_level, + sys.executable, + "core.py", + "validate", + "-lr", + str(rule_yml.resolve()), + "-d", + str(data_dir.resolve()), + "-dep", + str(env_file.resolve()), + "-of", + "CSV", + "-o", + str(output_path.resolve()), + "-p", + "disabled", + "-l", + log_level, ] try: @@ -151,7 +162,7 @@ def run_rule( log_level: str, capture_logs: bool, ): - rule_yml = find_rule_yml(rule_path) + rule_yml = find_rule_yml(rule_path) all_cases = get_test_cases(rule_path) if specific_case: @@ -168,28 +179,18 @@ def run_rule( for test_type in ("positive", "negative"): for case in all_cases[test_type]: any_ran = True - case_id = case["case_id"] - data_dir = case["data_dir"] + case_id = case["case_id"] + data_dir = case["data_dir"] output_path = next_results_path(case["results_dir"]) print(f"\n Running {test_type}/{case_id}...") - ok, output = run_engine(rule_yml, data_dir, output_path, log_level, capture_logs) - - json_path = Path(str(output_path) + ".json") - if ok and json_path.exists(): - print(f" Done — results written to {json_path}") - csv_path = output_path.with_suffix(".csv") - try: - proc = subprocess.run( - [sys.executable, str(CONVERT_SCRIPT), str(json_path), str(csv_path)], - capture_output=True, text=True - ) - if proc.returncode == 0: - print(f" Done — CSV written to {csv_path}") - else: - print(f" [WARN] CSV conversion failed: {proc.stderr.strip()}") - except Exception as e: - print(f" [WARN] CSV conversion error: {e}") + ok, output = run_engine( + rule_yml, data_dir, output_path, log_level, capture_logs + ) + + csv_path = Path(str(output_path) + ".csv") + if ok and csv_path.exists(): + print(f" Done — CSV written to {csv_path}") else: print(f" [ERROR] Engine failed for {test_type}/{case_id}") for line in output.splitlines(): @@ -205,6 +206,7 @@ def run_rule( # Prompts # --------------------------------------------------------------------------- + def prompt_rule_path() -> Path: print("\nEnter the path to your rule folder (e.g. Unpublished/CORE-000001).") print("Expected structure:") @@ -228,11 +230,7 @@ def prompt_rule_path() -> Path: def prompt_case(cases: Dict[str, List[dict]]) -> Optional[str]: - flat = [ - f"{t}/{c['case_id']}" - for t in ("positive", "negative") - for c in cases[t] - ] + flat = [f"{t}/{c['case_id']}" for t in ("positive", "negative") for c in cases[t]] if not flat: return None @@ -275,11 +273,12 @@ def prompt_capture_logs() -> bool: # Entry point # --------------------------------------------------------------------------- + def main(): - rule_path = prompt_rule_path() - cases = get_test_cases(rule_path) - specific = prompt_case(cases) - log_level = prompt_log_level() + rule_path = prompt_rule_path() + cases = get_test_cases(rule_path) + specific = prompt_case(cases) + log_level = prompt_log_level() capture_logs = prompt_capture_logs() run_rule(rule_path, specific, log_level, capture_logs) @@ -289,4 +288,4 @@ def main(): main() except KeyboardInterrupt: print("\nInterrupted.") - sys.exit(1) \ No newline at end of file + sys.exit(1)