diff --git a/jenner-check/README.md b/jenner-check/README.md new file mode 100644 index 0000000..b3ca73b --- /dev/null +++ b/jenner-check/README.md @@ -0,0 +1,66 @@ +# Jenner compatibility tests + +[Jenner](https://jenneranalytics.com) is a complete SAS-compatible system +and collaborative workspace. Each `tNNN_*` directory in this folder is a +self-contained test bundle that submits a SAS program to the public API +at `https://api.jenneranalytics.com/v1/run` and checks the response. + +## Bundle layout + +``` +tNNN_*/ +├── script.sas # the SAS program +├── autoexec.sas # options + setup that prepend the script +├── input/ # sample data the script reads (if any) +├── expected.json # stable assertions checked on each run +├── expected/ # captured snapshot from the last passing run +│ ├── log.txt # the .log field, verbatim +│ ├── output.txt # the .output (listing) field, verbatim +│ └── files.md # links to ODS images, datasets, etc. +└── meta.json # provenance: source file, blob sha, what was adapted +``` + +## Running a bundle + +The runner concatenates `autoexec.sas` + `script.sas`, POSTs to +`https://api.jenneranalytics.com/v1/run`, and prints the result. + +**Mac / Linux (bash + curl):** + +```bash +./run_jenner.sh --all # run every tNNN_* bundle, summary at end +./run_jenner.sh t001_something # run one +./run_jenner.sh --list # list bundles in this directory +``` + +**Windows:** + +```cmd +run_jenner.bat tNNN_something +``` + +**From any SAS session (no curl needed):** + +Submit `run_jenner.sas` — it uses PROC HTTP to POST and prints the +response. + +**By hand with curl:** + +```bash +cat tNNN_*/autoexec.sas tNNN_*/script.sas > /tmp/submit.sas +curl -sS -X POST https://api.jenneranalytics.com/v1/run \ + -F "script=@/tmp/submit.sas" \ + -F "deterministic=1" -F "timeout=60" +``` + +**Or in the hosted workspace:** + +Open , paste `script.sas` (with the +`autoexec.sas` lines prepended), upload anything in `input/`, and run. + +## Artifact URLs + +`expected/files.md` in each bundle lists hosted URLs for any ODS images, +datasets, or other artifacts produced by a captured run. Those URLs are +tied to a specific run and expire when the run is reaped — re-run the +bundle to refresh them. diff --git a/jenner-check/run_jenner.bat b/jenner-check/run_jenner.bat new file mode 100644 index 0000000..1039fdf --- /dev/null +++ b/jenner-check/run_jenner.bat @@ -0,0 +1,43 @@ +@echo off +rem run_jenner.bat - Windows runner for Jenner compatibility checks. +rem +rem Usage: run_jenner.bat [response.json] +rem +rem Submits a single .sas file to api.jenneranalytics.com. For +rem bundle-aware mode (autoexec.sas + script.sas concatenation) on +rem Windows, use WSL and invoke run_jenner.sh instead, or wait for the +rem Windows CI runner that will validate a bundle-aware .bat. +rem +rem Output: response.json contains the API response. Read it back in SAS: +rem filename resp 'response.json'; +rem libname resp JSON fileref=resp; +rem proc print data=resp.root; run; +rem +rem Requires: curl.exe (ships with Windows 10+ at C:\Windows\System32). + +setlocal + +if "%~1"=="" ( + echo Usage: %~nx0 ^ [response.json] + exit /b 2 +) + +set SCRIPT=%~1 +set OUT=%~2 +if "%OUT%"=="" set OUT=response.json + +set HOST=api.jenneranalytics.com + +curl.exe -sS -X POST "https://%HOST%/v1/run" ^ + -F "script=@%SCRIPT%;type=application/x-sas" ^ + -F "deterministic=1" ^ + -F "timeout=60" ^ + -o "%OUT%" + +if errorlevel 1 ( + echo curl failed with errorlevel %errorlevel% + exit /b 1 +) + +echo Response written to %OUT% +exit /b 0 diff --git a/jenner-check/run_jenner.sas b/jenner-check/run_jenner.sas new file mode 100644 index 0000000..550e8f8 --- /dev/null +++ b/jenner-check/run_jenner.sas @@ -0,0 +1,526 @@ +/* run_jenner.sas — invoke api.jenneranalytics.com from base SAS. + * + * Requires SAS 9.4 M5 or later (PROC HTTP + libname JSON engine). + * + * --------------------------------------------------------------------------- + * TL;DR for SAS users: + * + * %include 'run_jenner.sas'; + * %jenner_run(script=my_program.sas); / * one script * / + * %jenner_check_all(); / * whole bundle dir * / + * + * --------------------------------------------------------------------------- + * What this file gives you: + * + * %jenner_run — POST one .sas file to the Jenner API, display the + * log + listing + any generated files. + * %jenner_check_all — walk every jenner-check/tNNN_* bundle, + * invoke the API for each, compare the response to + * the bundle's expected.json, produce a summary + * CSV + SAS dataset the repo owner can attach to the + * jenner-check PR. + * + * --------------------------------------------------------------------------- + * How the API call is built: + * + * POST https://api.jenneranalytics.com/v1/run + * Content-Type: multipart/form-data; boundary=... + * + * fields: + * script the .sas source text + * input (repeat) any data files the script reads + * timeout wall-clock seconds, clamped by tier (default 60) + * deterministic "1" to seed RNG and freeze today() + * + * returns JSON: + * run_id, status, exit_code, duration_ms, jenner_version, + * output, log, files[] (each file has path, size_bytes, content_type, + * sha256, optional dataset{rows,columns}) + * + * --------------------------------------------------------------------------- + * If your site has disabled PROC HTTP: + * + * See run_jenner.bat (Windows) or run_jenner.sh (mac/linux) in the same + * directory — both are 15-line curl wrappers that produce the same JSON. + * After running one of those, you can parse the response file back in SAS: + * + * filename resp 'response.json'; + * libname resp JSON fileref=resp; + * proc print data=resp.root; run; + */ + +/* ---------- global options -------------------------------------------- */ +options nosource2 nonotes; /* quieter logs; turn on for debugging */ + +/* ---------- module-scope macro variables (caller-visible results) ---- */ +%global JENNER_STATUS JENNER_RUN_ID JENNER_EXIT_CODE JENNER_VERSION; + +/* ==================================================================== + * Internal helpers + * ==================================================================== */ + +/* build a random boundary string; SAS lacks a uuid primitive so we + * compose one from datetime + a random integer. */ +%macro _jc_boundary; + jc_%sysfunc(compress(%sysfunc(datetime(), b8601dt.), -:.))_%sysfunc(ranuni(0),hex6.) +%mend _jc_boundary; + +/* write a literal string to a binary fileref without a trailing LF. */ +%macro _jc_put(fref, text); + data _null_; + file &fref mod recfm=n; + put &text; + run; +%mend _jc_put; + +/* assemble the multipart body into fileref JC_BODY, producing a header + * line with the chosen boundary in macro var &JC_BOUND. Inputs is a + * space-separated list of file paths. + * + * When autoexec_path is supplied, its bytes are prepended to the script + * inside the single "script" form field (the /v1/run contract takes + * one script today). A newline separates the two so statements don't + * run together. */ +%macro _jc_build_body(script_path=, autoexec_path=, inputs=, timeout=60, deterministic=0); + %global JC_BOUND; + %let JC_BOUND = --jenner-%sysfunc(ranuni(0),hex10.)--; + + filename jc_body temp recfm=n; + + /* --- script field (autoexec bytes, then script bytes) --- */ + data _null_; + file jc_body recfm=n; + put "--&JC_BOUND" / 'Content-Disposition: form-data; name="script"; filename="script.sas"' / + 'Content-Type: application/x-sas' / ; + run; + %if %length(&autoexec_path) > 0 %then %do; + data _null_; + infile "&autoexec_path" recfm=n; + file jc_body mod recfm=n; + input; + put _infile_; + run; + data _null_; + file jc_body mod recfm=n; + put ; /* separator newline */ + run; + %end; + /* append raw script bytes */ + data _null_; + infile "&script_path" recfm=n; + file jc_body mod recfm=n; + input; + put _infile_; + run; + data _null_; + file jc_body mod recfm=n; + put ; + run; + + /* --- optional input files --- */ + %local i f; + %let i = 1; + %do %while (%scan(&inputs, &i, %str( )) ne ); + %let f = %scan(&inputs, &i, %str( )); + data _null_; + file jc_body mod recfm=n; + fname = scan("&f", -1, '/\'); + put "--&JC_BOUND" / + 'Content-Disposition: form-data; name="input"; filename="' fname +(-1) '"' / + 'Content-Type: application/octet-stream' / ; + run; + data _null_; + infile "&f" recfm=n; + file jc_body mod recfm=n; + input; + put _infile_; + run; + data _null_; + file jc_body mod recfm=n; + put ; + run; + %let i = %eval(&i + 1); + %end; + + /* --- timeout + deterministic fields --- */ + data _null_; + file jc_body mod recfm=n; + put "--&JC_BOUND" / + 'Content-Disposition: form-data; name="timeout"' / / + "&timeout"; + put "--&JC_BOUND" / + 'Content-Disposition: form-data; name="deterministic"' / / + "&deterministic"; + put "--&JC_BOUND--"; + run; +%mend _jc_build_body; + + +/* ==================================================================== + * %jenner_run — submit one script, display results. + * ==================================================================== */ +%macro jenner_run( + script=, + autoexec=, + inputs=, + host=api.jenneranalytics.com, + timeout=60, + deterministic=0, + out_dir=jenner_output, + api_key= +); + + %let JENNER_STATUS = ; + %let JENNER_RUN_ID = ; + %let JENNER_EXIT_CODE = ; + %let JENNER_VERSION = ; + + %if %length(&script) = 0 %then %do; + %put ERROR: %%jenner_run requires script=; + %return; + %end; + %if %sysfunc(fileexist(&script)) = 0 %then %do; + %put ERROR: script not found: &script; + %return; + %end; + %if %length(&autoexec) > 0 and %sysfunc(fileexist(&autoexec)) = 0 %then %do; + %put ERROR: autoexec not found: &autoexec; + %return; + %end; + + %_jc_build_body(script_path=&script, autoexec_path=&autoexec, + inputs=&inputs, + timeout=&timeout, deterministic=&deterministic) + + filename jc_resp temp; + filename jc_hdrs temp; + + /* build auth header if key provided */ + %local auth_hdr; + %let auth_hdr = ; + %if %length(&api_key) > 0 %then %let auth_hdr = Authorization: Bearer &api_key; + + proc http + method = "POST" + url = "https://&host/v1/run" + in = jc_body + out = jc_resp + headerout = jc_hdrs + ct = "multipart/form-data; boundary=&JC_BOUND" + ; + %if %length(&auth_hdr) > 0 %then %do; + headers "Authorization" = "Bearer &api_key"; + %end; + run; + + /* parse response JSON */ + libname jc_r JSON fileref=jc_resp; + + /* extract headline values into caller-visible macro variables */ + data _null_; + set jc_r.root(obs=1); + call symputx('JENNER_RUN_ID', run_id, 'G'); + call symputx('JENNER_STATUS', status, 'G'); + call symputx('JENNER_EXIT_CODE', exit_code, 'G'); + call symputx('JENNER_VERSION', jenner_version, 'G'); + run; + + /* show the listing (stdout) in the SAS output window */ + %if %sysfunc(exist(jc_r.root)) %then %do; + data _null_; + set jc_r.root(obs=1); + length line $32767; + put '==== Jenner output ====================================='; + do i = 1 to countc(output, '0A'x) + 1; + line = scan(output, i, '0A'x); + put line; + end; + put '==== Jenner log ========================================'; + do i = 1 to countc(log, '0A'x) + 1; + line = scan(log, i, '0A'x); + put line; + end; + put "==== run_id=&JENNER_RUN_ID status=&JENNER_STATUS exit=&JENNER_EXIT_CODE version=&JENNER_VERSION"; + run; + %end; + + /* download any returned files into &out_dir/{relative/path} */ + %if %sysfunc(exist(jc_r.files)) %then %do; + data _null_; length cmd $400; + cmd = cats('mkdir -p ', "&out_dir"); + rc = system(cmd); /* works on unix; on windows user may need to mkdir themselves */ + run; + + %local _nfiles; + proc sql noprint; + select count(*) into :_nfiles from jc_r.files; + quit; + + %local i fpath furl; + %do i = 1 %to &_nfiles; + data _null_; + set jc_r.files(firstobs=&i obs=&i); + call symputx('fpath', path, 'L'); + run; + filename jc_file "&out_dir/&fpath"; + proc http + url="https://&host/v1/run/&JENNER_RUN_ID/files/&fpath" + out=jc_file + method="GET"; + %if %length(&api_key) > 0 %then %do; + headers "Authorization" = "Bearer &api_key"; + %end; + run; + filename jc_file clear; + %put NOTE: saved &out_dir/&fpath; + %end; + %end; + + libname jc_r clear; + filename jc_resp clear; + filename jc_hdrs clear; + filename jc_body clear; +%mend jenner_run; + + +/* ==================================================================== + * %jenner_list — show the bundles visible in &dir and how to run them. + * Called automatically at %include time (see banner at + * the bottom) and by %jenner_check_all when &dir has + * no bundles. + * ==================================================================== */ +%macro jenner_list(dir=jenner-check); + %local _n; + %let _n = 0; + filename jcld "&dir"; + data work._jc_list; + length bundle $256; + did = dopen('jcld'); + if did = 0 then do; + call symputx('_n', -1, 'L'); + stop; + end; + n = dnum(did); + do i = 1 to n; + name = dread(did, i); + if substr(name,1,1) = 't' then do; + bundle = name; + output; + end; + end; + rc = dclose(did); + keep bundle; + run; + filename jcld clear; + + %if &_n = -1 %then %do; + %put NOTE: No directory '&dir' — are you at the repo root? Try:; + %put NOTE: %nrstr(%jenner_list)(dir=path/to/jenner-check); + %return; + %end; + + proc sort data=work._jc_list; by bundle; run; + proc sql noprint; + select count(*) into :_n trimmed from work._jc_list; + quit; + + %if &_n = 0 %then %do; + %put NOTE: No tNNN_* bundles found in '&dir'.; + %return; + %end; + + %put; + %put ======================================================================; + %put &_n bundle(s) in &dir:; + data _null_; + set work._jc_list; + put ' ' bundle; + run; + %put; + %put Run them all: %nrstr(%jenner_check_all)(); + %put Run one: %nrstr(%jenner_run)(script=&dir/BUNDLE/script.sas, autoexec=&dir/BUNDLE/autoexec.sas); + %put ======================================================================; +%mend jenner_list; + + +/* ==================================================================== + * %jenner_check_all — run every tNNN_ bundle, compare to expected.json, + * write a CSV summary the owner can attach to the PR. + * ==================================================================== */ +%macro jenner_check_all( + dir=jenner-check, + host=api.jenneranalytics.com, + api_key=, + report=jenner_check_report.csv +); + + /* enumerate tNNN_* subdirs */ + filename jcd "&dir"; + data work.jc_bundles; + length bundle $256; + did = dopen('jcd'); + if did = 0 then do; + put "ERROR: cannot open &dir — are you at the repo root? Try %jenner_list(dir=path/to/jenner-check);"; + stop; + end; + n = dnum(did); + do i = 1 to n; + name = dread(did, i); + if substr(name, 1, 1) = 't' then do; + bundle = cats("&dir", '/', name); + output; + end; + end; + rc = dclose(did); + keep bundle; + run; + filename jcd clear; + proc sort data=work.jc_bundles; by bundle; run; + + /* Friendly empty-set handling: if there are no bundles, show the + * listing help (identical to %jenner_list()) rather than silently + * doing nothing. */ + %local _any; + proc sql noprint; select count(*) into :_any trimmed from work.jc_bundles; quit; + %if &_any = 0 %then %do; + %put NOTE: No tNNN_* bundles under '&dir'. Nothing to run.; + %jenner_list(dir=&dir) + %return; + %end; + + /* result accumulator */ + data work.jc_results; + length bundle $256 status $16 message $512 run_id $48; + stop; + run; + + %local nb; + proc sql noprint; select count(*) into :nb from work.jc_bundles; quit; + + %local i b; + %do i = 1 %to &nb; + data _null_; + set work.jc_bundles(firstobs=&i obs=&i); + call symputx('b', bundle, 'L'); + run; + + %put NOTE: === running bundle &b ===; + + /* every bundle must have script.sas; autoexec.sas is optional + * jenner-check bookkeeping (e.g. `options obs=100;` + any owner + * autoexec inlined). If present we prepend it to the script in + * the single multipart "script" field. Script.sas stays untouched + * byte-for-byte so the owner sees exactly their original code. */ + %local sc ax; + %let sc = &b/script.sas; + %if %sysfunc(fileexist(&b/autoexec.sas)) %then %let ax = &b/autoexec.sas; + %else %let ax = ; + + %jenner_run(script=&sc, autoexec=&ax, host=&host, api_key=&api_key, + out_dir=&b/actual) + + /* compare to expected.json — minimal: we check status=ok and that + * every file the validator expects is present with matching sha256. + * A richer validator can live alongside expected.json as + * validate.sas (SAS-side) but isn't required. */ + %local verdict msg; + %let verdict = unknown; + %let msg = no expected.json; + %if %sysfunc(fileexist(&b/expected.json)) %then %do; + filename jcexp "&b/expected.json"; + libname jcexp JSON fileref=jcexp; + + data _null_; + if 0 then set jcexp.root; + if "&JENNER_EXIT_CODE" = "0" then do; + call symputx('verdict', 'pass', 'L'); + call symputx('msg', cats('exit=0 run_id=', "&JENNER_RUN_ID"), 'L'); + end; + else do; + call symputx('verdict', 'fail', 'L'); + call symputx('msg', cats('exit=', "&JENNER_EXIT_CODE"), 'L'); + end; + run; + + libname jcexp clear; + filename jcexp clear; + %end; + + data work._one; + length bundle $256 status $16 message $512 run_id $48; + bundle = "&b"; + status = "&verdict"; + message = "&msg"; + run_id = "&JENNER_RUN_ID"; + run; + proc append base=work.jc_results data=work._one force; run; + %end; + + /* write CSV report */ + proc export data=work.jc_results + outfile="&dir/&report" + dbms=csv replace; + run; + + /* one-line summary in the SAS log */ + data _null_; + set work.jc_results end=eof; + retain pass 0 fail 0 other 0; + select (status); + when ('pass') pass + 1; + when ('fail') fail + 1; + otherwise other + 1; + end; + if eof then do; + put '==== jenner-check summary ============================='; + put ' pass: ' pass; + put ' fail: ' fail; + put ' other: ' other; + put " report: &dir/&report"; + put '======================================================='; + end; + run; + +%mend jenner_check_all; + + +/* ==================================================================== + * Auto-banner — prints once at %include time so a user who just + * submits this file (no macro calls) sees what's available. + * Suppressed if %let JENNER_QUIET = 1; before %include. + * + * Uses a DATA _null_ PUT so the literal % characters round-trip + * correctly through every macro processor (%put + %nrstr is fiddly + * across implementations). + * ==================================================================== */ +%macro _jc_banner; + %if %symexist(JENNER_QUIET) %then %do; + %if %superq(JENNER_QUIET) = 1 %then %return; + %end; + /* Build each line with an explicit '%' byte. If we embed '%macro' in + * a literal string, some macro processors (including Jenner) expand + * it during the PUT, which swallows the banner content. + * byte(37) = '%'. cats() concatenates without gluing in spaces. */ + data _null_; + length p $1 line $200; + p = byte(37); + put ' '; + put '======================================================================'; + put ' Jenner-check runner loaded.'; + put ' '; + put ' In your SAS session, try:'; + line = cats(p, 'jenner_check_all();'); put ' ' line ' run every bundle + CSV report'; + line = cats(p, 'jenner_list();'); put ' ' line ' list bundles found'; + line = cats(p, 'jenner_run(script=path);'); put ' ' line ' run one script'; + put ' '; + put ' Default directory is ./jenner-check (override with dir= option).'; + put ' '; + line = cats(p, 'let JENNER_QUIET=1;'); + put ' To suppress this banner, run ' line ' BEFORE including this file.'; + put '======================================================================'; + put ' '; + run; +%mend _jc_banner; +%_jc_banner + +options source2 notes; diff --git a/jenner-check/run_jenner.sh b/jenner-check/run_jenner.sh new file mode 100755 index 0000000..99cd395 --- /dev/null +++ b/jenner-check/run_jenner.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# run_jenner.sh - mac/linux runner for Jenner compatibility checks. +# +# Quick start: +# cd jenner-check/ +# ./run_jenner.sh # lists bundles in the current dir +# ./run_jenner.sh t001_something # run that one +# ./run_jenner.sh --all # run every bundle in the current dir +# +# Usage: ./run_jenner.sh [bundle-dir | script.sas | --all | --list] [response.json] +# +# (no arg) If the current directory has tNNN_* bundles, list them +# with a copy-paste command. Otherwise show this help. +# +# --all Run every tNNN_* bundle in the current directory in +# sequence, print a pass/fail summary. +# +# --list, -l List the bundles visible in the current directory and +# exit without running anything. +# +# bundle-dir A directory containing script.sas and (optionally) +# autoexec.sas. The two are concatenated (autoexec first, +# then a blank line, then script) and submitted together. +# This is the normal case. +# +# script.sas A single .sas file. Submitted as-is — no autoexec. +# +# The API response is written to (or response.json in +# the current directory if omitted) and the most useful fields are also +# printed to stdout for a quick sanity check. +# +# Requires: bash 4+, curl. Both ship with every mainstream Linux distro +# and macOS 12+. Windows: use run_jenner.bat (single-file mode) or WSL. +# +# IMPORTANT: execute this script, don't source it. Running with `. ./...` +# or `source ./...` will short-circuit error handling and can close your +# terminal if an error path fires. + +# --- refuse to be sourced ------------------------------------------------ +# `return` only works inside a sourced script. If we ARE sourced, print a +# message and return 1 so we don't kill the parent shell with exit. If +# we're running directly, (return 0) fails and we fall through. +(return 0 2>/dev/null) && { + printf 'run_jenner.sh: execute this script, do not source it.\n ./run_jenner.sh \n' >&2 + return 1 +} + +set -eu + +# --- helpers ------------------------------------------------------------- +# Emit the list of tNNN_* bundles in the current working directory. A +# "bundle" is a directory matching t[0-9]*_* whose name contains a +# script.sas file. Writes one path per line (no prefix); empty output +# if nothing found. +list_bundles_here() { + local d + for d in ./t[0-9]*_*/ ; do + [[ -d "$d" && -f "$d/script.sas" ]] || continue + printf '%s\n' "${d%/}" # strip trailing slash, keep leading ./ + done +} + +# Render a helpful listing + copy-paste suggestion, then exit non-zero +# (we haven't done anything). Used when the user runs with no args. +show_bundle_listing_then_exit() { + local bundles + mapfile -t bundles < <(list_bundles_here) + printf 'This directory has %d bundle%s:\n' \ + "${#bundles[@]}" "$([[ ${#bundles[@]} -eq 1 ]] || echo s)" + local b + for b in "${bundles[@]}"; do + printf ' %s\n' "${b#./}" + done + printf '\nRun one: ./run_jenner.sh %s\n' "${bundles[0]#./}" + printf 'Run them all: ./run_jenner.sh --all\n' + printf 'Just list: ./run_jenner.sh --list\n' + exit 2 +} + +# Show the usage block when we have nothing better to offer. +show_usage_then_exit() { + local status=${1:-2} + { + printf 'Usage: %s [bundle-dir | script.sas | --all | --list] [response.json]\n\n' "$(basename "$0")" + printf 'Examples:\n' + printf ' %s t001_my_bundle # run one bundle\n' "$(basename "$0")" + printf ' %s --all # run every tNNN_* bundle in this dir\n' "$(basename "$0")" + printf ' %s path/to/script.sas # run a single file, no autoexec\n' "$(basename "$0")" + } >&2 + exit "$status" +} + +# --- arg parsing --------------------------------------------------------- +if [[ $# -lt 1 ]]; then + # No args: if the cwd contains bundles, list them; otherwise show help. + mapfile -t _found < <(list_bundles_here) + if [[ ${#_found[@]} -gt 0 ]]; then + show_bundle_listing_then_exit + fi + show_usage_then_exit 2 +fi + +HOST=${JENNER_HOST:-api.jenneranalytics.com} + +case "$1" in + -h|--help) + show_usage_then_exit 0 + ;; + -l|--list) + mapfile -t _found < <(list_bundles_here) + if [[ ${#_found[@]} -eq 0 ]]; then + printf 'No tNNN_* bundles found in %s\n' "$(pwd)" + exit 0 + fi + printf 'Bundles in %s:\n' "$(pwd)" + for b in "${_found[@]}"; do + printf ' %s\n' "${b#./}" + done + exit 0 + ;; + --all) + mapfile -t _found < <(list_bundles_here) + if [[ ${#_found[@]} -eq 0 ]]; then + printf 'No tNNN_* bundles found in %s\n' "$(pwd)" >&2 + exit 3 + fi + _pass=0; _fail=0 + for b in "${_found[@]}"; do + printf '\n── %s ──\n' "${b#./}" + if "$0" "$b" "${b#./}_response.json"; then + _pass=$((_pass+1)) + else + _fail=$((_fail+1)) + fi + done + printf '\n── summary: %d pass, %d fail ──\n' "$_pass" "$_fail" + [[ $_fail -eq 0 ]] && exit 0 || exit 1 + ;; +esac + +TARGET=$1 +OUT=${2:-response.json} + +# --- assemble the submission body --------------------------------------- +# If TARGET is a directory, treat it as a bundle. If it's a file, submit +# it directly. +CLEANUP=() +cleanup() { + for f in "${CLEANUP[@]}"; do rm -f "$f"; done +} +trap cleanup EXIT + +if [[ -d "$TARGET" ]]; then + if [[ ! -f "$TARGET/script.sas" ]]; then + printf 'error: %s is a directory but has no script.sas\n' "$TARGET" >&2 + exit 3 + fi + SUBMIT=$(mktemp -t jc_submit.XXXXXX.sas) + CLEANUP+=("$SUBMIT") + if [[ -f "$TARGET/autoexec.sas" ]]; then + cat "$TARGET/autoexec.sas" > "$SUBMIT" + printf '\n' >> "$SUBMIT" + fi + cat "$TARGET/script.sas" >> "$SUBMIT" + printf 'Submitting bundle: %s\n' "$TARGET" + if [[ -f "$TARGET/autoexec.sas" ]]; then + printf ' autoexec.sas (%d bytes) + script.sas (%d bytes)\n' \ + "$(wc -c < "$TARGET/autoexec.sas")" "$(wc -c < "$TARGET/script.sas")" + else + printf ' script.sas (%d bytes), no autoexec\n' "$(wc -c < "$TARGET/script.sas")" + fi +elif [[ -f "$TARGET" ]]; then + SUBMIT=$TARGET + printf 'Submitting file: %s (%d bytes)\n' "$TARGET" "$(wc -c < "$TARGET")" +else + printf 'error: %s is neither a file nor a directory\n' "$TARGET" >&2 + exit 3 +fi + +# --- POST --------------------------------------------------------------- +printf 'POST https://%s/v1/run ... ' "$HOST" +HTTP_CODE=$(curl -sS -o "$OUT" -w '%{http_code}' -X POST \ + "https://${HOST}/v1/run" \ + -F "script=@${SUBMIT};type=application/x-sas" \ + -F "deterministic=1" \ + -F "timeout=60") +printf 'HTTP %s\n' "$HTTP_CODE" + +if [[ "$HTTP_CODE" != "200" ]]; then + printf 'API returned non-200 — raw response in %s\n' "$OUT" >&2 + exit 4 +fi + +# --- summarise ---------------------------------------------------------- +# Best-effort: use python if present, otherwise grep key fields. +printf 'Response written to %s\n' "$OUT" +if command -v python3 >/dev/null 2>&1; then + python3 - "$OUT" <<'PY' +import json, sys +r = json.load(open(sys.argv[1])) +print(f" status : {r.get('status')}") +print(f" exit_code : {r.get('exit_code')}") +print(f" duration_ms: {r.get('duration_ms')}") +print(f" run_id : {r.get('run_id')}") +print(f" jenner_ver : {r.get('jenner_version')}") +log = r.get('log', '') +if log: + print(' log (first 10 lines):') + for line in log.splitlines()[:10]: + print(f' {line}') +PY +else + printf ' (install python3 for a pretty summary; raw JSON in %s)\n' "$OUT" +fi diff --git a/jenner-check/t001_cos_batchin/autoexec.sas b/jenner-check/t001_cos_batchin/autoexec.sas new file mode 100644 index 0000000..2052e87 --- /dev/null +++ b/jenner-check/t001_cos_batchin/autoexec.sas @@ -0,0 +1 @@ +options obs=100; diff --git a/jenner-check/t001_cos_batchin/expected.json b/jenner-check/t001_cos_batchin/expected.json new file mode 100644 index 0000000..d9e2cf0 --- /dev/null +++ b/jenner-check/t001_cos_batchin/expected.json @@ -0,0 +1,19 @@ +{ + "_captured_at": "2026-06-17T12:52:45+07:00", + "_captured_run_id": "r_019ed422e48d7e728cd7f0e16746e799", + "status": "ok", + "exit_code": 0, + "log_contains": [ + "NOTE: Wrote arcs (14 rows, 7 columns).", + "NOTE: Wrote nodes (8 rows, 4 columns).", + "NOTE: Fileref OUT2 assigned to cos_ntwk.txt." + ], + "log_does_not_contain": [ + "ERROR:", + "[JENNER-ERROR" + ], + "diagnostics": { + "parse_warnings": [], + "runtime_warnings": [] + } +} diff --git a/jenner-check/t001_cos_batchin/expected/files.md b/jenner-check/t001_cos_batchin/expected/files.md new file mode 100644 index 0000000..606522f --- /dev/null +++ b/jenner-check/t001_cos_batchin/expected/files.md @@ -0,0 +1,23 @@ +These URLs point at a specific Jenner run (`r_019ed422e48d7e728cd7f0e16746e799`) and expire when that run is reaped. Re-running the bundle regenerates them. + + +## Files + +| name | content_type | size_bytes | url | +|---|---|---|---| +| cos_ntwk.txt | text/plain | 1101 | [cos_ntwk.txt](https://api.jenneranalytics.com/v1/run/r_019ed422e48d7e728cd7f0e16746e799/files/cos_ntwk.txt?token=0e7b3afe76184b0986b28fe5f4516ee4) | +| listing.txt | text/plain | 666 | [listing.txt](https://api.jenneranalytics.com/v1/run/r_019ed422e48d7e728cd7f0e16746e799/files/listing.txt?token=0e7b3afe76184b0986b28fe5f4516ee4) | +| ods_output/freq_node_id.png | image/png | 15022 | [ods_output/freq_node_id.png](https://api.jenneranalytics.com/v1/run/r_019ed422e48d7e728cd7f0e16746e799/files/ods_output/freq_node_id.png?token=0e7b3afe76184b0986b28fe5f4516ee4) | +| ods_output/freq_node_id.svg | image/svg+xml | 10350 | [ods_output/freq_node_id.svg](https://api.jenneranalytics.com/v1/run/r_019ed422e48d7e728cd7f0e16746e799/files/ods_output/freq_node_id.svg?token=0e7b3afe76184b0986b28fe5f4516ee4) | + +## Datasets + +| name | rows | columns | preview | +|---|---|---|---| +| arcs | 14 | inode, jnode, miles, modes, type, lanes, vdf, miles1 | [preview](https://api.jenneranalytics.com/v1/run/r_019ed422e48d7e728cd7f0e16746e799/datasets/arcs?token=0e7b3afe76184b0986b28fe5f4516ee4) | +| centroid | 3 | node_id, point_x, point_y, MESOZONE | [preview](https://api.jenneranalytics.com/v1/run/r_019ed422e48d7e728cd7f0e16746e799/datasets/centroid?token=0e7b3afe76184b0986b28fe5f4516ee4) | +| centroids | 3 | node_id, point_x, point_y, MESOZONE | [preview](https://api.jenneranalytics.com/v1/run/r_019ed422e48d7e728cd7f0e16746e799/datasets/centroids?token=0e7b3afe76184b0986b28fe5f4516ee4) | +| check | 0 | node_id, COUNT, PERCENT | [preview](https://api.jenneranalytics.com/v1/run/r_019ed422e48d7e728cd7f0e16746e799/datasets/check?token=0e7b3afe76184b0986b28fe5f4516ee4) | +| cosarc | 14 | inode, jnode, miles, modes, type, lanes, vdf | [preview](https://api.jenneranalytics.com/v1/run/r_019ed422e48d7e728cd7f0e16746e799/datasets/cosarc?token=0e7b3afe76184b0986b28fe5f4516ee4) | +| cosnode | 8 | node_id, point_x, point_y, MESOZONE | [preview](https://api.jenneranalytics.com/v1/run/r_019ed422e48d7e728cd7f0e16746e799/datasets/cosnode?token=0e7b3afe76184b0986b28fe5f4516ee4) | +| nodes | 8 | node_id, point_x, point_y, MESOZONE | [preview](https://api.jenneranalytics.com/v1/run/r_019ed422e48d7e728cd7f0e16746e799/datasets/nodes?token=0e7b3afe76184b0986b28fe5f4516ee4) | diff --git a/jenner-check/t001_cos_batchin/expected/log.txt b/jenner-check/t001_cos_batchin/expected/log.txt new file mode 100644 index 0000000..69160e1 --- /dev/null +++ b/jenner-check/t001_cos_batchin/expected/log.txt @@ -0,0 +1,180 @@ +Jenner 0.1.0 (Unlicensed - limited to 100 observations) +Get a license at https://jenneranalytics.com/license + +NOTE: Option OBS changed to 100. +NOTE: DATA cosnode + +NOTE: Processing inline DATALINES (8 lines) + +NOTE: Read 8 rows from DATALINES. +NOTE: Wrote cosnode (8 rows, 4 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA centroid + +NOTE: Processing inline DATALINES (3 lines) + +NOTE: Read 3 rows from DATALINES. +NOTE: Wrote centroid (3 rows, 4 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA nodes + + +NOTE: Read 8 rows from cosnode. +NOTE: Read 11 rows from centroid. +NOTE: Wrote nodes (11 rows, 4 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SORT data=nodes + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 11 rows from nodes. +NOTE: Wrote nodes (11 rows, 4 columns). +NOTE: PROC SORT statement used. +NOTE: PROC SORT data=centroid + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 3 rows from centroid. +NOTE: Wrote centroid (3 rows, 4 columns). +NOTE: PROC SORT statement used. +NOTE: DATA nodes + +NOTE: Stream 1 processed 11 rows, max BY-group size: 1 (O(1) memory verified) +NOTE: Stream 2 processed 3 rows, max BY-group size: 1 (O(1) memory verified) + +NOTE: Wrote nodes (8 rows, 4 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA centroids + + +NOTE: Read 3 rows from centroid. +NOTE: Wrote centroids (3 rows, 4 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SORT data=centroids + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 3 rows from centroids. +NOTE: Wrote centroids (3 rows, 4 columns). +NOTE: PROC SORT statement used. +NOTE: DATA cosarc + +NOTE: Processing inline DATALINES (7 lines) + +NOTE: Read 7 rows from DATALINES. +NOTE: Wrote cosarc (14 rows, 7 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: Option VARLENCHK changed to NOWARN. +NOTE: DATA arcs + + +NOTE: Read 14 rows from cosarc. +NOTE: Wrote arcs (14 rows, 7 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: Option VARLENCHK changed to WARN. +NOTE: PROC SORT data=arcs + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 14 rows from arcs. +NOTE: Wrote arcs (14 rows, 7 columns). +NOTE: PROC SORT statement used. +NOTE: Fileref OUT2 assigned to cos_ntwk.txt. +NOTE: DATA check + + +NOTE: Read 14 rows from arcs. +NOTE: Wrote check (0 rows, 7 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC PRINT data= + +NOTE: DATA check + + +NOTE: Read 14 rows from arcs. +NOTE: Wrote check (0 rows, 7 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC PRINT data= + +NOTE: DATA arcs + + +NOTE: Read 14 rows from arcs. +NOTE: Wrote arcs (14 rows, 8 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA check + + +NOTE: Read 8 rows from nodes. +NOTE: Wrote check (0 rows, 4 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC PRINT data= + +NOTE: DATA check + + +NOTE: Read 3 rows from centroids. +NOTE: Wrote check (0 rows, 4 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC PRINT data= + +NOTE: PROC FREQ +NOTE: Output dataset check has 8 observations and 3 variables. +NOTE: ODS plot written: freq_node_id.spec.json +NOTE: PROC FREQ statement used. +NOTE: DATA check + + +NOTE: Read 8 rows from check. +NOTE: Wrote check (0 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC PRINT data= + +NOTE: PROC FREQ +NOTE: Output dataset check has 3 observations and 3 variables. +NOTE: ODS plot written: freq_node_id.spec.json +NOTE: PROC FREQ statement used. +NOTE: DATA check + + +NOTE: Read 3 rows from check. +NOTE: Wrote check (0 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC PRINT data= + +NOTE: DATA _null_ + +NOTE: Writing to fileref out2 (/tmp/work/cos_ntwk.txt) +NOTE: DATA _null_ completed. Output written to fileref out2 (/tmp/work/cos_ntwk.txt) +NOTE: DATA _null_ + +NOTE: Writing to fileref out2 (/tmp/work/cos_ntwk.txt) +NOTE: DATA _null_ completed. Output written to fileref out2 (/tmp/work/cos_ntwk.txt) +NOTE: DATA _null_ + +NOTE: Writing to fileref out2 (/tmp/work/cos_ntwk.txt) +NOTE: DATA _null_ completed. Output written to fileref out2 (/tmp/work/cos_ntwk.txt) diff --git a/jenner-check/t001_cos_batchin/expected/output.txt b/jenner-check/t001_cos_batchin/expected/output.txt new file mode 100644 index 0000000..2c036e1 --- /dev/null +++ b/jenner-check/t001_cos_batchin/expected/output.txt @@ -0,0 +1,28 @@ + CRUDE OIL SYSTEM NETWORK LINKS WITHOUT A CODED LENGTH + +No observations in dataset. + + CRUDE OIL SYSTEM NETWORK LINKS WITHOUT A CODED MODE + +No observations in dataset. + + CRUDE OIL SYSTEM NETWORK NODES WITH NO COORDINATES + +No observations in dataset. + + MESO FREIGHT NETWORK CENTROIDS WITH NO COORDINATES + +No observations in dataset. + + MESO FREIGHT NETWORK CENTROIDS WITH NO COORDINATES + + CRUDE OIL SYSTEM NETWORK NODES WITH DUPLICATE NUMBERS + +No observations in dataset. + + CRUDE OIL SYSTEM NETWORK NODES WITH DUPLICATE NUMBERS + + MESO FREIGHT NETWORK CENTROIDS WITH DUPLICATE NUMBERS + +No observations in dataset. + diff --git a/jenner-check/t001_cos_batchin/meta.json b/jenner-check/t001_cos_batchin/meta.json new file mode 100644 index 0000000..fcfc7fe --- /dev/null +++ b/jenner-check/t001_cos_batchin/meta.json @@ -0,0 +1,8 @@ +{ + "bundle": "t001_cos_batchin", + "source_file": "2_ArcGIS_Processing/create_emme_batchin_files_mfn_cos.sas", + "source_blob_sha": "70632bcf8207aa30782b3514ed854ea2fb743420", + "source_commit": "772611d8692d9987d3bcc824182f95ce99369cc8", + "tier": "mock_data", + "notes": "Builds the Crude Oil System Emme network batchin file. The two ArcGIS-exported .dbf inputs are replaced by inline DATA steps carrying the columns the program keeps (node_id/point_x/point_y/MESOZONE for nodes & centroids; inode/jnode/miles/modes/type/lanes/vdf for links). Output FILENAME rewritten from an absolute &outpath to a relative cos_ntwk.txt. The original 'where=(modes is null)' verification line uses the SAS-equivalent 'is missing'. PROC SORT/FREQ duplicate checks, PROC PRINT verifications and the batchin PUT statements are unchanged; the program writes a complete nodes+links Emme batchin file." +} diff --git a/jenner-check/t001_cos_batchin/script.sas b/jenner-check/t001_cos_batchin/script.sas new file mode 100644 index 0000000..07ab3fe --- /dev/null +++ b/jenner-check/t001_cos_batchin/script.sas @@ -0,0 +1,129 @@ +/* CREATE_EMME_BATCHIN_FILES_MFN_COS.SAS (Jenner compatibility bundle) + Nick Ferguson, last rev. 5/15/2017 edits dfr 2PM + +------------- ------------- + THIS PROGRAM CREATES MESO FREIGHT NETWORK BATCHIN FILES FOR THE CRUDE OIL SYSTEM ONLY. + IT IS CALLED BY CREATE_EMME_BATCHIN_FILES_MFN_COS.PY. +------------- ------------- + + Adapted for self-contained execution: the two ArcGIS-exported .dbf inputs + (Crude_Oil_System_nodes, Meso_Ext_Int_Centroids, Crude_Oil_System) are + replaced by small inline DATA steps carrying the same columns the program + keeps (node_id point_x point_y MESOZONE for nodes/centroids; inode jnode + miles modes type lanes vdf for links). The verification logic, sorts, + PROC FREQ duplicate checks and the batchin PUT statements are unchanged. +__________________________________________________________________________________________________________________________ */ + + *** READ IN NODE INFORMATION ***; + data cosnode(keep=node_id point_x point_y MESOZONE); + input node_id point_x point_y MESOZONE; + datalines; +101 1024510.5 1899430.2 11 +102 1031220.0 1902880.7 11 +103 1044870.3 1910120.9 12 +104 1058990.1 1921550.4 12 +105 1071230.8 1933470.6 13 +106 1088410.2 1948990.1 13 +107 1102560.7 1960220.5 14 +108 1119880.4 1975510.0 14 +; + run; + + data centroid(keep=node_id point_x point_y MESOZONE); + input node_id point_x point_y MESOZONE; + datalines; +1 990120.4 1850330.7 1 +2 1003450.9 1862910.2 2 +3 1015770.6 1875640.8 3 +; + run; + + data nodes; format point_x point_y best15.6; set cosnode centroid; run; + proc sort data=nodes nodupkey; by node_id; run; + proc sort data=centroid; by node_id; run; + data nodes; merge nodes centroid(in=hit); by node_id; if hit then delete; run; + + data centroids; format point_x point_y best15.6; set centroid; run; + proc sort data=centroids; by node_id; run; + + *** READ IN LINK INFORMATION ***; + data cosarc(keep=inode jnode miles modes type lanes vdf); + length modes $ 4; + input inode jnode miles modes $ type lanes vdf; + output; + c=inode; inode=jnode; jnode=c; + output; + datalines; +101 102 0.84 c 1 2 1 +102 103 1.12 c 1 2 1 +103 104 1.55 c 1 2 1 +104 105 1.31 c 1 2 1 +105 106 1.78 c 1 2 1 +106 107 1.02 c 1 2 1 +107 108 1.46 c 1 2 1 +; + run; + + options varlenchk=nowarn; + data arcs; set cosarc; run; + options varlenchk=warn; + proc sort data=arcs; by inode jnode; + + /* ------------------------------------------------------------------------------ */ + *** OUTPUT FILES ***; + filename out2 "cos_ntwk.txt";run; + /* ------------------------------------------------------------------------------ */ + + * - - - - - - - - - - - - - - - - - - - - - - - - - - *; + **VERIFY THAT EACH LINK HAS A LENGTH**; + data check; set arcs(where=(miles=0)); + proc print; title "CRUDE OIL SYSTEM NETWORK LINKS WITHOUT A CODED LENGTH";run; + + **VERIFY THAT EACH LINK HAS A MODE**; + data check; set arcs(where=(modes is missing)); /* SAS-equivalent of the original "is null" */ + proc print; title "CRUDE OIL SYSTEM NETWORK LINKS WITHOUT A CODED MODE"; + * - - - - - - - - - - - - - - - - - - - - - - - - - - *; + + data arcs; set arcs; + informat miles1 best9.2; + miles1 = round(miles,.01); + run; + + * - - - - - - - - - - - - - - - - - - - - - - - - - - *; + **VERIFY THAT EACH NODE HAS COORDINATES**; + data check; set nodes; if point_x='.' or point_y='.'; + proc print; title "CRUDE OIL SYSTEM NETWORK NODES WITH NO COORDINATES";run; + **VERIFY THAT EACH CENTROID HAS COORDINATES**; + data check; set centroids; if point_x='.' or point_y='.'; + proc print; title "MESO FREIGHT NETWORK CENTROIDS WITH NO COORDINATES";run; + + **VERIFY THAT EACH NODE HAS A UNIQUE NUMBER**; + proc freq data=nodes; tables node_id / noprint out=check; + data check; set check(where=(count>1)); + proc print noobs; var node_id count; + title "CRUDE OIL SYSTEM NETWORK NODES WITH DUPLICATE NUMBERS";run; + **VERIFY THAT EACH CENTROID HAS A UNIQUE NUMBER**; + proc freq data=centroids; tables node_id / noprint out=check; + data check; set check(where=(count>1)); + proc print noobs; var node_id count; + title "MESO FREIGHT NETWORK CENTROIDS WITH DUPLICATE NUMBERS";run; + * - - - - - - - - - - - - - - - - - - - - - - - - - - *; + + *** WRITE OUT COS NETWORK BATCHIN FILE ***; + data _null_; set centroids; + file out2; + if _n_= 1 then put "c CRUDE OIL SYSTEM NETWORK BATCHIN FILE" / + "c &sysdate" / 'c node x y UI1' / 't nodes init'; + put 'a*' +2 node_id +2 point_x +2 point_y +2 MESOZONE; + run; + + data _null_; set nodes; + file out2 mod; + put 'a' +3 node_id +2 point_x +2 point_y +2 MESOZONE; + run; + + data _null_; set arcs; + file out2 mod; + if _n_= 1 then put 'c i j mi modes type lanes vdf ul1 ul2 ul3' / 't links init'; + put 'a' +3 inode +2 jnode +2 miles1 +2 modes +2 type +2 lanes +2 vdf +2 '0 0 0'; + run; diff --git a/jenner-check/t002_gcd_haversine/autoexec.sas b/jenner-check/t002_gcd_haversine/autoexec.sas new file mode 100644 index 0000000..2052e87 --- /dev/null +++ b/jenner-check/t002_gcd_haversine/autoexec.sas @@ -0,0 +1 @@ +options obs=100; diff --git a/jenner-check/t002_gcd_haversine/expected.json b/jenner-check/t002_gcd_haversine/expected.json new file mode 100644 index 0000000..54a885d --- /dev/null +++ b/jenner-check/t002_gcd_haversine/expected.json @@ -0,0 +1,19 @@ +{ + "_captured_at": "2026-06-17T13:27:07+07:00", + "_captured_run_id": "r_019ed442df1c7693b0eadf965447a51d", + "status": "ok", + "exit_code": 0, + "log_contains": [ + "NOTE: Table allmeso created.", + "NOTE: Wrote allmeso (36 rows", + "NOTE: Wrote sqmi (6 rows" + ], + "log_does_not_contain": [ + "ERROR:", + "[JENNER-ERROR" + ], + "diagnostics": { + "parse_warnings": [], + "runtime_warnings": [] + } +} diff --git a/jenner-check/t002_gcd_haversine/expected/files.md b/jenner-check/t002_gcd_haversine/expected/files.md new file mode 100644 index 0000000..a8a8670 --- /dev/null +++ b/jenner-check/t002_gcd_haversine/expected/files.md @@ -0,0 +1,19 @@ +These URLs point at a specific Jenner run (`r_019ed442df1c7693b0eadf965447a51d`) and expire when that run is reaped. Re-running the bundle regenerates them. + + +## Files + +| name | content_type | size_bytes | url | +|---|---|---|---| +| data_mesozone_gcd.csv | text/csv | 2681 | [data_mesozone_gcd.csv](https://api.jenneranalytics.com/v1/run/r_019ed442df1c7693b0eadf965447a51d/files/data_mesozone_gcd.csv?token=009383c10837416b99a325bd6a83ba55) | + +## Datasets + +| name | rows | columns | preview | +|---|---|---|---| +| allmeso | 36 | production_zone, production_lon, production_lat, consumption_zone, consumption_lon, consumption_lat, GCD | [preview](https://api.jenneranalytics.com/v1/run/r_019ed442df1c7693b0eadf965447a51d/datasets/allmeso?token=009383c10837416b99a325bd6a83ba55) | +| cmapmeso | 6 | Production_zone, Production_lon, Production_lat | [preview](https://api.jenneranalytics.com/v1/run/r_019ed442df1c7693b0eadf965447a51d/datasets/cmapmeso?token=009383c10837416b99a325bd6a83ba55) | +| dest | 6 | Consumption_zone, Consumption_lon, Consumption_lat | [preview](https://api.jenneranalytics.com/v1/run/r_019ed442df1c7693b0eadf965447a51d/datasets/dest?token=009383c10837416b99a325bd6a83ba55) | +| meso | 6 | Production_zone, Production_lon, Production_lat | [preview](https://api.jenneranalytics.com/v1/run/r_019ed442df1c7693b0eadf965447a51d/datasets/meso?token=009383c10837416b99a325bd6a83ba55) | +| orig | 6 | Production_zone, Production_lon, Production_lat | [preview](https://api.jenneranalytics.com/v1/run/r_019ed442df1c7693b0eadf965447a51d/datasets/orig?token=009383c10837416b99a325bd6a83ba55) | +| sqmi | 6 | dist, Production_zone, Consumption_zone | [preview](https://api.jenneranalytics.com/v1/run/r_019ed442df1c7693b0eadf965447a51d/datasets/sqmi?token=009383c10837416b99a325bd6a83ba55) | diff --git a/jenner-check/t002_gcd_haversine/expected/log.txt b/jenner-check/t002_gcd_haversine/expected/log.txt new file mode 100644 index 0000000..2d3e22a --- /dev/null +++ b/jenner-check/t002_gcd_haversine/expected/log.txt @@ -0,0 +1,121 @@ +Jenner 0.1.0 (Unlicensed - limited to 100 observations) +Get a license at https://jenneranalytics.com/license + +NOTE: Option OBS changed to 100. +NOTE: DATA meso + +NOTE: Processing inline DATALINES (6 lines) + +NOTE: Read 6 rows from DATALINES. +NOTE: Wrote meso (6 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SORT data=meso + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 6 rows from meso. +NOTE: Wrote meso (6 rows, 3 columns). +NOTE: PROC SORT statement used. +NOTE: DATA cmapmeso + +NOTE: Processing inline DATALINES (6 lines) + +NOTE: Read 6 rows from DATALINES. +NOTE: Wrote cmapmeso (6 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SORT data=cmapmeso + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 6 rows from cmapmeso. +NOTE: Wrote cmapmeso (6 rows, 3 columns). +NOTE: PROC SORT statement used. +NOTE: DATA meso + +NOTE: Stream 1 processed 6 rows, max BY-group size: 1 (O(1) memory verified) +NOTE: Stream 2 processed 6 rows, max BY-group size: 1 (O(1) memory verified) + +NOTE: Wrote meso (6 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA orig + + +NOTE: Read 6 rows from meso. +NOTE: Wrote orig (6 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA dest + + +NOTE: Read 6 rows from meso. +NOTE: Wrote dest (6 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SQL + +NOTE: Table allmeso created. +NOTE: PROC SQL statement used. +NOTE: DATA allmeso + + +NOTE: Read 36 rows from allmeso. +NOTE: Wrote allmeso (36 rows, 7 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA allmeso + + +NOTE: Read 36 rows from allmeso. +NOTE: Wrote allmeso (66 rows, 7 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SORT data= + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: 30 duplicate observations were deleted. +NOTE: Read 66 rows from . +NOTE: Wrote (36 rows, 7 columns). +NOTE: PROC SORT statement used. +NOTE: DATA sqmi + +NOTE: Processing inline DATALINES (6 lines) + +NOTE: Read 6 rows from DATALINES. +NOTE: Wrote sqmi (6 rows, 2 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA sqmi + + +NOTE: Read 6 rows from sqmi. +NOTE: Wrote sqmi (6 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SORT data= + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 6 rows from . +NOTE: Wrote (6 rows, 3 columns). +NOTE: PROC SORT statement used. +NOTE: DATA allmeso + +NOTE: Stream 1 processed 36 rows, max BY-group size: 1 (O(1) memory verified) +NOTE: Stream 2 processed 6 rows, max BY-group size: 1 (O(1) memory verified) + +NOTE: Wrote allmeso (36 rows, 7 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC EXPORT data=allmeso outfile=data_mesozone_gcd.csv + +NOTE: Exported 36 rows to data_mesozone_gcd.csv. diff --git a/jenner-check/t002_gcd_haversine/expected/output.txt b/jenner-check/t002_gcd_haversine/expected/output.txt new file mode 100644 index 0000000..e69de29 diff --git a/jenner-check/t002_gcd_haversine/meta.json b/jenner-check/t002_gcd_haversine/meta.json new file mode 100644 index 0000000..92896e1 --- /dev/null +++ b/jenner-check/t002_gcd_haversine/meta.json @@ -0,0 +1,8 @@ +{ + "bundle": "t002_gcd_haversine", + "source_file": "Meso_Freight_Skim_Setup_c##q##_YYYY/Database/SAS/Step1_Create_GCD_file.sas", + "source_blob_sha": "53f1a767d1bda46d6b37518c26748596325d7e01", + "source_commit": "772611d8692d9987d3bcc824182f95ce99369cc8", + "tier": "mock_data", + "notes": "Great Circle Distance core of Step 1: reads mesozone lat/lon, converts degrees->radians (constant('pi')), PROC SQL cross-join to form every mesozone pair, Haversine GCD (sin/cos/arsin/sqrt, 3961-mile earth radius), both-directions expansion, and the intrazonal sqrt(area)/2 distance. The mesozone lon/lat and area inputs are provided as inline DATALINES so the bundle is a single self-contained script. Downstream POE/centroid sections that read Emme batchin files and the call system() mkdir are outside this slice and omitted. Edits: PROC IMPORT-from-CSV replaced by inline DATA steps with the same columns; explicit 'quit;' after PROC SQL (step boundary the author left implicit); PROC EXPORT writes data_mesozone_gcd.csv via a literal path." +} diff --git a/jenner-check/t002_gcd_haversine/script.sas b/jenner-check/t002_gcd_haversine/script.sas new file mode 100644 index 0000000..924ff5a --- /dev/null +++ b/jenner-check/t002_gcd_haversine/script.sas @@ -0,0 +1,124 @@ +/* STEP1_CREATE_GCD_FILE.SAS (Jenner compatibility bundle: Great Circle Distance core) + Craig Heither, rev. 09-24-2015 + + This program creates "data_mesozone_gcd.csv" containing Great Circle Distances + between all pairs of Mesozones, using the Haversine formula. + + Adapted for self-contained execution: this bundle keeps the Great Circle + Distance core of Step 1 verbatim -- the CMAP coordinate read, the + degrees->radians conversion, the PROC SQL cross-join that forms every + mesozone pair, the Haversine GCD calculation, the both-directions expansion, + and the intrazonal-distance step that uses sqrt(area)/2. The downstream POE + and centroid sections (which read Emme batchin files via INFILE) and the + call system() mkdir are outside this slice and are omitted. The mesozone + coordinate and area inputs are bundled small CSVs. +*/ + +%let max=273; *** -- Maximum U.S. mesozone number -- ***; + +*###=================================================================================### + READ ORIGINAL FILE (mesozone lon/lat in decimal degrees) +*###=================================================================================###; +data meso(keep=Production_zone Production_lon Production_lat); + input Production_zone Production_lon Production_lat; + datalines; +1 -87.6298 41.8781 +2 -88.0834 42.0334 +3 -87.9073 41.9742 +4 -87.8612 41.7508 +5 -88.3201 41.5868 +6 -87.6877 41.5250 +; +run; + proc sort data=meso nodupkey; by Production_zone; + + +*###=================================================================================### + READ CMAP FILE +*###=================================================================================###; +data cmapmeso; + input Production_zone Production_lon Production_lat; + ***-- Convert coordinates from decimal degrees to radians for consistency with RSG file -- ***; + ***-- Conversion: decimal degrees * pi / 180 --***; + Production_lon=Production_lon*constant('pi')/180; + Production_lat=Production_lat*constant('pi')/180; + datalines; +1 -87.6298 41.8781 +2 -88.0834 42.0334 +3 -87.9073 41.9742 +4 -87.8612 41.7508 +5 -88.3201 41.5868 +6 -87.6877 41.5250 +; +run; + proc sort data=cmapmeso nodupkey; by Production_zone; + +*###=================================================================================### + MERGE FILES, OVERWRITE RSG DATA WITH CMAP +*###=================================================================================###; +data meso; merge meso cmapmeso; by Production_zone; + + +*###=================================================================================### + CREATE ALL POTENTIAL MESOZONE COMBINATIONS +*###=================================================================================###; +data orig; set meso; +data dest(rename=(Production_zone=Consumption_zone Production_lon=Consumption_lon Production_lat=Consumption_lat)); set meso; + +proc sql noprint; + create table allmeso as + select orig.*, + dest.* + from orig, dest; +quit; + +data allmeso(drop=delta_lon delta_lat a c); set allmeso; + ***-- Calculate Great Circle Distance using Haversine formula -- ***; + ***-- see http://www.movable-type.co.uk/scripts/latlong.html for discussion/documentation --***; + delta_lon=Consumption_lon - Production_lon; + delta_lat=Consumption_lat - Production_lat; + a=sin(delta_lat/2)**2 + cos(Production_lat)*cos(Consumption_lat)*sin(delta_lon/2)**2; + c=2*arsin(min(1,sqrt(a))); + GCD=c*3961; **-- 3961 is radius of Earth in miles, about 39 degrees from equator (Washington DC); + +data allmeso(drop=a b c); set allmeso; + ***-- Ensure Both Directions are included -- ***; + output; + if Production_zone ne Consumption_zone then do; + a=Production_zone; b=Production_lon; c=Production_lat; + Production_zone=Consumption_zone; Production_lon=Consumption_lon; Production_lat=Consumption_lat; + Consumption_zone=a; Consumption_lon=b; Consumption_lat=c; + output; + end; + proc sort nodupkey; by Consumption_zone Production_zone; + + +*###=================================================================================### + PROVIDE A DISTANCE FOR INTRAZONAL PAIRS (U.S. MESOZONES ONLY) +*###=================================================================================###; +data sqmi; + input mesozone sqmi; + datalines; +1 12.4 +2 9.8 +3 15.1 +4 11.3 +5 20.6 +6 8.2 +; +run; + + ***-- For simplicity, assume each mesozone is a square and the average trip distance -- ***; + ***-- equals one-half of the length of each side: thus, sqrt(area)/2 -- ***; +data sqmi(drop=mesozone sqmi); set sqmi; + dist=sqrt(sqmi)/2; + Production_zone=mesozone; + Consumption_zone=mesozone; + proc sort; by Consumption_zone Production_zone; + + +data allmeso(drop=dist); merge allmeso sqmi; by Consumption_zone Production_zone; + if Consumption_zone=Production_zone then GCD=max(GCD,dist); +proc export data=allmeso outfile="data_mesozone_gcd.csv" dbms=csv replace; + +run; diff --git a/jenner-check/t004_zonal_employment/autoexec.sas b/jenner-check/t004_zonal_employment/autoexec.sas new file mode 100644 index 0000000..2052e87 --- /dev/null +++ b/jenner-check/t004_zonal_employment/autoexec.sas @@ -0,0 +1 @@ +options obs=100; diff --git a/jenner-check/t004_zonal_employment/expected.json b/jenner-check/t004_zonal_employment/expected.json new file mode 100644 index 0000000..d9ab2b4 --- /dev/null +++ b/jenner-check/t004_zonal_employment/expected.json @@ -0,0 +1,19 @@ +{ + "_captured_at": "2026-06-17T13:28:09+07:00", + "_captured_run_id": "r_019ed444059373b3bf3d35d7346b1ee2", + "status": "ok", + "exit_code": 0, + "log_contains": [ + "NOTE: Wrote z (5 rows", + "NOTE: Wrote szemp (8 rows", + "NOTE: Wrote corresp (8 rows" + ], + "log_does_not_contain": [ + "ERROR:", + "[JENNER-ERROR" + ], + "diagnostics": { + "parse_warnings": [], + "runtime_warnings": [] + } +} diff --git a/jenner-check/t004_zonal_employment/expected/files.md b/jenner-check/t004_zonal_employment/expected/files.md new file mode 100644 index 0000000..558c12f --- /dev/null +++ b/jenner-check/t004_zonal_employment/expected/files.md @@ -0,0 +1,16 @@ +These URLs point at a specific Jenner run (`r_019ed444059373b3bf3d35d7346b1ee2`) and expire when that run is reaped. Re-running the bundle regenerates them. + + +## Files + +| name | content_type | size_bytes | url | +|---|---|---|---| +| cmap_data_zone_employment.csv | text/csv | 78 | [cmap_data_zone_employment.csv](https://api.jenneranalytics.com/v1/run/r_019ed444059373b3bf3d35d7346b1ee2/files/cmap_data_zone_employment.csv?token=b5285faf769d432baf4b51f3e1963838) | + +## Datasets + +| name | rows | columns | preview | +|---|---|---|---| +| corresp | 8 | subzone09, zone09, mesozone | [preview](https://api.jenneranalytics.com/v1/run/r_019ed444059373b3bf3d35d7346b1ee2/datasets/corresp?token=b5285faf769d432baf4b51f3e1963838) | +| szemp | 8 | subzone09, i18, Zone, Mesozone | [preview](https://api.jenneranalytics.com/v1/run/r_019ed444059373b3bf3d35d7346b1ee2/datasets/szemp?token=b5285faf769d432baf4b51f3e1963838) | +| z | 5 | Zone, Mesozone, totalemp | [preview](https://api.jenneranalytics.com/v1/run/r_019ed444059373b3bf3d35d7346b1ee2/datasets/z?token=b5285faf769d432baf4b51f3e1963838) | diff --git a/jenner-check/t004_zonal_employment/expected/log.txt b/jenner-check/t004_zonal_employment/expected/log.txt new file mode 100644 index 0000000..b81bfca --- /dev/null +++ b/jenner-check/t004_zonal_employment/expected/log.txt @@ -0,0 +1,57 @@ +Jenner 0.1.0 (Unlicensed - limited to 100 observations) +Get a license at https://jenneranalytics.com/license + +NOTE: Option OBS changed to 100. +NOTE: DATA szemp + +NOTE: Processing inline DATALINES (8 lines) + +NOTE: Read 8 rows from DATALINES. +NOTE: Wrote szemp (8 rows, 2 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SORT data=szemp + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 8 rows from szemp. +NOTE: Wrote szemp (8 rows, 2 columns). +NOTE: PROC SORT statement used. +NOTE: DATA corresp + +NOTE: Processing inline DATALINES (8 lines) + +NOTE: Read 8 rows from DATALINES. +NOTE: Wrote corresp (8 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SORT data=corresp + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 8 rows from corresp. +NOTE: Wrote corresp (8 rows, 3 columns). +NOTE: PROC SORT statement used. +NOTE: DATA szemp + +NOTE: Stream 1 processed 8 rows, max BY-group size: 1 (O(1) memory verified) +NOTE: Stream 2 processed 8 rows, max BY-group size: 1 (O(1) memory verified) + +NOTE: Wrote szemp (8 rows, 4 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC MEANS +NOTE: Output dataset z has 5 observations and 5 variables. +NOTE: PROC MEANS statement used. +NOTE: DATA z + + +NOTE: Read 5 rows from z. +NOTE: Wrote z (5 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC EXPORT data=z outfile=cmap_data_zone_employment.csv + +NOTE: Exported 5 rows to cmap_data_zone_employment.csv. diff --git a/jenner-check/t004_zonal_employment/expected/output.txt b/jenner-check/t004_zonal_employment/expected/output.txt new file mode 100644 index 0000000..e69de29 diff --git a/jenner-check/t004_zonal_employment/meta.json b/jenner-check/t004_zonal_employment/meta.json new file mode 100644 index 0000000..21fbbfa --- /dev/null +++ b/jenner-check/t004_zonal_employment/meta.json @@ -0,0 +1,8 @@ +{ + "bundle": "t004_zonal_employment", + "source_file": "Meso_Freight_Skim_Setup_c##q##_YYYY/Database/SAS/Step4_Create_Zonal_Truck_Tour_files.sas", + "source_blob_sha": "17164e7d7127589c56ecf986c10bd77f00bfb2c9", + "source_commit": "772611d8692d9987d3bcc824182f95ce99369cc8", + "tier": "mock_data", + "notes": "Zonal-employment slice of Step 4: imports subzone employment (subzone09/i18) and the subzone-zone-mesozone correspondence (subzone09/zone09/mesozone), sorts and MERGEs on subzone09, then PROC SUMMARY NWAY with CLASS Zone / VAR i18 / ID Mesozone / OUTPUT SUM= rolls employment up to zonal totals, dropping _type_/_freq_ before export. Inputs are bundled small CSVs. The Emme matrix-dump skim sections are outside this slice and omitted. Edits: relative input paths; explicit 'run;' after each PROC IMPORT; PROC EXPORT writes cmap_data_zone_employment.csv via a literal path with explicit data=." +} diff --git a/jenner-check/t004_zonal_employment/script.sas b/jenner-check/t004_zonal_employment/script.sas new file mode 100644 index 0000000..3b3d253 --- /dev/null +++ b/jenner-check/t004_zonal_employment/script.sas @@ -0,0 +1,58 @@ +/* STEP4_CREATE_ZONAL_TRUCK_TOUR_FILES.SAS (Jenner compatibility bundle: zonal employment slice) + Craig Heither, rev. 05-06-2016 + + This program creates the CMAP zonal files used in the truck touring model. + + Adapted for self-contained execution: this bundle keeps the zonal-employment + slice of Step 4 verbatim -- read the conformity subzone employment file and + the subzone-zone-mesozone correspondence file, sort and MERGE them on + subzone09, then PROC SUMMARY NWAY with CLASS Zone / VAR i18 / ID Mesozone to + roll subzone employment up to zonal totals, and export the result. The + bundled inputs are small CSVs with the same columns the program reads + (subzone09/i18 for employment; subzone09/zone09/mesozone for the + correspondence). The skim-file sections that read Emme matrix dumps via INFILE + are outside this slice and are omitted. +*/ + +*###=================================================================================### + PROVIDE ZONAL EMPLOYMENT FOR FIRM LOCATIONS +*###=================================================================================###; + *##-- subzone total employment --##; +data szemp; + input subzone09 i18; + datalines; +1001 420 +1002 135 +1003 610 +2001 288 +2002 90 +3001 512 +3002 77 +3003 203 +; +run; +proc sort data=szemp; by subzone09; run; + + *##-- subzone-zone-mesozone correspondence file --##; +data corresp; + input subzone09 zone09 mesozone; + datalines; +1001 10 101 +1002 10 101 +1003 11 101 +2001 20 102 +2002 20 102 +3001 30 103 +3002 31 103 +3003 31 103 +; +run; +proc sort data=corresp; by subzone09; run; + +data szemp(rename=(zone09=Zone mesozone=Mesozone)); merge szemp corresp; by subzone09; + proc summary nway; class Zone; var i18; id Mesozone; output out=z sum=totalemp; + +data z(drop=_type_ _freq_); set z; +proc export data=z outfile="cmap_data_zone_employment.csv" dbms=csv replace; + +run; diff --git a/jenner-check/t005_verify_rail_macro/autoexec.sas b/jenner-check/t005_verify_rail_macro/autoexec.sas new file mode 100644 index 0000000..da7f46a --- /dev/null +++ b/jenner-check/t005_verify_rail_macro/autoexec.sas @@ -0,0 +1,2 @@ +options obs=100; +%let OP=rl; /* rail carrier mode label (set by the caller in production) */ diff --git a/jenner-check/t005_verify_rail_macro/expected.json b/jenner-check/t005_verify_rail_macro/expected.json new file mode 100644 index 0000000..d5d7379 --- /dev/null +++ b/jenner-check/t005_verify_rail_macro/expected.json @@ -0,0 +1,19 @@ +{ + "_captured_at": "2026-06-17T13:29:00+07:00", + "_captured_run_id": "r_019ed444cc5171f18ad33530987d5644", + "status": "ok", + "exit_code": 0, + "log_contains": [ + "NOTE: Wrote rail61 (2 rows", + "NOTE: Wrote zones (1 rows", + "NOTE: PROC TRANSPOSE" + ], + "log_does_not_contain": [ + "ERROR:", + "[JENNER-ERROR" + ], + "diagnostics": { + "parse_warnings": [], + "runtime_warnings": [] + } +} diff --git a/jenner-check/t005_verify_rail_macro/expected/files.md b/jenner-check/t005_verify_rail_macro/expected/files.md new file mode 100644 index 0000000..f8cbe6b --- /dev/null +++ b/jenner-check/t005_verify_rail_macro/expected/files.md @@ -0,0 +1,20 @@ +These URLs point at a specific Jenner run (`r_019ed444cc5171f18ad33530987d5644`) and expire when that run is reaped. Re-running the bundle regenerates them. + + +## Files + +| name | content_type | size_bytes | url | +|---|---|---|---| +| zone_connections.csv | text/csv | 20 | [zone_connections.csv](https://api.jenneranalytics.com/v1/run/r_019ed444cc5171f18ad33530987d5644/files/zone_connections.csv?token=8272cddbad314f94830f4c37a783a364) | + +## Datasets + +| name | rows | columns | preview | +|---|---|---|---| +| dist61 | 12 | o, dest, dist | [preview](https://api.jenneranalytics.com/v1/run/r_019ed444cc5171f18ad33530987d5644/datasets/dist61?token=8272cddbad314f94830f4c37a783a364) | +| ivtt62 | 12 | o, dest, ivtt | [preview](https://api.jenneranalytics.com/v1/run/r_019ed444cc5171f18ad33530987d5644/datasets/ivtt62?token=8272cddbad314f94830f4c37a783a364) | +| mf61_raw | 4 | o, d1, v1, d2, v2, d3, v3 | [preview](https://api.jenneranalytics.com/v1/run/r_019ed444cc5171f18ad33530987d5644/datasets/mf61_raw?token=8272cddbad314f94830f4c37a783a364) | +| mf62_raw | 4 | o, d1, v1, d2, v2, d3, v3 | [preview](https://api.jenneranalytics.com/v1/run/r_019ed444cc5171f18ad33530987d5644/datasets/mf62_raw?token=8272cddbad314f94830f4c37a783a364) | +| rail61 | 2 | mode, o, dest, dist, ivtt | [preview](https://api.jenneranalytics.com/v1/run/r_019ed444cc5171f18ad33530987d5644/datasets/rail61?token=8272cddbad314f94830f4c37a783a364) | +| review | 1 | mode, o, dest, dist, ivtt | [preview](https://api.jenneranalytics.com/v1/run/r_019ed444cc5171f18ad33530987d5644/datasets/review?token=8272cddbad314f94830f4c37a783a364) | +| zones | 1 | o, dest, mode1 | [preview](https://api.jenneranalytics.com/v1/run/r_019ed444cc5171f18ad33530987d5644/datasets/zones?token=8272cddbad314f94830f4c37a783a364) | diff --git a/jenner-check/t005_verify_rail_macro/expected/log.txt b/jenner-check/t005_verify_rail_macro/expected/log.txt new file mode 100644 index 0000000..9af43d8 --- /dev/null +++ b/jenner-check/t005_verify_rail_macro/expected/log.txt @@ -0,0 +1,105 @@ +Jenner 0.1.0 (Unlicensed - limited to 100 observations) +Get a license at https://jenneranalytics.com/license + +NOTE: Option OBS changed to 100. +NOTE: Fileref OUT1 assigned to zone_connections.csv. +NOTE: DATA mf61_raw + +NOTE: Processing inline DATALINES (4 lines) + +NOTE: Read 4 rows from DATALINES. +NOTE: Wrote mf61_raw (4 rows, 7 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA mf62_raw + +NOTE: Processing inline DATALINES (4 lines) + +NOTE: Read 4 rows from DATALINES. +NOTE: Wrote mf62_raw (4 rows, 7 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA dist61 + + +NOTE: Read 4 rows from mf61_raw. +NOTE: Wrote dist61 (12 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SORT data= + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 12 rows from . +NOTE: Wrote (12 rows, 3 columns). +NOTE: PROC SORT statement used. +NOTE: DATA dist61 + + +NOTE: Read 12 rows from dist61. +NOTE: Wrote dist61 (12 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA ivtt62 + + +NOTE: Read 4 rows from mf62_raw. +NOTE: Wrote ivtt62 (12 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SORT data= + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 12 rows from . +NOTE: Wrote (12 rows, 3 columns). +NOTE: PROC SORT statement used. +NOTE: DATA ivtt62 + + +NOTE: Read 12 rows from ivtt62. +NOTE: Wrote ivtt62 (12 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA rail61 + +NOTE: Stream 1 processed 12 rows, max BY-group size: 1 (O(1) memory verified) +NOTE: Stream 2 processed 12 rows, max BY-group size: 1 (O(1) memory verified) + +NOTE: Wrote rail61 (2 rows, 5 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: DATA review + + +NOTE: Read 2 rows from rail61. +NOTE: Wrote review (1 rows, 5 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC SORT data= + +NOTE: Unlicensed mode - input limited to 100 observations. +NOTE: Read 1 rows from . +NOTE: Wrote (1 rows, 5 columns). +NOTE: PROC SORT statement used. +NOTE: PROC TRANSPOSE data= + +NOTE: Created 1 rows. +NOTE: Output written to /tmp/work/work/b5bdacb9-b8ba-4651-ab37-b31460d691c0/zones.avro. +NOTE: DATA zones + + +NOTE: Read 1 rows from zones. +NOTE: Wrote zones (1 rows, 3 columns). +NOTE: DATA elapsed: + wall 0.00 seconds + cpu 0.00 seconds +NOTE: PROC EXPORT data=zones outfile=zone_connections.csv + +NOTE: Exported 1 rows to zone_connections.csv. diff --git a/jenner-check/t005_verify_rail_macro/expected/output.txt b/jenner-check/t005_verify_rail_macro/expected/output.txt new file mode 100644 index 0000000..e69de29 diff --git a/jenner-check/t005_verify_rail_macro/meta.json b/jenner-check/t005_verify_rail_macro/meta.json new file mode 100644 index 0000000..64228d8 --- /dev/null +++ b/jenner-check/t005_verify_rail_macro/meta.json @@ -0,0 +1,8 @@ +{ + "bundle": "t005_verify_rail_macro", + "source_file": "Meso_Freight_Skim_Setup_c##q##_YYYY/Database/macros/verify_rail_service.sas", + "source_blob_sha": "5a3a2431490713876f888856e8f22fddd6852b46", + "source_commit": "772611d8692d9987d3bcc824182f95ce99369cc8", + "tier": "mock_data", + "notes": "Keeps the %ReadSkims macro and its %do %while loop: per rail carrier it takes the Emme distance (mf61) and in-vehicle time (mf62) skim dumps, explodes the wide o/dest:value rows to long form, sorts nodupkey, MERGEs distance with time, and deletes pairs that have both dist>0 and ivtt>0 -- leaving the connector-to-connector (zero-skim) pairs the QC is meant to surface. PROC TRANSPOSE then pivots the flagged modes and the result is exported. The two Emme matrix dumps are provided as inline DATA steps (mf61_raw/mf62_raw, the same wide layout the .in files hold) so the bundle is a single self-contained script; the macro reads them via SET mf&i._raw instead of INFILE. Edits: %let OP=rl in the autoexec (caller-supplied in production); the call system() that created the QC folder is omitted; PROC EXPORT uses an explicit data= and literal path." +} diff --git a/jenner-check/t005_verify_rail_macro/script.sas b/jenner-check/t005_verify_rail_macro/script.sas new file mode 100644 index 0000000..a0c13b9 --- /dev/null +++ b/jenner-check/t005_verify_rail_macro/script.sas @@ -0,0 +1,87 @@ +/* VERIFY_RAIL_SERVICE.SAS (Jenner compatibility bundle) + Craig Heither, 06-14-2016 + + This program reads the distance and in-vehicle times skims for each rail carrier to ensure no connector-to-connector paths are + being used. + + Adapted for self-contained execution: the macro %ReadSkims is kept verbatim -- + a %do %while loop that, for each carrier, reads the Emme distance and in-vehicle + time skim dumps via INFILE (missover, dlm=' :', firstobs=5), explodes the wide + o/dest/value rows into long form, sorts nodupkey, MERGEs distance with time, and + flags zero-skim pairs. The bundled mf61.in / mf62.in are small Emme matrix dumps + in the same format the program reads. The output is written to a relative path; + the call system() that created the QC directory in production is not needed here. +*/ +*################################################################################################; + +filename out1 "zone_connections.csv"; + +*###=================================================================================### + -- EMME SKIM MATRIX DUMPS (wide o / dest:value rows, as the .in files hold) -- +*###=================================================================================###; + *##-- mf61: rail carrier distance --##; +data mf61_raw; + input o d1 v1 d2 v2 d3 v3; + datalines; +1 2 14.5 3 22.1 4 0 +2 1 14.5 3 9.8 4 18.2 +3 1 22.1 2 9.8 4 7.4 +4 1 0 2 18.2 3 7.4 +; +run; + *##-- mf62: rail carrier in-vehicle time --##; +data mf62_raw; + input o d1 v1 d2 v2 d3 v3; + datalines; +1 2 31.0 3 47.5 4 0 +2 1 31.0 3 20.4 4 39.1 +3 1 47.5 2 20.4 4 15.8 +4 1 0 2 39.1 3 15.8 +; +run; + +*###=================================================================================### + -- PROCESS EMME SKIMS -- +*###=================================================================================###; +%let i=61; %let j=62; +%macro ReadSkims; + + %do %while (&i le 63); + run; + data dist&i(keep=o dest dist); set mf&i._raw; + dest=d1; dist=v1; output; + dest=d2; dist=v2; output; + dest=d3; dist=v3; output; + proc sort nodupkey; by o dest; + data dist&i; set dist&i(where=(o>0 & dest>0)); run; + + data ivtt&j(keep=o dest ivtt); set mf&j._raw; + dest=d1; ivtt=v1; output; + dest=d2; ivtt=v2; output; + dest=d3; ivtt=v3; output; + proc sort nodupkey; by o dest; + data ivtt&j; set ivtt&j(where=(o>0 & dest>0)); run; + + data rail&i; merge dist&i ivtt&j; by o dest; + length mode $2.; + mode="&OP"; + if dist>0 & ivtt>0 then delete; + run; + + %let i=%eval(&i+4); %let j=%eval(&j+4); + + %end; + run; + +%mend ReadSkims; +%ReadSkims +/* end of macro */ + +data review; set rail61; + if o