From 25b476c5920c70311857a762efc9375af2bb3f93 Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Sun, 19 Apr 2026 16:17:47 -0400 Subject: [PATCH 1/9] Add threat-model skill plugin with TNF, TNA, SNO, and LVMS support Co-Authored-By: Claude Opus 4.6 --- .claude/skills/threat-model-lvms | 1 + .claude/skills/threat-model-sno | 1 + .claude/skills/threat-model-tna | 1 + .claude/skills/threat-model-tnf | 1 + .../threat-model/.claude-plugin/plugin.json | 10 + plugins/threat-model/README.md | 104 ++++ .../references/mitre-findings-template.md | 13 + .../references/mitre-reference.md | 171 ++++++ .../references/owasp-reference.md | 117 ++++ .../skills/lvms-threat-model/SKILL.md | 359 +++++++++++ .../lvms-threat-model/dfd-elements-lvms.md | 17 + .../skills/sno-threat-model/SKILL.md | 455 ++++++++++++++ .../sno-threat-model/dfd-elements-sno.md | 118 ++++ .../skills/tna-threat-model/SKILL.md | 558 +++++++++++++++++ .../tna-threat-model/dfd-elements-tna.md | 68 +++ .../skills/tnf-threat-model/SKILL.md | 570 ++++++++++++++++++ .../tnf-threat-model/dfd-elements-tnf.md | 120 ++++ 17 files changed, 2684 insertions(+) create mode 120000 .claude/skills/threat-model-lvms create mode 120000 .claude/skills/threat-model-sno create mode 120000 .claude/skills/threat-model-tna create mode 120000 .claude/skills/threat-model-tnf create mode 100644 plugins/threat-model/.claude-plugin/plugin.json create mode 100644 plugins/threat-model/README.md create mode 100644 plugins/threat-model/references/mitre-findings-template.md create mode 100644 plugins/threat-model/references/mitre-reference.md create mode 100644 plugins/threat-model/references/owasp-reference.md create mode 100644 plugins/threat-model/skills/lvms-threat-model/SKILL.md create mode 100644 plugins/threat-model/skills/lvms-threat-model/dfd-elements-lvms.md create mode 100644 plugins/threat-model/skills/sno-threat-model/SKILL.md create mode 100644 plugins/threat-model/skills/sno-threat-model/dfd-elements-sno.md create mode 100644 plugins/threat-model/skills/tna-threat-model/SKILL.md create mode 100644 plugins/threat-model/skills/tna-threat-model/dfd-elements-tna.md create mode 100644 plugins/threat-model/skills/tnf-threat-model/SKILL.md create mode 100644 plugins/threat-model/skills/tnf-threat-model/dfd-elements-tnf.md diff --git a/.claude/skills/threat-model-lvms b/.claude/skills/threat-model-lvms new file mode 120000 index 00000000..34a1f628 --- /dev/null +++ b/.claude/skills/threat-model-lvms @@ -0,0 +1 @@ +../../plugins/threat-model/skills/lvms-threat-model \ No newline at end of file diff --git a/.claude/skills/threat-model-sno b/.claude/skills/threat-model-sno new file mode 120000 index 00000000..7b03e4b0 --- /dev/null +++ b/.claude/skills/threat-model-sno @@ -0,0 +1 @@ +../../plugins/threat-model/skills/sno-threat-model \ No newline at end of file diff --git a/.claude/skills/threat-model-tna b/.claude/skills/threat-model-tna new file mode 120000 index 00000000..f1ef2498 --- /dev/null +++ b/.claude/skills/threat-model-tna @@ -0,0 +1 @@ +../../plugins/threat-model/skills/tna-threat-model \ No newline at end of file diff --git a/.claude/skills/threat-model-tnf b/.claude/skills/threat-model-tnf new file mode 120000 index 00000000..7fd69d32 --- /dev/null +++ b/.claude/skills/threat-model-tnf @@ -0,0 +1 @@ +../../plugins/threat-model/skills/tnf-threat-model \ No newline at end of file diff --git a/plugins/threat-model/.claude-plugin/plugin.json b/plugins/threat-model/.claude-plugin/plugin.json new file mode 100644 index 00000000..775bf5c8 --- /dev/null +++ b/plugins/threat-model/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "threat-model", + "version": "1.0.0", + "description": "Analyze PRs for security threats with STRIDE/DFD analysis, MITRE ATT&CK and OWASP mapping for OpenShift edge topologies (TNA, TNF, SNO, LVMS)", + "author": { + "name": "TNF Security" + }, + "license": "Apache-2.0", + "keywords": ["security", "threat-model", "stride", "mitre", "owasp", "openshift", "tnf", "tna", "sno", "lvms"] +} diff --git a/plugins/threat-model/README.md b/plugins/threat-model/README.md new file mode 100644 index 00000000..4e767758 --- /dev/null +++ b/plugins/threat-model/README.md @@ -0,0 +1,104 @@ +# Threat Model Plugin for Claude Code + +Security threat analysis for OpenShift PRs across multiple topologies (TNF, TNA, SNO, LVMS). + +## What It Does + +Analyzes pull requests for security threats against OpenShift clusters: + +- Fetches PR diffs from GitHub +- Runs ShellCheck on shell scripts +- Maps changes to Data Flow Diagram (DFD) elements +- Applies per-element STRIDE analysis +- Cross-references against formal threat models +- Maps findings to MITRE ATT&CK techniques and OWASP Top 10:2025 +- Generates formal threat analysis reports + +## Usage + +### TNF (Two-Node Fencing) + +```bash +/threat-model:tnf 2136 +/threat-model:tnf https://github.com/ClusterLabs/resource-agents/pull/2136 +/threat-model:tnf resource-agents 2136 +``` + +### TNA (Two-Node Arbiter) + +```bash +/threat-model:tna 1437 +/threat-model:tna https://github.com/openshift/cluster-etcd-operator/pull/1437 +/threat-model:tna installer 10403 +``` + +### SNO (Single Node OpenShift) + +```bash +/threat-model:sno 10498 +/threat-model:sno https://github.com/openshift/installer/pull/10498 +/threat-model:sno installer 10498 +``` + +### LVMS (LVM Storage) + +```bash +/threat-model:lvms 2271 +/threat-model:lvms https://github.com/openshift/lvm-operator/pull/2271 +/threat-model:lvms lvm-operator 2271 +``` + +> **Note**: The LVMS DFD model is not yet defined. The LVMS skill performs general security analysis, ShellCheck scanning, and MITRE/OWASP mapping. Full DFD/STRIDE analysis will be available once its DFD model is created. + +## Workspace Requirements + +The skill expects a workspace with a `repos/` directory containing cloned repositories. It auto-discovers the workspace root at runtime. + +### Recommended workspace layout + +```text +your-workspace/ +├── repos/ +│ ├── cluster-etcd-operator/ +│ ├── installer/ +│ ├── machine-config-operator/ +│ ├── resource-agents/ +│ ├── two-node-toolbox/ +│ │ └── docs/ +│ │ ├── TNF-THREAT-MODEL.md +│ │ └── TNA-THREAT-MODEL.md +│ └── ... +└── .claude/ + └── skills/ + ├── threat-model/ + ├── mitre-findings-tnf.md # Created automatically on first use + ├── mitre-findings-tna.md + ├── mitre-findings-sno.md + └── mitre-findings-lvms.md +``` + +### Optional dependencies + +- **ShellCheck** (`dnf install ShellCheck`) - for automated shell script analysis +- **gh** CLI - for fetching PR details from GitHub +- **Formal threat model files** - for DFD/STRIDE cross-referencing + +## What's Included + +| File | Purpose | +|------|---------| +| `skills/tnf-threat-model/SKILL.md` | TNF threat analysis skill | +| `skills/tnf-threat-model/dfd-elements-tnf.md` | TNF DFD element catalog | +| `skills/tna-threat-model/SKILL.md` | TNA threat analysis skill | +| `skills/tna-threat-model/dfd-elements-tna.md` | TNA DFD element catalog | +| `skills/sno-threat-model/SKILL.md` | SNO threat analysis skill | +| `skills/sno-threat-model/dfd-elements-sno.md` | SNO DFD element catalog (SNO-P1–P6, SNO-DS1–DS6, SNO-DF1–DF10, SNO-TB1–TB3) | +| `skills/lvms-threat-model/SKILL.md` | LVMS threat analysis skill | +| `skills/lvms-threat-model/dfd-elements-lvms.md` | LVMS DFD element catalog (placeholder) | +| `references/mitre-reference.md` | MITRE ATT&CK quick reference | +| `references/owasp-reference.md` | OWASP Top 10:2025 reference | +| `references/mitre-findings-template.md` | Cumulative findings tracker template | + +## License + +Apache-2.0 diff --git a/plugins/threat-model/references/mitre-findings-template.md b/plugins/threat-model/references/mitre-findings-template.md new file mode 100644 index 00000000..845a9a78 --- /dev/null +++ b/plugins/threat-model/references/mitre-findings-template.md @@ -0,0 +1,13 @@ +# MITRE ATT&CK Findings Tracker + +Cumulative security findings from PR threat analysis. + +## Legend + +**Severity**: Critical / High / Medium / Low / Info +**Status**: Open / Mitigated / Accepted / FalsePositive + +--- + + + diff --git a/plugins/threat-model/references/mitre-reference.md b/plugins/threat-model/references/mitre-reference.md new file mode 100644 index 00000000..662db209 --- /dev/null +++ b/plugins/threat-model/references/mitre-reference.md @@ -0,0 +1,171 @@ +# MITRE ATT&CK Quick Reference + +Common techniques for infrastructure and Kubernetes security. + +## Initial Access (TA0001) + +| ID | Technique | Indicators | +|----|-----------|------------| +| T1078 | Valid Accounts | Default creds, leaked tokens, service account abuse | +| T1190 | Exploit Public-Facing App | Unpatched CVEs, injection flaws | +| T1133 | External Remote Services | Exposed SSH, RDP, VNC, API endpoints | + +## Execution (TA0002) + +| ID | Technique | Indicators | +|----|-----------|------------| +| T1059 | Command/Script Interpreter | Shell exec, eval, unsanitized input to commands | +| T1609 | Container Admin Command | kubectl exec, docker exec, crictl | +| T1610 | Deploy Container | Malicious container images, privileged pods | + +## Persistence (TA0003) + +| ID | Technique | Indicators | +|----|-----------|------------| +| T1053 | Scheduled Task/Job | CronJobs, systemd timers | +| T1098 | Account Manipulation | Adding users, modifying RBAC | +| T1543 | Create/Modify System Process | Systemd services, init scripts | +| T1136 | Create Account | New ServiceAccounts, local users | + +## Privilege Escalation (TA0004) + +| ID | Technique | Indicators | +|----|-----------|------------| +| T1068 | Exploitation for Privilege Escalation | CVE exploits, kernel vulns | +| T1548 | Abuse Elevation Control | sudo, setuid, capabilities | +| T1611 | Escape to Host | Container breakout, hostPID, hostNetwork | + +## Defense Evasion (TA0005) + +| ID | Technique | Indicators | +|----|-----------|------------| +| T1070 | Indicator Removal | Log deletion, history clearing | +| T1562 | Impair Defenses | Disabling SELinux, seccomp, audit | +| T1036 | Masquerading | Renamed binaries, fake processes | + +## Credential Access (TA0006) + +| ID | Technique | Indicators | +|----|-----------|------------| +| T1552 | Unsecured Credentials | Hardcoded secrets, env vars, config files | +| T1528 | Steal Application Access Token | Token theft, SA token access | +| T1003 | OS Credential Dumping | /etc/shadow, memory scraping | +| T1555 | Credentials from Password Stores | Secret managers, keyrings | + +## Discovery (TA0007) + +| ID | Technique | Indicators | +|----|-----------|------------| +| T1083 | File and Directory Discovery | Filesystem enumeration | +| T1046 | Network Service Discovery | Port scanning, service probing | +| T1613 | Container and Resource Discovery | kubectl get, API enumeration | + +## Lateral Movement (TA0008) + +| ID | Technique | Indicators | +|----|-----------|------------| +| T1021 | Remote Services | SSH, WinRM, kubectl | +| T1550 | Use Alternate Auth Material | Token reuse, cert theft | + +## Impact (TA0040) + +| ID | Technique | Indicators | +|----|-----------|------------| +| T1485 | Data Destruction | rm -rf, etcd data deletion | +| T1486 | Data Encrypted for Impact | Ransomware patterns | +| T1489 | Service Stop | systemctl stop, kill processes | +| T1529 | System Shutdown/Reboot | STONITH abuse, power off | + +## TNF-Specific Techniques + +| ID | Technique | TNF Context | DFD Elements | +|----|-----------|-------------|--------------| +| T1552 | Unsecured Credentials | BMC credentials in install-config, secrets, CIB | DS1, DS2, DS3, DF1-DF9 | +| T1529 | System Shutdown | Malicious fencing, STONITH abuse | P6, P8, EE2 | +| T1489 | Service Stop | etcd/pacemaker service disruption | P7, DS5 | +| T1557 | Adversary-in-the-Middle | BMC MITM when cert disabled, Corosync interception | P8, DF10, EE2, EE3 | +| T1078 | Valid Accounts | BMC account compromise, predictable PCSD token | P3, P8, DS4, EE2 | +| T1059 | Command Interpreter | Shell injection via credentials, OCF agent scripts | P5, P7, DF9 | +| T1611 | Escape to Host | Privileged TNF setup/fencing containers with nsenter | P3, P4, P5 | +| T1562 | Impair Defenses | CIB manipulation to disable STONITH | DS3, P4 | + +## TNA-Specific Techniques + +| ID | Technique | TNA Context | DFD Elements | +|----|-----------|-------------|--------------| +| T1078 | Valid Accounts | Admin credential theft (kubeconfig) | TNA-EE1 | +| T1552 | Unsecured Credentials | Worker ignition token leak | TNA-DS6 | +| T1611 | Escape to Host | Container escape from pod to node root | TNA-P5 | +| T1562 | Impair Defenses | Arbiter taint removal disabling scheduling protection | TNA-P3 | +| T1489 | Service Stop | etcd quorum disruption (arbiter + 1 master) | TNA-DS5 | +| T1021 | Remote Services | Lateral movement from worker to control plane via pod network | TNA-P5, TNA-DS5 | + +## SNO-Specific Techniques + +| ID | Technique | SNO Context | DFD Elements | +|----|-----------|-------------|--------------| +| T1552 | Unsecured Credentials | Pull secret, offline token, kubeadmin-password on admin workstation | SNO-DS1, SNO-DS4 | +| T1611 | Escape to Host | Bootstrap-in-place agent runs privileged on bare metal | SNO-P5 | +| T1610 | Deploy Container | Workloads scheduled on master (no worker isolation) | SNO-P6 | +| T1562 | Impair Defenses | UnsafeScalingStrategy bypasses quorum safety checks | SNO-P4 | +| T1485 | Data Destruction | Single etcd member — node failure = total data loss | SNO-DS3 | +| T1195 | Supply Chain Compromise | Discovery ISO tampering before boot | SNO-DS2 | +| T1078 | Valid Accounts | Admin credential theft (kubeconfig) | SNO-EE1 | + +## LVMS-Specific Techniques + +> **Not yet defined.** This section will be populated once the LVMS DFD model is created. + +## TNF DFD Element to ATT&CK Mapping + +| DFD Element | Primary ATT&CK Techniques | Per-Element STRIDE IDs | +|-------------|--------------------------|----------------------| +| P1 (Installer) | T1552 | PE-P1-I-1, PE-P1-T-1 | +| P3 (Auth Job) | T1078, T1611 | PE-P3-S-1, PE-P3-E-1 | +| P4 (Setup Job) | T1611, T1562 | PE-P4-E-1, PE-P4-T-1 | +| P5 (Fencing Job) | T1059, T1552, T1611 | PE-P5-I-1, PE-P5-T-1, PE-P5-E-1 | +| P6 (fenced) | T1529 | PE-P6-S-1, PE-P6-D-1 | +| P7 (podman-etcd) | T1489, T1059 | PE-P7-T-1, PE-P7-D-1 | +| P8 (fence_redfish) | T1557, T1529, T1552 | PE-P8-S-1, PE-P8-I-1 | +| DS1 (install-config) | T1552 | PE-DS1-I-1 | +| DS2 (K8s Secrets) | T1552 | PE-DS2-I-1, PE-DS2-T-1 | +| DS3 (CIB) | T1552, T1562 | PE-DS3-I-1, PE-DS3-T-1 | +| DS4 (PCSD Token) | T1078 | PE-DS4-I-1 | +| DF9 (creds as CLI args) | T1552, T1059 | PE-DF9-I-1 | +| DF10 (Redfish HTTPS) | T1557 | PE-DF10-T-1, PE-DF10-I-1 | +| EE2 (BMC) | T1529, T1190 | PE-EE2-S-1, PE-EE2-S-2 | + +## TNA DFD Element to ATT&CK Mapping + +| DFD Element | Primary ATT&CK Techniques | Per-Element STRIDE IDs | +|-------------|--------------------------|----------------------| +| TNA-P1 (Installer) | T1552 | PE-TNA-P1-T-1, PE-TNA-P1-D-1 | +| TNA-P3 (MCO) | T1562 | PE-TNA-P3-T-1, PE-TNA-P3-D-1 | +| TNA-P4 (CEO) | T1489 | PE-TNA-P4-T-1, PE-TNA-P4-D-1 | +| TNA-P5 (Worker Kubelet) | T1021, T1611 | PE-TNA-P5-S-1, PE-TNA-P5-E-1 | +| TNA-DS5 (etcd Data) | T1552, T1489 | PE-TNA-DS5-T-1, PE-TNA-DS5-I-1, PE-TNA-DS5-D-1 | +| TNA-DS6 (Worker Ignition) | T1552 | PE-TNA-DS6-T-1, PE-TNA-DS6-I-1 | +| TNA-EE1 (Admin) | T1078 | PE-TNA-EE1-S-1, PE-TNA-EE1-R-1 | + +## SNO DFD Element to ATT&CK Mapping + +| DFD Element | Primary ATT&CK Techniques | Per-Element STRIDE IDs | +|-------------|--------------------------|----------------------| +| SNO-P1 (Installer) | T1552 | PE-SNO-P1-T-1, PE-SNO-P1-I-1 | +| SNO-P2 (Assisted Service) | T1078, T1552 | PE-SNO-P2-S-1, PE-SNO-P2-T-1 | +| SNO-P3 (MCO) | T1611 | PE-SNO-P3-T-1, PE-SNO-P3-E-1 | +| SNO-P4 (CEO) | T1562 | PE-SNO-P4-T-1, PE-SNO-P4-D-1 | +| SNO-P5 (Bootstrap Agent) | T1611, T1552 | PE-SNO-P5-E-1, PE-SNO-P5-I-1 | +| SNO-P6 (Kubelet) | T1610 | PE-SNO-P6-E-1, PE-SNO-P6-D-1 | +| SNO-DS1 (install-config) | T1552 | PE-SNO-DS1-I-1 | +| SNO-DS2 (Discovery ISO) | T1195 | PE-SNO-DS2-T-1, PE-SNO-DS2-I-1 | +| SNO-DS3 (etcd Data) | T1485, T1552 | PE-SNO-DS3-T-1, PE-SNO-DS3-I-1, PE-SNO-DS3-D-1 | +| SNO-DS4 (Credentials) | T1552 | PE-SNO-DS4-I-1 | +| SNO-EE1 (Admin) | T1078 | PE-SNO-EE1-S-1, PE-SNO-EE1-R-1 | +| SNO-EE2 (Assisted Service Cloud) | T1195 | PE-SNO-EE2-S-1 | + +## References + +- MITRE ATT&CK Enterprise: +- MITRE ATT&CK Containers: +- MITRE ATT&CK Mitigations: diff --git a/plugins/threat-model/references/owasp-reference.md b/plugins/threat-model/references/owasp-reference.md new file mode 100644 index 00000000..155af281 --- /dev/null +++ b/plugins/threat-model/references/owasp-reference.md @@ -0,0 +1,117 @@ +# OWASP Top 10:2025 Reference + +Quick reference for mapping findings to OWASP categories. + +Source: + +## OWASP Top 10:2025 Categories + +| ID | Category | Description | Common CWEs | +|----|----------|-------------|-------------| +| **A01** | Broken Access Control | Missing or improper access restrictions; SSRF now included | CWE-22, CWE-284, CWE-285, CWE-352, CWE-918 | +| **A02** | Security Misconfiguration | Insecure defaults, open cloud storage, verbose errors, missing hardening | CWE-16, CWE-209, CWE-548 | +| **A03** | Software Supply Chain Failures | Vulnerable dependencies, compromised build pipelines, untrusted sources | CWE-426, CWE-494, CWE-829 | +| **A04** | Cryptographic Failures | Weak crypto, exposed keys, missing encryption, improper certificate validation | CWE-259, CWE-327, CWE-328, CWE-330, CWE-331 | +| **A05** | Injection | SQL, NoSQL, OS command, LDAP, XSS injection | CWE-20, CWE-74, CWE-77, CWE-78, CWE-79, CWE-89 | +| **A06** | Insecure Design | Missing threat modeling, insecure architecture patterns | CWE-73, CWE-183, CWE-209, CWE-312 | +| **A07** | Authentication Failures | Broken auth, credential stuffing, weak passwords, session issues | CWE-287, CWE-384, CWE-522, CWE-798 | +| **A08** | Software or Data Integrity Failures | Code/data without integrity verification, insecure deserialization | CWE-345, CWE-353, CWE-426, CWE-502 | +| **A09** | Security Logging and Alerting Failures | Missing audit logs, unmonitored security events | CWE-117, CWE-223, CWE-532, CWE-778 | +| **A10** | Mishandling of Exceptional Conditions | Improper error handling, fail-open logic, unhandled exceptions | CWE-252, CWE-280, CWE-388, CWE-754, CWE-755 | + +--- + +## Pattern to OWASP Mapping + +| Security Pattern | OWASP | MITRE | CWE | +|-----------------|-------|-------|-----| +| **Command Injection** | A05 | T1059 | CWE-78 | +| Shell exec with unsanitized input | A05 | T1059 | CWE-78 | +| fmt.Sprintf() building shell commands | A05 | T1059 | CWE-78 | +| **Hardcoded Credentials** | A07 | T1552 | CWE-798 | +| Passwords in source code | A07 | T1552 | CWE-798 | +| API keys in config files | A07 | T1552 | CWE-798 | +| **Broken Access Control** | A01 | T1078 | CWE-284 | +| Missing authorization checks | A01 | T1078 | CWE-285 | +| Path traversal | A01 | T1083 | CWE-22 | +| SSRF | A01 | T1046 | CWE-918 | +| **Cryptographic Failures** | A04 | T1573 | CWE-327 | +| Weak algorithms (MD5, SHA1) | A04 | T1573 | CWE-328 | +| Disabled TLS verification | A04 | T1557 | CWE-295 | +| InsecureSkipVerify = true | A04 | T1557 | CWE-295 | +| **Security Misconfiguration** | A02 | T1562 | CWE-16 | +| Debug mode in production | A02 | T1562 | CWE-489 | +| Privileged containers | A02 | T1611 | CWE-250 | +| **Insecure Deserialization** | A08 | T1059 | CWE-502 | +| pickle.loads(), yaml.load() | A08 | T1059 | CWE-502 | +| **Logging Sensitive Data** | A09 | T1005 | CWE-532 | +| Credentials in logs | A09 | T1005 | CWE-532 | +| **Missing Error Handling** | A10 | - | CWE-754 | +| Unchecked error returns | A10 | - | CWE-252 | +| Fail-open logic | A10 | T1562 | CWE-636 | + +--- + +## TNF-Specific OWASP Mappings + +| TNF Component | Risk | OWASP | MITRE | CWE | DFD Elements | PE-* IDs | +|---------------|------|-------|-------|-----|--------------|----------| +| BMC credentials in install-config | Hardcoded secrets | A07 | T1552 | CWE-798 | P1, DS1, DF1, DF2 | PE-P1-I-1, PE-DS1-I-1 | +| BMC password in shell command | Command injection | A05 | T1059 | CWE-78 | P5, DF9 | PE-P5-T-1, PE-P5-I-1 | +| Credentials in CIB XML | Plaintext storage | A04 | T1552 | CWE-312 | DS3, DF7 | PE-DS3-I-1, PE-DF7-I-1 | +| InsecureSkipVerify on BMC | Crypto failure | A04 | T1557 | CWE-295 | P8, DF10 | PE-P8-S-1, PE-DF10-T-1 | +| Privileged TNF setup pods | Misconfiguration | A02 | T1611 | CWE-250 | P3, P4, P5 | PE-P4-E-1, PE-P5-E-1 | +| fencing-credentials Secret | Access control | A01 | T1552 | CWE-284 | DS2, DF4 | PE-DS2-I-1, PE-DS2-T-1 | +| Corosync unencrypted | Crypto failure | A04 | T1557 | CWE-319 | EE3, DF12 | PE-EE3-S-1 | +| PCS token generation | Auth weakness | A07 | T1078 | CWE-330 | P3, DS4, DF5 | PE-P3-S-1, PE-DS4-I-1 | +| Credentials in CLI args | Info exposure | A07 | T1552 | CWE-214 | P6, P8, DF9 | PE-DF9-I-1, PE-P8-I-1 | +| No fencing audit trail | Logging failure | A09 | - | CWE-778 | P5, P6 | PE-P5-R-1, PE-P1-R-1 | + +--- + +## TNA-Specific OWASP Mappings + +| TNA Component | Risk | OWASP | MITRE | CWE | DFD Elements | PE-* IDs | +|---------------|------|-------|-------|-----|--------------|----------| +| Arbiter taint as sole scheduling protection | Misconfiguration | A02 | T1562 | CWE-250 | TNA-P3 | PE-TNA-P3-T-1 | +| Worker ignition token | Credential exposure | A07 | T1552 | CWE-798 | TNA-DS6 | PE-TNA-DS6-I-1 | +| Worker lateral movement to control plane | Access control | A01 | T1021 | CWE-284 | TNA-P5, TNA-DS5 | PE-TNA-P5-E-1 | +| etcd data on compromised node | Crypto failure | A04 | T1552 | CWE-312 | TNA-DS5 | PE-TNA-DS5-I-1 | +| Rogue worker CSR approval | Auth failure | A07 | T1078 | CWE-287 | TNA-P5, TNA-DS6 | PE-TNA-P5-S-1 | +| No arbiter taint drift alert | Logging failure | A09 | - | CWE-778 | TNA-P3 | PE-TNA-P3-T-1 | + +--- + +## SNO-Specific OWASP Mappings + +| SNO Component | Risk | OWASP | MITRE | CWE | DFD Elements | PE-* IDs | +|---------------|------|-------|-------|-----|--------------|----------| +| install-config with pull secret + offline token | Credential exposure | A07 | T1552 | CWE-798 | SNO-DS1 | PE-SNO-DS1-I-1 | +| Single-member etcd (no quorum) | Data loss / total compromise | A06 | T1485 | CWE-312 | SNO-DS3 | PE-SNO-DS3-I-1, PE-SNO-DS3-D-1 | +| UnsafeScalingStrategy bypasses quorum checks | Insecure design | A06 | T1562 | CWE-636 | SNO-P4 | PE-SNO-P4-D-1 | +| Bootstrap-in-place runs privileged on bare metal | Misconfiguration | A02 | T1611 | CWE-250 | SNO-P5 | PE-SNO-P5-E-1 | +| Master schedulable (workloads on control plane) | Access control | A01 | T1610 | CWE-284 | SNO-P6 | PE-SNO-P6-E-1 | +| Kubeconfig + kubeadmin-password on admin workstation | Credential exposure | A07 | T1552 | CWE-522 | SNO-DS4 | PE-SNO-DS4-I-1 | +| Discovery ISO integrity | Supply chain | A03 | T1195 | CWE-494 | SNO-DS2 | PE-SNO-DS2-T-1 | + +--- + +## LVMS-Specific OWASP Mappings + +| LVMS Component | Risk | OWASP | MITRE | CWE | DFD Elements | PE-* IDs | +|----------------|------|-------|-------|-----|--------------|----------| +| TBD | TBD | TBD | TBD | TBD | TBD | TBD | + +--- + +## OWASP Cheat Sheets + +| Topic | URL | +|-------|-----| +| OS Command Injection | | +| Secrets Management | | +| Input Validation | | +| Cryptographic Storage | | +| Error Handling | | +| Docker Security | | +| Kubernetes Security | | diff --git a/plugins/threat-model/skills/lvms-threat-model/SKILL.md b/plugins/threat-model/skills/lvms-threat-model/SKILL.md new file mode 100644 index 00000000..04a24bf8 --- /dev/null +++ b/plugins/threat-model/skills/lvms-threat-model/SKILL.md @@ -0,0 +1,359 @@ +--- +name: threat-model:lvms +description: Analyze a PR for LVMS (LVM Storage) security threats with STRIDE/DFD analysis, MITRE ATT&CK and OWASP mapping +disable-model-invocation: true +allowed-tools: Read, Grep, Glob, Write, Edit, Bash, WebFetch +argument-hint: "" +--- + +# LVMS PR Threat Analysis + +Analyze a pull request for security threats against the **LVMS (LVM Storage)** operator, map to MITRE ATT&CK, and generate a formal report. + +> **Note**: The LVMS DFD model (`dfd-elements-lvms.md`) is not yet defined. This skill will perform general security analysis, ShellCheck scanning, and MITRE/OWASP mapping. DFD element mapping and STRIDE cross-referencing will be available once the DFD model is created. + +## Reference Files + +Bundled with this skill: +- `dfd-elements-lvms.md` — LVMS DFD element catalog (placeholder — not yet modeled) + +Shared references (in `../../references/`): +- `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings +- `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references +- `mitre-findings-template.md` — Template for cumulative findings tracker + +Discovered at runtime from the workspace: +- `$THREAT_MODEL_DIR/LVMS-THREAT-MODEL.md` — LVMS formal threat model (when available) +- `$FINDINGS_FILE` — LVMS findings tracker (created from template on first use) + +## Workspace Discovery + +Before starting analysis, discover the workspace layout. + +### Discovery Steps + +1. **Find workspace root**: Walk upward from `$PWD` until a directory containing `repos/` is found. If no parent qualifies, fall back to checking whether the current git repo sits inside a `repos/` directory: + + ```bash + d="$PWD" + while [ "$d" != "/" ]; do + if [ -d "$d/repos" ]; then + echo "$d" + break + fi + d="$(dirname "$d")" + done + if [ "$d" = "/" ]; then + repo_root="$(git rev-parse --show-toplevel 2>/dev/null || true)" + if [ -n "$repo_root" ] && [ "$(basename "$(dirname "$repo_root")")" = "repos" ]; then + echo "$(dirname "$(dirname "$repo_root")")" + fi + fi + ``` + +2. **Set workspace paths**: Once the workspace root (`WORKSPACE`) is found: + - **Repos directory**: `$WORKSPACE/repos/` + - **Threat model**: Look for `LVMS-THREAT-MODEL.md` in: + - `$WORKSPACE/repos/lvm-operator/docs/` + - `$WORKSPACE/docs/` + - The current directory + - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). + - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-lvms.md` — initialized from `../../references/mitre-findings-template.md` on first use. + +3. **Validate workspace**: Warn the user if: + - No `repos/` directory is found + - Required repos for the target PR are not cloned locally + - DFD model is not yet defined (analysis proceeds without DFD mapping) + +### Path Variables Reference + +| Variable | Description | Example | +|----------|-------------|---------| +| `$WORKSPACE` | Root directory containing `repos/` | `/home/user/Projects/lvm-workspace` | +| `$REPOS` | Repos directory | `$WORKSPACE/repos` | +| `$THREAT_MODEL_DIR` | Directory containing formal threat model | `$REPOS/lvm-operator/docs` | +| `$REPORT_DIR` | Directory for generated reports | Same as `$THREAT_MODEL_DIR` or `$WORKSPACE/reports` | +| `$FINDINGS_FILE` | LVMS findings tracker | `$WORKSPACE/.claude/skills/threat-model/mitre-findings-lvms.md` | + +### Findings File + +Each threat-model skill writes to its own findings file (`mitre-findings-tnf.md`, `mitre-findings-tna.md`, `mitre-findings-sno.md`, `mitre-findings-lvms.md`), so no file locking is required during concurrent execution. + +**Append protocol** (use in step 12): + +```bash +FINDINGS_FILE="$WORKSPACE/.claude/skills/threat-model/mitre-findings-lvms.md" + +mkdir -p "$(dirname "$FINDINGS_FILE")" +cp -n "RESOLVED_TEMPLATE_PATH" "$FINDINGS_FILE" + +cat >> "$FINDINGS_FILE" <<'FINDINGS_BLOCK' + +## LVMS — REPO PR #NUMBER (YYYY-MM-DD) + +| Technique ID | Technique Name | Finding | Severity | Status | Notes | +|--------------|----------------|---------|----------|--------|-------| +| T#### | Name | VULN-# | Severity | Open | Description | + +--- +FINDINGS_BLOCK +``` + +Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `../../references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. + +## Input Formats + +### Option 1: PR Number Only + +```text +/threat-model:lvms 2271 +``` + +Detects the repository from the current working directory. + +### Option 2: GitHub PR URL + +```text +/threat-model:lvms https://github.com/openshift/lvm-operator/pull/2271 +``` + +### Option 3: Explicit repo and PR + +```text +/threat-model:lvms lvm-operator 2271 +``` + +## Parsing Logic + +1. **If input is a URL** (contains `github.com`): + - Extract org/repo/PR from: `https://github.com///pull/` + +2. **If input is a single number**: + - Detect repo from current directory path + - Look for pattern `repos//` in the working directory + +3. **If input is ` `**: + - Use provided repo name + - Look up org from the repository mapping table + +## Repository Mapping + +| Repo | GitHub Org | +|------|------------| +| lvm-operator | openshift | +| origin | openshift | + +## Instructions + +1. **Discover workspace** using the Workspace Discovery steps above +2. **Parse input** to determine org, repo, and PR number +3. **Fetch PR details** using `gh pr view --repo /` or WebFetch +4. **Get changed files** with `gh pr diff --repo /` or WebFetch +5. **Run ShellCheck** on any shell scripts in the changed files (see Automated Scanner section) +6. **Analyze all changes** for security-relevant patterns (see Security Patterns) +7. **Map to DFD elements** — if `dfd-elements-lvms.md` has been populated, map changed files to affected DFD elements. If not yet modeled, skip and note in the report. +8. **Apply per-element STRIDE** to affected elements (if DFD is available) and cross-reference against `$THREAT_MODEL_DIR/LVMS-THREAT-MODEL.md` (if found) +9. **Combine findings** from ShellCheck + AI analysis + DFD/STRIDE analysis +10. **Map findings to MITRE ATT&CK** techniques (see `../../references/mitre-reference.md`) +11. **Generate report** at `$REPORT_DIR/` +12. **Append findings to tracker** — follow the Append Protocol to write a findings block to `$FINDINGS_FILE` + +--- + +## Automated Scanner: ShellCheck + +ShellCheck is available in RHEL/Fedora repos (`dnf install ShellCheck`) - no external downloads required. + +### Installation Check + +```bash +command -v shellcheck >/dev/null && echo "shellcheck: installed" || echo "shellcheck: NOT installed (run: dnf install ShellCheck)" +``` + +### Running ShellCheck + +```bash +shellcheck -f json +shellcheck -S warning +shellcheck -s bash +``` + +### Security-Relevant ShellCheck Codes + +| Code | Severity | Security Relevance | MITRE | +|------|----------|-------------------|-------| +| SC2086 | Warning | Unquoted variable - command injection risk | T1059 | +| SC2091 | Warning | Command in $() used as condition - injection | T1059 | +| SC2046 | Warning | Unquoted command substitution | T1059 | +| SC2012 | Info | Parsing ls output - can be exploited | T1059 | +| SC2029 | Warning | ssh command with unescaped variables | T1059 | +| SC2087 | Warning | Unquoted heredoc - variable expansion | T1059 | +| SC2155 | Warning | Declare/assign separately to avoid masking errors | - | +| SC2164 | Warning | cd without error-exit guard - path traversal risk | T1083 | + +--- + +## Optional External Scanners + +| Tool | Source | Risks | Mitigations | +|------|--------|-------|-------------| +| **Semgrep** | pip/GitHub | Fetches rules from semgrep.dev; may send telemetry | Use `--offline` mode with local rules | +| **Gitleaks** | GitHub releases | Binary from external source | Verify checksums; use container image | +| **gosec** | GitHub/go install | Binary from external source | Verify checksums; audit source | + +--- + +## Security Patterns to Detect + +| Category | Patterns | MITRE | Severity | +|----------|----------|-------|----------| +| Command Injection | shell exec, os.system, subprocess, fmt.Sprintf with shell | T1059 | Critical | +| Credentials | hardcoded secrets, API keys, tokens, passwords in code | T1552 | Critical | +| Privilege Escalation | setuid, capabilities, privileged containers, sudo, nsenter | T1548 | High | +| Authentication | auth bypass, weak validation, token handling flaws | T1078 | High | +| Crypto Weakness | weak algorithms, hardcoded keys, disabled TLS verify | T1573 | High | +| Path Traversal | unsanitized file paths, symlink attacks | T1083 | Medium | +| Container Escape | host mounts, hostPID, hostNetwork, privileged mode | T1611 | Critical | +| Logging Exposure | sensitive data in logs, credential printing | T1005 | Medium | +| SSRF/Network | unvalidated URLs, exposed internal endpoints | T1046 | Medium | +| Deserialization | unsafe unmarshal, pickle, yaml.load | T1059 | High | + +## LVMS DFD Element Mapping + +> **Not yet modeled.** Once `dfd-elements-lvms.md` is populated with DFD elements, add code path mapping and trust boundary crossing tables here. + +When the DFD is available, the analysis should follow the same STRIDE methodology as TNF/TNA: +- Map changed files to affected DFD elements +- Apply per-element STRIDE questions +- Cross-reference against `LVMS-THREAT-MODEL.md` + +--- + +## Report Naming Convention + +- **Full threat model**: `PR-THREAT-MODEL-.md` +- **Individual vuln**: `VULN-PR-.md` + +## Report Format: Threat Model + +```markdown +# PR # Threat Analysis: + +**Document Version**: 1.0 +**Date**: YYYY-MM-DD +**Classification**: Internal - Security Sensitive +**Repository**: +**Topology**: LVMS +**PR Author**: +**PR URL**: + +--- + +## Executive Summary + +[Brief overview of the PR and key security findings] + +### Findings Summary + +| Severity | Count | Summary | +|----------|-------|---------| +| Critical | X | [brief] | +| High | X | [brief] | +| Medium | X | [brief] | +| Low | X | [brief] | + +--- + +## Change Overview + +[What this PR does, its purpose, and security-relevant changes] + +--- + +## Affected Files + +| File | Changes | Security Relevance | +|------|---------|-------------------| +| path/to/file.go | +X/-Y lines | [relevance] | + +--- + +## DFD Impact Analysis + +> DFD model not yet defined for LVMS. This section will be populated once `dfd-elements-lvms.md` is created. + +--- + +## Threat Analysis + +### VULN-1: [Vulnerability Title] + +**Severity**: Critical/High/Medium/Low +**OWASP**: A##:2025 - Category Name +**MITRE ATT&CK**: T#### - Technique Name +**CWE**: CWE-### + +#### Affected Code + +**File**: `path/to/file.go:line` + +#### Description + +[Detailed description] + +#### Attack Vector + +[How this could be exploited] + +#### Impact + +- **Confidentiality**: [impact] +- **Integrity**: [impact] +- **Availability**: [impact] + +#### Recommended Fix + +[Code showing the fix] + +--- + +## OWASP & MITRE ATT&CK Mapping + +| Finding | OWASP | MITRE | CWE | Status | +|---------|-------|-------|-----|--------| +| VULN-1 | A05:2025 Injection | T1059 | CWE-78 | Open | + +--- + +## Risk Assessment + +| Finding | Likelihood | Impact | Risk | +|---------|------------|--------|------| +| VULN-1 | High | Critical | Critical | + +--- + +## Recommendations + +### For Developers (Code Changes) + +1. [Recommendations] + +### For Customers (Deployment & Operations) + +1. [Recommendations] + +--- + +## References + +- [OWASP Top 10:2025](https://owasp.org/Top10/2025/) +- [MITRE ATT&CK](https://attack.mitre.org/) +``` + +## Available Repositories + +| Repo | Org | Focus Areas | +|------|-----|-------------| +| lvm-operator | openshift | CSI driver, VG management, privilege, LVM commands | +| origin | openshift | Test code security | diff --git a/plugins/threat-model/skills/lvms-threat-model/dfd-elements-lvms.md b/plugins/threat-model/skills/lvms-threat-model/dfd-elements-lvms.md new file mode 100644 index 00000000..26f7e16e --- /dev/null +++ b/plugins/threat-model/skills/lvms-threat-model/dfd-elements-lvms.md @@ -0,0 +1,17 @@ +# LVMS (LVM Storage) DFD Elements + +This file will contain the Data Flow Diagram element catalog for the LVMS topology. + +## Status + +**Not yet defined.** This is a placeholder for future DFD modeling. + +## Expected Structure + +Once modeled, this file should define: + +- **Processes (P#)**: Components involved in LVMS operation (operator, vg-manager, topolvm-controller, topolvm-node) +- **Data Stores (DS#)**: LVM volume groups, PVs, thin pools, device state +- **Data Flows (DF#)**: Communication paths between components (CSI gRPC, k8s API, LVM commands) +- **Trust Boundaries (TB#)**: Security isolation boundaries (k8s API, node host, LVM subsystem) +- **External Entities (EE#)**: Users, workloads consuming PVCs, block devices diff --git a/plugins/threat-model/skills/sno-threat-model/SKILL.md b/plugins/threat-model/skills/sno-threat-model/SKILL.md new file mode 100644 index 00000000..04a52744 --- /dev/null +++ b/plugins/threat-model/skills/sno-threat-model/SKILL.md @@ -0,0 +1,455 @@ +--- +name: threat-model:sno +description: Analyze a PR for SNO (Single Node OpenShift) security threats with STRIDE/DFD analysis, MITRE ATT&CK and OWASP mapping +disable-model-invocation: true +allowed-tools: Read, Grep, Glob, Write, Edit, Bash, WebFetch +argument-hint: "" +--- + +# SNO PR Threat Analysis + +Analyze a pull request for security threats against the **SNO (Single Node OpenShift)** topology, map to MITRE ATT&CK, and generate a formal report. + +This skill focuses on SNO-specific DFD elements, trust boundaries, and code paths. For TNF analysis, use `/threat-model:tnf`. For TNA, use `/threat-model:tna`. + +## Reference Files + +Bundled with this skill: +- `dfd-elements-sno.md` — SNO DFD element catalog (SNO-P1–P6, SNO-DS1–DS6, SNO-DF1–DF10, SNO-TB1–TB3) + +Shared references (in `../../references/`): +- `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings +- `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references +- `mitre-findings-template.md` — Template for cumulative findings tracker + +Discovered at runtime from the workspace: +- `$THREAT_MODEL_DIR/SNO-THREAT-MODEL.md` — SNO formal threat model (when available) +- `$FINDINGS_FILE` — SNO findings tracker (created from template on first use) + +## Workspace Discovery + +Before starting analysis, discover the workspace layout. + +### Discovery Steps + +1. **Find workspace root**: Walk upward from `$PWD` until a directory containing `repos/` is found. If no parent qualifies, fall back to checking whether the current git repo sits inside a `repos/` directory: + + ```bash + d="$PWD" + while [ "$d" != "/" ]; do + if [ -d "$d/repos" ]; then + echo "$d" + break + fi + d="$(dirname "$d")" + done + if [ "$d" = "/" ]; then + repo_root="$(git rev-parse --show-toplevel 2>/dev/null || true)" + if [ -n "$repo_root" ] && [ "$(basename "$(dirname "$repo_root")")" = "repos" ]; then + echo "$(dirname "$(dirname "$repo_root")")" + fi + fi + ``` + +2. **Set workspace paths**: Once the workspace root (`WORKSPACE`) is found: + - **Repos directory**: `$WORKSPACE/repos/` + - **Threat model**: Look for `SNO-THREAT-MODEL.md` in: + - `$WORKSPACE/repos/sno-deploy/docs/` + - `$WORKSPACE/docs/` + - The current directory + - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). + - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-sno.md` — initialized from `../../references/mitre-findings-template.md` on first use. + +3. **Validate workspace**: Warn the user if: + - No `repos/` directory is found + - Required repos for the target PR are not cloned locally + - Formal threat model file (`SNO-THREAT-MODEL.md`) is not found (analysis can still proceed, but cross-referencing will be skipped) + +### Path Variables Reference + +| Variable | Description | Example | +|----------|-------------|---------| +| `$WORKSPACE` | Root directory containing `repos/` | `/home/user/Projects/sno-dev-env` | +| `$REPOS` | Repos directory | `$WORKSPACE/repos` | +| `$THREAT_MODEL_DIR` | Directory containing formal threat model | `$REPOS/sno-deploy/docs` | +| `$REPORT_DIR` | Directory for generated reports | Same as `$THREAT_MODEL_DIR` or `$WORKSPACE/reports` | +| `$FINDINGS_FILE` | SNO findings tracker | `$WORKSPACE/.claude/skills/threat-model/mitre-findings-sno.md` | + +### Findings File + +Each threat-model skill writes to its own findings file (`mitre-findings-tnf.md`, `mitre-findings-tna.md`, `mitre-findings-sno.md`, `mitre-findings-lvms.md`), so no file locking is required during concurrent execution. + +**Append protocol** (use in step 12): + +```bash +FINDINGS_FILE="$WORKSPACE/.claude/skills/threat-model/mitre-findings-sno.md" + +mkdir -p "$(dirname "$FINDINGS_FILE")" +cp -n "RESOLVED_TEMPLATE_PATH" "$FINDINGS_FILE" + +cat >> "$FINDINGS_FILE" <<'FINDINGS_BLOCK' + +## SNO — REPO PR #NUMBER (YYYY-MM-DD) + +| Technique ID | Technique Name | Finding | Severity | Status | Notes | +|--------------|----------------|---------|----------|--------|-------| +| T#### | Name | VULN-# | Severity | Open | Description | + +--- +FINDINGS_BLOCK +``` + +Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `../../references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. + +## Input Formats + +### Option 1: PR Number Only + +```text +/threat-model:sno 10498 +``` + +Detects the repository from the current working directory. + +### Option 2: GitHub PR URL + +```text +/threat-model:sno https://github.com/openshift/installer/pull/10498 +``` + +### Option 3: Explicit repo and PR + +```text +/threat-model:sno installer 10498 +``` + +## Parsing Logic + +1. **If input is a URL** (contains `github.com`): + - Extract org/repo/PR from: `https://github.com///pull/` + +2. **If input is a single number**: + - Detect repo from current directory path + - Look for pattern `repos//` in the working directory + +3. **If input is ` `**: + - Use provided repo name + - Look up org from the repository mapping table + +## Repository Mapping + +| Repo | GitHub Org | +|------|------------| +| installer | openshift | +| machine-config-operator | openshift | +| cluster-etcd-operator | openshift | +| assisted-service | openshift | +| origin | openshift | +| dev-scripts | openshift-metal3 | +| release | openshift | + +## Instructions + +1. **Discover workspace** using the Workspace Discovery steps above +2. **Parse input** to determine org, repo, and PR number +3. **Fetch PR details** using `gh pr view --repo /` or WebFetch +4. **Get changed files** with `gh pr diff --repo /` or WebFetch +5. **Run ShellCheck** on any shell scripts in the changed files (see Automated Scanner section) +6. **Analyze all changes** for security-relevant patterns (see Security Patterns) +7. **Map to DFD elements** — identify which DFD elements are affected using the SNO mapping table below and `dfd-elements-sno.md` +8. **Apply per-element STRIDE** to affected elements and cross-reference against `$THREAT_MODEL_DIR/SNO-THREAT-MODEL.md` (if found) +9. **Combine findings** from ShellCheck + AI analysis + DFD/STRIDE analysis +10. **Map findings to MITRE ATT&CK** techniques (see `../../references/mitre-reference.md`) +11. **Generate report** at `$REPORT_DIR/` +12. **Append findings to tracker** — follow the Append Protocol to write a findings block to `$FINDINGS_FILE` + +--- + +## Automated Scanner: ShellCheck + +ShellCheck is available in RHEL/Fedora repos (`dnf install ShellCheck`) - no external downloads required. + +### Installation Check + +```bash +command -v shellcheck >/dev/null && echo "shellcheck: installed" || echo "shellcheck: NOT installed (run: dnf install ShellCheck)" +``` + +### Running ShellCheck + +```bash +shellcheck -f json +shellcheck -S warning +shellcheck -s bash +``` + +### Security-Relevant ShellCheck Codes + +| Code | Severity | Security Relevance | MITRE | +|------|----------|-------------------|-------| +| SC2086 | Warning | Unquoted variable - command injection risk | T1059 | +| SC2091 | Warning | Command in $() used as condition - injection | T1059 | +| SC2046 | Warning | Unquoted command substitution | T1059 | +| SC2012 | Info | Parsing ls output - can be exploited | T1059 | +| SC2029 | Warning | ssh command with unescaped variables | T1059 | +| SC2087 | Warning | Unquoted heredoc - variable expansion | T1059 | +| SC2155 | Warning | Declare/assign separately to avoid masking errors | - | +| SC2164 | Warning | cd without error-exit guard - path traversal risk | T1083 | + +--- + +## Optional External Scanners + +| Tool | Source | Risks | Mitigations | +|------|--------|-------|-------------| +| **Semgrep** | pip/GitHub | Fetches rules from semgrep.dev; may send telemetry | Use `--offline` mode with local rules | +| **Gitleaks** | GitHub releases | Binary from external source | Verify checksums; use container image | +| **gosec** | GitHub/go install | Binary from external source | Verify checksums; audit source | + +--- + +## Security Patterns to Detect + +| Category | Patterns | MITRE | Severity | +|----------|----------|-------|----------| +| Command Injection | shell exec, os.system, subprocess, fmt.Sprintf with shell | T1059 | Critical | +| Credentials | hardcoded secrets, API keys, tokens, passwords in code | T1552 | Critical | +| Privilege Escalation | setuid, capabilities, privileged containers, sudo, nsenter | T1548 | High | +| Authentication | auth bypass, weak validation, token handling flaws | T1078 | High | +| Crypto Weakness | weak algorithms, hardcoded keys, disabled TLS verify | T1573 | High | +| Path Traversal | unsanitized file paths, symlink attacks | T1083 | Medium | +| Container Escape | host mounts, hostPID, hostNetwork, privileged mode | T1611 | Critical | +| Logging Exposure | sensitive data in logs, credential printing | T1005 | Medium | +| SSRF/Network | unvalidated URLs, exposed internal endpoints | T1046 | Medium | +| Deserialization | unsafe unmarshal, pickle, yaml.load | T1059 | High | + +## SNO DFD Element Mapping + +See `dfd-elements-sno.md` for the full element catalog. + +### Code Path to DFD Element + +| Code Path Pattern | DFD Element | STRIDE Focus | +|-------------------|-------------|--------------| +| `installer/pkg/types/installconfig.go` (IsSingleNodeOpenShift, BootstrapInPlace) | SNO-P1 (Installer) | T, D | +| `installer/pkg/asset/machines/master.go` (SingleReplicaTopologyMode) | SNO-P1 | T, D | +| `installer/pkg/types/validation/installconfig.go` (BootstrapInPlace) | SNO-P1 | T | +| `installer/data/data/bootstrap/bootstrap-in-place/` | SNO-P5 (Bootstrap Agent) | T, I, E | +| `assisted-service/internal/common/common.go` (IsSingleNodeCluster) | SNO-P2 (Assisted Service) | T | +| `assisted-service/internal/cluster/validator.go` (SNO validations) | SNO-P2 | S, T | +| `assisted-service/internal/host/validator.go` (SNO host checks) | SNO-P2 | T | +| `cluster-etcd-operator/pkg/operator/ceohelpers/bootstrap.go` (UnsafeScalingStrategy) | SNO-P4 (CEO) | T, D | +| `cluster-etcd-operator/pkg/operator/ceohelpers/control_plane_topology.go` | SNO-P4 | T, D | +| `machine-config-operator/` (MachineConfig, kubelet config) | SNO-P3 (MCO) | T, E | +| `sno-deploy/day_two/templates/` (DU policy generation, workload partitioning) | SNO-P3 (MCO), SNO-DS6 | T | +| `origin/test/extended/` (SNO test code) | Test | - | + +### Trust Boundary Crossings + +When a PR modifies code that crosses a trust boundary, apply additional scrutiny: + +| Boundary Crossing | Code Indicators | Key Threats | +|-------------------|-----------------|-------------| +| SNO-TB1->SNO-TB2 (Admin -> Assisted Service) | install-config, offline-token, pull-secret, API calls to console.redhat.com | I (credential exposure), T (config tampering) | +| SNO-TB2->SNO-TB3 (Assisted Service -> SNO Node) | Discovery ISO generation, ignition delivery, host inventory | T (ISO tampering), I (ignition secrets), E (privileged bootstrap) | +| SNO-TB1->SNO-TB3 (Admin -> SNO Node) | oc/kubectl, kubeconfig, kubeadmin-password | S (admin impersonation), I (credential theft) | + +### Per-Element STRIDE for PR Analysis + +For each affected DFD element, ask these questions: + +**Processes (all 6 STRIDE categories)**: +- **S**: Can the process be impersonated? Are auth checks adequate? +- **T**: Can inputs/outputs be modified? Is data validated? +- **R**: Are actions auditable? Are logs adequate and redacted? +- **I**: Does it handle secrets? Are they protected in transit/at rest? +- **D**: Can it be crashed or blocked? What happens on failure? (Critical for SNO — no failover) +- **E**: Does it run with minimal privilege? Can it be abused for escalation? + +**Data Stores (T, I, D)**: +- **T**: Can stored data be modified by unauthorized parties? +- **I**: Is sensitive data encrypted? Who can read it? +- **D**: Can the store be corrupted or deleted? (Single etcd member — total loss) + +**Data Flows (T, I, D)**: +- **T**: Can data in transit be modified? Is integrity verified? +- **I**: Is the channel encrypted? Are credentials visible? +- **D**: Can the flow be interrupted or flooded? + +**External Entities (S, R)**: +- **S**: Can the entity be impersonated? Is authentication enforced? +- **R**: Can the entity deny having performed an action? Are interactions logged? + +### Cross-Referencing the Threat Model + +After identifying per-element threats, check against `$THREAT_MODEL_DIR/SNO-THREAT-MODEL.md`: + +1. Search for relevant `PE-SNO--*` IDs in the Per-Element STRIDE Analysis section +2. If a PR introduces a **new** threat not covered by existing PE-* entries, flag it as a gap +3. If a PR **mitigates** an existing PE-* threat, note it as a positive finding +4. If a PR **worsens** an existing PE-* threat, flag with elevated severity + +If the formal threat model file is not found, skip cross-referencing and note this in the report. + +--- + +## Report Naming Convention + +- **Full threat model**: `PR-THREAT-MODEL-.md` +- **Individual vuln**: `VULN-PR-.md` + +## Report Format: Threat Model + +```markdown +# PR # Threat Analysis: + +**Document Version**: 1.0 +**Date**: YYYY-MM-DD +**Classification**: Internal - Security Sensitive +**Repository**: +**Topology**: SNO +**PR Author**: +**PR URL**: + +--- + +## Executive Summary + +[Brief overview of the PR and key security findings] + +### Findings Summary + +| Severity | Count | Summary | +|----------|-------|---------| +| Critical | X | [brief] | +| High | X | [brief] | +| Medium | X | [brief] | +| Low | X | [brief] | + +--- + +## Change Overview + +[What this PR does, its purpose, and security-relevant changes] + +--- + +## Affected Files + +| File | Changes | Security Relevance | +|------|---------|-------------------| +| path/to/file.go | +X/-Y lines | [relevance] | + +--- + +## DFD Impact Analysis + +This PR affects the following elements in the SNO Data Flow Diagram +(see SNO-THREAT-MODEL.md): + +### Affected DFD Elements + +| Element | Name | Impact | Trust Boundary | +|---------|------|--------|----------------| +| SNO-P# | [process name] | [what changed] | SNO-TB# | +| SNO-DS# | [store name] | [what changed] | SNO-TB# | + +### Trust Boundary Crossings + +[Describe any trust boundaries crossed by the changed code] + +### Per-Element STRIDE + +| Element | S | T | R | I | D | E | Notes | +|---------|---|---|---|---|---|---|-------| +| SNO-P# | - | - | - | - | - | - | [Processes: all 6] | +| SNO-DS# | N/A | - | N/A | - | - | N/A | [Data Stores: T, I, D] | + +**Legend**: **X** = new threat found, **~** = existing threat modified, **-** = no impact, N/A = not applicable + +### Threat Model Cross-Reference + +| PR Finding | Existing PE-* ID | Status | +|------------|-----------------|--------| +| [finding] | PE-SNO-XX-X-X | Matches existing / New gap / Mitigated | + +--- + +## Threat Analysis + +### VULN-1: [Vulnerability Title] + +**Severity**: Critical/High/Medium/Low +**OWASP**: A##:2025 - Category Name +**MITRE ATT&CK**: T#### - Technique Name +**CWE**: CWE-### + +#### Affected Code + +**File**: `path/to/file.go:line` + +#### Description + +[Detailed description] + +#### Attack Vector + +[How this could be exploited] + +#### Impact + +- **Confidentiality**: [impact] +- **Integrity**: [impact] +- **Availability**: [impact] + +#### Recommended Fix + +[Code showing the fix] + +--- + +## OWASP & MITRE ATT&CK Mapping + +| Finding | OWASP | MITRE | CWE | Status | +|---------|-------|-------|-----|--------| +| VULN-1 | A05:2025 Injection | T1059 | CWE-78 | Open | + +--- + +## Risk Assessment + +| Finding | Likelihood | Impact | Risk | +|---------|------------|--------|------| +| VULN-1 | High | Critical | Critical | + +--- + +## Recommendations + +### For Developers (Code Changes) + +1. [Recommendations] + +### For Customers (Deployment & Operations) + +1. [Recommendations] + +--- + +## References + +- [OWASP Top 10:2025](https://owasp.org/Top10/2025/) +- [MITRE ATT&CK](https://attack.mitre.org/) +``` + +## Available Repositories + +| Repo | Org | Focus Areas | +|------|-----|-------------| +| installer | openshift | Install config, ignition | +| machine-config-operator | openshift | Node config, privilege escalation | +| cluster-etcd-operator | openshift | Etcd management | +| assisted-service | openshift | API security, credential handling | +| origin | openshift | Test code security | +| dev-scripts | openshift-metal3 | Shell scripts, credential handling | +| release | openshift | Release artifacts, CI/CD, manifests | diff --git a/plugins/threat-model/skills/sno-threat-model/dfd-elements-sno.md b/plugins/threat-model/skills/sno-threat-model/dfd-elements-sno.md new file mode 100644 index 00000000..f742d96b --- /dev/null +++ b/plugins/threat-model/skills/sno-threat-model/dfd-elements-sno.md @@ -0,0 +1,118 @@ +# SNO DFD Element Reference + +> **Topology**: SNO (Single Node OpenShift) only. For TNF elements see dfd-elements-tnf.md; for TNA see dfd-elements-tna.md. + +Quick reference for mapping PR changes to Data Flow Diagram elements defined in +the SNO threat model. + +> **ID Namespace**: SNO elements use `SNO-` prefixed IDs (SNO-P1, SNO-DS1, etc.) to avoid ambiguity with TNF/TNA element IDs. + +## Processes + +| ID | Name | Code Reference | STRIDE | +|----|------|---------------|--------| +| SNO-P1 | Installer (bootstrap-in-place) | `installer/pkg/types/installconfig.go` (IsSingleNodeOpenShift, BootstrapInPlace), `installer/pkg/asset/machines/master.go` (SingleReplicaTopologyMode) | S, T, R, I, D, E | +| SNO-P2 | Assisted Service API | `assisted-service/internal/common/common.go` (IsSingleNodeCluster), `assisted-service/internal/cluster/validator.go` | S, T, R, I, D, E | +| SNO-P3 | MCO / MCD | `machine-config-operator/` (MachineConfig delivery, kubelet config) | S, T, R, I, D, E | +| SNO-P4 | CEO (single-member etcd) | `cluster-etcd-operator/pkg/operator/ceohelpers/bootstrap.go` (SingleReplicaTopologyMode, UnsafeScalingStrategy) | S, T, R, I, D, E | +| SNO-P5 | Bootstrap-in-Place Agent | `installer/data/data/bootstrap/bootstrap-in-place/` (install-to-disk.service) | S, T, R, I, D, E | +| SNO-P6 | Kubelet (master + worker role) | Single node runs both control-plane and worker workloads | S, T, R, I, D, E | + +## Data Stores + +| ID | Name | Location | STRIDE | +|----|------|----------|--------| +| SNO-DS1 | install-config | `~/.sno-deploy//` or installer workdir — contains pull secret, SSH key, offline token | T, I, D | +| SNO-DS2 | Discovery ISO | Generated by Assisted Service, booted on target node | T, I, D | +| SNO-DS3 | etcd Data (single member) | Single etcd pod on the SNO node — all K8s secrets, no quorum redundancy | T, I, D | +| SNO-DS4 | Kubeconfig / Credentials | `~/.sno-deploy//creds/` — kubeconfig, kubeadmin-password | T, I, D | +| SNO-DS5 | Ignition Config | Bootstrap-in-place ignition written to installation disk | T, I, D | +| SNO-DS6 | MachineConfig State | On-disk machine config applied by MCD | T, I, D | + +## Data Flows + +| ID | From | To | Protocol | STRIDE | +|----|------|----|----------|--------| +| SNO-DF1 | SNO-EE1 (Admin) | SNO-P1 (Installer) | CLI / install-config YAML | T, I | +| SNO-DF2 | SNO-P7 (aicli) | SNO-P2 (Assisted Service) | HTTPS REST API (console.redhat.com); offline token passed as `AI_OFFLINETOKEN` env var | T, I, D | +| SNO-DF3 | SNO-P2 (Assisted Service) | SNO-DS2 (Discovery ISO) | ISO generation + download | T, I | +| SNO-DF4 | SNO-DS2 (Discovery ISO) | SNO-P5 (Bootstrap Agent) | Boot from ISO, discover hardware | T, I | +| SNO-DF5 | SNO-P5 (Bootstrap Agent) | SNO-P2 (Assisted Service) | HTTPS (host inventory, progress) | T, I | +| SNO-DF6 | SNO-P5 (Bootstrap Agent) | SNO-DS5 (Ignition) | Write ignition to installation disk | T, I | +| SNO-DF7 | SNO-P4 (CEO) | SNO-DS3 (etcd) | localhost gRPC (single member, no peer traffic) | T, I, D | +| SNO-DF8 | SNO-P3 (MCO) | SNO-P6 (Kubelet) | MachineConfig delivery | T, I | +| SNO-DF9 | SNO-EE1 (Admin) | SNO-P6 (Kubelet) | oc/kubectl via API server | T, I | +| SNO-DF10 | SNO-P3 (MCO) | SNO-DS6 (MachineConfig) | Workload partitioning + RT kernel configs written to disk | T, I | + +## External Entities + +| ID | Name | Protocol | STRIDE | +|----|------|----------|--------| +| SNO-EE1 | User / Cluster Admin | oc/kubectl, sno-deploy CLI | S, R | +| SNO-EE2 | Assisted Installer Service (console.redhat.com) | HTTPS REST API | S, R | + +## Trust Boundaries + +| ID | Boundary | Elements Inside | +|----|----------|----------------| +| SNO-TB1 | Admin Workstation | SNO-EE1, SNO-DS1, SNO-DS4 | +| SNO-TB2 | Assisted Service (cloud) | SNO-P2, SNO-EE2 | +| SNO-TB3 | SNO Node (single node = control plane + worker) | SNO-P3, SNO-P4, SNO-P5, SNO-P6, SNO-DS3, SNO-DS5, SNO-DS6 | + +--- + +## High-Risk Elements + +Elements with the most significant threats: + +| Element | Key Risks | Notes | +|---------|-----------|-------| +| SNO-DS3 (etcd Data) | Single-member etcd — no quorum redundancy; node compromise exposes all K8s secrets | No HA failover; total data loss on node failure | +| SNO-P5 (Bootstrap Agent) | Runs privileged on bare metal; writes ignition to disk; trusts Assisted Service API | install-to-disk.service runs as root | +| SNO-DS1 (install-config) | Contains pull secret, SSH key, offline token in plaintext on admin workstation | `~/.sno-deploy/` directory | +| SNO-P6 (Kubelet) | Master + worker on same node — no workload isolation; control-plane compromise = full cluster compromise | Workloads schedulable on master by design (ShouldMastersBeSchedulable returns true) | +| SNO-P4 (CEO) | UnsafeScalingStrategy — bypasses quorum safety checks entirely | "Not officially tested or supported" per code comments | + +--- + +## SNO-Specific Characteristics + +SNO differs fundamentally from multi-node topologies (TNF, TNA): + +- **Single point of failure**: One node is the entire cluster — no HA, no quorum, no failover +- **Bootstrap-in-place**: No separate bootstrap node; the target node bootstraps itself via `install-to-disk.service` +- **Master is schedulable**: `ShouldMastersBeSchedulable()` always returns true for SNO — workloads run on control plane +- **UnsafeScalingStrategy**: CEO uses scaling strategy that bypasses all quorum/fault-tolerance checks +- **Assisted Installer**: Cluster creation goes through `console.redhat.com` Assisted Service API (external trust boundary) +- **No VIPs required**: SNO skips API/Ingress VIP validation since there's only one node +- **Validation suppressions**: SNO skips MTU, majority-group connectivity, and multi-host validations +- **Workload partitioning**: DU configuration pins management workloads to specific CPU cores via MachineConfig; RT kernel applied as MachineConfig + +## SNO Does NOT Have + +- No multi-node quorum or HA failover +- No Pacemaker / Corosync / STONITH / fencing (TNF-specific) +- No arbiter node (TNA-specific) +- No worker nodes (master runs all workloads) +- No separate bootstrap node (uses bootstrap-in-place) +- No peer etcd traffic (single member, localhost only) +- No VIP management (single node = single IP) + +Any PR analysis mentioning these components is **not applicable** to SNO topology. + +--- + +## Developer Tooling Risks (not customer-facing) + +The `sno-deploy/` scripts in edge-tooling use **aicli** (`quay.io/karmab/aicli:latest`) to automate SNO cluster creation for dev/test. This is **not** part of any product install path — customers use the installer, assisted-service UI, or agent-based installer directly. + +These risks only affect developers running `sno-deploy/`: + +| Risk | Detail | +|------|--------| +| TLS verification disabled | `verify_ssl = False` + `urllib3.disable_warnings` — token exchange to `sso.redhat.com` vulnerable to MITM | +| Tokens cached in plaintext | Bearer token → `~/.aicli/token.txt`, offline token → `~/.aicli/offlinetoken.txt` | +| Offline token in env var | `AI_OFFLINETOKEN` visible via `podman inspect` or `/proc//environ` | +| Container runs as root | No `USER` directive in Dockerfile | +| Unpinned image tag | `:latest` with no digest verification; Alpine base + pip deps also unpinned | +| Unsupported tool | README states "not supported in any way by Red Hat" | diff --git a/plugins/threat-model/skills/tna-threat-model/SKILL.md b/plugins/threat-model/skills/tna-threat-model/SKILL.md new file mode 100644 index 00000000..d497fe30 --- /dev/null +++ b/plugins/threat-model/skills/tna-threat-model/SKILL.md @@ -0,0 +1,558 @@ +--- +name: threat-model:tna +description: Analyze a PR for TNA (Two-Node Arbiter) security threats with STRIDE/DFD analysis, MITRE ATT&CK and OWASP mapping +disable-model-invocation: true +allowed-tools: Read, Grep, Glob, Write, Edit, Bash, WebFetch +argument-hint: "" +--- + +# TNA PR Threat Analysis + +Analyze a pull request for security threats against the **TNA (Two-Node Arbiter)** topology, map to MITRE ATT&CK, and generate a formal report. + +This skill focuses on TNA-specific DFD elements, trust boundaries, and code paths. For TNF analysis, use `/threat-model:tnf`. + +## Reference Files + +Bundled with this skill: +- `dfd-elements-tna.md` — TNA DFD element catalog (TNA-P1, TNA-P3–P5, TNA-DS5–DS6, TNA-TB1–TB3) + +Shared references (in `../../references/`): +- `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings +- `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references +- `mitre-findings-template.md` — Template for cumulative findings tracker + +Discovered at runtime from the workspace: +- `$THREAT_MODEL_DIR/TNA-THREAT-MODEL.md` — TNA formal threat model with DFD and per-element STRIDE analysis +- `$FINDINGS_FILE` — TNA findings tracker (created from template on first use) + +## Workspace Discovery + +Before starting analysis, discover the workspace layout. + +### Discovery Steps + +1. **Find workspace root**: Walk upward from `$PWD` until a directory containing `repos/` is found. If no parent qualifies, fall back to checking whether the current git repo sits inside a `repos/` directory: + + ```bash + d="$PWD" + while [ "$d" != "/" ]; do + if [ -d "$d/repos" ]; then + echo "$d" + break + fi + d="$(dirname "$d")" + done + if [ "$d" = "/" ]; then + repo_root="$(git rev-parse --show-toplevel 2>/dev/null || true)" + if [ -n "$repo_root" ] && [ "$(basename "$(dirname "$repo_root")")" = "repos" ]; then + echo "$(dirname "$(dirname "$repo_root")")" + fi + fi + ``` + +2. **Set workspace paths**: Once the workspace root (`WORKSPACE`) is found: + - **Repos directory**: `$WORKSPACE/repos/` + - **Threat model**: Look for `TNA-THREAT-MODEL.md` in: + - `$WORKSPACE/repos/two-node-toolbox/docs/` + - `$WORKSPACE/docs/` + - The current directory + - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). + - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-tna.md` — initialized from `../../references/mitre-findings-template.md` on first use. + +3. **Validate workspace**: Warn the user if: + - No `repos/` directory is found + - Required repos for the target PR are not cloned locally + - Threat model reference file is not found (analysis can still proceed, but DFD cross-referencing will be skipped) + +### Path Variables Reference + +| Variable | Description | Example | +|----------|-------------|---------| +| `$WORKSPACE` | Root directory containing `repos/` | `/home/user/Projects/tna-dev-env` | +| `$REPOS` | Repos directory | `$WORKSPACE/repos` | +| `$THREAT_MODEL_DIR` | Directory containing formal threat model | `$REPOS/two-node-toolbox/docs` | +| `$REPORT_DIR` | Directory for generated reports | Same as `$THREAT_MODEL_DIR` or `$WORKSPACE/reports` | +| `$FINDINGS_FILE` | TNA findings tracker | `$WORKSPACE/.claude/skills/threat-model/mitre-findings-tna.md` | + +### Findings File + +Each threat-model skill writes to its own findings file (`mitre-findings-tnf.md`, `mitre-findings-tna.md`, `mitre-findings-sno.md`, `mitre-findings-lvms.md`), so no file locking is required during concurrent execution. + +**Append protocol** (use in step 12): + +```bash +FINDINGS_FILE="$WORKSPACE/.claude/skills/threat-model/mitre-findings-tna.md" + +mkdir -p "$(dirname "$FINDINGS_FILE")" +cp -n "RESOLVED_TEMPLATE_PATH" "$FINDINGS_FILE" + +cat >> "$FINDINGS_FILE" <<'FINDINGS_BLOCK' + +## TNA — REPO PR #NUMBER (YYYY-MM-DD) + +| Technique ID | Technique Name | Finding | Severity | Status | Notes | +|--------------|----------------|---------|----------|--------|-------| +| T#### | Name | VULN-# | Severity | Open | Description | + +--- +FINDINGS_BLOCK +``` + +Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `../../references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. + +## Input Formats + +### Option 1: PR Number Only + +```text +/threat-model:tna 1437 +``` + +Detects the repository from the current working directory. Must be inside a repo under `$REPOS//`. + +### Option 2: GitHub PR URL + +```text +/threat-model:tna https://github.com/openshift/cluster-etcd-operator/pull/1437 +``` + +Extracts org, repo, and PR number from the URL automatically. + +### Option 3: Explicit repo and PR + +```text +/threat-model:tna cluster-etcd-operator 1437 +``` + +Specify repo name and PR number explicitly. + +## Parsing Logic + +1. **If input is a URL** (contains `github.com`): + - Extract org/repo/PR from: `https://github.com///pull/` + +2. **If input is a single number**: + - Detect repo from current directory path + - Look for pattern `repos//` in the working directory + - Use the repo's configured remote to determine the org + +3. **If input is ` `**: + - Use provided repo name + - Look up org from the repository mapping table + +## Repository Mapping + +| Repo | GitHub Org | +|------|------------| +| assisted-service | openshift | +| cluster-etcd-operator | openshift | +| machine-config-operator | openshift | +| installer | openshift | +| cluster-baremetal-operator | openshift | +| origin | openshift | +| dev-scripts | openshift-metal3 | +| release | openshift | +| enhancements | openshift | +| openshift-docs | openshift | + +## Instructions + +1. **Discover workspace** using the Workspace Discovery steps above +2. **Parse input** to determine org, repo, and PR number +3. **Fetch PR details** using `gh pr view --repo /` or WebFetch +4. **Get changed files** with `gh pr diff --repo /` or WebFetch +5. **Run ShellCheck** on any shell scripts in the changed files (see Automated Scanner section) +6. **Analyze all changes** for security-relevant patterns (see Security Patterns) +7. **Map to DFD elements** — identify which DFD elements are affected using the TNA mapping table below and `dfd-elements-tna.md` +8. **Apply per-element STRIDE** to affected elements and cross-reference against `$THREAT_MODEL_DIR/TNA-THREAT-MODEL.md` (if found) +9. **Combine findings** from ShellCheck + AI analysis + DFD/STRIDE analysis +10. **Map findings to MITRE ATT&CK** techniques (see `../../references/mitre-reference.md`) +11. **Generate report** at `$REPORT_DIR/` +12. **Append findings to tracker** — follow the Append Protocol to write a findings block to `$FINDINGS_FILE` + +--- + +## Automated Scanner: ShellCheck + +ShellCheck is available in RHEL/Fedora repos (`dnf install ShellCheck`) - no external downloads required. + +### Installation Check + +```bash +command -v shellcheck >/dev/null && echo "shellcheck: installed" || echo "shellcheck: NOT installed (run: dnf install ShellCheck)" +``` + +### Running ShellCheck + +```bash +shellcheck -f json +shellcheck -S warning +shellcheck -s bash +``` + +### Security-Relevant ShellCheck Codes + +| Code | Severity | Security Relevance | MITRE | +|------|----------|-------------------|-------| +| SC2086 | Warning | Unquoted variable - command injection risk | T1059 | +| SC2091 | Warning | Command in $() used as condition - injection | T1059 | +| SC2046 | Warning | Unquoted command substitution | T1059 | +| SC2012 | Info | Parsing ls output - can be exploited | T1059 | +| SC2029 | Warning | ssh command with unescaped variables | T1059 | +| SC2087 | Warning | Unquoted heredoc - variable expansion | T1059 | +| SC2155 | Warning | Declare/assign separately to avoid masking errors | - | +| SC2164 | Warning | cd without without error-exit guard - path traversal risk | T1083 | + +### Include in Report + +Add ShellCheck results under Automated Scanner Results: + +```markdown +## Automated Scanner Results + +### ShellCheck + +**Tool**: ShellCheck (from RHEL repos) +**Version**: X.X.X + +| Code | Severity | File | Line | Message | +|------|----------|------|------|---------| +| SC2086 | warning | script.sh | 42 | Double quote to prevent globbing and word splitting | +``` + +If ShellCheck is not installed, note: *Not installed. Install with: `dnf install ShellCheck`* +If no shell scripts in PR, note: *No shell scripts in this PR - skipped.* + +--- + +## Optional External Scanners + +The following scanners provide additional coverage but require **external downloads**. Use at your own discretion. + +| Tool | Source | Risks | Mitigations | +|------|--------|-------|-------------| +| **Semgrep** | pip/GitHub | Fetches rules from semgrep.dev; may send telemetry | Use `--offline` mode with local rules | +| **Gitleaks** | GitHub releases | Binary from external source | Verify checksums; use container image | +| **gosec** | GitHub/go install | Binary from external source | Verify checksums; audit source | + +```bash +command -v semgrep >/dev/null && echo "semgrep: installed" || echo "semgrep: not installed (external)" +command -v gitleaks >/dev/null && echo "gitleaks: installed" || echo "gitleaks: not installed (external)" +command -v gosec >/dev/null && echo "gosec: installed" || echo "gosec: not installed (external)" +``` + +--- + +## Security Patterns to Detect + +| Category | Patterns | MITRE | Severity | +|----------|----------|-------|----------| +| Command Injection | shell exec, os.system, subprocess, fmt.Sprintf with shell | T1059 | Critical | +| Credentials | hardcoded secrets, API keys, tokens, passwords in code | T1552 | Critical | +| Privilege Escalation | setuid, capabilities, privileged containers, sudo, nsenter | T1548 | High | +| Authentication | auth bypass, weak validation, token handling flaws | T1078 | High | +| Crypto Weakness | weak algorithms, hardcoded keys, disabled TLS verify | T1573 | High | +| Path Traversal | unsanitized file paths, symlink attacks | T1083 | Medium | +| Container Escape | host mounts, hostPID, hostNetwork, privileged mode | T1611 | Critical | +| Logging Exposure | sensitive data in logs, credential printing | T1005 | Medium | +| SSRF/Network | unvalidated URLs, exposed internal endpoints | T1046 | Medium | +| Deserialization | unsafe unmarshal, pickle, yaml.load | T1059 | High | + +## TNA DFD Element Mapping + +See `dfd-elements-tna.md` for the full element catalog. + +### Code Path to DFD Element + +| Code Path Pattern | DFD Element | STRIDE Focus | +|-------------------|-------------|--------------| +| `installer/pkg/asset/machines/arbiter*` | TNA-P1 (Installer) | T, D | +| `installer/pkg/asset/ignition/machine/arbiter*` | TNA-P1, TNA-DS6 | T, I | +| `installer/pkg/types/installconfig.go` (IsArbiterEnabled) | TNA-P1 | T, D | +| `installer/pkg/types/validation/installconfig.go` (arbiter) | TNA-P1 | T | +| `assisted-service/internal/common/common.go` (arbiter) | TNA-P1 | T | +| `assisted-service/internal/cluster/validator.go` (arbiter role) | TNA-P1 | S, T | +| `machine-config-operator/manifests/arbiter*` | TNA-P3 (MCO) | T, D | +| `machine-config-operator/templates/arbiter/` | TNA-P3 | T, E | +| `cluster-etcd-operator/pkg/operator/ceohelpers/control_plane_topology.go` | TNA-P4 (CEO) | T, D | +| `cluster-etcd-operator/pkg/operator/ceohelpers/multiselector_lister.go` | TNA-P4 | T, D | +| `cluster-etcd-operator/pkg/operator/configobservation/*replicas*` | TNA-P4 | T, D | +| `origin/test/extended/two_node/arbiter_topology.go` | Test | - | +| `origin/test/extended/two_node/tna_recovery.go` | Test | - | + +### Trust Boundary Crossings + +When a PR modifies code that crosses a trust boundary, apply additional scrutiny: + +| Boundary Crossing | Code Indicators | Key Threats | +|-------------------|-----------------|-------------| +| TNA-TB1->TNA-TB2 (Admin -> K8s API) | install-config, oc commands | S (admin impersonation), T (config tampering) | +| TNA-TB2 internal (MCO -> kubelet) | arbiter MCP, kubelet config, taint | T (taint removal), D (misconfiguration) | +| TNA-TB2->TNA-TB3 (K8s API -> Worker) | CSR approval, ignition endpoint | S (rogue CSR), E (lateral movement) | + +### Per-Element STRIDE for PR Analysis + +For each affected DFD element, ask these questions: + +**Processes (all 6 STRIDE categories)**: +- **S**: Can the process be impersonated? Are auth checks adequate? +- **T**: Can inputs/outputs be modified? Is data validated? +- **R**: Are actions auditable? Are logs adequate and redacted? +- **I**: Does it handle secrets? Are they protected in transit/at rest? +- **D**: Can it be crashed or blocked? What happens on failure? +- **E**: Does it run with minimal privilege? Can it be abused for escalation? + +**Data Stores (T, I, D)**: +- **T**: Can stored data be modified by unauthorized parties? +- **I**: Is sensitive data encrypted? Who can read it? +- **D**: Can the store be corrupted or deleted? + +**Data Flows (T, I, D)**: +- **T**: Can data in transit be modified? Is integrity verified? +- **I**: Is the channel encrypted? Are credentials visible? +- **D**: Can the flow be interrupted or flooded? + +**External Entities (S, R)**: +- **S**: Can the entity be impersonated? Is authentication enforced? +- **R**: Can the entity deny having performed an action? Are interactions logged? + +### Cross-Referencing the Threat Model + +After identifying per-element threats, check against `$THREAT_MODEL_DIR/TNA-THREAT-MODEL.md`: + +1. Search for relevant `PE--*` IDs in the Per-Element STRIDE Analysis section +2. If a PR introduces a **new** threat not covered by existing PE-* entries, flag it as a gap +3. If a PR **mitigates** an existing PE-* threat, note it as a positive finding +4. If a PR **worsens** an existing PE-* threat, flag with elevated severity + +If the formal threat model file is not found, skip cross-referencing and note this in the report. + +--- + +## Report Naming Convention + +- **Full threat model**: `PR-THREAT-MODEL-.md` +- **Individual vuln**: `VULN-PR-.md` + +## Report Format: Threat Model + +```markdown +# PR # Threat Analysis: + +**Document Version**: 1.0 +**Date**: YYYY-MM-DD +**Classification**: Internal - Security Sensitive +**Repository**: +**Topology**: TNA +**PR Author**: +**PR URL**: + +--- + +## Executive Summary + +[Brief overview of the PR and key security findings] + +### Findings Summary + +| Severity | Count | Summary | +|----------|-------|---------| +| Critical | X | [brief] | +| High | X | [brief] | +| Medium | X | [brief] | +| Low | X | [brief] | + +--- + +## Change Overview + +[What this PR does, its purpose, and security-relevant changes] + +--- + +## Affected Files + +| File | Changes | Security Relevance | +|------|---------|-------------------| +| path/to/file.go | +X/-Y lines | [relevance] | + +--- + +## DFD Impact Analysis + +This PR affects the following elements in the TNA Data Flow Diagram +(see TNA-THREAT-MODEL.md): + +### Affected DFD Elements + +| Element | Name | Impact | Trust Boundary | +|---------|------|--------|----------------| +| P# | [process name] | [what changed] | TB# | +| DS# | [store name] | [what changed] | TB# | +| DF# | [flow description] | [what changed] | TB#->TB# | + +### Trust Boundary Crossings + +[Describe any trust boundaries crossed by the changed code] + +### Per-Element STRIDE + +| Element | S | T | R | I | D | E | Notes | +|---------|---|---|---|---|---|---|-------| +| P# | - | - | - | - | - | - | [Processes: all 6] | +| DS# | N/A | - | N/A | - | - | N/A | [Data Stores: T, I, D] | +| DF# | N/A | - | N/A | - | - | N/A | [Data Flows: T, I, D] | +| EE# | - | N/A | - | N/A | N/A | N/A | [External Entities: S, R] | + +**Legend**: **X** = new threat found, **~** = existing threat modified, **-** = no impact, N/A = not applicable + +### Threat Model Cross-Reference + +| PR Finding | Existing PE-* ID | Status | +|------------|-----------------|--------| +| [finding] | PE-XX-X-X | Matches existing / New gap / Mitigated | + +--- + +## Threat Analysis + +### VULN-1: [Vulnerability Title] + +**Severity**: Critical/High/Medium/Low +**OWASP**: A##:2025 - Category Name +**MITRE ATT&CK**: T#### - Technique Name +**CWE**: CWE-### + +#### Affected Code + +**File**: `path/to/file.go:line` + +#### Description + +[Detailed description of the vulnerability] + +#### Attack Vector + +[How this could be exploited] + +#### Impact + +- **Confidentiality**: [impact] +- **Integrity**: [impact] +- **Availability**: [impact] + +#### Recommended Fix + +[Code showing the fix] + +--- + +## OWASP & MITRE ATT&CK Mapping + +| Finding | OWASP | MITRE | CWE | Status | +|---------|-------|-------|-----|--------| +| VULN-1 | A05:2025 Injection | T1059 | CWE-78 | Open | + +--- + +## Risk Assessment + +| Finding | Likelihood | Impact | Risk | +|---------|------------|--------|------| +| VULN-1 | High | Critical | Critical | + +--- + +## Recommendations + +### For Developers (Code Changes) + +#### Before Merge + +1. [Code fix or change required in this PR] + +#### After Merge + +1. [Follow-up code improvement, test addition, or refactor] + +### For Customers (Deployment & Operations) + +#### Configuration Hardening + +1. [Cluster configuration or hardening recommendation] + +#### Operational Practices + +1. [Monitoring, incident response, or day-2 operational guidance] + +--- + +## References + +- [OWASP Top 10:2025](https://owasp.org/Top10/2025/) +- [MITRE ATT&CK](https://attack.mitre.org/) +- [Relevant CVEs, CWEs, documentation] +``` + +## Report Format: Individual Vulnerability (for Critical/High findings) + +```markdown +# Security Ticket: [Vulnerability Title] + +**Ticket ID**: VULN-PR- +**Severity**: CRITICAL/HIGH +**Component**: +**Status**: Open +**Created**: YYYY-MM-DD +**PR**: # + +## Summary + +[One paragraph summary] + +## Affected Code + +**File**: `path/to/file.go:lines` + +## Exploitation + +### Attack Flow + +[ASCII diagram or description of attack flow] + +### Exploit Examples + +[Code examples showing exploitation] + +## Impact + +[Detailed impact analysis] + +## Recommended Fix + +### For Developers + +[Code showing the fix with explanation] + +### For Customers + +[Deployment hardening, configuration changes, or monitoring guidance] + +## References + +- [CWE, OWASP, other references] +``` + +## Available Repositories + +These are the repos expected in `$REPOS/`: + +| Repo | Org | Focus Areas | +|------|-----|-------------| +| assisted-service | openshift | API security, credential handling | +| cluster-etcd-operator | openshift | Topology detection, etcd config | +| machine-config-operator | openshift | Arbiter MCP, node config | +| installer | openshift | Arbiter install config, ignition | +| origin | openshift | Test code security | +| dev-scripts | openshift-metal3 | Shell scripts, credential handling | diff --git a/plugins/threat-model/skills/tna-threat-model/dfd-elements-tna.md b/plugins/threat-model/skills/tna-threat-model/dfd-elements-tna.md new file mode 100644 index 00000000..a0544aa0 --- /dev/null +++ b/plugins/threat-model/skills/tna-threat-model/dfd-elements-tna.md @@ -0,0 +1,68 @@ +# TNA DFD Element Reference + +> **Topology**: TNA (Two-Node with Arbiter) only. For TNF elements, see dfd-elements-tnf.md. + +Quick reference for mapping PR changes to Data Flow Diagram elements defined in +the TNA formal threat model (see `TNA-THREAT-MODEL.md` in the two-node-toolbox docs directory). + +> **ID Namespace**: TNA elements use `TNA-` prefixed IDs (TNA-P1, TNA-DS5, etc.) to avoid ambiguity with TNF element IDs (e.g., TNF P3 = Auth Job vs TNA-P3 = MCO). + +## Processes + +| ID | Name | Code Reference | STRIDE | +|----|------|---------------|--------| +| TNA-P1 | Installer (arbiter topology) | `installer/pkg/asset/machines/arbiter.go`, `installer/pkg/types/installconfig.go` | S, T, R, I, D, E | +| TNA-P3 | MCO (arbiter config) | `machine-config-operator/manifests/arbiter.machineconfigpool.yaml` | S, T, R, I, D, E | +| TNA-P4 | CEO (standard etcd) | `cluster-etcd-operator/pkg/operator/ceohelpers/control_plane_topology.go` | S, T, R, I, D, E | +| TNA-P5 | Worker Kubelet (optional, OCP 4.22+) | Worker node kubelet | S, T, R, I, D, E | + +## Data Stores + +| ID | Name | Location | STRIDE | +|----|------|----------|--------| +| TNA-DS5 | etcd Data | etcd pods on 2 masters + arbiter | T, I, D | +| TNA-DS6 | Worker Ignition / Credentials | Worker ignition endpoint | T, I, D | + +## External Entities + +| ID | Name | Protocol | STRIDE | +|----|------|----------|--------| +| TNA-EE1 | User / Cluster Admin | oc/kubectl | S, R | + +## Trust Boundaries + +| ID | Boundary | Elements Inside | +|----|----------|----------------| +| TNA-TB1 | Admin Network | TNA-EE1 | +| TNA-TB2 | Kubernetes API | TNA-P1, TNA-P3, TNA-P4, TNA-DS5 | +| TNA-TB3 | Worker Compute (optional) | TNA-P5, TNA-DS6 | + +--- + +## High-Risk Elements + +Elements with the most significant threats (from TNA-THREAT-MODEL.md): + +| Element | Key Risks | Related Threats | +|---------|-----------|-----------------| +| TNA-P3 (MCO) | Arbiter taint removal -> workload scheduling -> quorum loss | T-2, D-1 | +| TNA-DS5 (etcd Data) | Node compromise exposes all K8s secrets | I-1, T-1 | +| TNA-P5 (Worker Kubelet) | Lateral movement from worker to control plane | E-2 | + +--- + +## TNA Does NOT Have + +TNA uses standard Kubernetes etcd (3-member quorum via arbiter) and does **not** include any RHEL-HA / Pacemaker components. The following TNF elements have **no equivalent** in TNA: + +- No Pacemaker / Corosync / STONITH / fencing +- No BMC credentials or fencing-credentials secrets +- No podman-etcd OCF agent +- No PCSD authentication +- No privileged TNF setup jobs (TNF processes P2: CEO Controller, P3: Auth Job, P4: Setup Job, P5: Fencing Job handle Pacemaker setup and fencing and do not exist in TNA). Note: TNA reuses IDs P3–P5 for different components (TNA-P3: MCO, TNA-P4: CEO, TNA-P5: Worker Kubelet) +- No CIB (Cluster Information Base) +- No fence_redfish +- No Corosync network (UDP 5404-5406) +- No BMC network trust boundary + +Any PR analysis mentioning these components is **not applicable** to TNA topology. diff --git a/plugins/threat-model/skills/tnf-threat-model/SKILL.md b/plugins/threat-model/skills/tnf-threat-model/SKILL.md new file mode 100644 index 00000000..c49d0e3c --- /dev/null +++ b/plugins/threat-model/skills/tnf-threat-model/SKILL.md @@ -0,0 +1,570 @@ +--- +name: threat-model:tnf +description: Analyze a PR for TNF (Two-Node Fencing) security threats with STRIDE/DFD analysis, MITRE ATT&CK and OWASP mapping +disable-model-invocation: true +allowed-tools: Read, Grep, Glob, Write, Edit, Bash, WebFetch +argument-hint: "" +--- + +# TNF PR Threat Analysis + +Analyze a pull request for security threats against the **TNF (Two-Node Fencing)** topology, map to MITRE ATT&CK, and generate a formal report. + +This skill focuses on TNF-specific DFD elements, trust boundaries, and code paths. For TNA analysis, use `/threat-model:tna`. + +## Reference Files + +Bundled with this skill: +- `dfd-elements-tnf.md` — TNF DFD element catalog (P1-P8, DS1-DS5, DF1-DF12, TB1-TB6) + +Shared references (in `../../references/`): +- `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings +- `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references +- `mitre-findings-template.md` — Template for cumulative findings tracker + +Discovered at runtime from the workspace: +- `$THREAT_MODEL_DIR/TNF-THREAT-MODEL.md` — TNF formal threat model with DFD and per-element STRIDE analysis +- `$FINDINGS_FILE` — TNF findings tracker (created from template on first use) + +## Workspace Discovery + +Before starting analysis, discover the workspace layout. + +### Discovery Steps + +1. **Find workspace root**: Walk upward from `$PWD` until a directory containing `repos/` is found. If no parent qualifies, fall back to checking whether the current git repo sits inside a `repos/` directory: + ```bash + d="$PWD" + while [ "$d" != "/" ]; do + if [ -d "$d/repos" ]; then + echo "$d" + break + fi + d="$(dirname "$d")" + done + if [ "$d" = "/" ]; then + repo_root="$(git rev-parse --show-toplevel 2>/dev/null || true)" + if [ -n "$repo_root" ] && [ "$(basename "$(dirname "$repo_root")")" = "repos" ]; then + echo "$(dirname "$(dirname "$repo_root")")" + fi + fi + ``` + +2. **Set workspace paths**: Once the workspace root (`WORKSPACE`) is found: + - **Repos directory**: `$WORKSPACE/repos/` + - **Threat model**: Look for `TNF-THREAT-MODEL.md` in: + - `$WORKSPACE/repos/two-node-toolbox/docs/` + - `$WORKSPACE/docs/` + - The current directory + - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). + - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-tnf.md` — initialized from `../../references/mitre-findings-template.md` on first use. + +3. **Validate workspace**: Warn the user if: + - No `repos/` directory is found + - Required repos for the target PR are not cloned locally + - Threat model reference file is not found (analysis can still proceed, but DFD cross-referencing will be skipped) + +### Path Variables Reference + +| Variable | Description | Example | +|----------|-------------|---------| +| `$WORKSPACE` | Root directory containing `repos/` | `/home/user/Projects/tnf-dev-env` | +| `$REPOS` | Repos directory | `$WORKSPACE/repos` | +| `$THREAT_MODEL_DIR` | Directory containing formal threat model | `$REPOS/two-node-toolbox/docs` | +| `$REPORT_DIR` | Directory for generated reports | Same as `$THREAT_MODEL_DIR` or `$WORKSPACE/reports` | +| `$FINDINGS_FILE` | TNF findings tracker | `$WORKSPACE/.claude/skills/threat-model/mitre-findings-tnf.md` | + +### Findings File + +Each threat-model skill writes to its own findings file (`mitre-findings-tnf.md`, `mitre-findings-tna.md`, `mitre-findings-sno.md`, `mitre-findings-lvms.md`), so no file locking is required during concurrent execution. + +**Append protocol** (use in step 12): + +```bash +FINDINGS_FILE="$WORKSPACE/.claude/skills/threat-model/mitre-findings-tnf.md" + +mkdir -p "$(dirname "$FINDINGS_FILE")" +cp -n "RESOLVED_TEMPLATE_PATH" "$FINDINGS_FILE" + +cat >> "$FINDINGS_FILE" <<'FINDINGS_BLOCK' + +## TNF — REPO PR #NUMBER (YYYY-MM-DD) + +| Technique ID | Technique Name | Finding | Severity | Status | Notes | +|--------------|----------------|---------|----------|--------|-------| +| T#### | Name | VULN-# | Severity | Open | Description | + +--- +FINDINGS_BLOCK +``` + +Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `../../references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. + +## Input Formats + +### Option 1: PR Number Only + +```text +/threat-model:tnf 2136 +``` + +Detects the repository from the current working directory. Must be inside a repo under `$REPOS//`. + +### Option 2: GitHub PR URL + +```text +/threat-model:tnf https://github.com/ClusterLabs/resource-agents/pull/2136 +``` + +Extracts org, repo, and PR number from the URL automatically. + +### Option 3: Explicit repo and PR + +```text +/threat-model:tnf resource-agents 2136 +``` + +Specify repo name and PR number explicitly. + +## Parsing Logic + +1. **If input is a URL** (contains `github.com`): + - Extract org/repo/PR from: `https://github.com///pull/` + +2. **If input is a single number**: + - Detect repo from current directory path + - Look for pattern `repos//` in the working directory + - Use the repo's configured remote to determine the org + +3. **If input is ` `**: + - Use provided repo name + - Look up org from the repository mapping table + +## Repository Mapping + +| Repo | GitHub Org | +|------|------------| +| assisted-service | openshift | +| cluster-etcd-operator | openshift | +| machine-config-operator | openshift | +| installer | openshift | +| cluster-baremetal-operator | openshift | +| resource-agents | ClusterLabs | +| origin | openshift | +| dev-scripts | openshift-metal3 | +| release | openshift | +| enhancements | openshift | +| openshift-docs | openshift | +| pacemaker | ClusterLabs | + +## Instructions + +1. **Discover workspace** using the Workspace Discovery steps above +2. **Parse input** to determine org, repo, and PR number +3. **Fetch PR details** using `gh pr view --repo /` or WebFetch +4. **Get changed files** with `gh pr diff --repo /` or WebFetch +5. **Run ShellCheck** on any shell scripts in the changed files (see Automated Scanner section) +6. **Analyze all changes** for security-relevant patterns (see Security Patterns) +7. **Map to DFD elements** — identify which DFD elements are affected using the TNF mapping table below and `dfd-elements-tnf.md` +8. **Apply per-element STRIDE** to affected elements and cross-reference against `$THREAT_MODEL_DIR/TNF-THREAT-MODEL.md` (if found) +9. **Combine findings** from ShellCheck + AI analysis + DFD/STRIDE analysis +10. **Map findings to MITRE ATT&CK** techniques (see `../../references/mitre-reference.md`) +11. **Generate report** at `$REPORT_DIR/` +12. **Append findings to tracker** — follow the Append Protocol to write a findings block to `$FINDINGS_FILE` + +--- + +## Automated Scanner: ShellCheck + +ShellCheck is available in RHEL/Fedora repos (`dnf install ShellCheck`) - no external downloads required. + +### Installation Check + +```bash +command -v shellcheck >/dev/null && echo "shellcheck: installed" || echo "shellcheck: NOT installed (run: dnf install ShellCheck)" +``` + +### Running ShellCheck + +```bash +shellcheck -f json +shellcheck -S warning +shellcheck -s bash +``` + +### Security-Relevant ShellCheck Codes + +| Code | Severity | Security Relevance | MITRE | +|------|----------|-------------------|-------| +| SC2086 | Warning | Unquoted variable - command injection risk | T1059 | +| SC2091 | Warning | Command in $() used as condition - injection | T1059 | +| SC2046 | Warning | Unquoted command substitution | T1059 | +| SC2012 | Info | Parsing ls output - can be exploited | T1059 | +| SC2029 | Warning | ssh command with unescaped variables | T1059 | +| SC2087 | Warning | Unquoted heredoc - variable expansion | T1059 | +| SC2155 | Warning | Declare/assign separately to avoid masking errors | - | +| SC2164 | Warning | cd without error-exit guard - path traversal risk | T1083 | + +### Include in Report + +Add ShellCheck results under Automated Scanner Results: + +```markdown +## Automated Scanner Results + +### ShellCheck + +**Tool**: ShellCheck (from RHEL repos) +**Version**: X.X.X + +| Code | Severity | File | Line | Message | +|------|----------|------|------|---------| +| SC2086 | warning | podman-etcd | 42 | Double quote to prevent globbing and word splitting | +``` + +If ShellCheck is not installed, note: *Not installed. Install with: `dnf install ShellCheck`* +If no shell scripts in PR, note: *No shell scripts in this PR - skipped.* + +--- + +## Optional External Scanners + +The following scanners provide additional coverage but require **external downloads**. Use at your own discretion. + +| Tool | Source | Risks | Mitigations | +|------|--------|-------|-------------| +| **Semgrep** | pip/GitHub | Fetches rules from semgrep.dev; may send telemetry | Use `--offline` mode with local rules | +| **Gitleaks** | GitHub releases | Binary from external source | Verify checksums; use container image | +| **gosec** | GitHub/go install | Binary from external source | Verify checksums; audit source | + +```bash +command -v semgrep >/dev/null && echo "semgrep: installed" || echo "semgrep: not installed (external)" +command -v gitleaks >/dev/null && echo "gitleaks: installed" || echo "gitleaks: not installed (external)" +command -v gosec >/dev/null && echo "gosec: installed" || echo "gosec: not installed (external)" +``` + +--- + +## Security Patterns to Detect + +| Category | Patterns | MITRE | Severity | +|----------|----------|-------|----------| +| Command Injection | shell exec, os.system, subprocess, fmt.Sprintf with shell | T1059 | Critical | +| Credentials | hardcoded secrets, API keys, tokens, passwords in code | T1552 | Critical | +| Privilege Escalation | setuid, capabilities, privileged containers, sudo, nsenter | T1548 | High | +| Authentication | auth bypass, weak validation, token handling flaws | T1078 | High | +| Crypto Weakness | weak algorithms, hardcoded keys, disabled TLS verify | T1573 | High | +| Path Traversal | unsanitized file paths, symlink attacks | T1083 | Medium | +| Container Escape | host mounts, hostPID, hostNetwork, privileged mode | T1611 | Critical | +| Logging Exposure | sensitive data in logs, credential printing | T1005 | Medium | +| SSRF/Network | unvalidated URLs, exposed internal endpoints | T1046 | Medium | +| Deserialization | unsafe unmarshal, pickle, yaml.load | T1059 | High | + +## TNF DFD Element Mapping + +See `dfd-elements-tnf.md` for the full element catalog. + +### Code Path to DFD Element + +| Code Path Pattern | DFD Element | STRIDE Focus | +|-------------------|-------------|--------------| +| `assisted-service/internal/installcfg/` | P1 (Installer) | I, T, R | +| `assisted-service/internal/bminventory/` | P1 (Installer) | I, S, T | +| `assisted-service/models/fencing*` | P1 (Installer), DF1 | I, T | +| `cluster-etcd-operator/pkg/tnf/operator/` | P2 (CEO Controller) | S, D, E | +| `cluster-etcd-operator/pkg/tnf/auth/` | P3 (Auth Job) | S, E | +| `cluster-etcd-operator/pkg/tnf/setup/` | P4 (Setup Job) | T, I, E, D | +| `cluster-etcd-operator/pkg/tnf/fencing/` | P5 (Fencing Job) | I, T, R, E | +| `cluster-etcd-operator/pkg/tnf/pkg/pcs/fencing*` | P5, DF7, DF9 | I, T | +| `cluster-etcd-operator/pkg/tnf/pkg/pcs/cluster*` | P4, DS3 | T, D | +| `cluster-etcd-operator/pkg/tnf/pkg/tools/secrets*` | DS2, DF4 | I, T | +| `cluster-etcd-operator/pkg/tnf/pkg/tools/redact*` | P5, DF9 | I, R | +| `cluster-etcd-operator/pkg/tnf/pkg/exec/` | P3-P5 (nsenter) | E | +| `cluster-etcd-operator/bindata/tnfdeployment/job*` | P3-P5 (container spec) | E | +| `pacemaker/daemons/fenced/` | P6 (fenced) | S, I, D | +| `resource-agents/heartbeat/podman-etcd` | P7 (OCF Agent) | T, D, I | +| `resource-agents/heartbeat/podman` | P7 (OCF Agent) | T, D | +| `machine-config-operator/templates/*two-node*` | DS4 (PCSD setup) | T, E | +| `installer/pkg/asset/agent/manifests/fencing*` | P1, DS1, DF1, DF2 | I, T | + +### Trust Boundary Crossings + +When a PR modifies code that crosses a trust boundary, apply additional scrutiny: + +| Boundary Crossing | Code Indicators | Key Threats | +|-------------------|-----------------|-------------| +| TB2->TB3 (K8s -> Privileged Container) | Job specs, SA tokens, secret reads | E (escape), I (secret leak) | +| TB3->TB4 (Container -> Host) | nsenter calls, hostPID, privileged | E (host access), T (CIB tamper) | +| TB4->TB5 (Host -> BMC) | fence_redfish calls, Redfish URLs | S (MITM), I (credential exposure) | +| TB2->TB4 (Secrets -> CIB) | Secret->pcs command pipeline | I (plaintext creds in XML) | +| TB6 (Inter-Node) | Corosync config, PCSD auth | S (spoofing), D (quorum loss) | + +### Per-Element STRIDE for PR Analysis + +For each affected DFD element, ask these questions: + +**Processes (all 6 STRIDE categories)**: +- **S**: Can the process be impersonated? Are auth checks adequate? +- **T**: Can inputs/outputs be modified? Is data validated? +- **R**: Are actions auditable? Are logs adequate and redacted? +- **I**: Does it handle secrets? Are they protected in transit/at rest? +- **D**: Can it be crashed or blocked? What happens on failure? +- **E**: Does it run with minimal privilege? Can it be abused for escalation? + +**Data Stores (T, I, D)**: +- **T**: Can stored data be modified by unauthorized parties? +- **I**: Is sensitive data encrypted? Who can read it? +- **D**: Can the store be corrupted or deleted? + +**Data Flows (T, I, D)**: +- **T**: Can data in transit be modified? Is integrity verified? +- **I**: Is the channel encrypted? Are credentials visible? +- **D**: Can the flow be interrupted or flooded? + +**External Entities (S, R)**: +- **S**: Can the entity be impersonated? Is authentication enforced? +- **R**: Can the entity deny having performed an action? Are interactions logged? + +### Cross-Referencing the Threat Model + +After identifying per-element threats, check against `$THREAT_MODEL_DIR/TNF-THREAT-MODEL.md`: + +1. Search for relevant `PE--*` IDs in the Per-Element STRIDE Analysis section +2. If a PR introduces a **new** threat not covered by existing PE-* entries, flag it as a gap +3. If a PR **mitigates** an existing PE-* threat, note it as a positive finding +4. If a PR **worsens** an existing PE-* threat, flag with elevated severity + +If the formal threat model file is not found, skip cross-referencing and note this in the report. + +--- + +## Report Naming Convention + +- **Full threat model**: `PR-THREAT-MODEL-.md` +- **Individual vuln**: `VULN-PR-.md` + +## Report Format: Threat Model + +```markdown +# PR # Threat Analysis: + +**Document Version**: 1.0 +**Date**: YYYY-MM-DD +**Classification**: Internal - Security Sensitive +**Repository**: +**Topology**: TNF +**PR Author**: +**PR URL**: + +--- + +## Executive Summary + +[Brief overview of the PR and key security findings] + +### Findings Summary + +| Severity | Count | Summary | +|----------|-------|---------| +| Critical | X | [brief] | +| High | X | [brief] | +| Medium | X | [brief] | +| Low | X | [brief] | + +--- + +## Change Overview + +[What this PR does, its purpose, and security-relevant changes] + +--- + +## Affected Files + +| File | Changes | Security Relevance | +|------|---------|-------------------| +| path/to/file.go | +X/-Y lines | [relevance] | + +--- + +## DFD Impact Analysis + +This PR affects the following elements in the TNF Data Flow Diagram +(see TNF-THREAT-MODEL.md): + +### Affected DFD Elements + +| Element | Name | Impact | Trust Boundary | +|---------|------|--------|----------------| +| P# | [process name] | [what changed] | TB# | +| DS# | [store name] | [what changed] | TB# | +| DF# | [flow description] | [what changed] | TB#->TB# | + +### Trust Boundary Crossings + +[Describe any trust boundaries crossed by the changed code] + +### Per-Element STRIDE + +| Element | S | T | R | I | D | E | Notes | +|---------|---|---|---|---|---|---|-------| +| P# | - | - | - | - | - | - | [Processes: all 6] | +| DS# | N/A | - | N/A | - | - | N/A | [Data Stores: T, I, D] | +| DF# | N/A | - | N/A | - | - | N/A | [Data Flows: T, I, D] | +| EE# | - | N/A | - | N/A | N/A | N/A | [External Entities: S, R] | + +**Legend**: **X** = new threat found, **~** = existing threat modified, **-** = no impact, N/A = not applicable + +### Threat Model Cross-Reference + +| PR Finding | Existing PE-* ID | Status | +|------------|-----------------|--------| +| [finding] | PE-XX-X-X | Matches existing / New gap / Mitigated | + +--- + +## Threat Analysis + +### VULN-1: [Vulnerability Title] + +**Severity**: Critical/High/Medium/Low +**OWASP**: A##:2025 - Category Name +**MITRE ATT&CK**: T#### - Technique Name +**CWE**: CWE-### + +#### Affected Code + +**File**: `path/to/file.go:line` + +#### Description + +[Detailed description of the vulnerability] + +#### Attack Vector + +[How this could be exploited] + +#### Impact + +- **Confidentiality**: [impact] +- **Integrity**: [impact] +- **Availability**: [impact] + +#### Recommended Fix + +[Code showing the fix] + +--- + +## OWASP & MITRE ATT&CK Mapping + +| Finding | OWASP | MITRE | CWE | Status | +|---------|-------|-------|-----|--------| +| VULN-1 | A05:2025 Injection | T1059 | CWE-78 | Open | + +--- + +## Risk Assessment + +| Finding | Likelihood | Impact | Risk | +|---------|------------|--------|------| +| VULN-1 | High | Critical | Critical | + +--- + +## Recommendations + +### For Developers (Code Changes) + +#### Before Merge + +1. [Code fix or change required in this PR] + +#### After Merge + +1. [Follow-up code improvement, test addition, or refactor] + +### For Customers (Deployment & Operations) + +#### Configuration Hardening + +1. [Cluster configuration or hardening recommendation] + +#### Operational Practices + +1. [Monitoring, incident response, or day-2 operational guidance] + +--- + +## References + +- [OWASP Top 10:2025](https://owasp.org/Top10/2025/) +- [MITRE ATT&CK](https://attack.mitre.org/) +- [Relevant CVEs, CWEs, documentation] +``` + +## Report Format: Individual Vulnerability (for Critical/High findings) + +```markdown +# Security Ticket: [Vulnerability Title] + +**Ticket ID**: VULN-PR- +**Severity**: CRITICAL/HIGH +**Component**: +**Status**: Open +**Created**: YYYY-MM-DD +**PR**: # + +## Summary + +[One paragraph summary] + +## Affected Code + +**File**: `path/to/file.go:lines` + +## Exploitation + +### Attack Flow + +[ASCII diagram or description of attack flow] + +### Exploit Examples + +[Code examples showing exploitation] + +## Impact + +[Detailed impact analysis] + +## Recommended Fix + +### For Developers + +[Code showing the fix with explanation] + +### For Customers + +[Deployment hardening, configuration changes, or monitoring guidance] + +## References + +- [CWE, OWASP, other references] +``` + +## Available Repositories + +These are the repos expected in `$REPOS/`: + +| Repo | Org | Focus Areas | +|------|-----|-------------| +| assisted-service | openshift | API security, credential handling | +| cluster-etcd-operator | openshift | Privilege, shell injection, secrets | +| machine-config-operator | openshift | Node config, privilege escalation | +| installer | openshift | Install config, credential storage | +| cluster-baremetal-operator | openshift | BMC access, metal3 security | +| resource-agents | ClusterLabs | OCF scripts, shell injection | +| two-node-toolbox | (internal) | Deployment scripts, credentials | +| origin | openshift | Test code security | +| dev-scripts | openshift-metal3 | Shell scripts, credential handling | +| pacemaker | ClusterLabs | Fencing daemon, cluster auth | diff --git a/plugins/threat-model/skills/tnf-threat-model/dfd-elements-tnf.md b/plugins/threat-model/skills/tnf-threat-model/dfd-elements-tnf.md new file mode 100644 index 00000000..5cddeccf --- /dev/null +++ b/plugins/threat-model/skills/tnf-threat-model/dfd-elements-tnf.md @@ -0,0 +1,120 @@ +# TNF DFD Element Reference + +> **Topology**: TNF (Two-Node with Fencing) only. For TNA elements, see dfd-elements-tna.md. + +Quick reference for mapping PR changes to Data Flow Diagram elements defined in +the TNF formal threat model (see `TNF-THREAT-MODEL.md` in the two-node-toolbox docs directory). + +## Processes + +| ID | Name | Code Reference | STRIDE | +|----|------|---------------|--------| +| P1 | Installer / Assisted Service | `assisted-service/internal/installcfg/builder/builder.go` | S, T, R, I, D, E | +| P2 | CEO TNF Controller | `cluster-etcd-operator/pkg/tnf/operator/starter.go` | S, T, R, I, D, E | +| P3 | TNF Auth Job | `cluster-etcd-operator/pkg/tnf/auth/runner.go` | S, T, R, I, D, E | +| P4 | TNF Setup Job | `cluster-etcd-operator/pkg/tnf/setup/runner.go` | S, T, R, I, D, E | +| P5 | TNF Fencing Job | `cluster-etcd-operator/pkg/tnf/fencing/runner.go` | S, T, R, I, D, E | +| P6 | Pacemaker fenced | `pacemaker/daemons/fenced/` | S, T, R, I, D, E | +| P7 | podman-etcd OCF Agent | `resource-agents/heartbeat/podman-etcd` | S, T, R, I, D, E | +| P8 | fence_redfish | `/usr/sbin/fence_redfish` (RPM) | S, T, R, I, D, E | + +## Data Stores + +| ID | Name | Location | STRIDE | +|----|------|----------|--------| +| DS1 | install-config.yaml | Installer host filesystem | T, I, D | +| DS2 | K8s Secrets | `openshift-etcd` namespace | T, I, D | +| DS3 | Pacemaker CIB | `/var/lib/pacemaker/cib/cib.xml` | T, I, D | +| DS4 | PCSD Token | `/var/lib/pcsd/token` | T, I, D | +| DS5 | etcd Data | etcd containers / persistent storage | T, I, D | + +## Data Flows + +| ID | From | To | Data | STRIDE | +|----|------|----|------|--------| +| DF1 | EE1 | P1 | BMC credentials | T, I, D | +| DF2 | P1 | DS1 | Credentials in install-config | T, I, D | +| DF3 | P1 | DS2 | Credentials as K8s Secrets | T, I, D | +| DF4 | DS2 | P5 | Secret read by fencing job | T, I, D | +| DF5 | P3 | DS4 | ClusterID as PCSD token | T, I, D | +| DF6 | P4 | DS3 | Cluster + etcd config to CIB | T, I, D | +| DF7 | P4/P5 | DS3 | STONITH credentials to CIB | T, I, D | +| DF8 | DS3 | P6 | Credentials read for fencing | T, I, D | +| DF9 | P6 | P8 | Credentials as CLI args | T, I, D | +| DF10 | P8 | EE2 | HTTPS Basic Auth to BMC | T, I, D | +| DF11 | P7 | DS5 | etcd container lifecycle | T, I, D | +| DF12 | EE3 | P4/P6 | Membership + CIB replication | T, I, D | + +## External Entities + +| ID | Name | Protocol | STRIDE | +|----|------|----------|--------| +| EE1 | User / Cluster Admin | REST API / YAML | S, R | +| EE2 | BMC Controllers | Redfish HTTPS | S, R | +| EE3 | Corosync Network | UDP 5404-5406 | S, R | + +## Trust Boundaries + +| ID | Boundary | Elements Inside | +|----|----------|----------------| +| TB1 | External Network | EE1, EE2 | +| TB2 | Kubernetes API | P1, P2, DS2 | +| TB3 | Privileged Container (nsenter) | P3, P4, P5 | +| TB4 | Host / Pacemaker | P6, P7, P8, DS3, DS4, DS5 | +| TB5 | BMC Network | EE2 | +| TB6 | Inter-Node (Corosync) | EE3 | + +--- + +## High-Risk Elements + +Elements with the most existing per-element threats (from TNF-THREAT-MODEL.md): + +| Element | Threat Count | Key Risks | Related VULNs | +|---------|-------------|-----------|---------------| +| P5 (Fencing Job) | 4 | Credential exposure, shell injection, privilege | VULN-1, VULN-3 | +| P4 (Setup Job) | 4 | CIB tampering, credential storage, privilege | VULN-3, VULN-4 | +| P8 (fence_redfish) | 4 | MITM, credential exposure, BMC spoofing | VULN-1, VULN-2 | +| DS3 (CIB) | 3 | Plaintext credentials, fencing disable, corruption | VULN-4 | +| P6 (fenced) | 4 | Spoofed requests, credential relay, malicious fencing | VULN-1 | +| P3 (Auth Job) | 3 | Predictable token, privilege escalation | VULN-3, VULN-5 | +| DF9 (P6->P8) | 2 | Credentials in /proc/cmdline | VULN-1 | +| DF10 (P8->EE2) | 3 | MITM, credential interception | VULN-2 | + +--- + +## Credential Flow Path + +The full path credentials take through the system (highest-risk data flow): + +```text +EE1 (Admin) --DF1--> P1 (Installer) --DF2--> DS1 (install-config) [plaintext on disk] + --DF3--> DS2 (K8s Secret) [base64 in etcd] + | + DF4 + | + v + P5 (Fencing Job) + | + DF7 + | + v + DS3 (CIB) [plaintext XML] + | + DF8 + | + v + P6 (fenced) + | + DF9 + | + v + P8 (fence_redfish) [CLI args visible] + | + DF10 + | + v + EE2 (BMC) [HTTPS Basic Auth] +``` + +Any PR touching code along this path requires careful credential handling review. From bbfce725377c07c52ae34711e0b943a1afddeebd Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Sun, 26 Apr 2026 08:02:22 -0400 Subject: [PATCH 2/9] fix: add missing blank lines to pass markdownlint (MD031/MD032) Co-Authored-By: Claude Opus 4.6 --- plugins/threat-model/skills/lvms-threat-model/SKILL.md | 4 ++++ plugins/threat-model/skills/sno-threat-model/SKILL.md | 7 +++++++ plugins/threat-model/skills/tna-threat-model/SKILL.md | 7 +++++++ plugins/threat-model/skills/tnf-threat-model/SKILL.md | 8 ++++++++ 4 files changed, 26 insertions(+) diff --git a/plugins/threat-model/skills/lvms-threat-model/SKILL.md b/plugins/threat-model/skills/lvms-threat-model/SKILL.md index 04a24bf8..97dd65f2 100644 --- a/plugins/threat-model/skills/lvms-threat-model/SKILL.md +++ b/plugins/threat-model/skills/lvms-threat-model/SKILL.md @@ -15,14 +15,17 @@ Analyze a pull request for security threats against the **LVMS (LVM Storage)** o ## Reference Files Bundled with this skill: + - `dfd-elements-lvms.md` — LVMS DFD element catalog (placeholder — not yet modeled) Shared references (in `../../references/`): + - `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings - `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references - `mitre-findings-template.md` — Template for cumulative findings tracker Discovered at runtime from the workspace: + - `$THREAT_MODEL_DIR/LVMS-THREAT-MODEL.md` — LVMS formal threat model (when available) - `$FINDINGS_FILE` — LVMS findings tracker (created from template on first use) @@ -223,6 +226,7 @@ shellcheck -s bash > **Not yet modeled.** Once `dfd-elements-lvms.md` is populated with DFD elements, add code path mapping and trust boundary crossing tables here. When the DFD is available, the analysis should follow the same STRIDE methodology as TNF/TNA: + - Map changed files to affected DFD elements - Apply per-element STRIDE questions - Cross-reference against `LVMS-THREAT-MODEL.md` diff --git a/plugins/threat-model/skills/sno-threat-model/SKILL.md b/plugins/threat-model/skills/sno-threat-model/SKILL.md index 04a52744..5edd3966 100644 --- a/plugins/threat-model/skills/sno-threat-model/SKILL.md +++ b/plugins/threat-model/skills/sno-threat-model/SKILL.md @@ -15,14 +15,17 @@ This skill focuses on SNO-specific DFD elements, trust boundaries, and code path ## Reference Files Bundled with this skill: + - `dfd-elements-sno.md` — SNO DFD element catalog (SNO-P1–P6, SNO-DS1–DS6, SNO-DF1–DF10, SNO-TB1–TB3) Shared references (in `../../references/`): + - `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings - `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references - `mitre-findings-template.md` — Template for cumulative findings tracker Discovered at runtime from the workspace: + - `$THREAT_MODEL_DIR/SNO-THREAT-MODEL.md` — SNO formal threat model (when available) - `$FINDINGS_FILE` — SNO findings tracker (created from template on first use) @@ -259,6 +262,7 @@ When a PR modifies code that crosses a trust boundary, apply additional scrutiny For each affected DFD element, ask these questions: **Processes (all 6 STRIDE categories)**: + - **S**: Can the process be impersonated? Are auth checks adequate? - **T**: Can inputs/outputs be modified? Is data validated? - **R**: Are actions auditable? Are logs adequate and redacted? @@ -267,16 +271,19 @@ For each affected DFD element, ask these questions: - **E**: Does it run with minimal privilege? Can it be abused for escalation? **Data Stores (T, I, D)**: + - **T**: Can stored data be modified by unauthorized parties? - **I**: Is sensitive data encrypted? Who can read it? - **D**: Can the store be corrupted or deleted? (Single etcd member — total loss) **Data Flows (T, I, D)**: + - **T**: Can data in transit be modified? Is integrity verified? - **I**: Is the channel encrypted? Are credentials visible? - **D**: Can the flow be interrupted or flooded? **External Entities (S, R)**: + - **S**: Can the entity be impersonated? Is authentication enforced? - **R**: Can the entity deny having performed an action? Are interactions logged? diff --git a/plugins/threat-model/skills/tna-threat-model/SKILL.md b/plugins/threat-model/skills/tna-threat-model/SKILL.md index d497fe30..1362429e 100644 --- a/plugins/threat-model/skills/tna-threat-model/SKILL.md +++ b/plugins/threat-model/skills/tna-threat-model/SKILL.md @@ -15,14 +15,17 @@ This skill focuses on TNA-specific DFD elements, trust boundaries, and code path ## Reference Files Bundled with this skill: + - `dfd-elements-tna.md` — TNA DFD element catalog (TNA-P1, TNA-P3–P5, TNA-DS5–DS6, TNA-TB1–TB3) Shared references (in `../../references/`): + - `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings - `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references - `mitre-findings-template.md` — Template for cumulative findings tracker Discovered at runtime from the workspace: + - `$THREAT_MODEL_DIR/TNA-THREAT-MODEL.md` — TNA formal threat model with DFD and per-element STRIDE analysis - `$FINDINGS_FILE` — TNA findings tracker (created from template on first use) @@ -296,6 +299,7 @@ When a PR modifies code that crosses a trust boundary, apply additional scrutiny For each affected DFD element, ask these questions: **Processes (all 6 STRIDE categories)**: + - **S**: Can the process be impersonated? Are auth checks adequate? - **T**: Can inputs/outputs be modified? Is data validated? - **R**: Are actions auditable? Are logs adequate and redacted? @@ -304,16 +308,19 @@ For each affected DFD element, ask these questions: - **E**: Does it run with minimal privilege? Can it be abused for escalation? **Data Stores (T, I, D)**: + - **T**: Can stored data be modified by unauthorized parties? - **I**: Is sensitive data encrypted? Who can read it? - **D**: Can the store be corrupted or deleted? **Data Flows (T, I, D)**: + - **T**: Can data in transit be modified? Is integrity verified? - **I**: Is the channel encrypted? Are credentials visible? - **D**: Can the flow be interrupted or flooded? **External Entities (S, R)**: + - **S**: Can the entity be impersonated? Is authentication enforced? - **R**: Can the entity deny having performed an action? Are interactions logged? diff --git a/plugins/threat-model/skills/tnf-threat-model/SKILL.md b/plugins/threat-model/skills/tnf-threat-model/SKILL.md index c49d0e3c..2fea7e91 100644 --- a/plugins/threat-model/skills/tnf-threat-model/SKILL.md +++ b/plugins/threat-model/skills/tnf-threat-model/SKILL.md @@ -15,14 +15,17 @@ This skill focuses on TNF-specific DFD elements, trust boundaries, and code path ## Reference Files Bundled with this skill: + - `dfd-elements-tnf.md` — TNF DFD element catalog (P1-P8, DS1-DS5, DF1-DF12, TB1-TB6) Shared references (in `../../references/`): + - `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings - `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references - `mitre-findings-template.md` — Template for cumulative findings tracker Discovered at runtime from the workspace: + - `$THREAT_MODEL_DIR/TNF-THREAT-MODEL.md` — TNF formal threat model with DFD and per-element STRIDE analysis - `$FINDINGS_FILE` — TNF findings tracker (created from template on first use) @@ -33,6 +36,7 @@ Before starting analysis, discover the workspace layout. ### Discovery Steps 1. **Find workspace root**: Walk upward from `$PWD` until a directory containing `repos/` is found. If no parent qualifies, fall back to checking whether the current git repo sits inside a `repos/` directory: + ```bash d="$PWD" while [ "$d" != "/" ]; do @@ -304,6 +308,7 @@ When a PR modifies code that crosses a trust boundary, apply additional scrutiny For each affected DFD element, ask these questions: **Processes (all 6 STRIDE categories)**: + - **S**: Can the process be impersonated? Are auth checks adequate? - **T**: Can inputs/outputs be modified? Is data validated? - **R**: Are actions auditable? Are logs adequate and redacted? @@ -312,16 +317,19 @@ For each affected DFD element, ask these questions: - **E**: Does it run with minimal privilege? Can it be abused for escalation? **Data Stores (T, I, D)**: + - **T**: Can stored data be modified by unauthorized parties? - **I**: Is sensitive data encrypted? Who can read it? - **D**: Can the store be corrupted or deleted? **Data Flows (T, I, D)**: + - **T**: Can data in transit be modified? Is integrity verified? - **I**: Is the channel encrypted? Are credentials visible? - **D**: Can the flow be interrupted or flooded? **External Entities (S, R)**: + - **S**: Can the entity be impersonated? Is authentication enforced? - **R**: Can the entity deny having performed an action? Are interactions logged? From a74506d08019414a82466903085baed444236f62 Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Thu, 7 May 2026 16:35:40 -0400 Subject: [PATCH 3/9] Shorten threat-model skill directory names and extract shared report templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename skill directories from *-threat-model to shorter names (e.g., tnf-threat-model → tnf) and extract duplicated report templates into a shared references/report-templates.md, reducing SKILL.md sizes by ~756 lines. Co-Authored-By: Claude Opus 4.6 --- .../references/report-templates.md | 220 ++++++++++++++++ .../{lvms-threat-model => lvms}/SKILL.md | 137 +--------- .../dfd-elements-lvms.md | 0 .../skills/{sno-threat-model => sno}/SKILL.md | 169 +------------ .../dfd-elements-sno.md | 0 .../skills/{tna-threat-model => tna}/SKILL.md | 235 +---------------- .../dfd-elements-tna.md | 0 .../skills/{tnf-threat-model => tnf}/SKILL.md | 239 +----------------- .../dfd-elements-tnf.md | 0 9 files changed, 244 insertions(+), 756 deletions(-) create mode 100644 plugins/threat-model/references/report-templates.md rename plugins/threat-model/skills/{lvms-threat-model => lvms}/SKILL.md (76%) rename plugins/threat-model/skills/{lvms-threat-model => lvms}/dfd-elements-lvms.md (100%) rename plugins/threat-model/skills/{sno-threat-model => sno}/SKILL.md (75%) rename plugins/threat-model/skills/{sno-threat-model => sno}/dfd-elements-sno.md (100%) rename plugins/threat-model/skills/{tna-threat-model => tna}/SKILL.md (71%) rename plugins/threat-model/skills/{tna-threat-model => tna}/dfd-elements-tna.md (100%) rename plugins/threat-model/skills/{tnf-threat-model => tnf}/SKILL.md (71%) rename plugins/threat-model/skills/{tnf-threat-model => tnf}/dfd-elements-tnf.md (100%) diff --git a/plugins/threat-model/references/report-templates.md b/plugins/threat-model/references/report-templates.md new file mode 100644 index 00000000..7fd0ebec --- /dev/null +++ b/plugins/threat-model/references/report-templates.md @@ -0,0 +1,220 @@ +# Report Templates + +Shared report naming conventions and output templates used by all threat-model skills. +Each skill substitutes its own topology name (TNA, TNF, SNO, LVMS) where indicated. + +--- + +## Report Naming Convention + +- **Full threat model**: `PR-THREAT-MODEL-.md` +- **Individual vuln**: `VULN-PR-.md` + +## Report Format: Threat Model + +```markdown +# PR # Threat Analysis: + +**Document Version**: 1.0 +**Date**: YYYY-MM-DD +**Classification**: Internal - Security Sensitive +**Repository**: +**Topology**: +**PR Author**: +**PR URL**: + +--- + +## Executive Summary + +[Brief overview of the PR and key security findings] + +### Findings Summary + +| Severity | Count | Summary | +|----------|-------|---------| +| Critical | X | [brief] | +| High | X | [brief] | +| Medium | X | [brief] | +| Low | X | [brief] | + +--- + +## Change Overview + +[What this PR does, its purpose, and security-relevant changes] + +--- + +## Affected Files + +| File | Changes | Security Relevance | +|------|---------|-------------------| +| path/to/file.go | +X/-Y lines | [relevance] | + +--- + +## DFD Impact Analysis + +This PR affects the following elements in the Data Flow Diagram +(see -THREAT-MODEL.md): + +### Affected DFD Elements + +| Element | Name | Impact | Trust Boundary | +|---------|------|--------|----------------| +| P# | [process name] | [what changed] | TB# | +| DS# | [store name] | [what changed] | TB# | +| DF# | [flow description] | [what changed] | TB#->TB# | + +### Trust Boundary Crossings + +[Describe any trust boundaries crossed by the changed code] + +### Per-Element STRIDE + +| Element | S | T | R | I | D | E | Notes | +|---------|---|---|---|---|---|---|-------| +| P# | - | - | - | - | - | - | [Processes: all 6] | +| DS# | N/A | - | N/A | - | - | N/A | [Data Stores: T, I, D] | +| DF# | N/A | - | N/A | - | - | N/A | [Data Flows: T, I, D] | +| EE# | - | N/A | - | N/A | N/A | N/A | [External Entities: S, R] | + +**Legend**: **X** = new threat found, **~** = existing threat modified, **-** = no impact, N/A = not applicable + +### Threat Model Cross-Reference + +| PR Finding | Existing PE-* ID | Status | +|------------|-----------------|--------| +| [finding] | PE-XX-X-X | Matches existing / New gap / Mitigated | + +--- + +## Threat Analysis + +### VULN-1: [Vulnerability Title] + +**Severity**: Critical/High/Medium/Low +**OWASP**: A##:2025 - Category Name +**MITRE ATT&CK**: T#### - Technique Name +**CWE**: CWE-### + +#### Affected Code + +**File**: `path/to/file.go:line` + +#### Description + +[Detailed description of the vulnerability] + +#### Attack Vector + +[How this could be exploited] + +#### Impact + +- **Confidentiality**: [impact] +- **Integrity**: [impact] +- **Availability**: [impact] + +#### Recommended Fix + +[Code showing the fix] + +--- + +## OWASP & MITRE ATT&CK Mapping + +| Finding | OWASP | MITRE | CWE | Status | +|---------|-------|-------|-----|--------| +| VULN-1 | A05:2025 Injection | T1059 | CWE-78 | Open | + +--- + +## Risk Assessment + +| Finding | Likelihood | Impact | Risk | +|---------|------------|--------|------| +| VULN-1 | High | Critical | Critical | + +--- + +## Recommendations + +### For Developers (Code Changes) + +#### Before Merge + +1. [Code fix or change required in this PR] + +#### After Merge + +1. [Follow-up code improvement, test addition, or refactor] + +### For Customers (Deployment & Operations) + +#### Configuration Hardening + +1. [Cluster configuration or hardening recommendation] + +#### Operational Practices + +1. [Monitoring, incident response, or day-2 operational guidance] + +--- + +## References + +- [OWASP Top 10:2025](https://owasp.org/Top10/2025/) +- [MITRE ATT&CK](https://attack.mitre.org/) +- [Relevant CVEs, CWEs, documentation] +``` + +## Report Format: Individual Vulnerability (for Critical/High findings) + +```markdown +# Security Ticket: [Vulnerability Title] + +**Ticket ID**: VULN-PR- +**Severity**: CRITICAL/HIGH +**Component**: +**Status**: Open +**Created**: YYYY-MM-DD +**PR**: # + +## Summary + +[One paragraph summary] + +## Affected Code + +**File**: `path/to/file.go:lines` + +## Exploitation + +### Attack Flow + +[ASCII diagram or description of attack flow] + +### Exploit Examples + +[Code examples showing exploitation] + +## Impact + +[Detailed impact analysis] + +## Recommended Fix + +### For Developers + +[Code showing the fix with explanation] + +### For Customers + +[Deployment hardening, configuration changes, or monitoring guidance] + +## References + +- [CWE, OWASP, other references] +``` diff --git a/plugins/threat-model/skills/lvms-threat-model/SKILL.md b/plugins/threat-model/skills/lvms/SKILL.md similarity index 76% rename from plugins/threat-model/skills/lvms-threat-model/SKILL.md rename to plugins/threat-model/skills/lvms/SKILL.md index 97dd65f2..c753d49d 100644 --- a/plugins/threat-model/skills/lvms-threat-model/SKILL.md +++ b/plugins/threat-model/skills/lvms/SKILL.md @@ -18,7 +18,7 @@ Bundled with this skill: - `dfd-elements-lvms.md` — LVMS DFD element catalog (placeholder — not yet modeled) -Shared references (in `../../references/`): +Shared references (in `$PLUGIN_DIR/references/`): - `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings - `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references @@ -61,7 +61,7 @@ Before starting analysis, discover the workspace layout. - `$WORKSPACE/docs/` - The current directory - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). - - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-lvms.md` — initialized from `../../references/mitre-findings-template.md` on first use. + - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-lvms.md` — initialized from `$PLUGIN_DIR/references/mitre-findings-template.md` on first use. 3. **Validate workspace**: Warn the user if: - No `repos/` directory is found @@ -102,7 +102,7 @@ cat >> "$FINDINGS_FILE" <<'FINDINGS_BLOCK' FINDINGS_BLOCK ``` -Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `../../references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. +Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `$PLUGIN_DIR/references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. ## Input Formats @@ -157,7 +157,7 @@ Detects the repository from the current working directory. 7. **Map to DFD elements** — if `dfd-elements-lvms.md` has been populated, map changed files to affected DFD elements. If not yet modeled, skip and note in the report. 8. **Apply per-element STRIDE** to affected elements (if DFD is available) and cross-reference against `$THREAT_MODEL_DIR/LVMS-THREAT-MODEL.md` (if found) 9. **Combine findings** from ShellCheck + AI analysis + DFD/STRIDE analysis -10. **Map findings to MITRE ATT&CK** techniques (see `../../references/mitre-reference.md`) +10. **Map findings to MITRE ATT&CK** techniques (see `$PLUGIN_DIR/references/mitre-reference.md`) 11. **Generate report** at `$REPORT_DIR/` 12. **Append findings to tracker** — follow the Append Protocol to write a findings block to `$FINDINGS_FILE` @@ -233,131 +233,6 @@ When the DFD is available, the analysis should follow the same STRIDE methodolog --- -## Report Naming Convention +## Report Output -- **Full threat model**: `PR-THREAT-MODEL-.md` -- **Individual vuln**: `VULN-PR-.md` - -## Report Format: Threat Model - -```markdown -# PR # Threat Analysis: - -**Document Version**: 1.0 -**Date**: YYYY-MM-DD -**Classification**: Internal - Security Sensitive -**Repository**: -**Topology**: LVMS -**PR Author**: -**PR URL**: - ---- - -## Executive Summary - -[Brief overview of the PR and key security findings] - -### Findings Summary - -| Severity | Count | Summary | -|----------|-------|---------| -| Critical | X | [brief] | -| High | X | [brief] | -| Medium | X | [brief] | -| Low | X | [brief] | - ---- - -## Change Overview - -[What this PR does, its purpose, and security-relevant changes] - ---- - -## Affected Files - -| File | Changes | Security Relevance | -|------|---------|-------------------| -| path/to/file.go | +X/-Y lines | [relevance] | - ---- - -## DFD Impact Analysis - -> DFD model not yet defined for LVMS. This section will be populated once `dfd-elements-lvms.md` is created. - ---- - -## Threat Analysis - -### VULN-1: [Vulnerability Title] - -**Severity**: Critical/High/Medium/Low -**OWASP**: A##:2025 - Category Name -**MITRE ATT&CK**: T#### - Technique Name -**CWE**: CWE-### - -#### Affected Code - -**File**: `path/to/file.go:line` - -#### Description - -[Detailed description] - -#### Attack Vector - -[How this could be exploited] - -#### Impact - -- **Confidentiality**: [impact] -- **Integrity**: [impact] -- **Availability**: [impact] - -#### Recommended Fix - -[Code showing the fix] - ---- - -## OWASP & MITRE ATT&CK Mapping - -| Finding | OWASP | MITRE | CWE | Status | -|---------|-------|-------|-----|--------| -| VULN-1 | A05:2025 Injection | T1059 | CWE-78 | Open | - ---- - -## Risk Assessment - -| Finding | Likelihood | Impact | Risk | -|---------|------------|--------|------| -| VULN-1 | High | Critical | Critical | - ---- - -## Recommendations - -### For Developers (Code Changes) - -1. [Recommendations] - -### For Customers (Deployment & Operations) - -1. [Recommendations] - ---- - -## References - -- [OWASP Top 10:2025](https://owasp.org/Top10/2025/) -- [MITRE ATT&CK](https://attack.mitre.org/) -``` - -## Available Repositories - -| Repo | Org | Focus Areas | -|------|-----|-------------| -| lvm-operator | openshift | CSI driver, VG management, privilege, LVM commands | -| origin | openshift | Test code security | +Use report templates from `$PLUGIN_DIR/references/report-templates.md`. Set `` to **LVMS** when filling in the templates. diff --git a/plugins/threat-model/skills/lvms-threat-model/dfd-elements-lvms.md b/plugins/threat-model/skills/lvms/dfd-elements-lvms.md similarity index 100% rename from plugins/threat-model/skills/lvms-threat-model/dfd-elements-lvms.md rename to plugins/threat-model/skills/lvms/dfd-elements-lvms.md diff --git a/plugins/threat-model/skills/sno-threat-model/SKILL.md b/plugins/threat-model/skills/sno/SKILL.md similarity index 75% rename from plugins/threat-model/skills/sno-threat-model/SKILL.md rename to plugins/threat-model/skills/sno/SKILL.md index 5edd3966..b018e089 100644 --- a/plugins/threat-model/skills/sno-threat-model/SKILL.md +++ b/plugins/threat-model/skills/sno/SKILL.md @@ -18,7 +18,7 @@ Bundled with this skill: - `dfd-elements-sno.md` — SNO DFD element catalog (SNO-P1–P6, SNO-DS1–DS6, SNO-DF1–DF10, SNO-TB1–TB3) -Shared references (in `../../references/`): +Shared references (in `$PLUGIN_DIR/references/`): - `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings - `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references @@ -61,7 +61,7 @@ Before starting analysis, discover the workspace layout. - `$WORKSPACE/docs/` - The current directory - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). - - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-sno.md` — initialized from `../../references/mitre-findings-template.md` on first use. + - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-sno.md` — initialized from `$PLUGIN_DIR/references/mitre-findings-template.md` on first use. 3. **Validate workspace**: Warn the user if: - No `repos/` directory is found @@ -102,7 +102,7 @@ cat >> "$FINDINGS_FILE" <<'FINDINGS_BLOCK' FINDINGS_BLOCK ``` -Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `../../references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. +Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `$PLUGIN_DIR/references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. ## Input Formats @@ -162,7 +162,7 @@ Detects the repository from the current working directory. 7. **Map to DFD elements** — identify which DFD elements are affected using the SNO mapping table below and `dfd-elements-sno.md` 8. **Apply per-element STRIDE** to affected elements and cross-reference against `$THREAT_MODEL_DIR/SNO-THREAT-MODEL.md` (if found) 9. **Combine findings** from ShellCheck + AI analysis + DFD/STRIDE analysis -10. **Map findings to MITRE ATT&CK** techniques (see `../../references/mitre-reference.md`) +10. **Map findings to MITRE ATT&CK** techniques (see `$PLUGIN_DIR/references/mitre-reference.md`) 11. **Generate report** at `$REPORT_DIR/` 12. **Append findings to tracker** — follow the Append Protocol to write a findings block to `$FINDINGS_FILE` @@ -300,163 +300,6 @@ If the formal threat model file is not found, skip cross-referencing and note th --- -## Report Naming Convention +## Report Output -- **Full threat model**: `PR-THREAT-MODEL-.md` -- **Individual vuln**: `VULN-PR-.md` - -## Report Format: Threat Model - -```markdown -# PR # Threat Analysis: - -**Document Version**: 1.0 -**Date**: YYYY-MM-DD -**Classification**: Internal - Security Sensitive -**Repository**: -**Topology**: SNO -**PR Author**: -**PR URL**: - ---- - -## Executive Summary - -[Brief overview of the PR and key security findings] - -### Findings Summary - -| Severity | Count | Summary | -|----------|-------|---------| -| Critical | X | [brief] | -| High | X | [brief] | -| Medium | X | [brief] | -| Low | X | [brief] | - ---- - -## Change Overview - -[What this PR does, its purpose, and security-relevant changes] - ---- - -## Affected Files - -| File | Changes | Security Relevance | -|------|---------|-------------------| -| path/to/file.go | +X/-Y lines | [relevance] | - ---- - -## DFD Impact Analysis - -This PR affects the following elements in the SNO Data Flow Diagram -(see SNO-THREAT-MODEL.md): - -### Affected DFD Elements - -| Element | Name | Impact | Trust Boundary | -|---------|------|--------|----------------| -| SNO-P# | [process name] | [what changed] | SNO-TB# | -| SNO-DS# | [store name] | [what changed] | SNO-TB# | - -### Trust Boundary Crossings - -[Describe any trust boundaries crossed by the changed code] - -### Per-Element STRIDE - -| Element | S | T | R | I | D | E | Notes | -|---------|---|---|---|---|---|---|-------| -| SNO-P# | - | - | - | - | - | - | [Processes: all 6] | -| SNO-DS# | N/A | - | N/A | - | - | N/A | [Data Stores: T, I, D] | - -**Legend**: **X** = new threat found, **~** = existing threat modified, **-** = no impact, N/A = not applicable - -### Threat Model Cross-Reference - -| PR Finding | Existing PE-* ID | Status | -|------------|-----------------|--------| -| [finding] | PE-SNO-XX-X-X | Matches existing / New gap / Mitigated | - ---- - -## Threat Analysis - -### VULN-1: [Vulnerability Title] - -**Severity**: Critical/High/Medium/Low -**OWASP**: A##:2025 - Category Name -**MITRE ATT&CK**: T#### - Technique Name -**CWE**: CWE-### - -#### Affected Code - -**File**: `path/to/file.go:line` - -#### Description - -[Detailed description] - -#### Attack Vector - -[How this could be exploited] - -#### Impact - -- **Confidentiality**: [impact] -- **Integrity**: [impact] -- **Availability**: [impact] - -#### Recommended Fix - -[Code showing the fix] - ---- - -## OWASP & MITRE ATT&CK Mapping - -| Finding | OWASP | MITRE | CWE | Status | -|---------|-------|-------|-----|--------| -| VULN-1 | A05:2025 Injection | T1059 | CWE-78 | Open | - ---- - -## Risk Assessment - -| Finding | Likelihood | Impact | Risk | -|---------|------------|--------|------| -| VULN-1 | High | Critical | Critical | - ---- - -## Recommendations - -### For Developers (Code Changes) - -1. [Recommendations] - -### For Customers (Deployment & Operations) - -1. [Recommendations] - ---- - -## References - -- [OWASP Top 10:2025](https://owasp.org/Top10/2025/) -- [MITRE ATT&CK](https://attack.mitre.org/) -``` - -## Available Repositories - -| Repo | Org | Focus Areas | -|------|-----|-------------| -| installer | openshift | Install config, ignition | -| machine-config-operator | openshift | Node config, privilege escalation | -| cluster-etcd-operator | openshift | Etcd management | -| assisted-service | openshift | API security, credential handling | -| origin | openshift | Test code security | -| dev-scripts | openshift-metal3 | Shell scripts, credential handling | -| release | openshift | Release artifacts, CI/CD, manifests | +Use report templates from `$PLUGIN_DIR/references/report-templates.md`. Set `` to **SNO** when filling in the templates. diff --git a/plugins/threat-model/skills/sno-threat-model/dfd-elements-sno.md b/plugins/threat-model/skills/sno/dfd-elements-sno.md similarity index 100% rename from plugins/threat-model/skills/sno-threat-model/dfd-elements-sno.md rename to plugins/threat-model/skills/sno/dfd-elements-sno.md diff --git a/plugins/threat-model/skills/tna-threat-model/SKILL.md b/plugins/threat-model/skills/tna/SKILL.md similarity index 71% rename from plugins/threat-model/skills/tna-threat-model/SKILL.md rename to plugins/threat-model/skills/tna/SKILL.md index 1362429e..09829141 100644 --- a/plugins/threat-model/skills/tna-threat-model/SKILL.md +++ b/plugins/threat-model/skills/tna/SKILL.md @@ -18,7 +18,7 @@ Bundled with this skill: - `dfd-elements-tna.md` — TNA DFD element catalog (TNA-P1, TNA-P3–P5, TNA-DS5–DS6, TNA-TB1–TB3) -Shared references (in `../../references/`): +Shared references (in `$PLUGIN_DIR/references/`): - `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings - `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references @@ -61,7 +61,7 @@ Before starting analysis, discover the workspace layout. - `$WORKSPACE/docs/` - The current directory - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). - - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-tna.md` — initialized from `../../references/mitre-findings-template.md` on first use. + - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-tna.md` — initialized from `$PLUGIN_DIR/references/mitre-findings-template.md` on first use. 3. **Validate workspace**: Warn the user if: - No `repos/` directory is found @@ -102,7 +102,7 @@ cat >> "$FINDINGS_FILE" <<'FINDINGS_BLOCK' FINDINGS_BLOCK ``` -Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `../../references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. +Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `$PLUGIN_DIR/references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. ## Input Formats @@ -170,7 +170,7 @@ Specify repo name and PR number explicitly. 7. **Map to DFD elements** — identify which DFD elements are affected using the TNA mapping table below and `dfd-elements-tna.md` 8. **Apply per-element STRIDE** to affected elements and cross-reference against `$THREAT_MODEL_DIR/TNA-THREAT-MODEL.md` (if found) 9. **Combine findings** from ShellCheck + AI analysis + DFD/STRIDE analysis -10. **Map findings to MITRE ATT&CK** techniques (see `../../references/mitre-reference.md`) +10. **Map findings to MITRE ATT&CK** techniques (see `$PLUGIN_DIR/references/mitre-reference.md`) 11. **Generate report** at `$REPORT_DIR/` 12. **Append findings to tracker** — follow the Append Protocol to write a findings block to `$FINDINGS_FILE` @@ -337,229 +337,6 @@ If the formal threat model file is not found, skip cross-referencing and note th --- -## Report Naming Convention +## Report Output -- **Full threat model**: `PR-THREAT-MODEL-.md` -- **Individual vuln**: `VULN-PR-.md` - -## Report Format: Threat Model - -```markdown -# PR # Threat Analysis: - -**Document Version**: 1.0 -**Date**: YYYY-MM-DD -**Classification**: Internal - Security Sensitive -**Repository**: -**Topology**: TNA -**PR Author**: -**PR URL**: - ---- - -## Executive Summary - -[Brief overview of the PR and key security findings] - -### Findings Summary - -| Severity | Count | Summary | -|----------|-------|---------| -| Critical | X | [brief] | -| High | X | [brief] | -| Medium | X | [brief] | -| Low | X | [brief] | - ---- - -## Change Overview - -[What this PR does, its purpose, and security-relevant changes] - ---- - -## Affected Files - -| File | Changes | Security Relevance | -|------|---------|-------------------| -| path/to/file.go | +X/-Y lines | [relevance] | - ---- - -## DFD Impact Analysis - -This PR affects the following elements in the TNA Data Flow Diagram -(see TNA-THREAT-MODEL.md): - -### Affected DFD Elements - -| Element | Name | Impact | Trust Boundary | -|---------|------|--------|----------------| -| P# | [process name] | [what changed] | TB# | -| DS# | [store name] | [what changed] | TB# | -| DF# | [flow description] | [what changed] | TB#->TB# | - -### Trust Boundary Crossings - -[Describe any trust boundaries crossed by the changed code] - -### Per-Element STRIDE - -| Element | S | T | R | I | D | E | Notes | -|---------|---|---|---|---|---|---|-------| -| P# | - | - | - | - | - | - | [Processes: all 6] | -| DS# | N/A | - | N/A | - | - | N/A | [Data Stores: T, I, D] | -| DF# | N/A | - | N/A | - | - | N/A | [Data Flows: T, I, D] | -| EE# | - | N/A | - | N/A | N/A | N/A | [External Entities: S, R] | - -**Legend**: **X** = new threat found, **~** = existing threat modified, **-** = no impact, N/A = not applicable - -### Threat Model Cross-Reference - -| PR Finding | Existing PE-* ID | Status | -|------------|-----------------|--------| -| [finding] | PE-XX-X-X | Matches existing / New gap / Mitigated | - ---- - -## Threat Analysis - -### VULN-1: [Vulnerability Title] - -**Severity**: Critical/High/Medium/Low -**OWASP**: A##:2025 - Category Name -**MITRE ATT&CK**: T#### - Technique Name -**CWE**: CWE-### - -#### Affected Code - -**File**: `path/to/file.go:line` - -#### Description - -[Detailed description of the vulnerability] - -#### Attack Vector - -[How this could be exploited] - -#### Impact - -- **Confidentiality**: [impact] -- **Integrity**: [impact] -- **Availability**: [impact] - -#### Recommended Fix - -[Code showing the fix] - ---- - -## OWASP & MITRE ATT&CK Mapping - -| Finding | OWASP | MITRE | CWE | Status | -|---------|-------|-------|-----|--------| -| VULN-1 | A05:2025 Injection | T1059 | CWE-78 | Open | - ---- - -## Risk Assessment - -| Finding | Likelihood | Impact | Risk | -|---------|------------|--------|------| -| VULN-1 | High | Critical | Critical | - ---- - -## Recommendations - -### For Developers (Code Changes) - -#### Before Merge - -1. [Code fix or change required in this PR] - -#### After Merge - -1. [Follow-up code improvement, test addition, or refactor] - -### For Customers (Deployment & Operations) - -#### Configuration Hardening - -1. [Cluster configuration or hardening recommendation] - -#### Operational Practices - -1. [Monitoring, incident response, or day-2 operational guidance] - ---- - -## References - -- [OWASP Top 10:2025](https://owasp.org/Top10/2025/) -- [MITRE ATT&CK](https://attack.mitre.org/) -- [Relevant CVEs, CWEs, documentation] -``` - -## Report Format: Individual Vulnerability (for Critical/High findings) - -```markdown -# Security Ticket: [Vulnerability Title] - -**Ticket ID**: VULN-PR- -**Severity**: CRITICAL/HIGH -**Component**: -**Status**: Open -**Created**: YYYY-MM-DD -**PR**: # - -## Summary - -[One paragraph summary] - -## Affected Code - -**File**: `path/to/file.go:lines` - -## Exploitation - -### Attack Flow - -[ASCII diagram or description of attack flow] - -### Exploit Examples - -[Code examples showing exploitation] - -## Impact - -[Detailed impact analysis] - -## Recommended Fix - -### For Developers - -[Code showing the fix with explanation] - -### For Customers - -[Deployment hardening, configuration changes, or monitoring guidance] - -## References - -- [CWE, OWASP, other references] -``` - -## Available Repositories - -These are the repos expected in `$REPOS/`: - -| Repo | Org | Focus Areas | -|------|-----|-------------| -| assisted-service | openshift | API security, credential handling | -| cluster-etcd-operator | openshift | Topology detection, etcd config | -| machine-config-operator | openshift | Arbiter MCP, node config | -| installer | openshift | Arbiter install config, ignition | -| origin | openshift | Test code security | -| dev-scripts | openshift-metal3 | Shell scripts, credential handling | +Use report templates from `$PLUGIN_DIR/references/report-templates.md`. Set `` to **TNA** when filling in the templates. diff --git a/plugins/threat-model/skills/tna-threat-model/dfd-elements-tna.md b/plugins/threat-model/skills/tna/dfd-elements-tna.md similarity index 100% rename from plugins/threat-model/skills/tna-threat-model/dfd-elements-tna.md rename to plugins/threat-model/skills/tna/dfd-elements-tna.md diff --git a/plugins/threat-model/skills/tnf-threat-model/SKILL.md b/plugins/threat-model/skills/tnf/SKILL.md similarity index 71% rename from plugins/threat-model/skills/tnf-threat-model/SKILL.md rename to plugins/threat-model/skills/tnf/SKILL.md index 2fea7e91..b009e910 100644 --- a/plugins/threat-model/skills/tnf-threat-model/SKILL.md +++ b/plugins/threat-model/skills/tnf/SKILL.md @@ -18,7 +18,7 @@ Bundled with this skill: - `dfd-elements-tnf.md` — TNF DFD element catalog (P1-P8, DS1-DS5, DF1-DF12, TB1-TB6) -Shared references (in `../../references/`): +Shared references (in `$PLUGIN_DIR/references/`): - `mitre-reference.md` — MITRE ATT&CK lookup with DFD element mappings - `owasp-reference.md` — OWASP Top 10:2025 mapping with DFD element cross-references @@ -61,7 +61,7 @@ Before starting analysis, discover the workspace layout. - `$WORKSPACE/docs/` - The current directory - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). - - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-tnf.md` — initialized from `../../references/mitre-findings-template.md` on first use. + - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-tnf.md` — initialized from `$PLUGIN_DIR/references/mitre-findings-template.md` on first use. 3. **Validate workspace**: Warn the user if: - No `repos/` directory is found @@ -102,7 +102,7 @@ cat >> "$FINDINGS_FILE" <<'FINDINGS_BLOCK' FINDINGS_BLOCK ``` -Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `../../references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. +Substitute `RESOLVED_TEMPLATE_PATH` with the absolute path to `$PLUGIN_DIR/references/mitre-findings-template.md` (resolved from this skill's directory). Fill in `REPO`, `NUMBER`, `YYYY-MM-DD`, and the table rows from the current analysis. ## Input Formats @@ -172,7 +172,7 @@ Specify repo name and PR number explicitly. 7. **Map to DFD elements** — identify which DFD elements are affected using the TNF mapping table below and `dfd-elements-tnf.md` 8. **Apply per-element STRIDE** to affected elements and cross-reference against `$THREAT_MODEL_DIR/TNF-THREAT-MODEL.md` (if found) 9. **Combine findings** from ShellCheck + AI analysis + DFD/STRIDE analysis -10. **Map findings to MITRE ATT&CK** techniques (see `../../references/mitre-reference.md`) +10. **Map findings to MITRE ATT&CK** techniques (see `$PLUGIN_DIR/references/mitre-reference.md`) 11. **Generate report** at `$REPORT_DIR/` 12. **Append findings to tracker** — follow the Append Protocol to write a findings block to `$FINDINGS_FILE` @@ -346,233 +346,6 @@ If the formal threat model file is not found, skip cross-referencing and note th --- -## Report Naming Convention +## Report Output -- **Full threat model**: `PR-THREAT-MODEL-.md` -- **Individual vuln**: `VULN-PR-.md` - -## Report Format: Threat Model - -```markdown -# PR # Threat Analysis: - -**Document Version**: 1.0 -**Date**: YYYY-MM-DD -**Classification**: Internal - Security Sensitive -**Repository**: -**Topology**: TNF -**PR Author**: -**PR URL**: - ---- - -## Executive Summary - -[Brief overview of the PR and key security findings] - -### Findings Summary - -| Severity | Count | Summary | -|----------|-------|---------| -| Critical | X | [brief] | -| High | X | [brief] | -| Medium | X | [brief] | -| Low | X | [brief] | - ---- - -## Change Overview - -[What this PR does, its purpose, and security-relevant changes] - ---- - -## Affected Files - -| File | Changes | Security Relevance | -|------|---------|-------------------| -| path/to/file.go | +X/-Y lines | [relevance] | - ---- - -## DFD Impact Analysis - -This PR affects the following elements in the TNF Data Flow Diagram -(see TNF-THREAT-MODEL.md): - -### Affected DFD Elements - -| Element | Name | Impact | Trust Boundary | -|---------|------|--------|----------------| -| P# | [process name] | [what changed] | TB# | -| DS# | [store name] | [what changed] | TB# | -| DF# | [flow description] | [what changed] | TB#->TB# | - -### Trust Boundary Crossings - -[Describe any trust boundaries crossed by the changed code] - -### Per-Element STRIDE - -| Element | S | T | R | I | D | E | Notes | -|---------|---|---|---|---|---|---|-------| -| P# | - | - | - | - | - | - | [Processes: all 6] | -| DS# | N/A | - | N/A | - | - | N/A | [Data Stores: T, I, D] | -| DF# | N/A | - | N/A | - | - | N/A | [Data Flows: T, I, D] | -| EE# | - | N/A | - | N/A | N/A | N/A | [External Entities: S, R] | - -**Legend**: **X** = new threat found, **~** = existing threat modified, **-** = no impact, N/A = not applicable - -### Threat Model Cross-Reference - -| PR Finding | Existing PE-* ID | Status | -|------------|-----------------|--------| -| [finding] | PE-XX-X-X | Matches existing / New gap / Mitigated | - ---- - -## Threat Analysis - -### VULN-1: [Vulnerability Title] - -**Severity**: Critical/High/Medium/Low -**OWASP**: A##:2025 - Category Name -**MITRE ATT&CK**: T#### - Technique Name -**CWE**: CWE-### - -#### Affected Code - -**File**: `path/to/file.go:line` - -#### Description - -[Detailed description of the vulnerability] - -#### Attack Vector - -[How this could be exploited] - -#### Impact - -- **Confidentiality**: [impact] -- **Integrity**: [impact] -- **Availability**: [impact] - -#### Recommended Fix - -[Code showing the fix] - ---- - -## OWASP & MITRE ATT&CK Mapping - -| Finding | OWASP | MITRE | CWE | Status | -|---------|-------|-------|-----|--------| -| VULN-1 | A05:2025 Injection | T1059 | CWE-78 | Open | - ---- - -## Risk Assessment - -| Finding | Likelihood | Impact | Risk | -|---------|------------|--------|------| -| VULN-1 | High | Critical | Critical | - ---- - -## Recommendations - -### For Developers (Code Changes) - -#### Before Merge - -1. [Code fix or change required in this PR] - -#### After Merge - -1. [Follow-up code improvement, test addition, or refactor] - -### For Customers (Deployment & Operations) - -#### Configuration Hardening - -1. [Cluster configuration or hardening recommendation] - -#### Operational Practices - -1. [Monitoring, incident response, or day-2 operational guidance] - ---- - -## References - -- [OWASP Top 10:2025](https://owasp.org/Top10/2025/) -- [MITRE ATT&CK](https://attack.mitre.org/) -- [Relevant CVEs, CWEs, documentation] -``` - -## Report Format: Individual Vulnerability (for Critical/High findings) - -```markdown -# Security Ticket: [Vulnerability Title] - -**Ticket ID**: VULN-PR- -**Severity**: CRITICAL/HIGH -**Component**: -**Status**: Open -**Created**: YYYY-MM-DD -**PR**: # - -## Summary - -[One paragraph summary] - -## Affected Code - -**File**: `path/to/file.go:lines` - -## Exploitation - -### Attack Flow - -[ASCII diagram or description of attack flow] - -### Exploit Examples - -[Code examples showing exploitation] - -## Impact - -[Detailed impact analysis] - -## Recommended Fix - -### For Developers - -[Code showing the fix with explanation] - -### For Customers - -[Deployment hardening, configuration changes, or monitoring guidance] - -## References - -- [CWE, OWASP, other references] -``` - -## Available Repositories - -These are the repos expected in `$REPOS/`: - -| Repo | Org | Focus Areas | -|------|-----|-------------| -| assisted-service | openshift | API security, credential handling | -| cluster-etcd-operator | openshift | Privilege, shell injection, secrets | -| machine-config-operator | openshift | Node config, privilege escalation | -| installer | openshift | Install config, credential storage | -| cluster-baremetal-operator | openshift | BMC access, metal3 security | -| resource-agents | ClusterLabs | OCF scripts, shell injection | -| two-node-toolbox | (internal) | Deployment scripts, credentials | -| origin | openshift | Test code security | -| dev-scripts | openshift-metal3 | Shell scripts, credential handling | -| pacemaker | ClusterLabs | Fencing daemon, cluster auth | +Use report templates from `$PLUGIN_DIR/references/report-templates.md`. Set `` to **TNF** when filling in the templates. diff --git a/plugins/threat-model/skills/tnf-threat-model/dfd-elements-tnf.md b/plugins/threat-model/skills/tnf/dfd-elements-tnf.md similarity index 100% rename from plugins/threat-model/skills/tnf-threat-model/dfd-elements-tnf.md rename to plugins/threat-model/skills/tnf/dfd-elements-tnf.md From 8477584a96555849b61fc400130afb3fce16f5a2 Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Thu, 11 Jun 2026 14:36:20 -0400 Subject: [PATCH 4/9] Support REPORT_DIR env var override in threat-model skills Allow the report output directory to be set externally via $REPORT_DIR, skipping workspace discovery for report path. Enables eval harnesses and CI to control where reports are written. Co-Authored-By: Claude Opus 4.6 --- plugins/threat-model/skills/lvms/SKILL.md | 2 +- plugins/threat-model/skills/sno/SKILL.md | 2 +- plugins/threat-model/skills/tna/SKILL.md | 2 +- plugins/threat-model/skills/tnf/SKILL.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/threat-model/skills/lvms/SKILL.md b/plugins/threat-model/skills/lvms/SKILL.md index c753d49d..8295e495 100644 --- a/plugins/threat-model/skills/lvms/SKILL.md +++ b/plugins/threat-model/skills/lvms/SKILL.md @@ -60,7 +60,7 @@ Before starting analysis, discover the workspace layout. - `$WORKSPACE/repos/lvm-operator/docs/` - `$WORKSPACE/docs/` - The current directory - - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). + - **Report output**: If `$REPORT_DIR` is already set in the environment, use it directly. Otherwise, write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-lvms.md` — initialized from `$PLUGIN_DIR/references/mitre-findings-template.md` on first use. 3. **Validate workspace**: Warn the user if: diff --git a/plugins/threat-model/skills/sno/SKILL.md b/plugins/threat-model/skills/sno/SKILL.md index b018e089..e5e300b0 100644 --- a/plugins/threat-model/skills/sno/SKILL.md +++ b/plugins/threat-model/skills/sno/SKILL.md @@ -60,7 +60,7 @@ Before starting analysis, discover the workspace layout. - `$WORKSPACE/repos/sno-deploy/docs/` - `$WORKSPACE/docs/` - The current directory - - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). + - **Report output**: If `$REPORT_DIR` is already set in the environment, use it directly. Otherwise, write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-sno.md` — initialized from `$PLUGIN_DIR/references/mitre-findings-template.md` on first use. 3. **Validate workspace**: Warn the user if: diff --git a/plugins/threat-model/skills/tna/SKILL.md b/plugins/threat-model/skills/tna/SKILL.md index 09829141..3520619a 100644 --- a/plugins/threat-model/skills/tna/SKILL.md +++ b/plugins/threat-model/skills/tna/SKILL.md @@ -60,7 +60,7 @@ Before starting analysis, discover the workspace layout. - `$WORKSPACE/repos/two-node-toolbox/docs/` - `$WORKSPACE/docs/` - The current directory - - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). + - **Report output**: If `$REPORT_DIR` is already set in the environment, use it directly. Otherwise, write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-tna.md` — initialized from `$PLUGIN_DIR/references/mitre-findings-template.md` on first use. 3. **Validate workspace**: Warn the user if: diff --git a/plugins/threat-model/skills/tnf/SKILL.md b/plugins/threat-model/skills/tnf/SKILL.md index b009e910..5ebf3943 100644 --- a/plugins/threat-model/skills/tnf/SKILL.md +++ b/plugins/threat-model/skills/tnf/SKILL.md @@ -60,7 +60,7 @@ Before starting analysis, discover the workspace layout. - `$WORKSPACE/repos/two-node-toolbox/docs/` - `$WORKSPACE/docs/` - The current directory - - **Report output**: Write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). + - **Report output**: If `$REPORT_DIR` is already set in the environment, use it directly. Otherwise, write reports to the same directory where the threat model is found. If not found, write to `$WORKSPACE/reports/` (create if needed). - **Findings tracker**: `$WORKSPACE/.claude/skills/threat-model/mitre-findings-tnf.md` — initialized from `$PLUGIN_DIR/references/mitre-findings-template.md` on first use. 3. **Validate workspace**: Warn the user if: From 4ede53f656765c50dd6c8642898c92a91b332d97 Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Sat, 6 Jun 2026 16:01:55 -0400 Subject: [PATCH 5/9] OCPEDGE-2727: Add eval harness configs for cluster-diagnostic and threat-model skills Add evaluation configs, test cases, and README for two skills: - cluster-diagnostic: 5 cases covering validate and recovery-guide modes - threat-model-tnf: 5 cases covering PR security analysis Co-Authored-By: Claude Opus 4.6 --- plugins/two-node/evals/README.md | 50 ++++ plugins/two-node/evals/cluster-diagnostic.md | 83 ++++++ .../two-node/evals/cluster-diagnostic.yaml | 218 ++++++++++++++++ .../annotations.yaml | 9 + .../input.yaml | 4 + .../annotations.yaml | 5 + .../case-002-validate-safe-redfish/input.yaml | 5 + .../annotations.yaml | 5 + .../input.yaml | 2 + .../annotations.yaml | 5 + .../case-004-recovery-standby/input.yaml | 2 + .../annotations.yaml | 6 + .../case-005-validate-pcs-standby/input.yaml | 4 + plugins/two-node/evals/threat-model-tnf.md | 101 +++++++ plugins/two-node/evals/threat-model-tnf.yaml | 247 ++++++++++++++++++ .../annotations.yaml | 18 ++ .../case-001-shell-script-k8s-api/input.yaml | 3 + .../annotations.yaml | 21 ++ .../input.yaml | 3 + .../annotations.yaml | 16 ++ .../case-003-mac-fencing-lookup/input.yaml | 3 + .../annotations.yaml | 9 + .../input.yaml | 3 + .../annotations.yaml | 15 ++ .../case-005-tnf-retry-bugfix/input.yaml | 3 + 25 files changed, 840 insertions(+) create mode 100644 plugins/two-node/evals/README.md create mode 100644 plugins/two-node/evals/cluster-diagnostic.md create mode 100644 plugins/two-node/evals/cluster-diagnostic.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-001-validate-sequential-shutdown/annotations.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-001-validate-sequential-shutdown/input.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-002-validate-safe-redfish/annotations.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-002-validate-safe-redfish/input.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-003-recovery-full-shutdown/annotations.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-003-recovery-full-shutdown/input.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-004-recovery-standby/annotations.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-004-recovery-standby/input.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-005-validate-pcs-standby/annotations.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-005-validate-pcs-standby/input.yaml create mode 100644 plugins/two-node/evals/threat-model-tnf.md create mode 100644 plugins/two-node/evals/threat-model-tnf.yaml create mode 100644 plugins/two-node/evals/threat-model-tnf/cases/case-001-shell-script-k8s-api/annotations.yaml create mode 100644 plugins/two-node/evals/threat-model-tnf/cases/case-001-shell-script-k8s-api/input.yaml create mode 100644 plugins/two-node/evals/threat-model-tnf/cases/case-002-credential-rotation-script/annotations.yaml create mode 100644 plugins/two-node/evals/threat-model-tnf/cases/case-002-credential-rotation-script/input.yaml create mode 100644 plugins/two-node/evals/threat-model-tnf/cases/case-003-mac-fencing-lookup/annotations.yaml create mode 100644 plugins/two-node/evals/threat-model-tnf/cases/case-003-mac-fencing-lookup/input.yaml create mode 100644 plugins/two-node/evals/threat-model-tnf/cases/case-004-trivial-indentation-fix/annotations.yaml create mode 100644 plugins/two-node/evals/threat-model-tnf/cases/case-004-trivial-indentation-fix/input.yaml create mode 100644 plugins/two-node/evals/threat-model-tnf/cases/case-005-tnf-retry-bugfix/annotations.yaml create mode 100644 plugins/two-node/evals/threat-model-tnf/cases/case-005-tnf-retry-bugfix/input.yaml diff --git a/plugins/two-node/evals/README.md b/plugins/two-node/evals/README.md new file mode 100644 index 00000000..57094e77 --- /dev/null +++ b/plugins/two-node/evals/README.md @@ -0,0 +1,50 @@ +# Evaluation Configs + +Automated quality testing for two-node plugin skills using the +[agent-eval-harness](https://github.com/opendatahub-io/agent-eval-harness) +Claude Code plugin. + +## Available Evals + +| Config | Skill | Modes Tested | Cases | +|--------|-------|--------------|-------| +| `cluster-diagnostic.yaml` | `two-node:cluster-diagnostic` | validate, recovery-guide | 5 | +| `threat-model-tnf.yaml` | `threat-model:tnf` | PR analysis | 5 | + +## Running Locally + +```bash +# Install the eval harness plugin first +/plugin marketplace add opendatahub-skills/agent-eval-harness + +# Run an eval +/eval-run --model claude-opus-4-6 --config evals/cluster-diagnostic.yaml +``` + +## Running in CI + +Comment `/test eval-cluster-diagnostic` on a PR to trigger the eval job. +The CI workflow is defined in +[openshift/release](https://github.com/openshift/release) under +`ci-operator/config/openshift-eng/edge-tooling/`. + +## Directory Structure + +``` +evals/ +├── .yaml # Eval config (judges, thresholds, schema) +├── .md # Cached skill analysis +└── / + └── cases/ + └── case-NNN-/ + ├── input.yaml # Test input + └── annotations.yaml # Expected outcomes +``` + +## Adding a New Eval + +1. `/eval-analyze --skill --config evals/.yaml` +2. `/eval-dataset --config evals/.yaml` +3. `/eval-run --model claude-opus-4-6 --config evals/.yaml` +4. `/eval-review --run-id --config evals/.yaml` +5. Commit the config, analysis, and cases. Run artifacts are ephemeral. diff --git a/plugins/two-node/evals/cluster-diagnostic.md b/plugins/two-node/evals/cluster-diagnostic.md new file mode 100644 index 00000000..345a131a --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic.md @@ -0,0 +1,83 @@ +--- +# Auto-generated by /eval-analyze — edit to override +skill: two-node:cluster-diagnostic +analyzed_at: 2026-06-05T23:00:00Z +skill_hash: bb04c2fed029 + +# Discovered skill capabilities +execution_mode: case +headless: true +dry_run: false + +# Suggested judges (summary from analysis) +suggested_judges: + - name: budget_check + type: builtin + description: "Cost stays within $3.00 per case" + - name: severity_classification + type: check + description: "Validate mode assigns correct BLOCKER/WARNING/INFO severity" + - name: procedure_completeness + type: check + description: "Recovery-guide mode returns bash commands, verification steps, parameter templates" + - name: forbidden_recommendations + type: check + description: "Never recommends pcs standby, sequential shutdown, or shutdown -h" + - name: knowledge_base_accuracy + type: llm + description: "Response accurately reflects TNF knowledge base content" +--- + +# Skill Analysis + +The `two-node:cluster-diagnostic` skill diagnoses TNF (Two-Node Fencing) +cluster issues across 4 modes: diagnose (live SSH), validate (check proposed +procedures), recovery-guide (return correct procedures), and game (interactive +training). The skill encodes 7 validated bare metal test scenarios (HPE ProLiant +e920t, OCP 4.22.0-rc.3) into a knowledge base. + +**Eval scope**: Only `validate` and `recovery-guide` modes are testable in eval +because `diagnose` requires live SSH access and `game` requires interactive +AskUserQuestion handling. Game mode can be tested with tool interception but +adds complexity. + +## Inputs + +Each test case has `input.yaml` with: +- `command_input`: Full argument string (e.g., `validate "cordon, drain, shutdown"`, + `recovery-guide full-shutdown`) +- `mode`: Which mode is being tested (`validate`, `recovery-guide`, `game`) + +And `annotations.yaml` with expected outcomes: +- `expected_blockers`: List of BLOCKER findings expected (validate mode) +- `expected_warnings`: List of WARNING findings expected +- `expected_scenario`: Scenario name (recovery-guide mode) +- `should_reject`: Whether the procedure should be rejected (validate mode) + +## Outputs + +All output is conversational — the skill writes nothing to disk. Judges use +`{{ conversation }}` to evaluate the assistant's response text. + +## Pipeline Flow + +1. Parse argument to determine mode +2. Read `cluster-knowledge-base.md` (800+ lines with 7 failure modes, severity + table, correct procedures, edge cases) +3. For validate: parse procedure text → check each step against 7 failure modes + → report BLOCKER/WARNING/INFO with explanations +4. For recovery-guide: look up scenario → return step-by-step bash commands with + parameter templates and verification steps +5. For game: read `game-mode.md` → present questions via AskUserQuestion → score + +## Quality Criteria + +**Deterministic** (code-checkable): +- Severity classification matches knowledge base table +- Never recommends pcs standby, sequential shutdown, or shutdown -h +- Recovery procedures include bash commands and verification steps + +**LLM judgment** (requires reasoning): +- Response accurately reflects TNF architecture facts +- Failure mode explanations reference correct root causes +- Recovery procedures match validated bare metal test results diff --git a/plugins/two-node/evals/cluster-diagnostic.yaml b/plugins/two-node/evals/cluster-diagnostic.yaml new file mode 100644 index 00000000..4385e9b1 --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic.yaml @@ -0,0 +1,218 @@ +name: cluster-diagnostic-eval +description: Evaluate the cluster-diagnostic skill across validate, recovery-guide, and game modes +skill: two-node:cluster-diagnostic + +execution: + mode: case + arguments: "{command_input}" + timeout: 300 + max_budget_usd: 3.0 + +runner: + type: claude-code + plugin_dirs: + - plugins/two-node + +models: + skill: claude-opus-4-6 + judge: claude-opus-4-6 + hook: claude-sonnet-4-6 + +permissions: + allow: [] + deny: [] + +mlflow: + experiment: cluster-diagnostic-eval + +dataset: + path: cluster-diagnostic/cases + schema: | + Each case directory contains: + - input.yaml: YAML file with: + - 'command_input' (string): The full argument string passed to the skill. + For validate mode: 'validate ' + For recovery-guide mode: 'recovery-guide ' + For game mode: 'game' (requires AskUserQuestion interception) + - 'mode' (string): One of 'validate', 'recovery-guide', 'game' + Used by annotation-aware judges to apply mode-specific checks. + - annotations.yaml: Expected outcomes for the test case: + - 'mode' (string): validate | recovery-guide | game + - 'expected_blockers' (list): BLOCKER findings expected (validate mode) + - 'expected_warnings' (list): WARNING findings expected (validate mode) + - 'expected_scenario' (string): scenario name (recovery-guide mode) + - 'should_reject' (bool): whether the procedure should be rejected (validate mode) + + Note: diagnose mode is excluded from eval because it requires live SSH + access to cluster nodes. Test validate and recovery-guide modes which + operate on text input against the knowledge base. + +inputs: + tools: + - match: Questions asked to the user via AskUserQuestion (game mode) + prompt: | + Answer based on the test case context in input.yaml and answers.yaml. + For game mode selection, pick 'quiz' unless answers.yaml specifies otherwise. + For quiz/scenario/rapid-fire answers, use answers.yaml guidance. + Default: pick the first option. + +outputs: + - path: output + schema: | + This skill produces conversation output only — no files are written to disk. + Judges should use {{ conversation }} to evaluate the assistant's response text. + + For validate mode: expect a findings list with BLOCKER/WARNING/INFO severity + classifications, each referencing a failure mode from the knowledge base. + + For recovery-guide mode: expect step-by-step markdown with bash commands + using parameter templates ($BMC_USER, $BMC_PASS, etc.) and verification steps. + + For game mode: expect interactive questions, scoring, and a final rating + (Novice/Operator/Expert/TNF Master). + +traces: + stdout: true + stderr: true + events: true + metrics: true + +judges: + - name: budget_check + builtin: cost_budget + arguments: + max_cost_usd: 3.0 + + - name: severity_classification + description: | + For validate mode: checks that BLOCKER/WARNING/INFO severity is correctly + assigned. Sequential shutdown and pcs standby must be BLOCKER. ForceOff + must be INFO or WARNING, not BLOCKER. + if: "annotations.get('mode') == 'validate'" + check: | + conversation = outputs.get("conversation", "") + ann = outputs.get("annotations", {}) + expected_blockers = ann.get("expected_blockers", []) + should_reject = ann.get("should_reject", False) + + if not conversation: + return (False, "No conversation output found") + + conv_upper = conversation.upper() + has_blocker = "BLOCKER" in conv_upper + + if should_reject and not has_blocker: + return (False, "Procedure should have been rejected with BLOCKER but no BLOCKER found") + + if not should_reject and has_blocker: + return (False, "Procedure should NOT have BLOCKER findings but BLOCKER was found") + + found_blockers = [] + for b in expected_blockers: + if b.lower() in conversation.lower(): + found_blockers.append(b) + + if expected_blockers and not found_blockers: + return (False, f"Expected blockers {expected_blockers} not found in output") + + return (True, f"Severity classification correct. Blockers found: {found_blockers}") + + - name: procedure_completeness + description: | + For recovery-guide mode: checks that the returned procedure includes + bash commands, verification steps, and parameter templates. + if: "annotations.get('mode') == 'recovery-guide'" + check: | + conversation = outputs.get("conversation", "") + + if not conversation: + return (False, "No conversation output found") + + checks = { + "bash_commands": any(marker in conversation for marker in ["```bash", "```sh", "curl ", "pcs ", "oc "]), + "has_verification": any(w in conversation.lower() for w in ["verify", "confirm", "check", "poll", "wait"]), + "has_parameters": any(p in conversation for p in ["$BMC", "$NODE", "BMC_USER", "BMC_PASS", "BMC_HOST"]), + } + + passed = sum(checks.values()) + total = len(checks) + failed = [k for k, v in checks.items() if not v] + + if passed == total: + return (True, f"All {total} completeness checks passed") + else: + return (False, f"{passed}/{total} checks passed. Missing: {failed}") + + - name: forbidden_recommendations + description: | + Checks that the skill never recommends procedures that violate known + failure modes: pcs node standby, sequential shutdown, shutdown -h. + check: | + conversation = outputs.get("conversation", "") + ann = outputs.get("annotations", {}) + mode = ann.get("mode", "") + + if not conversation: + return (False, "No conversation output found") + + # Only check recommendations sections, not quoted failure descriptions + forbidden = [] + + # Check if skill RECOMMENDS (not just mentions) dangerous procedures + conv_lower = conversation.lower() + + recommend_sections = [] + for marker in ["recommend", "suggested", "recovery", "procedure", "steps to"]: + idx = conv_lower.find(marker) + if idx >= 0: + recommend_sections.append(conversation[idx:idx+500]) + + for section in recommend_sections: + sec_lower = section.lower() + if "pcs node standby" in sec_lower and "never" not in sec_lower and "do not" not in sec_lower: + forbidden.append("pcs node standby recommended") + if "shutdown -h 1" in sec_lower and "never" not in sec_lower and "do not" not in sec_lower: + forbidden.append("shutdown -h 1 recommended") + + if forbidden: + return (False, f"Forbidden recommendations found: {forbidden}") + return (True, "No forbidden procedures recommended") + + - name: knowledge_base_accuracy + description: | + LLM judge that evaluates whether the skill's response accurately + reflects the TNF knowledge base content — correct failure modes, + proper severity classification reasoning, and accurate recovery procedures. + prompt: | + Evaluate whether this cluster-diagnostic skill response is accurate + and complete for the given mode. + + ## Skill Response + {{ conversation }} + + ## Test Case Annotations + {{ annotations }} + + ## Scoring Criteria + + Score 1-5: + - 5: Response is fully accurate, references correct failure modes, + severity is properly justified, procedures match tested bare metal results + - 4: Minor omissions but no inaccuracies, severity is correct + - 3: Mostly accurate but missing important details or has minor inaccuracy + - 2: Contains inaccurate claims about TNF behavior or recommends untested procedures + - 1: Fundamentally incorrect — wrong failure modes, wrong severity, dangerous recommendations + + Return a JSON object: {"score": <1-5>, "rationale": ""} + +thresholds: + budget_check: + min_pass_rate: 1.0 + severity_classification: + min_pass_rate: 0.8 + procedure_completeness: + min_pass_rate: 0.8 + forbidden_recommendations: + min_pass_rate: 1.0 + knowledge_base_accuracy: + min_mean: 3.5 diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-001-validate-sequential-shutdown/annotations.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-001-validate-sequential-shutdown/annotations.yaml new file mode 100644 index 00000000..217fedab --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-001-validate-sequential-shutdown/annotations.yaml @@ -0,0 +1,9 @@ +mode: validate +expected_blockers: + - sequential shutdown + - shutdown -h +expected_warnings: + - cordon + - drain +expected_scenario: null +should_reject: true diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-001-validate-sequential-shutdown/input.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-001-validate-sequential-shutdown/input.yaml new file mode 100644 index 00000000..4186c5e7 --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-001-validate-sequential-shutdown/input.yaml @@ -0,0 +1,4 @@ +command_input: >- + validate "cordon all nodes, drain workloads, then shut down each node + one at a time using shutdown -h 1 via oc debug" +mode: validate diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-002-validate-safe-redfish/annotations.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-002-validate-safe-redfish/annotations.yaml new file mode 100644 index 00000000..c762f6b6 --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-002-validate-safe-redfish/annotations.yaml @@ -0,0 +1,5 @@ +mode: validate +expected_blockers: [] +expected_warnings: [] +expected_scenario: null +should_reject: false diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-002-validate-safe-redfish/input.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-002-validate-safe-redfish/input.yaml new file mode 100644 index 00000000..13b06b84 --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-002-validate-safe-redfish/input.yaml @@ -0,0 +1,5 @@ +command_input: >- + validate "Send Redfish GracefulShutdown to both nodes simultaneously + using curl, poll PowerState until Off, then send On to both nodes + to restart" +mode: validate diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-003-recovery-full-shutdown/annotations.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-003-recovery-full-shutdown/annotations.yaml new file mode 100644 index 00000000..4b741409 --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-003-recovery-full-shutdown/annotations.yaml @@ -0,0 +1,5 @@ +mode: recovery-guide +expected_blockers: [] +expected_warnings: [] +expected_scenario: full-shutdown +should_reject: false diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-003-recovery-full-shutdown/input.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-003-recovery-full-shutdown/input.yaml new file mode 100644 index 00000000..58326c61 --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-003-recovery-full-shutdown/input.yaml @@ -0,0 +1,2 @@ +command_input: recovery-guide full-shutdown +mode: recovery-guide diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-004-recovery-standby/annotations.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-004-recovery-standby/annotations.yaml new file mode 100644 index 00000000..66d323bb --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-004-recovery-standby/annotations.yaml @@ -0,0 +1,5 @@ +mode: recovery-guide +expected_blockers: [] +expected_warnings: [] +expected_scenario: standby +should_reject: false diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-004-recovery-standby/input.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-004-recovery-standby/input.yaml new file mode 100644 index 00000000..dda90015 --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-004-recovery-standby/input.yaml @@ -0,0 +1,2 @@ +command_input: recovery-guide standby +mode: recovery-guide diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-005-validate-pcs-standby/annotations.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-005-validate-pcs-standby/annotations.yaml new file mode 100644 index 00000000..f0ee61ee --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-005-validate-pcs-standby/annotations.yaml @@ -0,0 +1,6 @@ +mode: validate +expected_blockers: + - pcs node standby +expected_warnings: [] +expected_scenario: null +should_reject: true diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-005-validate-pcs-standby/input.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-005-validate-pcs-standby/input.yaml new file mode 100644 index 00000000..9ef90652 --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-005-validate-pcs-standby/input.yaml @@ -0,0 +1,4 @@ +command_input: >- + validate "Put both nodes in standby using pcs node standby --all, + wait for resources to stop, then power off the servers" +mode: validate diff --git a/plugins/two-node/evals/threat-model-tnf.md b/plugins/two-node/evals/threat-model-tnf.md new file mode 100644 index 00000000..4dc11d23 --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf.md @@ -0,0 +1,101 @@ +--- +# Auto-generated by /eval-analyze — edit to override +skill: threat-model:tnf +analyzed_at: 2026-06-05T00:00:00Z +skill_hash: ca8e410b0d9b + +# Discovered skill capabilities +execution_mode: case +headless: true +dry_run: false + +# Suggested judges (summary from analysis) +suggested_judges: + - name: budget_check + type: builtin + description: "Cost stays under $8 per invocation" + - name: report_exists + type: check + description: "PR-THREAT-MODEL-.md file was generated" + - name: report_sections_complete + type: check + description: "All 9 required sections present in report" + - name: dfd_elements_mapped + type: check + description: "DFD element IDs (P/DS/DF/EE/TB) referenced in report" + - name: stride_matrix_present + type: check + description: "Per-element STRIDE matrix has X/~/-/N/A markers" + - name: mitre_techniques_assigned + type: check + description: "MITRE ATT&CK technique IDs (T####) are present" + - name: threat_analysis_quality + type: llm + description: "Overall quality: severity accuracy, DFD mapping, STRIDE completeness, recommendations" + - name: findings_tracker_updated + type: check + description: "Cumulative findings tracker was appended" +--- + +## Skill Analysis + +The `threat-model:tnf` skill performs security threat analysis on GitHub PRs affecting the TNF (Two-Node Fencing) OpenShift topology. It combines three approaches: + +1. **Automated scanning** — runs ShellCheck on shell scripts in the PR diff +2. **Pattern detection** — searches for command injection, credential exposure, privilege escalation, and 7 other security pattern categories +3. **Formal threat modeling** — maps code changes to TNF DFD elements (8 processes, 5 data stores, 12 data flows, 3 external entities, 6 trust boundaries), applies per-element STRIDE analysis, and cross-references against the formal TNF threat model + +Output is a formal threat model report with MITRE ATT&CK technique mappings, OWASP Top 10:2025 categorization, risk assessment, and actionable recommendations for developers and customers. + +## Inputs + +Each test case provides a single PR identifier via `input.yaml`: + +- **`pr_input`** — the PR to analyze, in one of three formats: + - PR number only: `2156` (repo detected from working directory) + - GitHub URL: `https://github.com/ClusterLabs/resource-agents/pull/2156` + - Repo + number: `resource-agents 2156` + +The PR must be a real, accessible GitHub PR. The skill uses `gh pr view` and `gh pr diff` to fetch PR data. + +Optional fields: `repo` (repository name), `org` (GitHub organization). + +## Outputs + +The skill writes to a `reports/` directory (resolved via workspace discovery): + +- **`PR-THREAT-MODEL-.md`** — main threat model report (~200-500 lines) +- **`VULN-PR-.md`** — individual vulnerability tickets (Critical/High only, optional) + +It also appends to a cumulative findings tracker at `$WORKSPACE/.claude/skills/threat-model/mitre-findings-tnf.md`. + +## Pipeline Flow + +1. **Workspace discovery** — walk up from CWD looking for `repos/` directory; set WORKSPACE, REPOS, THREAT_MODEL_DIR, REPORT_DIR, FINDINGS_FILE +2. **Parse input** — extract org, repo, PR number from the three input formats +3. **Fetch PR** — `gh pr view` for metadata, `gh pr diff` for the full diff +4. **ShellCheck** — run on any .sh files; map security codes (SC2086→T1059) to MITRE +5. **Pattern analysis** — search diff for 10 security pattern categories +6. **DFD mapping** — match code paths to TNF elements using the mapping table in `dfd-elements-tnf.md` +7. **STRIDE analysis** — per-element threat assessment; cross-reference against TNF-THREAT-MODEL.md if available +8. **Combine findings** — deduplicate, assign VULN-N IDs, determine severity +9. **MITRE/OWASP mapping** — assign technique IDs and OWASP categories using reference files +10. **Generate report** — write markdown report using report-templates.md format +11. **Append tracker** — add findings block to cumulative mitre-findings-tnf.md + +## Quality Criteria + +A **good** report: +- Correctly identifies all affected DFD elements from the code paths in the PR +- Applies STRIDE systematically to each element (all 6 categories for processes, T/I/D for stores and flows) +- Assigns accurate severity levels matching MITRE/OWASP standards +- Identifies trust boundary crossings (especially TB3→TB4, TB4→TB5) +- Provides specific, actionable recommendations with code-level guidance +- Maps findings to correct MITRE techniques (T1059 for injection, T1552 for credentials, T1611 for container escape) + +A **bad** report: +- Misses affected DFD elements or assigns wrong elements to code paths +- Has incomplete STRIDE matrix (missing categories or missing rationale) +- Over/under-rates severity (e.g., calling a minor code quality issue "Critical") +- Provides vague recommendations ("improve security") without specific guidance +- Missing sections or incorrect report structure diff --git a/plugins/two-node/evals/threat-model-tnf.yaml b/plugins/two-node/evals/threat-model-tnf.yaml new file mode 100644 index 00000000..672480b4 --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf.yaml @@ -0,0 +1,247 @@ +name: threat-model-tnf-eval +description: Evaluate the threat-model:tnf skill — PR security analysis with STRIDE/DFD, MITRE ATT&CK, and OWASP mapping for TNF topology +skill: threat-model:tnf + +execution: + mode: case + arguments: "{pr_input}" + timeout: 600 + max_budget_usd: 8.0 + +runner: + type: claude-code + plugin_dirs: + - plugins/threat-model + +models: + skill: claude-opus-4-6 + judge: claude-opus-4-6 + +permissions: + allow: [] + deny: [] + +mlflow: + experiment: threat-model-tnf-eval + +dataset: + path: threat-model-tnf/cases + schema: | + Each case directory contains: + - input.yaml: YAML file with fields: + - 'pr_input': the PR identifier to analyze — one of: + - A PR number (e.g., '2156') + - A GitHub URL (e.g., 'https://github.com/ClusterLabs/resource-agents/pull/2156') + - A 'repo number' pair (e.g., 'resource-agents 2156') + [EXTERNAL: GitHub] — must be a real, accessible PR on GitHub + - 'repo' (optional): repository name for context (e.g., 'resource-agents') + - 'org' (optional): GitHub org (e.g., 'ClusterLabs', 'openshift') + - reference.md (optional): gold-standard threat model report for comparison. + Uses the report template format: Executive Summary, DFD Impact Analysis, + Per-Element STRIDE matrix, Threat Analysis (VULN-N sections), MITRE/OWASP + mapping, Risk Assessment, and Recommendations. + - annotations.yaml (optional): expected metadata for outcome-aware scoring: + - 'expected_vuln_count': expected number of findings + - 'expected_severities': list of expected severity levels + - 'affected_dfd_elements': list of expected DFD element IDs (e.g., ['P5', 'P7', 'DS3']) + - 'expected_mitre_techniques': list of expected MITRE technique IDs + - 'has_shell_scripts': whether the PR contains shell scripts for ShellCheck + - 'has_trust_boundary_crossing': whether the PR crosses trust boundaries + +outputs: + - path: reports + schema: | + The skill writes threat model reports as markdown files: + - PR-THREAT-MODEL-.md: main report with sections: + - Document header (version, date, classification, repo, topology, author, URL) + - Executive Summary with findings count table (Critical/High/Medium/Low) + - Change Overview describing the PR and security-relevant changes + - Affected Files table (file path, line changes, security relevance) + - DFD Impact Analysis: + - Affected DFD Elements table (Element ID, Name, Impact, Trust Boundary) + - Trust Boundary Crossings narrative + - Per-Element STRIDE matrix (S/T/R/I/D/E per element, X/~/-/N/A) + - Threat Model Cross-Reference table (PE-* IDs if formal model exists) + - Automated Scanner Results (ShellCheck table or "skipped" note) + - Threat Analysis: per-VULN section with Severity, OWASP, MITRE, CWE, + Affected Code, Description, Attack Vector, Impact (CIA), Recommended Fix + - OWASP & MITRE ATT&CK Mapping table + - Risk Assessment table (Likelihood, Impact, Risk) + - Recommendations (Developers: Before/After Merge; Customers: Config/Ops) + - References + - VULN-PR-.md (optional): individual vulnerability tickets + for Critical/High findings only + +traces: + stdout: true + stderr: true + events: true + metrics: true + +judges: + - name: budget_check + builtin: cost_budget + arguments: + max_cost_usd: 8.0 + + - name: report_exists + description: Verify that the main threat model report markdown file was generated + check: | + files = outputs.get("files", {}) + reports = [k for k in files if "THREAT-MODEL" in k and k.endswith(".md")] + if not reports: + return (False, "No threat model report file found") + return (True, f"Report generated: {reports[0]}") + + - name: report_sections_complete + description: Verify all required report sections are present in the generated report + check: | + files = outputs.get("files", {}) + reports = {k: v for k, v in files.items() if "THREAT-MODEL" in k and k.endswith(".md")} + if not reports: + return (False, "No report file found") + content = list(reports.values())[0] + required = [ + "Executive Summary", + "Change Overview", + "Affected Files", + "DFD Impact Analysis", + "STRIDE", + "Threat Analysis", + "MITRE", + "Risk Assessment", + "Recommendations", + ] + missing = [s for s in required if s not in content] + if missing: + return (False, f"Missing sections: {', '.join(missing)}") + return (True, f"All {len(required)} required sections present") + + - name: dfd_elements_mapped + description: Verify that DFD elements (P1-P8, DS1-DS5, DF1-DF12) are referenced in the report + check: | + import re + files = outputs.get("files", {}) + reports = {k: v for k, v in files.items() if "THREAT-MODEL" in k and k.endswith(".md")} + if not reports: + return (False, "No report file found") + content = list(reports.values())[0] + elements = re.findall(r'\b(P[1-8]|DS[1-5]|DF(?:1[0-2]|[1-9])|EE[1-3]|TB[1-6])\b', content) + unique = set(elements) + if not unique: + return (False, "No DFD elements found in report") + return (True, f"DFD elements referenced: {sorted(unique)}") + + - name: stride_matrix_present + description: Verify the per-element STRIDE matrix is populated with X, ~, or - markers + check: | + import re + files = outputs.get("files", {}) + reports = {k: v for k, v in files.items() if "THREAT-MODEL" in k and k.endswith(".md")} + if not reports: + return (False, "No report file found") + content = list(reports.values())[0] + stride_section = content.split("Per-Element STRIDE") + if len(stride_section) < 2: + return (False, "No Per-Element STRIDE section found") + markers = re.findall(r'\b[XxNn/Aa~-]\b', stride_section[1][:2000]) + if len(markers) < 3: + return (False, f"STRIDE matrix appears empty or minimal ({len(markers)} markers)") + return (True, f"STRIDE matrix populated ({len(markers)} cell markers found)") + + - name: mitre_techniques_assigned + description: Verify MITRE ATT&CK technique IDs (T####) are present and mapped to findings + check: | + import re + files = outputs.get("files", {}) + reports = {k: v for k, v in files.items() if "THREAT-MODEL" in k and k.endswith(".md")} + if not reports: + return (False, "No report file found") + content = list(reports.values())[0] + techniques = set(re.findall(r'T\d{4}', content)) + if not techniques: + return (False, "No MITRE ATT&CK technique IDs found") + return (True, f"MITRE techniques: {sorted(techniques)}") + + - name: threat_analysis_quality + description: | + LLM judge assessing overall threat analysis quality: severity accuracy, + DFD mapping correctness, STRIDE completeness, and recommendation actionability + prompt: | + You are evaluating a TNF (Two-Node Fencing) PR threat analysis report. + + ## Report output: + + {{ outputs }} + + ## Skill conversation: + + {{ conversation }} + + ## Evaluation criteria + + Score on a 1-5 scale across these dimensions, then give an overall score: + + **1. Severity accuracy** — do assigned severities (Critical/High/Medium/Low) match the actual risk? + - Critical: RCE, credential exposure at high-trust boundary (P5/P6/P8), STONITH bypass + - High: command injection with exploitation path, new credential dependency, missing validation on network boundary + - Medium: fail-open behavior, non-critical info disclosure, potential race condition + - Low: minor code quality, non-exploitable pattern + + **2. DFD mapping correctness** — are code changes correctly mapped to TNF DFD elements (P1-P8, DS1-DS5, DF1-DF12)? + - Code paths should match the element mapping table (e.g., cluster-etcd-operator/pkg/tnf/fencing/ → P5) + - Trust boundary crossings should be identified (TB2→TB3, TB3→TB4, TB4→TB5) + + **3. STRIDE completeness** — is each affected element analyzed across all applicable STRIDE categories? + - Processes: all 6 (S,T,R,I,D,E) + - Data Stores: T,I,D + - Data Flows: T,I,D + - External Entities: S,R + + **4. MITRE/OWASP accuracy** — are technique assignments correct? + - T1059 for command injection, T1552 for credential exposure, T1611 for container escape + - OWASP categories should match (A05 for injection, A07 for auth failures) + + **5. Recommendation quality** — are recommendations specific and actionable? + - Developer recommendations should include code-level guidance + - Customer recommendations should include hardening or monitoring steps + - Vague recommendations ("improve security") score low + + ## Scoring + Score 1: Report is missing major sections, contains incorrect mappings, or has no useful findings + Score 2: Report exists but has significant gaps — missing STRIDE analysis, wrong DFD elements, or vague recommendations + Score 3: Adequate report covering basics — correct elements identified, some STRIDE analysis, generic recommendations + Score 4: Good report — accurate DFD mapping, thorough STRIDE, relevant MITRE techniques, specific recommendations + Score 5: Excellent — comprehensive coverage, all trust boundaries analyzed, accurate severity, actionable recommendations with code examples + + Respond with a single integer score (1-5) on the first line, then explain your reasoning. + + - name: findings_tracker_updated + description: Verify the findings tracker was appended with new entries (checks conversation for append confirmation) + check: | + conv = outputs.get("conversation", "") + files = outputs.get("files", {}) + tracker_files = [k for k in files if "mitre-findings" in k.lower()] + if tracker_files: + return (True, f"Findings tracker file found: {tracker_files[0]}") + if "findings" in conv.lower() and ("append" in conv.lower() or "tracker" in conv.lower()): + return (True, "Findings tracker update mentioned in conversation") + return (False, "No evidence of findings tracker update") + +thresholds: + budget_check: + min_pass_rate: 1.0 + report_exists: + min_pass_rate: 1.0 + report_sections_complete: + min_pass_rate: 1.0 + dfd_elements_mapped: + min_pass_rate: 1.0 + stride_matrix_present: + min_pass_rate: 0.8 + mitre_techniques_assigned: + min_pass_rate: 1.0 + threat_analysis_quality: + min_mean: 3.5 + findings_tracker_updated: + min_pass_rate: 0.8 diff --git a/plugins/two-node/evals/threat-model-tnf/cases/case-001-shell-script-k8s-api/annotations.yaml b/plugins/two-node/evals/threat-model-tnf/cases/case-001-shell-script-k8s-api/annotations.yaml new file mode 100644 index 00000000..751425b7 --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf/cases/case-001-shell-script-k8s-api/annotations.yaml @@ -0,0 +1,18 @@ +description: > + Shell script PR that adds kubeconfig-based K8s API access to the podman-etcd + OCF agent. Introduces new trust boundary crossing (TB4→TB2) and credential + dependency. Should trigger ShellCheck analysis and identify credential exposure. +has_shell_scripts: true +has_trust_boundary_crossing: true +expected_severities: + - High + - Medium + - Low +affected_dfd_elements: + - P7 + - DS5 + - DF11 +expected_mitre_techniques: + - T1552 + - T1078 + - T1005 diff --git a/plugins/two-node/evals/threat-model-tnf/cases/case-001-shell-script-k8s-api/input.yaml b/plugins/two-node/evals/threat-model-tnf/cases/case-001-shell-script-k8s-api/input.yaml new file mode 100644 index 00000000..37361080 --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf/cases/case-001-shell-script-k8s-api/input.yaml @@ -0,0 +1,3 @@ +pr_input: "https://github.com/ClusterLabs/resource-agents/pull/2156" +repo: resource-agents +org: ClusterLabs diff --git a/plugins/two-node/evals/threat-model-tnf/cases/case-002-credential-rotation-script/annotations.yaml b/plugins/two-node/evals/threat-model-tnf/cases/case-002-credential-rotation-script/annotations.yaml new file mode 100644 index 00000000..7d6e2436 --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf/cases/case-002-credential-rotation-script/annotations.yaml @@ -0,0 +1,21 @@ +description: > + Adds a TNF fencing credentials rotation script. This touches the full credential + flow path (DS2→P5→DS3) and should identify high-severity findings around + credential handling, STONITH configuration, and BMC access. Complex case with + multiple DFD elements affected. +has_shell_scripts: true +has_trust_boundary_crossing: true +expected_severities: + - Critical + - High + - Medium +affected_dfd_elements: + - P5 + - DS2 + - DS3 + - DF4 + - DF7 +expected_mitre_techniques: + - T1552 + - T1059 + - T1529 diff --git a/plugins/two-node/evals/threat-model-tnf/cases/case-002-credential-rotation-script/input.yaml b/plugins/two-node/evals/threat-model-tnf/cases/case-002-credential-rotation-script/input.yaml new file mode 100644 index 00000000..cd609686 --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf/cases/case-002-credential-rotation-script/input.yaml @@ -0,0 +1,3 @@ +pr_input: "cluster-etcd-operator 1611" +repo: cluster-etcd-operator +org: openshift diff --git a/plugins/two-node/evals/threat-model-tnf/cases/case-003-mac-fencing-lookup/annotations.yaml b/plugins/two-node/evals/threat-model-tnf/cases/case-003-mac-fencing-lookup/annotations.yaml new file mode 100644 index 00000000..b62f1b9c --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf/cases/case-003-mac-fencing-lookup/annotations.yaml @@ -0,0 +1,16 @@ +description: > + Adds MAC-address based fencing credentials lookup. Introduces a new data flow + for credential resolution and modifies the fencing job's credential discovery + path. Tests DFD mapping for P5 and the credential pipeline. +has_shell_scripts: false +has_trust_boundary_crossing: true +expected_severities: + - High + - Medium +affected_dfd_elements: + - P5 + - DS2 + - DF4 +expected_mitre_techniques: + - T1552 + - T1078 diff --git a/plugins/two-node/evals/threat-model-tnf/cases/case-003-mac-fencing-lookup/input.yaml b/plugins/two-node/evals/threat-model-tnf/cases/case-003-mac-fencing-lookup/input.yaml new file mode 100644 index 00000000..34abba7b --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf/cases/case-003-mac-fencing-lookup/input.yaml @@ -0,0 +1,3 @@ +pr_input: "https://github.com/openshift/cluster-etcd-operator/pull/1600" +repo: cluster-etcd-operator +org: openshift diff --git a/plugins/two-node/evals/threat-model-tnf/cases/case-004-trivial-indentation-fix/annotations.yaml b/plugins/two-node/evals/threat-model-tnf/cases/case-004-trivial-indentation-fix/annotations.yaml new file mode 100644 index 00000000..7a2fc3ac --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf/cases/case-004-trivial-indentation-fix/annotations.yaml @@ -0,0 +1,9 @@ +description: > + Trivial indentation fix in nfsserver — not TNF-specific, no shell scripts + relevant to TNF. Should produce a report with minimal or no security findings. + Edge case testing the skill's handling of low-risk, non-TNF PRs. +has_shell_scripts: false +has_trust_boundary_crossing: false +expected_severities: [] +affected_dfd_elements: [] +expected_mitre_techniques: [] diff --git a/plugins/two-node/evals/threat-model-tnf/cases/case-004-trivial-indentation-fix/input.yaml b/plugins/two-node/evals/threat-model-tnf/cases/case-004-trivial-indentation-fix/input.yaml new file mode 100644 index 00000000..f883d7a3 --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf/cases/case-004-trivial-indentation-fix/input.yaml @@ -0,0 +1,3 @@ +pr_input: "https://github.com/ClusterLabs/resource-agents/pull/2168" +repo: resource-agents +org: ClusterLabs diff --git a/plugins/two-node/evals/threat-model-tnf/cases/case-005-tnf-retry-bugfix/annotations.yaml b/plugins/two-node/evals/threat-model-tnf/cases/case-005-tnf-retry-bugfix/annotations.yaml new file mode 100644 index 00000000..d9d3e7b2 --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf/cases/case-005-tnf-retry-bugfix/annotations.yaml @@ -0,0 +1,15 @@ +description: > + Bug fix gating dual-replica setup and adding retry logic in TNF pipeline. + Modifies P4 (Setup Job) behavior. Tests whether the skill correctly identifies + denial-of-service risk from retry logic changes and setup gate modifications. + Uses bare PR number format — tests repo auto-detection from CWD. +has_shell_scripts: false +has_trust_boundary_crossing: false +expected_severities: + - Medium + - Low +affected_dfd_elements: + - P4 + - P2 +expected_mitre_techniques: + - T1499 diff --git a/plugins/two-node/evals/threat-model-tnf/cases/case-005-tnf-retry-bugfix/input.yaml b/plugins/two-node/evals/threat-model-tnf/cases/case-005-tnf-retry-bugfix/input.yaml new file mode 100644 index 00000000..2ccb92a2 --- /dev/null +++ b/plugins/two-node/evals/threat-model-tnf/cases/case-005-tnf-retry-bugfix/input.yaml @@ -0,0 +1,3 @@ +pr_input: "1620" +repo: cluster-etcd-operator +org: openshift From dd1d402765bf955f098ee8999039cbf1371dd90a Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Mon, 8 Jun 2026 08:54:31 -0400 Subject: [PATCH 6/9] Fix dataset.path to use repo-root-relative paths The eval harness resolves dataset.path from the repo root, not relative to the config file. Both configs were using short relative paths that broke when running from different working directories. Co-Authored-By: Claude Opus 4.6 --- plugins/two-node/evals/cluster-diagnostic.yaml | 2 +- plugins/two-node/evals/threat-model-tnf.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/two-node/evals/cluster-diagnostic.yaml b/plugins/two-node/evals/cluster-diagnostic.yaml index 4385e9b1..35d8c75a 100644 --- a/plugins/two-node/evals/cluster-diagnostic.yaml +++ b/plugins/two-node/evals/cluster-diagnostic.yaml @@ -26,7 +26,7 @@ mlflow: experiment: cluster-diagnostic-eval dataset: - path: cluster-diagnostic/cases + path: plugins/two-node/evals/cluster-diagnostic/cases schema: | Each case directory contains: - input.yaml: YAML file with: diff --git a/plugins/two-node/evals/threat-model-tnf.yaml b/plugins/two-node/evals/threat-model-tnf.yaml index 672480b4..2d5f3b1b 100644 --- a/plugins/two-node/evals/threat-model-tnf.yaml +++ b/plugins/two-node/evals/threat-model-tnf.yaml @@ -25,7 +25,7 @@ mlflow: experiment: threat-model-tnf-eval dataset: - path: threat-model-tnf/cases + path: plugins/two-node/evals/threat-model-tnf/cases schema: | Each case directory contains: - input.yaml: YAML file with fields: From c63b86f25c4a58f1727f19db99864967c1c0e580 Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Sun, 7 Jun 2026 16:56:43 -0400 Subject: [PATCH 7/9] Update evals README with detailed pipeline steps Co-Authored-By: Claude Opus 4.6 --- plugins/two-node/evals/README.md | 41 ++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/plugins/two-node/evals/README.md b/plugins/two-node/evals/README.md index 57094e77..bf30e5a6 100644 --- a/plugins/two-node/evals/README.md +++ b/plugins/two-node/evals/README.md @@ -14,13 +14,15 @@ Claude Code plugin. ## Running Locally ```bash -# Install the eval harness plugin first +# Install the eval harness plugin /plugin marketplace add opendatahub-skills/agent-eval-harness -# Run an eval +# Run an existing eval /eval-run --model claude-opus-4-6 --config evals/cluster-diagnostic.yaml ``` +To create a new eval, see [Adding a New Eval](#adding-a-new-eval) below. + ## Running in CI Comment `/test eval-cluster-diagnostic` on a PR to trigger the eval job. @@ -43,8 +45,33 @@ evals/ ## Adding a New Eval -1. `/eval-analyze --skill --config evals/.yaml` -2. `/eval-dataset --config evals/.yaml` -3. `/eval-run --model claude-opus-4-6 --config evals/.yaml` -4. `/eval-review --run-id --config evals/.yaml` -5. Commit the config, analysis, and cases. Run artifacts are ephemeral. +1. **Analyze the skill** — reads SKILL.md, designs judges, writes the eval config + ``` + /eval-analyze --skill --config evals/.yaml + ``` + +2. **Generate test cases** — creates `input.yaml` + `annotations.yaml` per case + ``` + /eval-dataset --config evals/.yaml + ``` + +3. **Run the eval** — executes the skill against each case, scores with judges, generates HTML report + ``` + /eval-run --model claude-opus-4-6 --config evals/.yaml + ``` + +4. **Review results** — walk through cases, collect human feedback + ``` + /eval-review --run-id --config evals/.yaml + ``` + +5. **(Optional) Optimize** — auto-fix SKILL.md based on judge failures, re-run to verify + ``` + /eval-optimize --config evals/.yaml + ``` + +6. **Commit and CI** + - Commit `evals/.yaml`, `evals/.md`, and `evals//cases/` to this repo + - Add a CI entry in [openshift/release](https://github.com/openshift/release) + pointing `EVAL_CONFIG` to the yaml path + - PR reviewers can then trigger the eval with `/test eval-` From 11404a4217370862095fc451b1d2e80a2878764c Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Thu, 11 Jun 2026 11:33:19 -0400 Subject: [PATCH 8/9] Add game mode eval case and improve judges - Add case-006-game-quiz with quiz mode test case and answers - Add warning_classification judge for expected WARNING findings - Add game_mode_scoring judge for rating/score validation - Fix forbidden_recommendations to check 'shutdown -h' (not 'shutdown -h 1') - Update severity_classification description for clarity - Drop models.skill default (let CLI --model flag control it) - Simplify schema note to only exclude diagnose mode Co-Authored-By: Claude Opus 4.6 --- .../two-node/evals/cluster-diagnostic.yaml | 63 ++++++++++++++++--- .../cases/case-006-game-quiz/annotations.yaml | 5 ++ .../cases/case-006-game-quiz/answers.yaml | 6 ++ .../cases/case-006-game-quiz/input.yaml | 2 + 4 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/annotations.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/answers.yaml create mode 100644 plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/input.yaml diff --git a/plugins/two-node/evals/cluster-diagnostic.yaml b/plugins/two-node/evals/cluster-diagnostic.yaml index 35d8c75a..ec29c31e 100644 --- a/plugins/two-node/evals/cluster-diagnostic.yaml +++ b/plugins/two-node/evals/cluster-diagnostic.yaml @@ -14,7 +14,6 @@ runner: - plugins/two-node models: - skill: claude-opus-4-6 judge: claude-opus-4-6 hook: claude-sonnet-4-6 @@ -44,8 +43,7 @@ dataset: - 'should_reject' (bool): whether the procedure should be rejected (validate mode) Note: diagnose mode is excluded from eval because it requires live SSH - access to cluster nodes. Test validate and recovery-guide modes which - operate on text input against the knowledge base. + access to cluster nodes. inputs: tools: @@ -85,9 +83,9 @@ judges: - name: severity_classification description: | - For validate mode: checks that BLOCKER/WARNING/INFO severity is correctly - assigned. Sequential shutdown and pcs standby must be BLOCKER. ForceOff - must be INFO or WARNING, not BLOCKER. + For validate mode: checks that expected BLOCKER findings are present + and procedures with blockers are rejected. Sequential shutdown and + pcs standby must be BLOCKER. if: "annotations.get('mode') == 'validate'" check: | conversation = outputs.get("conversation", "") @@ -117,6 +115,30 @@ judges: return (True, f"Severity classification correct. Blockers found: {found_blockers}") + - name: warning_classification + description: | + For validate mode: checks that expected WARNING findings are present + in the output. Verifies the skill identifies non-blocking issues. + if: "annotations.get('mode') == 'validate'" + check: | + conversation = outputs.get("conversation", "") + ann = outputs.get("annotations", {}) + expected_warnings = ann.get("expected_warnings", []) + + if not conversation: + return (False, "No conversation output found") + + if not expected_warnings: + return (True, "No warnings expected for this case") + + conv_lower = conversation.lower() + found = [w for w in expected_warnings if w.lower() in conv_lower] + missing = [w for w in expected_warnings if w.lower() not in conv_lower] + + if missing: + return (False, f"Expected warnings not found: {missing}. Found: {found}") + return (True, f"All expected warnings found: {found}") + - name: procedure_completeness description: | For recovery-guide mode: checks that the returned procedure includes @@ -171,13 +193,36 @@ judges: sec_lower = section.lower() if "pcs node standby" in sec_lower and "never" not in sec_lower and "do not" not in sec_lower: forbidden.append("pcs node standby recommended") - if "shutdown -h 1" in sec_lower and "never" not in sec_lower and "do not" not in sec_lower: + if "shutdown -h" in sec_lower and "never" not in sec_lower and "do not" not in sec_lower: forbidden.append("shutdown -h 1 recommended") if forbidden: return (False, f"Forbidden recommendations found: {forbidden}") return (True, "No forbidden procedures recommended") + - name: game_mode_scoring + description: | + For game mode: checks that the skill produces a score and a + final rating (Novice/Operator/Expert/TNF Master). + if: "annotations.get('mode') == 'game'" + check: | + conversation = outputs.get("conversation", "") + + if not conversation: + return (False, "No conversation output found") + + conv_lower = conversation.lower() + ratings = ["novice", "operator", "expert", "tnf master"] + found_rating = [r for r in ratings if r in conv_lower] + + has_score = any(w in conv_lower for w in ["score", "points", "/"]) + + if not found_rating: + return (False, "No rating (Novice/Operator/Expert/TNF Master) found") + if not has_score: + return (False, "No score or points found in output") + return (True, f"Game completed with rating: {found_rating[0]}") + - name: knowledge_base_accuracy description: | LLM judge that evaluates whether the skill's response accurately @@ -210,9 +255,13 @@ thresholds: min_pass_rate: 1.0 severity_classification: min_pass_rate: 0.8 + warning_classification: + min_pass_rate: 0.8 procedure_completeness: min_pass_rate: 0.8 forbidden_recommendations: min_pass_rate: 1.0 + game_mode_scoring: + min_pass_rate: 1.0 knowledge_base_accuracy: min_mean: 3.5 diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/annotations.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/annotations.yaml new file mode 100644 index 00000000..3b953d22 --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/annotations.yaml @@ -0,0 +1,5 @@ +mode: game +expected_blockers: [] +expected_warnings: [] +expected_scenario: null +should_reject: false diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/answers.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/answers.yaml new file mode 100644 index 00000000..44678d18 --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/answers.yaml @@ -0,0 +1,6 @@ +game_mode: quiz +answer_correctly: true +difficulty_guidance: > + Answer TNF knowledge questions accurately based on the + cluster-knowledge-base content. Pick the most correct option + for each question. diff --git a/plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/input.yaml b/plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/input.yaml new file mode 100644 index 00000000..d7e8057b --- /dev/null +++ b/plugins/two-node/evals/cluster-diagnostic/cases/case-006-game-quiz/input.yaml @@ -0,0 +1,2 @@ +command_input: "game" +mode: game From d5874c44a4dae40bf6c7c7445e9ee5a8200b20eb Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Thu, 11 Jun 2026 14:39:58 -0400 Subject: [PATCH 9/9] Set REPORT_DIR in threat-model eval and improve README framing - Add execution.env.REPORT_DIR to threat-model-tnf.yaml so reports are written to the eval workspace instead of external paths - Reframe README: scoring not testing, scenarios not test cases - Update cluster-diagnostic case count to 6 (game mode added) Co-Authored-By: Claude Opus 4.6 --- plugins/two-node/evals/README.md | 12 ++++++++---- plugins/two-node/evals/threat-model-tnf.yaml | 2 ++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/plugins/two-node/evals/README.md b/plugins/two-node/evals/README.md index bf30e5a6..45bfdeb6 100644 --- a/plugins/two-node/evals/README.md +++ b/plugins/two-node/evals/README.md @@ -1,14 +1,18 @@ # Evaluation Configs -Automated quality testing for two-node plugin skills using the +Automated quality scoring for two-node plugin skills using the [agent-eval-harness](https://github.com/opendatahub-io/agent-eval-harness) Claude Code plugin. +Evals measure skill quality on a spectrum (judges score 1-5, not +pass/fail) — they catch regressions and drift, not exact-match +correctness. + ## Available Evals | Config | Skill | Modes Tested | Cases | |--------|-------|--------------|-------| -| `cluster-diagnostic.yaml` | `two-node:cluster-diagnostic` | validate, recovery-guide | 5 | +| `cluster-diagnostic.yaml` | `two-node:cluster-diagnostic` | validate, recovery-guide, game | 6 | | `threat-model-tnf.yaml` | `threat-model:tnf` | PR analysis | 5 | ## Running Locally @@ -39,7 +43,7 @@ evals/ └── / └── cases/ └── case-NNN-/ - ├── input.yaml # Test input + ├── input.yaml # Scenario input └── annotations.yaml # Expected outcomes ``` @@ -50,7 +54,7 @@ evals/ /eval-analyze --skill --config evals/.yaml ``` -2. **Generate test cases** — creates `input.yaml` + `annotations.yaml` per case +2. **Generate scenarios** — creates `input.yaml` + `annotations.yaml` per case ``` /eval-dataset --config evals/.yaml ``` diff --git a/plugins/two-node/evals/threat-model-tnf.yaml b/plugins/two-node/evals/threat-model-tnf.yaml index 2d5f3b1b..f788bc7d 100644 --- a/plugins/two-node/evals/threat-model-tnf.yaml +++ b/plugins/two-node/evals/threat-model-tnf.yaml @@ -7,6 +7,8 @@ execution: arguments: "{pr_input}" timeout: 600 max_budget_usd: 8.0 + env: + REPORT_DIR: "reports/" runner: type: claude-code