diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..b2be90a --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,9 @@ +title = "Nightward gitleaks config" + +[[allowlists]] +description = "Synthetic provider parser fixtures with fake redacted values" +paths = [ + '''^testdata/providers/gitleaks\.json$''', + '''^testdata/providers/trufflehog\.jsonl$''', + '''^testdata/providers/trivy\.json$''', +] diff --git a/Makefile b/Makefile index 7a030a7..f2113f0 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ CARGO_AUDIT_VERSION ?= 0.22.1 CARGO_DENY_VERSION ?= 0.19.4 CARGO_LLVM_COV_VERSION ?= 0.8.5 -.PHONY: doctor install-dev-tools test test-fast test-security test-ux test-release test-local test-prepush test-release-install fmt clippy cargo-test cargo-nextest cargo-doc cargo-audit cargo-deny cargo-llvm-cov coverage-check fuzz-smoke test-junit trunk-check trunk-fix trunk-flaky-validate ci-scripts-test gitleaks raycast-install raycast-test raycast-test-junit raycast-audit raycast-lint raycast-build raycast-store-check raycast-verify npm-package-install npm-package-test npm-package-audit npm-package-pack npm-package-verify docs-reference docs-reference-check docs-freshness docs-qa site-install site-audit site-build site-verify demo-assets tui-media release-snapshot verify build install-local clean-reports +.PHONY: doctor install-dev-tools test test-fast test-security test-ux test-release test-local test-prepush test-release-install fmt clippy cargo-test cargo-nextest cargo-doc cargo-audit cargo-deny cargo-llvm-cov coverage-check fuzz-smoke test-junit trunk-check trunk-fix trunk-flaky-validate ci-scripts-test gitleaks raycast-install raycast-test raycast-test-junit raycast-audit raycast-lint raycast-build raycast-store-check raycast-verify npm-package-install npm-package-test npm-package-audit npm-package-pack npm-package-verify docs-reference docs-reference-check docs-freshness docs-qa demo-ids-check site-install site-audit site-build site-verify demo-assets tui-media release-snapshot verify build install-local clean-reports doctor: bash scripts/dev-doctor.sh @@ -149,7 +149,10 @@ docs-reference-check: docs-freshness: node scripts/check-docs-freshness.mjs -docs-qa: docs-reference-check docs-freshness +docs-qa: docs-reference-check docs-freshness demo-ids-check + +demo-ids-check: + node scripts/check-demo-ids.mjs site-verify: docs-qa site-install site-audit site-build diff --git a/crates/nightward-core/src/analysis.rs b/crates/nightward-core/src/analysis.rs index 0752dec..3d13ffd 100644 --- a/crates/nightward-core/src/analysis.rs +++ b/crates/nightward-core/src/analysis.rs @@ -324,7 +324,7 @@ fn append_provider_signals(out: &mut Report, scan: &ScanReport, options: &Option path: root.display().to_string(), message: format!("{provider} provider execution failed."), evidence: redact_text(&error.to_string()), - severity: RiskLevel::Low, + severity: RiskLevel::High, category: SignalCategory::Unknown, }, ); diff --git a/crates/nightward-core/src/fixplan.rs b/crates/nightward-core/src/fixplan.rs index 5446b31..8fc4af3 100644 --- a/crates/nightward-core/src/fixplan.rs +++ b/crates/nightward-core/src/fixplan.rs @@ -1,3 +1,4 @@ +use crate::inventory::redact_text; use crate::{Finding, FixKind, Report as ScanReport, RiskLevel}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -154,11 +155,15 @@ fn action_from_finding(finding: &Finding) -> Action { steps: if finding.fix_steps.is_empty() { vec![ "Inspect the redacted finding evidence.".to_string(), - finding.recommended_action.clone(), + redact_text(&finding.recommended_action), "Re-run Nightward and compare the next report.".to_string(), ] } else { - finding.fix_steps.clone() + finding + .fix_steps + .iter() + .map(|step| redact_text(step)) + .collect() }, preview, } @@ -166,10 +171,10 @@ fn action_from_finding(finding: &Finding) -> Action { fn preview_for(finding: &Finding) -> String { let Some(hint) = &finding.patch_hint else { - return format!( + return redact_text(&format!( "# plan-only review\n# {}\n# {}", finding.path, finding.recommended_action - ); + )); }; match hint.kind { Some(FixKind::ExternalizeSecret) => { @@ -178,19 +183,22 @@ fn preview_for(finding: &Finding) -> String { } else { &hint.env_key }; - format!( + redact_text(&format!( "- inline secret value in {}\n+ external reference to ${}\n# review required before editing", finding.path, key - ) + )) } - Some(FixKind::PinPackage) => format!( + Some(FixKind::PinPackage) if !hint.package.is_empty() => redact_text(&format!( "- {}\n+ {}@\n# choose and review an explicit version", hint.package, hint.package - ), + )), + Some(FixKind::PinPackage) => { + "# choose and review an explicit package version manually".to_string() + } Some(FixKind::NarrowFilesystem) => { "- broad filesystem path\n+ ".to_string() } - _ => format!("# review required for {}", finding.path), + _ => redact_text(&format!("# review required for {}", finding.path)), } } diff --git a/crates/nightward-core/src/inventory.rs b/crates/nightward-core/src/inventory.rs index 6cf9cba..b7b2810 100644 --- a/crates/nightward-core/src/inventory.rs +++ b/crates/nightward-core/src/inventory.rs @@ -1,15 +1,19 @@ use crate::rules; use crate::{AdapterStatus, Classification, Finding, FixKind, Item, PatchHint, Report, RiskLevel}; -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; use chrono::{DateTime, Utc}; use regex::Regex; use serde_json::Value; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::fs; +use std::io::Read; +use std::net::IpAddr; use std::path::{Path, PathBuf}; +use std::str::FromStr; const MAX_CONFIG_BYTES: u64 = 2 * 1024 * 1024; +const MAX_REPORT_BYTES: u64 = 16 * 1024 * 1024; #[derive(Debug, Clone)] struct Adapter { @@ -163,11 +167,50 @@ pub fn scan_workspace(workspace: impl AsRef) -> Result { } pub fn load_report(path: impl AsRef) -> Result { - let text = fs::read_to_string(path.as_ref()) + let text = read_bounded_regular_file(path.as_ref(), MAX_REPORT_BYTES) .with_context(|| format!("read {}", path.as_ref().display()))?; serde_json::from_str(&text).context("parse Nightward report JSON") } +pub fn load_report_summary(path: impl AsRef) -> Result { + let text = read_bounded_regular_file(path.as_ref(), MAX_REPORT_BYTES) + .with_context(|| format!("read {}", path.as_ref().display()))?; + let value: Value = serde_json::from_str(&text).context("parse Nightward report JSON")?; + Ok(value + .get("summary") + .and_then(|summary| summary.get("total_findings")) + .and_then(Value::as_u64) + .unwrap_or(0) as usize) +} + +fn read_bounded_regular_file(path: &Path, max_bytes: u64) -> Result { + let meta = fs::symlink_metadata(path).with_context(|| format!("stat {}", path.display()))?; + if !meta.file_type().is_file() { + return Err(anyhow!("{} is not a regular file", path.display())); + } + if meta.file_type().is_symlink() { + return Err(anyhow!("{} is a symlink", path.display())); + } + if meta.len() > max_bytes { + return Err(anyhow!( + "{} exceeds {} byte report size cap", + path.display(), + max_bytes + )); + } + let mut file = fs::File::open(path)?; + let mut bytes = Vec::with_capacity(meta.len() as usize); + file.by_ref().take(max_bytes + 1).read_to_end(&mut bytes)?; + if bytes.len() as u64 > max_bytes { + return Err(anyhow!( + "{} exceeds {} byte report size cap", + path.display(), + max_bytes + )); + } + String::from_utf8(bytes).context("report is not valid UTF-8") +} + pub fn write_report(path: impl AsRef, report: &Report) -> Result<()> { if let Some(parent) = path.as_ref().parent() { fs::create_dir_all(parent)?; @@ -432,6 +475,16 @@ fn inspect_server(report: &mut Report, tool: &str, path: &Path, server: &str, co } if let Some(package) = unpinned_package(&command, &args) { + let patch_hint = package.map(|package| PatchHint { + kind: Some(FixKind::PinPackage), + package, + env_key: String::new(), + header_key: String::new(), + inline_secret: false, + direct_command: String::new(), + direct_args: Vec::new(), + replacement: String::new(), + }); push_finding( report, tool, @@ -446,16 +499,7 @@ fn inspect_server(report: &mut Report, tool: &str, path: &Path, server: &str, co &evidence, "Replace unversioned or @latest package references with a reviewed explicit version.", FixKind::PinPackage, - Some(PatchHint { - kind: Some(FixKind::PinPackage), - package, - env_key: String::new(), - header_key: String::new(), - inline_secret: false, - direct_command: String::new(), - direct_args: Vec::new(), - replacement: String::new(), - }), + patch_hint, ); } @@ -711,7 +755,7 @@ fn object_entries(config: &Value, keys: &[&str]) -> Vec<(String, String)> { out } -fn unpinned_package(command: &str, args: &[String]) -> Option { +fn unpinned_package(command: &str, args: &[String]) -> Option> { let command_base = command.rsplit('/').next().unwrap_or(command); if !matches!( command_base, @@ -726,9 +770,31 @@ fn unpinned_package(command: &str, args: &[String]) -> Option { if package_has_version_pin(arg) { return None; } - return Some(arg.clone()); + return Some(safe_package_label(arg)); + } + Some(safe_package_label(command_base)) +} + +fn safe_package_label(value: &str) -> Option { + let package = value.trim(); + if package.is_empty() + || package.len() > 128 + || package.contains('=') + || package.contains("://") + || package.contains('?') + || package.contains('#') + || package.starts_with('$') + || secret_key(package) + || secret_value(package) + || looks_opaque_provider_token(package) + { + return None; } - Some(command.to_string()) + let package_re = Regex::new( + r"^(?:@[A-Za-z0-9._~-]+/[A-Za-z0-9._~-]+|[A-Za-z0-9._~-]+)(?:/[A-Za-z0-9._~-]+)?$", + ) + .expect("valid regex"); + package_re.is_match(package).then(|| package.to_string()) } fn package_has_version_pin(package: &str) -> bool { @@ -757,9 +823,10 @@ fn secret_key(key: &str) -> bool { } fn secret_value(value: &str) -> bool { - Regex::new(r"\b(sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|npm_[A-Za-z0-9]{20,}|xox[abprs]-[A-Za-z0-9-]{20,})\b") + Regex::new(r"\b(sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|npm_[A-Za-z0-9]{20,}|xox[abprs]-[A-Za-z0-9-]{20,}|eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})\b") .expect("valid regex") .is_match(value) + || looks_opaque_provider_token(value) } fn env_reference(value: &str) -> bool { @@ -790,20 +857,129 @@ pub fn redact_text(value: &str) -> String { r#"(?i)([\w.-]*{sensitive_key}[\w.-]*\s*[:=]\s*)(?:\$\{{[A-Za-z_][A-Za-z0-9_]*\}}|[^\s\r\n,}}]+(?:\s+[^\s\r\n,}}]+)?)"# )) .expect("valid regex"); - let provider = Regex::new(r"(?i)\b(?:Bearer\s+[-A-Za-z0-9._~+/=]{8,}|sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|npm_[A-Za-z0-9]{20,}|xox[abprs]-[A-Za-z0-9-]{20,}|eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})\b") + let sensitive_flag = Regex::new(&format!( + r#"(?i)(--?[\w.-]*{sensitive_key}[\w.-]*)(?:=|\s+)(?:\$\{{[A-Za-z_][A-Za-z0-9_]*\}}|"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|[^\s\r\n,}}]+)"# + )) + .expect("valid regex"); + let provider = Regex::new(r"(?i)\b(?:(?:Basic|Bearer)\s+[-A-Za-z0-9._~+/=]{8,}|sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|npm_[A-Za-z0-9]{20,}|xox[abprs]-[A-Za-z0-9-]{20,}|eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})\b") .expect("valid regex"); let redacted = double_quoted.replace_all(value, "$1[redacted]$2"); let redacted = single_quoted.replace_all(&redacted, "$1[redacted]$2"); let redacted = provider.replace_all(&redacted, "[redacted]"); - assignment + let redacted = sensitive_flag.replace_all(&redacted, "$1 [redacted]"); + let redacted = assignment .replace_all(&redacted, "$1[redacted]") - .to_string() + .to_string(); + redacted + .split_inclusive(char::is_whitespace) + .map(redact_opaque_token_part) + .collect() } fn local_endpoint(value: &str) -> bool { - Regex::new(r"(?i)\b(localhost|127\.0\.0\.1|0\.0\.0\.0|10\.\d+\.\d+\.\d+|192\.168\.\d+\.\d+|172\.(1[6-9]|2\d|3[01])\.\d+\.\d+)\b") - .expect("valid regex") - .is_match(value) + value + .split(|ch: char| ch.is_whitespace() || matches!(ch, '"' | '\'' | ',' | ';')) + .any(token_references_local_endpoint) +} + +fn token_references_local_endpoint(token: &str) -> bool { + let token = token.trim_matches(|ch: char| matches!(ch, '"' | '\'' | '`' | ',' | ';')); + if token.is_empty() { + return false; + } + let host = host_candidate(token); + is_local_host(&host) +} + +fn host_candidate(token: &str) -> String { + let mut candidate = token; + if let Some(index) = candidate.find("://") { + candidate = &candidate[index + 3..]; + } else if let Some(index) = candidate.rfind('=') { + candidate = &candidate[index + 1..]; + } + if let Some(index) = candidate.rfind('@') { + candidate = &candidate[index + 1..]; + } + if let Some(stripped) = candidate.strip_prefix('[') { + return stripped.split(']').next().unwrap_or_default().to_string(); + } + candidate + .split(['/', '?', '#']) + .next() + .unwrap_or_default() + .split(':') + .next() + .unwrap_or_default() + .to_string() +} + +fn is_local_host(host: &str) -> bool { + let host = host + .trim_matches(|ch| matches!(ch, '[' | ']' | '"' | '\'' | '`' | ',' | ';')) + .trim_end_matches('.') + .to_ascii_lowercase() + .replace("%25", "%"); + let host_without_zone = host.split('%').next().unwrap_or(&host); + if host_without_zone == "localhost" + || host_without_zone == "0.0.0.0" + || host_without_zone.ends_with(".localhost") + || host_without_zone.ends_with(".local") + { + return true; + } + let Ok(ip) = IpAddr::from_str(host_without_zone) else { + return false; + }; + match ip { + IpAddr::V4(ip) => ip.is_loopback() || ip.is_private() || ip.is_unspecified(), + IpAddr::V6(ip) => { + ip.is_loopback() + || ip.is_unspecified() + || ip.is_unicast_link_local() + || (ip.segments()[0] & 0xfe00) == 0xfc00 + } + } +} + +fn redact_opaque_token_part(part: &str) -> String { + let trailing_ws_len = part + .chars() + .rev() + .take_while(|ch| ch.is_whitespace()) + .map(char::len_utf8) + .sum::(); + let (token, whitespace) = part.split_at(part.len().saturating_sub(trailing_ws_len)); + if looks_opaque_provider_token(token) { + format!("[redacted]{whitespace}") + } else if let Some((key, value)) = token.split_once('=') { + if looks_opaque_provider_token(value) { + format!("{key}=[redacted]{whitespace}") + } else { + part.to_string() + } + } else { + part.to_string() + } +} + +fn looks_opaque_provider_token(value: &str) -> bool { + let trimmed = value.trim_matches(|ch: char| { + matches!( + ch, + '"' | '\'' | '`' | ',' | '.' | ':' | ';' | ')' | '(' | '[' | ']' + ) + }); + trimmed.len() >= 36 + && !trimmed.contains('/') + && !trimmed.contains('\\') + && !trimmed.contains('@') + && !trimmed.contains('.') + && trimmed + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-') + && trimmed.chars().any(|ch| ch.is_ascii_digit()) + && trimmed.chars().any(|ch| ch.is_ascii_alphabetic()) } fn broad_filesystem(args: &[String]) -> bool { @@ -942,6 +1118,79 @@ mod tests { assert!(!redacted.contains(&token)); } + #[test] + fn redacts_sensitive_flag_values_in_space_delimited_args() { + let token = ["sk-", "live", "-validation-123456789"].concat(); + let redacted = redact_text(&format!("--api-key {token} --profile dev")); + + assert_eq!(redacted, "--api-key [redacted] --profile dev"); + assert!(!redacted.contains(&token)); + } + + #[test] + fn redacts_sensitive_flag_values_in_equals_args() { + let token = ["ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"].concat(); + let redacted = redact_text(&format!("--authorization={token} --profile dev")); + + assert_eq!(redacted, "--authorization [redacted] --profile dev"); + assert!(!redacted.contains(&token)); + } + + #[test] + fn redacts_opaque_provider_tokens_without_hiding_local_paths() { + let token = ["provider", "Token", "Value1234567890abcdefABCDEF"].concat(); + let path = "/Users/example/Library/Application Support/Claude/claude_desktop_config.json"; + let redacted = redact_text(&format!("opaque={token} path={path}")); + + assert!(!redacted.contains(&token)); + assert!(redacted.contains(path)); + } + + #[test] + fn unpinned_package_does_not_surface_secret_like_package_specs() { + let dir = tempfile::tempdir().unwrap(); + fs::write( + dir.path().join(".mcp.json"), + r#"{"mcpServers":{"demo":{"command":"npx","args":["--package","api_key=super-secret-token-12345"]}}}"#, + ) + .unwrap(); + + let report = scan_workspace(dir.path()).unwrap(); + let finding = report + .findings + .iter() + .find(|finding| finding.rule == "mcp_unpinned_package") + .expect("unpinned package finding"); + + assert!(finding.patch_hint.is_none()); + assert!(!finding.evidence.contains("super-secret-token-12345")); + assert!(!finding.fix_summary.contains("super-secret-token-12345")); + assert!(finding + .fix_steps + .iter() + .all(|step| !step.contains("super-secret-token-12345"))); + } + + #[test] + fn local_endpoint_detects_root_dot_and_scoped_ipv6_urls() { + assert!(local_endpoint("url=http://localhost.:8787/mcp")); + assert!(local_endpoint("url=http://user:pass@localhost.:8787/mcp")); + assert!(local_endpoint("url=http://[fe80::1%25en0]:8787/mcp")); + assert!(local_endpoint("url=http://[::1]:8787/mcp")); + } + + #[test] + fn load_report_rejects_oversized_regular_files() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("huge.json"); + let file = fs::File::create(&path).unwrap(); + file.set_len(MAX_REPORT_BYTES + 1).unwrap(); + + let error = load_report(&path).expect_err("oversized report should fail"); + + assert!(format!("{error:#}").contains("report size cap")); + } + #[test] fn parses_toml_mcp_servers() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/nightward-core/src/policy.rs b/crates/nightward-core/src/policy.rs index 73dbc67..a73bf71 100644 --- a/crates/nightward-core/src/policy.rs +++ b/crates/nightward-core/src/policy.rs @@ -452,6 +452,21 @@ mod tests { assert!(sarif_text.contains("\"provider\":\"gitleaks\"")); } + #[test] + fn default_policy_blocks_high_provider_execution_failures() { + let scan = ScanReport::empty("home".to_string(), String::new(), "home".to_string()); + let mut analysis = analysis_report_with_signal(RiskLevel::High); + analysis.summary.provider_warnings = 1; + analysis.signals[0].provider = "gitleaks".to_string(); + analysis.signals[0].rule = "gitleaks/provider_execution_failed".to_string(); + let config = PolicyConfig::default(); + + let report = check(&scan, &config, Some(&analysis)); + + assert!(!report.passed); + assert_eq!(report.analysis_violation_count, 1); + } + #[test] fn check_ignores_rules_by_rule_key() { let mut scan = ScanReport::empty("home".to_string(), String::new(), "home".to_string()); diff --git a/crates/nightward-core/src/providers.rs b/crates/nightward-core/src/providers.rs index d15ad80..fb18d37 100644 --- a/crates/nightward-core/src/providers.rs +++ b/crates/nightward-core/src/providers.rs @@ -8,6 +8,7 @@ use std::env; use std::io::Read; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::thread; use std::time::Duration; use wait_timeout::ChildExt; @@ -161,30 +162,26 @@ pub fn run_provider(name: &str, root: &Path) -> Result> { .stderr(Stdio::piped()) .spawn() .with_context(|| format!("spawn provider {name}"))?; + let stdout_handle = child + .stdout + .take() + .map(|stream| thread::spawn(move || read_stream_capped(stream, stdout_cap))); + let stderr_handle = child + .stderr + .take() + .map(|stream| thread::spawn(move || read_stream_capped(stream, stderr_cap))); let status = match child.wait_timeout(timeout)? { Some(status) => status, None => { let _ = child.kill(); let _ = child.wait(); + let _ = join_stream(stdout_handle); + let _ = join_stream(stderr_handle); return Err(anyhow!("provider timed out after {:?}", timeout)); } }; - let mut stdout = Vec::new(); - let mut stderr = Vec::new(); - if let Some(mut stream) = child.stdout.take() { - let _ = stream - .by_ref() - .take(stdout_cap as u64 + 1) - .read_to_end(&mut stdout); - } - if let Some(mut stream) = child.stderr.take() { - let _ = stream - .by_ref() - .take(stderr_cap as u64 + 1) - .read_to_end(&mut stderr); - } - let (stdout, stdout_truncated) = capped_string(stdout, stdout_cap); - let (stderr, _) = capped_string(stderr, stderr_cap); + let (stdout, stdout_truncated) = join_stream(stdout_handle); + let (stderr, _) = join_stream(stderr_handle); if stdout_truncated { return Err(anyhow!("provider stdout exceeded {stdout_cap} byte cap")); } @@ -194,6 +191,34 @@ pub fn run_provider(name: &str, root: &Path) -> Result> { parse_provider_output(name, root, &stdout) } +fn read_stream_capped(mut stream: impl Read, cap: usize) -> (String, bool) { + let mut out = Vec::with_capacity(cap.min(64 * 1024)); + let mut truncated = false; + let mut buf = [0_u8; 8192]; + loop { + let Ok(read) = stream.read(&mut buf) else { + break; + }; + if read == 0 { + break; + } + let remaining = cap.saturating_sub(out.len()); + if remaining > 0 { + out.extend_from_slice(&buf[..read.min(remaining)]); + } + if read > remaining { + truncated = true; + } + } + (redact_text(&String::from_utf8_lossy(&out)), truncated) +} + +fn join_stream(handle: Option>) -> (String, bool) { + handle + .and_then(|handle| handle.join().ok()) + .unwrap_or_default() +} + pub fn parse_provider_output( name: &str, root: &Path, @@ -619,15 +644,6 @@ fn provider_stderr_cap() -> usize { .unwrap_or(DEFAULT_STDERR_CAP) } -fn capped_string(bytes: Vec, cap: usize) -> (String, bool) { - let truncated = bytes.len() > cap; - let mut value = String::from_utf8_lossy(&bytes[..bytes.len().min(cap)]).to_string(); - if truncated { - value.push_str("\n[provider output truncated]"); - } - (redact_text(&value), truncated) -} - fn first_line(value: &str) -> String { value .lines() diff --git a/crates/nightward-core/src/schedule.rs b/crates/nightward-core/src/schedule.rs index 91f774b..03f1c1c 100644 --- a/crates/nightward-core/src/schedule.rs +++ b/crates/nightward-core/src/schedule.rs @@ -1,3 +1,4 @@ +use crate::inventory::load_report_summary; use chrono::{DateTime, Utc}; use serde::Serialize; use std::fs; @@ -76,22 +77,21 @@ fn history(dir: &Path) -> Vec { if path.extension().and_then(|ext| ext.to_str()) != Some("json") { continue; } + let Ok(file_type) = entry.file_type() else { + continue; + }; + if !file_type.is_file() || file_type.is_symlink() { + continue; + } let Ok(meta) = entry.metadata() else { continue; }; let Ok(modified) = meta.modified() else { continue; }; - let findings = fs::read_to_string(&path) - .ok() - .and_then(|text| serde_json::from_str::(&text).ok()) - .and_then(|value| { - value - .get("summary") - .and_then(|summary| summary.get("total_findings")) - .and_then(serde_json::Value::as_u64) - }) - .unwrap_or(0) as usize; + let Ok(findings) = load_report_summary(&path) else { + continue; + }; out.push(ReportHistoryEntry { report_name: entry.file_name().to_string_lossy().to_string(), path: path.display().to_string(), @@ -101,3 +101,50 @@ fn history(dir: &Path) -> Vec { } out } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn history_skips_oversized_reports_before_counting_findings() { + let home = tempfile::tempdir().expect("temp home"); + let dir = report_dir(home.path()); + fs::create_dir_all(&dir).expect("report dir"); + fs::write( + dir.join("small.json"), + r#"{"summary":{"total_findings":3}}"#, + ) + .expect("small report"); + let huge = fs::File::create(dir.join("huge.json")).expect("huge report"); + huge.set_len(17 * 1024 * 1024).expect("set len"); + + let status = status(home.path()); + + assert_eq!(status.history.len(), 1); + assert_eq!(status.history[0].report_name, "small.json"); + assert_eq!(status.history[0].findings, 3); + } + + #[cfg(unix)] + #[test] + fn history_skips_json_fifos() { + use std::os::unix::fs::FileTypeExt; + use std::process::Command; + + let home = tempfile::tempdir().expect("temp home"); + let dir = report_dir(home.path()); + fs::create_dir_all(&dir).expect("report dir"); + let fifo = dir.join("pipe.json"); + let mkfifo_status = Command::new("mkfifo").arg(&fifo).status().expect("mkfifo"); + assert!(mkfifo_status.success()); + assert!(fs::symlink_metadata(&fifo) + .expect("fifo metadata") + .file_type() + .is_fifo()); + + let status = status(home.path()); + + assert!(status.history.is_empty()); + } +} diff --git a/crates/nightward-core/tests/provider_contracts.rs b/crates/nightward-core/tests/provider_contracts.rs index 4a8c5b3..6639706 100644 --- a/crates/nightward-core/tests/provider_contracts.rs +++ b/crates/nightward-core/tests/provider_contracts.rs @@ -126,6 +126,26 @@ fn provider_stdout_cap_fails_closed_before_parsing() { assert_eq!(error.to_string(), "provider stdout exceeded 16 byte cap"); } +#[cfg(unix)] +#[test] +fn provider_large_stdout_is_drained_until_cap_error() { + let _guard = EnvRestore::set(&[ + ("PATH", None), + ("NIGHTWARD_PROVIDER_TIMEOUT_MS", Some("1000")), + ("NIGHTWARD_PROVIDER_STDOUT_CAP", Some("16")), + ]); + let dir = tempfile::tempdir().expect("temp dir"); + write_executable( + dir.path().join("gitleaks"), + "#!/bin/sh\n/usr/bin/head -c 262144 /dev/zero | /usr/bin/tr '\\0' a\n", + ); + std::env::set_var("PATH", dir.path()); + + let error = run_provider("gitleaks", dir.path()).expect_err("output cap"); + + assert_eq!(error.to_string(), "provider stdout exceeded 16 byte cap"); +} + #[cfg(unix)] fn write_executable(path: impl AsRef, body: &str) { use std::os::unix::fs::PermissionsExt; diff --git a/integrations/raycast/src/format.ts b/integrations/raycast/src/format.ts index beca4a8..5e3a9f1 100644 --- a/integrations/raycast/src/format.ts +++ b/integrations/raycast/src/format.ts @@ -13,8 +13,10 @@ import type { const secretAssignmentPattern = /((?:token|secret|password|passwd|api[_-]?key|auth|authorization|credential|private[_-]?key)[\w.-]*\s*[:=]\s*)(["']?)(?:\$\{[A-Za-z_][A-Za-z0-9_]*\}|[^"',\s}]+)/gi; +const sensitiveFlagPattern = + /(--?[\w.-]*(?:token|secret|password|passwd|api[_-]?key|auth|authorization|credential|private[_-]?key)[\w.-]*)(?:=|\s+)(?:\$\{[A-Za-z_][A-Za-z0-9_]*\}|"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|[^\s\r\n,}]+)/gi; const providerTokenPattern = - /\b(?:Bearer\s+[-A-Za-z0-9._~+/=]{8,}|sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|npm_[A-Za-z0-9]{20,}|xox[abprs]-[A-Za-z0-9-]{20,}|eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})\b/g; + /\b(?:(?:Basic|Bearer)\s+[-A-Za-z0-9._~+/=]{8,}|sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|npm_[A-Za-z0-9]{20,}|xox[abprs]-[A-Za-z0-9-]{20,}|eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})\b/g; const riskRank: Record = { info: 0, @@ -28,11 +30,9 @@ export function redactText(value: string | undefined): string { if (!value) return ""; const redacted = value .replace(providerTokenPattern, "[redacted]") + .replace(sensitiveFlagPattern, "$1 [redacted]") .replace(secretAssignmentPattern, "$1$2[redacted]"); - return redacted - .split(/(\s+)/) - .map((part) => (looksOpaqueProviderToken(part) ? "[redacted]" : part)) - .join(""); + return redacted.split(/(\s+)/).map(redactOpaqueTokenPart).join(""); } export function severityColor(severity: RiskLevel): string { @@ -548,6 +548,17 @@ function looksOpaqueProviderToken(value: string): boolean { return /\d/.test(trimmed) && /[A-Za-z]/.test(trimmed); } +function redactOpaqueTokenPart(part: string): string { + if (looksOpaqueProviderToken(part)) return "[redacted]"; + const equalsIndex = part.indexOf("="); + if (equalsIndex > 0) { + const key = part.slice(0, equalsIndex); + const value = part.slice(equalsIndex + 1); + if (looksOpaqueProviderToken(value)) return `${key}=[redacted]`; + } + return part; +} + const markdownSpecialChars = new Set([ "\\", "`", diff --git a/integrations/raycast/test/format.test.ts b/integrations/raycast/test/format.test.ts index 4ea5f09..cc4c3c3 100644 --- a/integrations/raycast/test/format.test.ts +++ b/integrations/raycast/test/format.test.ts @@ -60,6 +60,21 @@ test("redacts env reference assignments without trailing braces", () => { assert.equal(output, "env.API_TOKEN=[redacted]"); }); +test("redacts basic auth and sensitive flag values", () => { + const basic = "dXNlcjpwYXNz"; + const apiKey = "sk-" + "live-validation-123456789"; + const output = redactText( + `Authorization: Basic ${basic} --api-key ${apiKey} --profile dev`, + ); + + assert.equal( + output, + "Authorization: [redacted] --api-key [redacted] --profile dev", + ); + assert.doesNotMatch(output, /dXNlcjpwYXNz/); + assert.doesNotMatch(output, /live-validation/); +}); + test("finding markdown keeps guidance while avoiding secret values", () => { const keyName = "API_" + "KEY"; const token = "sk-" + "1234567890abcdef"; diff --git a/packages/npm/bin/nightward.mjs b/packages/npm/bin/nightward.mjs index 17d9aff..e8884f0 100755 --- a/packages/npm/bin/nightward.mjs +++ b/packages/npm/bin/nightward.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { createHash } from "node:crypto"; import { createWriteStream, existsSync, realpathSync } from "node:fs"; -import { chmod, mkdir, readFile, rm } from "node:fs/promises"; +import { chmod, copyFile, lstat, mkdir, readFile, rm } from "node:fs/promises"; import { get } from "node:https"; import { homedir, platform as osPlatform, tmpdir } from "node:os"; import path from "node:path"; @@ -62,6 +62,27 @@ export async function verifyArchiveChecksum(file, expected) { } } +async function verifiedArchive(archive, url, expected) { + if (existsSync(archive)) { + try { + await verifyArchiveChecksum(archive, expected); + return archive; + } catch { + await rm(archive, { force: true }); + } + } + await download(url, archive); + await verifyArchiveChecksum(archive, expected); + return archive; +} + +async function assertRegularBinary(binary) { + const info = await lstat(binary); + if (!info.isFile() || info.isSymbolicLink()) { + throw new Error(`cached binary is not a regular file: ${binary}`); + } +} + export function cacheRoot() { if (process.env.NIGHTWARD_NPM_CACHE) { return process.env.NIGHTWARD_NPM_CACHE; @@ -91,6 +112,11 @@ export function cachedBinaryPath(command = commandName(), version = releaseVersi } async function download(url, destination, redirects = 0) { + if (url.startsWith("file://")) { + await mkdir(path.dirname(destination), { recursive: true }); + await copyFile(fileURLToPath(url), destination); + return; + } if (redirects > 5) { throw new Error(`too many redirects while downloading ${url}`); } @@ -162,10 +188,6 @@ export async function ensureBinary(command = commandName()) { const target = targetFor(); const binary = cachedBinaryPath(command, version, target); - if (existsSync(binary)) { - return binary; - } - const asset = assetName(version, target); const baseURL = releaseBaseURL(version); const archive = path.join(cacheRoot(), version, `${target.os}-${target.arch}`, asset); @@ -176,9 +198,10 @@ export async function ensureBinary(command = commandName()) { throw new Error(`checksums.txt does not include ${asset}`); } - await download(`${baseURL}/${asset}`, archive); - await verifyArchiveChecksum(archive, expected); + await verifiedArchive(archive, `${baseURL}/${asset}`, expected); + await rm(binary, { force: true }); await extractArchive(archive, installDir, target); + await assertRegularBinary(binary); await chmod(binary, 0o755); return binary; } diff --git a/packages/npm/test/launcher.test.mjs b/packages/npm/test/launcher.test.mjs index 2e0250c..5dfc9cb 100644 --- a/packages/npm/test/launcher.test.mjs +++ b/packages/npm/test/launcher.test.mjs @@ -1,8 +1,8 @@ -import { chmod, mkdtemp, symlink, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { spawnSync } from "node:child_process"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import test from "node:test"; import assert from "node:assert/strict"; @@ -10,8 +10,10 @@ import { assetName, cachedBinaryPath, commandName, + ensureBinary, parseChecksums, releaseBaseURL, + sha256, targetFor, verifyArchiveChecksum } from "../bin/nightward.mjs"; @@ -88,3 +90,71 @@ console.log("fake-nightward " + process.argv.slice(2).join(" ")); assert.equal(result.status, 0, result.stderr); assert.equal(result.stdout.trim(), "fake-nightward --version"); }); + +test("cache hits are re-extracted from a verified archive before execution", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "nightward-npm-cache-")); + const release = path.join(dir, "release"); + const packageDir = path.join(dir, "package"); + const cache = path.join(dir, "cache"); + const version = "9.9.9"; + const target = targetFor(process.platform, process.arch); + const asset = assetName(version, target); + await mkdir(release, { recursive: true }); + await mkdir(packageDir, { recursive: true }); + const command = process.platform === "win32" ? "nightward.exe" : "nightward"; + const binary = path.join(packageDir, command); + await writeFile(binary, "#!/usr/bin/env node\nconsole.log('GOOD_CACHE_BINARY');\n"); + await chmod(binary, 0o755); + const archive = path.join(release, asset); + const archiveResult = process.platform === "win32" + ? spawnSync("powershell", [ + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-Command", + "Compress-Archive -LiteralPath $args[0] -DestinationPath $args[1] -Force", + binary, + archive + ], { encoding: "utf8" }) + : spawnSync("tar", ["-czf", archive, "-C", packageDir, command], { + encoding: "utf8" + }); + assert.equal(archiveResult.status, 0, archiveResult.stderr); + await writeFile( + path.join(release, "checksums.txt"), + `${await sha256(archive)} ${asset}\n` + ); + + const previous = { + cache: process.env.NIGHTWARD_NPM_CACHE, + version: process.env.NIGHTWARD_NPM_VERSION, + base: process.env.NIGHTWARD_NPM_DOWNLOAD_BASE + }; + process.env.NIGHTWARD_NPM_CACHE = cache; + process.env.NIGHTWARD_NPM_VERSION = version; + process.env.NIGHTWARD_NPM_DOWNLOAD_BASE = pathToFileURL(release).href; + try { + const cached = await ensureBinary("nightward"); + await writeFile(cached, "#!/usr/bin/env node\nconsole.log('POISONED_CACHE_BINARY');\n"); + await chmod(cached, 0o755); + + const repaired = await ensureBinary("nightward"); + const contents = await readFile(repaired, "utf8"); + + assert.equal(repaired, cached); + assert.match(contents, /GOOD_CACHE_BINARY/); + assert.doesNotMatch(contents, /POISONED_CACHE_BINARY/); + } finally { + restoreEnv("NIGHTWARD_NPM_CACHE", previous.cache); + restoreEnv("NIGHTWARD_NPM_VERSION", previous.version); + restoreEnv("NIGHTWARD_NPM_DOWNLOAD_BASE", previous.base); + } +}); + +function restoreEnv(key, value) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } +} diff --git a/scripts/check-demo-ids.mjs b/scripts/check-demo-ids.mjs new file mode 100644 index 0000000..4250eda --- /dev/null +++ b/scripts/check-demo-ids.mjs @@ -0,0 +1,46 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const scanPath = join(repoRoot, "site", "public", "demo", "nightward-sample-scan.json"); +const report = JSON.parse(readFileSync(scanPath, "utf8")); +let failures = 0; + +for (const item of report.items || []) { + const expected = stableId(["item", item.tool || "", item.path || ""]); + if (item.id !== expected) { + console.error(`item id mismatch for ${item.path}: expected ${expected}, got ${item.id}`); + failures += 1; + } +} + +for (const finding of report.findings || []) { + const expected = `${finding.rule}-${stableId([ + finding.rule || "", + finding.tool || "", + finding.path || "", + finding.server || "", + finding.evidence || "", + ])}`; + if (finding.id !== expected) { + console.error(`finding id mismatch for ${finding.rule}: expected ${expected}, got ${finding.id}`); + failures += 1; + } +} + +if (failures > 0) { + process.exit(1); +} +console.log("demo sample ids match scrubbed paths."); + +function stableId(parts) { + const hash = createHash("sha256"); + for (const part of parts) { + hash.update(String(part)); + hash.update("\0"); + } + return hash.digest("hex").slice(0, 12); +} diff --git a/scripts/generate-demo-assets.mjs b/scripts/generate-demo-assets.mjs index 227738e..70c0be2 100755 --- a/scripts/generate-demo-assets.mjs +++ b/scripts/generate-demo-assets.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { createHash } from "node:crypto"; import { execFileSync, spawn } from "node:child_process"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir, userInfo } from "node:os"; @@ -146,13 +147,32 @@ function deterministicReport(rawReport) { report.hostname = publicHost; report.home = publicHome; for (const item of report.items || []) { + item.id = stableId(["item", item.tool || "", item.path || ""]); if (item.mod_time) { item.mod_time = generatedAt; } } + for (const finding of report.findings || []) { + finding.id = `${finding.rule}-${stableId([ + finding.rule || "", + finding.tool || "", + finding.path || "", + finding.server || "", + finding.evidence || "", + ])}`; + } return report; } +function stableId(parts) { + const hash = createHash("sha256"); + for (const part of parts) { + hash.update(String(part)); + hash.update("\0"); + } + return hash.digest("hex").slice(0, 12); +} + function findChrome() { if (process.env.NIGHTWARD_CHROME && existsSync(process.env.NIGHTWARD_CHROME)) { return process.env.NIGHTWARD_CHROME; diff --git a/scripts/test-release-scripts.sh b/scripts/test-release-scripts.sh index 7480e0d..a0a03ab 100755 --- a/scripts/test-release-scripts.sh +++ b/scripts/test-release-scripts.sh @@ -14,6 +14,11 @@ if "${repo_root}/scripts/verify-npm-release.sh" "v0.1.0" >/dev/null 2>&1; then echo "expected verify-npm-release.sh to reject v-prefixed npm version" >&2 exit 1 fi +grep -q "npm install --global --prefix" "${repo_root}/scripts/verify-npm-release.sh" +if grep -q "ln -s" "${repo_root}/scripts/verify-npm-release.sh"; then + echo "expected verify-npm-release.sh to use npm-created bin links" >&2 + exit 1 +fi if "${repo_root}/scripts/validate-release-ref.sh" "latest" >/dev/null 2>&1; then echo "expected validate-release-ref.sh to reject non-semver tag" >&2 diff --git a/scripts/verify-npm-release.sh b/scripts/verify-npm-release.sh index 0148007..7d23aba 100755 --- a/scripts/verify-npm-release.sh +++ b/scripts/verify-npm-release.sh @@ -22,13 +22,8 @@ console.log(`${metadata.name}@${metadata.version} ${metadata.dist.integrity}`); ' "${metadata}" "${package}" "${version}" tarball="$(npm pack "${package}@${version}" --silent --pack-destination "${tmp_dir}")" -tar -xzf "${tmp_dir}/${tarball}" -C "${tmp_dir}" -package_dir="${tmp_dir}/package" prefix="${tmp_dir}/prefix" -mkdir -p "${prefix}/bin" -chmod 0755 "${package_dir}/bin/nightward.mjs" -ln -s "${package_dir}/bin/nightward.mjs" "${prefix}/bin/nightward" -ln -s "${package_dir}/bin/nightward.mjs" "${prefix}/bin/nw" +npm install --global --prefix "${prefix}" --ignore-scripts --no-audit "${tmp_dir}/${tarball}" PATH="${prefix}/bin:${PATH}" nightward --version | grep -Fx "${version}" PATH="${prefix}/bin:${PATH}" nw --version | grep -Fx "${version}" diff --git a/site/public/demo/nightward-sample-scan.json b/site/public/demo/nightward-sample-scan.json index 26cbe39..bc54baf 100644 --- a/site/public/demo/nightward-sample-scan.json +++ b/site/public/demo/nightward-sample-scan.json @@ -33,7 +33,7 @@ }, "items": [ { - "id": "d02aa351e661", + "id": "47fe604fd067", "tool": "Codex", "path": "/tmp/nightward-fixture-home/.codex/config.toml", "kind": "file", @@ -48,7 +48,7 @@ ], "findings": [ { - "id": "mcp_unpinned_package-23f4e5dd9f16", + "id": "mcp_unpinned_package-4aa22da6f0fe", "tool": "Codex", "path": "/tmp/nightward-fixture-home/.codex/config.toml", "server": "demo", @@ -77,7 +77,7 @@ } }, { - "id": "mcp_secret_env-2b536e661bca", + "id": "mcp_secret_env-ac18995a3bb4", "tool": "Codex", "path": "/tmp/nightward-fixture-home/.codex/config.toml", "server": "demo", @@ -106,7 +106,7 @@ } }, { - "id": "mcp_broad_filesystem-8dca8b68a5eb", + "id": "mcp_broad_filesystem-750be3ad0300", "tool": "Codex", "path": "/tmp/nightward-fixture-home/.codex/config.toml", "server": "demo", @@ -131,7 +131,7 @@ ] }, { - "id": "mcp_server_review-b836ab7c82bf", + "id": "mcp_server_review-aa2fd568f642", "tool": "Codex", "path": "/tmp/nightward-fixture-home/.codex/config.toml", "server": "demo",