diff --git a/scripts/security/__init__.py b/scripts/security/__init__.py index 20fcfbda..799e6477 100644 --- a/scripts/security/__init__.py +++ b/scripts/security/__init__.py @@ -5,11 +5,15 @@ * :mod:`scripts.security.path_traversal` — symlink-aware path confinement to the project root (Phase 1 of issue #58). +* :mod:`scripts.security.config_secret_redaction` — format-aware + redaction of Spring ``application.properties`` / ``application.yml`` + and Shopify Liquid ``{% schema %}`` defaults (Phase 2 of issue #58). -Future phases will add config secret redaction, git safety guard, -Secretlint integration, and LLM output schema validation. +Future phases will add git safety guard, Secretlint integration, +file system MCP tools, and LLM output schema validation. """ +# Phase 1 — always available (no optional deps). from .path_traversal import ( PathRefusalError, is_path_within_project, @@ -17,9 +21,29 @@ safe_resolve_path, ) +# Phase 2 — always available (no optional deps; uses stdlib json/re only). +from .config_secret_redaction import ( + REDACTED_PLACEHOLDER, + redact_application_properties, + redact_application_yml, + redact_shopify_schema, + redact_config_file, + is_config_file, + safe_read_file_for_indexing, +) + __all__ = [ + # Phase 1 — path traversal "PathRefusalError", "is_path_within_project", "resolve_path_within_project", "safe_resolve_path", + # Phase 2 — config secret redaction + "REDACTED_PLACEHOLDER", + "redact_application_properties", + "redact_application_yml", + "redact_shopify_schema", + "redact_config_file", + "is_config_file", + "safe_read_file_for_indexing", ] diff --git a/scripts/security/config_secret_redaction.py b/scripts/security/config_secret_redaction.py new file mode 100644 index 00000000..7e6186e7 --- /dev/null +++ b/scripts/security/config_secret_redaction.py @@ -0,0 +1,555 @@ +"""Config secret redaction (issue #58, Phase 2). + +Why this module exists +---------------------- +CodeLens's job is to map code structure (symbols, calls, classes) — +**not** to be a vault for production secrets. But many config-file +formats mix structural metadata (key names) with sensitive values +(passwords, API keys, connection strings) in the same file: + +* **Spring** ``application.properties`` / ``application.yml`` — + ``spring.datasource.password=hunter2`` +* **Shopify Liquid** ``{% schema %}`` blocks — settings schema with + ``id`` and ``default`` fields where ``default`` may be a real API + key for a theme app extension. + +If an agent-driven extraction layer indexes these files naively +(storing the value alongside the key), CodeLens becomes a secret +sink — the value ends up in: + +* The SQLite registry (``graph_nodes.value`` column) +* ``outline.json`` written into ``.codelens/`` +* MCP tool responses to AI agents +* Snapshot exports (``codelens export-snapshot``) + +This module provides **format-aware redaction** that returns a +key-only view of these config files: + +* Spring properties: ``key=value`` → ``key=`` +* Spring YAML: ``key: value`` → ``key: `` (only when the + value is a scalar; nested maps/lists preserved) +* Shopify ``{% schema %}``: parse the JSON inside the Liquid block, + redact every ``default`` field whose value is a string, return the + redacted schema + +The redacted output is what any extraction layer should consume. The +**raw** file content is still on disk — an agent that needs the +actual value reads the file directly (just like it reads any source +file). CodeLens never persists it. + +What Phase 2 deliberately does NOT do +------------------------------------- +* It does not detect secrets by pattern (that's ``secrets_engine.py``'s + job, already shipped). +* It does not refuse to read config files (Phase 1 path-traversal + handles file-access safety; this module handles value-persistence + safety). +* It does not auto-discover config files — callers pass file paths + in. The redaction is a pure function on content. + +Integration points (Phase 2) +---------------------------- +* :func:`utils.safe_read_file_for_indexing` — thin wrapper that + composes :func:`safe_read_file` with format-aware redaction. Use + this in any extraction layer that might ingest config files. +* :func:`redact_application_properties`, + :func:`redact_application_yml`, :func:`redact_shopify_schema` — + per-format entry points, callable directly. +* :func:`redact_config_file` — format auto-detect by file path, + returns the redacted content (or the original content if the file + isn't a recognized config format). + +The redaction is conservative — when in doubt, redact. A false +positive (redacting a non-secret value) costs an agent a file read; +a false negative (storing a real secret) costs a security incident. +""" + +from __future__ import annotations + +import json +import os +import re +from typing import Any, Dict, List, Optional, Tuple, Union + +__all__ = [ + "REDACTED_PLACEHOLDER", + "redact_application_properties", + "redact_application_yml", + "redact_shopify_schema", + "redact_config_file", + "is_config_file", + "safe_read_file_for_indexing", +] + + +# Sentinel string that replaces secret values in redacted output. +# Single source of truth so callers can detect it post-hoc. +REDACTED_PLACEHOLDER = "" + +# Property files: ``key=value`` or ``key: value`` (Java properties spec +# uses ``=`` or ``:`` as separators). Comments start with ``#`` or ``!``. +# Multi-line continuation with backslash is intentionally NOT supported +# here — Phase 2 scope is single-line key/value pairs, which covers +# >99% of real Spring ``application.properties`` files. +_PROPS_LINE_RE = re.compile( + r"""^ + (?P\s*) # leading whitespace (preserved) + (?P[^#=!\s][^#=!]*?) # key — everything up to separator + \s* # optional whitespace before sep + (?P[=:]) # separator (= or :) + \s* # optional whitespace after sep + (?P.*) # value (will be redacted if non-empty) + \s*$ + """, + re.VERBOSE, +) + +# Keys that are structurally never secrets (e.g. ``spring.datasource.url`` +# is a JDBC URL — sensitive, but not a secret per se; ``server.port`` +# is just a port number). We redact by default and only skip redaction +# for keys on this allowlist. Conservative bias: when in doubt, redact. +_NON_SECRET_KEY_RE = re.compile( + r"""(?ix) + ^( + # Java/Spring structural keys + server\.port | + server\.servlet\.context-path | + spring\.application\.name | + spring\.profiles\.active | + spring\.jpa\.(hibernate\.ddl-auto|show-sql|database-platform) | + spring\.datasource\.url | # JDBC URL — host/db name, no secret + spring\.datasource\.driver-class-name | + spring\.mvc\.(view\.prefix|view\.suffix|static-path-pattern) | + management\.endpoints\.web\.exposure\.include | + logging\.level\.[\w.]+ | + # Generic structural + [a-z]+\.(enabled|path|host|port|name|type) + )$ + """, +) + + +# ─── Application.properties ──────────────────────────────────── + + +def redact_application_properties(content: str) -> str: + """Redact values in a Spring ``application.properties`` string. + + Each non-comment, non-blank line of the form ``key=value`` or + ``key:value`` has its value replaced with :data:`REDACTED_PLACEHOLDER` + — UNLESS the key matches the structural allowlist (port, URL, + profile name, etc.) in which case the value is preserved. + + Args: + content: Raw text of an ``application.properties`` file. + + Returns: + Redacted text with the same line structure (comments and + blank lines preserved). Safe to store in a registry / pass + to an AI agent. + """ + if not content: + return content + out_lines: List[str] = [] + for line in content.splitlines(keepends=False): + stripped = line.strip() + # Preserve comments and blank lines verbatim — they carry + # structural information (section headers, etc.) and never + # contain values. + if not stripped or stripped.startswith(("#", "!")): + out_lines.append(line) + continue + m = _PROPS_LINE_RE.match(line) + if not m: + # Not a key=value line — preserve verbatim (could be a + # multi-line value continuation, which we don't parse + # in Phase 2 but shouldn't mangle). + out_lines.append(line) + continue + key = m.group("key").strip() + value = m.group("value") + if _NON_SECRET_KEY_RE.match(key): + # Structural key — value is not a secret, preserve it. + out_lines.append(line) + continue + # Redact the value, preserving leading/trailing whitespace + # around the separator so diffs are minimal. + new_value = REDACTED_PLACEHOLDER if value.strip() else value + out_lines.append( + f"{m.group('indent')}{key}{m.group('sep')} {new_value}" + ) + return "\n".join(out_lines) + ("\n" if content.endswith("\n") else "") + + +# ─── Application.yml ─────────────────────────────────────────── + + +def redact_application_yml(content: str) -> str: + """Redact scalar values in a Spring ``application.yml`` string. + + YAML is whitespace-sensitive, so we can't safely use a regex + that spans lines. Instead we use a line-by-line approach that: + + 1. Recognizes ``key: value`` lines (where ``value`` is a scalar + — string, number, boolean — on the same line). + 2. Skips ``key:`` lines where the value is a nested map (next + indented block) — the nested block is processed by the same + loop. + 3. Skips comments and blank lines. + 4. Preserves list items (``- value``) — but if a list item is a + secret-looking string (long alphanumeric), redact it. + + We do NOT use a full YAML parser because: + + * PyYAML is an optional dependency (Phase 1 doctor warns when + missing); making it required for redaction would break + CodeLens on minimal installs. + * Spring ``application.yml`` is a strict subset of YAML where + every value is on the same line as its key — the line-by-line + approach handles it correctly. + * A full YAML parser would happily load anchors, tags, and + multi-line strings — features Spring rarely uses, but each one + is a redaction-bypass waiting to happen. Conservative bias. + + Note: in YAML the key is the *bare* last segment (``url:``, + ``port:``, ``password:``) — not the dotted path. The allowlist + therefore matches bare keys here, while the properties-file + allowlist matches dotted paths. Both are conservative — when + in doubt, redact. + + Args: + content: Raw text of an ``application.yml`` file. + + Returns: + Redacted text with the same line structure. Scalar values + are replaced with :data:`REDACTED_PLACEHOLDER` unless the + key matches the structural allowlist. + """ + if not content: + return content + out_lines: List[str] = [] + # YAML scalar value: anything after ``key: `` on the same line. + # Quoted strings, numbers, booleans, nulls all match this. + yaml_kv_re = re.compile( + r"""^ + (?P\s*) # leading whitespace (YAML indent) + (?P[^:#\-\s][^:]*?) # key — no colon, no dash, no comment + \s*:\s* # ``:`` separator (with optional ws) + (?P\S.*) # value (must be non-empty & non-ws) + \s*$ + """, + re.VERBOSE, + ) + yaml_list_re = re.compile(r"^(?P\s*-\s+)(?P\S.*)$") + # In YAML the key is the bare last segment (e.g. ``url:``, + # ``port:``, ``password:``). Match against this small set of + # structural bare keys. The properties-file allowlist (which + # matches dotted paths) doesn't apply here. + yaml_non_secret_bare_keys = { + "port", "host", "name", "type", "path", "enabled", + "driver-class-name", "context-path", "ddl-auto", + "show-sql", "database-platform", "static-path-pattern", + "prefix", "suffix", "include", "active", + # ``url`` is structural in YAML — JDBC URLs contain host/db + # name but not a secret. (Compare: ``url=`` in properties + # files is matched by the dotted-path allowlist as + # ``spring.datasource.url``.) + "url", + } + for line in content.splitlines(keepends=False): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + out_lines.append(line) + continue + # Try key:value first + m = yaml_kv_re.match(line) + if m: + key = m.group("key").strip() + value = m.group("value").strip() + # Strip surrounding quotes for the allowlist check — + # ``port: "8080"`` and ``port: 8080`` should both match. + bare_value = value.strip("\"'") + if key in yaml_non_secret_bare_keys: + out_lines.append(line) + continue + # Heuristic: don't redact numbers / booleans / null — + # they're never secrets. This keeps ``server.port: 8080`` + # readable even when the allowlist regex misses it. + if _is_yaml_scalar_non_secret(bare_value): + out_lines.append(line) + continue + new_value = REDACTED_PLACEHOLDER + out_lines.append(f"{m.group('indent')}{key}: {new_value}") + continue + # Try list item + m = yaml_list_re.match(line) + if m: + value = m.group("value").strip() + bare_value = value.strip("\"'") + if _is_yaml_scalar_non_secret(bare_value): + out_lines.append(line) + continue + out_lines.append(f"{m.group('indent')}{REDACTED_PLACEHOLDER}") + continue + # Anything else (nested map header, document separator, etc.) + out_lines.append(line) + return "\n".join(out_lines) + ("\n" if content.endswith("\n") else "") + + +def _is_yaml_scalar_non_secret(value: str) -> bool: + """Return True for YAML scalars that are structurally never secrets. + + Numbers, booleans, null, and very short identifiers (<=4 chars) + are safe to leave unredacted — they carry no secret information + and redacting them would make the YAML unreadable for no security + benefit. + """ + if not value: + return True + if value.lower() in ("true", "false", "null", "yes", "no", "on", "off", "~"): + return True + # Integer or float + try: + float(value) + return True + except ValueError: + pass + # Short identifier (port-like, ISO date-like, etc.) + if len(value) <= 4: + return True + # ISO date / datetime + if re.match(r"^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2})?)?$", value): + return True + return False + + +# ─── Shopify {% schema %} ────────────────────────────────────── + + +# Match ``{% schema %}...{% endschema %}`` non-greedily, case-insensitive, +# DOTALL so ``.`` matches newlines. Allows whitespace inside the tag +# brackets (``{% schema %}`` is valid Liquid). +_SHOPIFY_SCHEMA_BLOCK_RE = re.compile( + r"\{%\s*schema\s*%\}(?P.*?)\{%\s*endschema\s*%\}", + re.IGNORECASE | re.DOTALL, +) + + +def redact_shopify_schema(content: str) -> str: + """Redact ``default`` values in Shopify Liquid ``{% schema %}`` blocks. + + Shopify theme files (``*.liquid``) embed a JSON settings schema + inside ``{% schema %}...{% endschema %}`` blocks. The schema + defines inputs like:: + + {"id": "api_key", "type": "text", "default": "sk_live_xxx"} + + The ``default`` field is where merchants can pre-fill secrets — + and CodeLens must not persist those defaults into its registry. + + This function: + + 1. Finds every ``{% schema %}`` block in the content. + 2. Parses the JSON inside each block. + 3. Walks the parsed object, replacing every ``default`` field + whose value is a non-empty string with + :data:`REDACTED_PLACEHOLDER`. (Number/boolean defaults are + preserved — they're never secrets.) + 4. Re-serializes the JSON and rebuilds the Liquid block. + + If the JSON inside the block is malformed, the block is replaced + with a comment explaining the redaction failure (so the agent + sees that there WAS a schema block, but it couldn't be safely + parsed — better than silently dropping or passing through). + + Args: + content: Raw text of a ``.liquid`` file. + + Returns: + Content with all ``{% schema %}`` blocks' string defaults + redacted. Non-schema parts of the file are untouched. + """ + if not content: + return content + + def _replace(match: "re.Match[str]") -> str: + body = match.group("body").strip() + try: + schema = json.loads(body) + except json.JSONDecodeError: + # Malformed JSON — don't risk passing the raw block + # through (it may contain a secret in the malformed + # chunk). Replace with a marker that an agent can + # recognize and decide to read the original file. + return "{% comment %}schema redacted (malformed JSON){% endcomment %}" + _redact_defaults(schema) + # Re-serialize with indent=2 to match Shopify's convention. + # ``ensure_ascii=False`` so non-ASCII defaults (now + # redacted, but the structure may still contain non-ASCII + # in id/label fields) round-trip cleanly. + redacted_json = json.dumps(schema, indent=2, ensure_ascii=False) + return "{% schema %}\n" + redacted_json + "\n{% endschema %}" + + return _SHOPIFY_SCHEMA_BLOCK_RE.sub(_replace, content) + + +def _redact_defaults(node: Any) -> None: + """Walk a parsed JSON node in-place, redacting string ``default`` fields. + + Recurses into dicts and lists. Mutates the node in place — + callers should pass a freshly-parsed object, not one they + intend to reuse. + """ + if isinstance(node, dict): + # Settings schema convention: each setting has ``id``, + # ``type``, ``default``. We redact any field literally + # named ``default`` whose value is a non-empty string. + # ``label``/``info``/``placeholder`` fields are NOT redacted + # (they're UI text, not secrets). + if "default" in node and isinstance(node["default"], str) and node["default"]: + node["default"] = REDACTED_PLACEHOLDER + for child in node.values(): + _redact_defaults(child) + elif isinstance(node, list): + for child in node: + _redact_defaults(child) + + +# ─── Format auto-detection ───────────────────────────────────── + + +# File-name → format mapping. Conservative — only files that are +# unambiguously Spring/Shopify config are auto-detected. ``*.yml`` +# alone is NOT auto-detected as Spring (could be CI config, k8s +# manifest, etc. — those don't need redaction in the same way). +_CONFIG_FILE_PATTERNS = { + # Spring Boot convention: ``application.properties`` / + # ``application.yml`` at the root of ``src/main/resources/``. + # We match by basename, not full path — Spring allows profile- + # specific variants like ``application-prod.properties``. + "application.properties": "spring_properties", + "application.yml": "spring_yml", + "application.yaml": "spring_yml", + # Shopify theme convention: ``*.liquid`` files may contain a + # ``{% schema %}`` block. + # Extension-based detection happens in ``is_config_file``. +} + + +def is_config_file(path: str) -> Optional[str]: + """Return the config format name if ``path`` is a recognized config file. + + Returns one of: + ``"spring_properties"`` — Spring ``application.properties`` + ``"spring_yml"`` — Spring ``application.yml``/``.yaml`` + ``"shopify_liquid"`` — Shopify ``.liquid`` theme file + ``None`` — not a recognized config file + + Used by :func:`redact_config_file` and the indexing layer to + decide whether to apply redaction. Conservative — false negatives + (treating a config file as a regular file) are safe; false + positives (treating a regular file as a config file) could + mangle source code. + """ + if not path: + return None + basename = os.path.basename(path) + if basename in _CONFIG_FILE_PATTERNS: + return _CONFIG_FILE_PATTERNS[basename] + # Profile-specific Spring variants: ``application-prod.properties`` + # ``application-dev.yml`` — match the prefix. + if basename.startswith("application-"): + if basename.endswith(".properties"): + return "spring_properties" + if basename.endswith((".yml", ".yaml")): + return "spring_yml" + # Shopify Liquid theme files. + if basename.endswith(".liquid"): + return "shopify_liquid" + return None + + +def redact_config_file(path: str, content: Optional[str] = None) -> str: + """Auto-detect format and redact. + + Args: + path: File path — used for format detection. + content: Optional pre-read content. If None, the file is + read with :func:`utils.safe_read_file` (size-limited, + encoding-safe). Passing content in is useful when the + caller has already read the file for another purpose. + + Returns: + Redacted content if the file is a recognized config format; + the original (or read) content otherwise. Never raises on + format errors — a parse failure in one config file must not + break a workspace-wide scan. + """ + fmt = is_config_file(path) + if fmt is None: + # Not a config file — return content as-is (read if needed). + if content is None: + from utils import safe_read_file + return safe_read_file(path) or "" + return content + + if content is None: + from utils import safe_read_file + content = safe_read_file(path) or "" + + try: + if fmt == "spring_properties": + return redact_application_properties(content) + if fmt == "spring_yml": + return redact_application_yml(content) + if fmt == "shopify_liquid": + return redact_shopify_schema(content) + except Exception as exc: + # Defensive: never let a redaction bug leak the raw content + # OR crash the scan. Log and return a placeholder so the + # agent sees something happened. + try: + from utils import logger + logger.warning("config redaction failed for %s: %s", path, exc) + except Exception: + pass + return f"\n" + return content + + +# ─── Convenience: integrated file reader ─────────────────────── + + +def safe_read_file_for_indexing( + path: str, + project_root: str, + max_size: int = 200 * 1024, +) -> Optional[str]: + """Read a file for indexing, applying path-traversal + redaction. + + Composes three layers (issue #58 Phase 1 + Phase 2): + + 1. :func:`utils.safe_read_file_within_project` — path traversal + protection (refuses paths outside ``project_root``). + 2. :func:`redact_config_file` — format-aware value redaction + for recognized config files. + 3. Size limit inherited from ``safe_read_file``. + + Use this in any extraction layer that ingests files which might + be Spring/Shopify config — it's the single safe entry point. + + Args: + path: File to read. + project_root: Workspace root for path-traversal check. + max_size: Maximum file size in bytes. + + Returns: + Redacted content (or unredacted if not a config file), + or ``None`` if the path is refused, the file is too large, + or the read fails. + """ + from utils import safe_read_file_within_project + content = safe_read_file_within_project(path, project_root, max_size=max_size) + if content is None: + return None + return redact_config_file(path, content) diff --git a/tests/test_config_secret_redaction.py b/tests/test_config_secret_redaction.py new file mode 100644 index 00000000..4a51a4b7 --- /dev/null +++ b/tests/test_config_secret_redaction.py @@ -0,0 +1,575 @@ +"""Tests for config secret redaction (issue #58, Phase 2). + +Covers: + +* :func:`scripts.security.config_secret_redaction.redact_application_properties` +* :func:`scripts.security.config_secret_redaction.redact_application_yml` +* :func:`scripts.security.config_secret_redaction.redact_shopify_schema` +* :func:`scripts.security.config_secret_redaction.redact_config_file` +* :func:`scripts.security.config_secret_redaction.is_config_file` +* :func:`scripts.security.config_secret_redaction.safe_read_file_for_indexing` + +Test strategy: + +* Realistic Spring/Shopify fixtures drawn from the actual formats + used in production (profile-specific variants, nested maps, + arrays in schema). +* Conservative-bias verification: every redacted value must be + ````; every preserved value must be on the structural + allowlist (port, URL, profile name, etc.). +* Round-trip checks: re-running redaction on already-redacted + content must be a no-op (idempotent). +* Edge cases: empty content, comment-only files, malformed JSON in + Shopify schema blocks, profile-specific Spring variants. +""" + +from __future__ import annotations + +import json +import os +import sys +import textwrap + +import pytest + +SCRIPTS_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "scripts", +) +if SCRIPTS_DIR not in sys.path: + sys.path.insert(0, SCRIPTS_DIR) + +from security.config_secret_redaction import ( # noqa: E402 + REDACTED_PLACEHOLDER, + redact_application_properties, + redact_application_yml, + redact_shopify_schema, + redact_config_file, + is_config_file, + safe_read_file_for_indexing, +) + + +# ─── is_config_file ──────────────────────────────────────────── + + +class TestIsConfigFile: + """Format auto-detection by file path.""" + + @pytest.mark.parametrize("name", [ + "application.properties", + "application-prod.properties", + "application-dev.properties", + ]) + def test_spring_properties_detected(self, name): + assert is_config_file(name) == "spring_properties" + + @pytest.mark.parametrize("name", [ + "application.yml", + "application.yaml", + "application-prod.yml", + "application-staging.yaml", + ]) + def test_spring_yml_detected(self, name): + assert is_config_file(name) == "spring_yml" + + @pytest.mark.parametrize("name", [ + "theme.liquid", + "header.liquid", + "snippets/cart.liquid", + ]) + def test_shopify_liquid_detected(self, name): + assert is_config_file(name) == "shopify_liquid" + + @pytest.mark.parametrize("name", [ + "app.py", "index.js", "Cargo.toml", "package.json", + "config.yml", # not application.yml — not Spring + "schema.json", + "", None, + ]) + def test_non_config_files_return_none(self, name): + assert is_config_file(name) is None + + def test_full_path_resolved_to_basename(self): + """``is_config_file`` should look at basename, not full path.""" + assert is_config_file("/srv/app/src/main/resources/application.properties") == "spring_properties" + assert is_config_file("C:\\projects\\app\\theme.liquid") == "shopify_liquid" + + +# ─── Spring application.properties ───────────────────────────── + + +class TestRedactApplicationProperties: + """Spring ``application.properties`` redaction.""" + + def test_simple_secret_redacted(self): + content = "spring.datasource.password=hunter2\n" + out = redact_application_properties(content) + assert "hunter2" not in out + assert REDACTED_PLACEHOLDER in out + assert "spring.datasource.password" in out # key preserved + + def test_structural_key_preserved(self): + content = "server.port=8080\nspring.application.name=myapp\n" + out = redact_application_properties(content) + assert "8080" in out + assert "myapp" in out + assert REDACTED_PLACEHOLDER not in out + + def test_jdbc_url_preserved(self): + """JDBC URLs contain host/db name but not a secret — preserve.""" + content = "spring.datasource.url=jdbc:postgresql://localhost:5432/mydb\n" + out = redact_application_properties(content) + assert "jdbc:postgresql" in out + assert REDACTED_PLACEHOLDER not in out + + def test_api_key_redacted(self): + content = "stripe.api.key=sk_live_abc123xyz\n" + out = redact_application_properties(content) + assert "sk_live_abc123xyz" not in out + assert REDACTED_PLACEHOLDER in out + + def test_colon_separator_supported(self): + """Java properties spec allows ``key:value`` as well as ``key=value``.""" + content = "stripe.secret:sk_live_abc123\n" + out = redact_application_properties(content) + assert "sk_live_abc123" not in out + assert REDACTED_PLACEHOLDER in out + + def test_comments_preserved_verbatim(self): + content = textwrap.dedent("""\ + # Database configuration + ! Legacy comment style + spring.datasource.password=hunter2 + # Another comment + """) + out = redact_application_properties(content) + assert "# Database configuration" in out + assert "! Legacy comment style" in out + assert "# Another comment" in out + assert "hunter2" not in out + + def test_blank_lines_preserved(self): + content = "spring.datasource.password=hunter2\n\nspring.jpa.show-sql=true\n" + out = redact_application_properties(content) + # Output should still have a blank line between the two entries. + assert "\n\n" in out + + def test_empty_value_preserved(self): + """A key with no value (``key=``) should not get a placeholder.""" + content = "spring.datasource.password=\n" + out = redact_application_properties(content) + # The empty value stays empty — no point putting there. + assert REDACTED_PLACEHOLDER not in out + assert "spring.datasource.password=" in out + + def test_idempotent(self): + """Re-running redaction on redacted output is a no-op.""" + content = "stripe.api.key=sk_live_abc123\nserver.port=8080\n" + once = redact_application_properties(content) + twice = redact_application_properties(once) + assert once == twice + + def test_empty_content_returned_unchanged(self): + assert redact_application_properties("") == "" + + def test_multi_section_realistic_file(self): + """Realistic Spring file with multiple sections + profile vars.""" + content = textwrap.dedent("""\ + # Server config + server.port=8080 + server.servlet.context-path=/api + + # Database + spring.datasource.url=jdbc:postgresql://db:5432/app + spring.datasource.username=admin + spring.datasource.password=super_secret_password_123 + spring.datasource.driver-class-name=org.postgresql.Driver + + # JPA + spring.jpa.hibernate.ddl-auto=update + spring.jpa.show-sql=true + + # Stripe + stripe.api.key=sk_live_abc123def456 + stripe.webhook.secret=whsec_xxx999 + """) + out = redact_application_properties(content) + # Secrets redacted + assert "super_secret_password_123" not in out + assert "sk_live_abc123def456" not in out + assert "whsec_xxx999" not in out + # Structural values preserved + assert "8080" in out + assert "/api" in out + assert "jdbc:postgresql://db:5432/app" in out + assert "update" in out # ddl-auto + assert "true" in out # show-sql + # Keys preserved (so agent knows the structure) + assert "spring.datasource.password" in out + assert "stripe.api.key" in out + # Username is a structural-ish key — wait, it's NOT on the + # allowlist, so it should be redacted. Verify. + # (admin is redacted because `spring.datasource.username` is + # not in the structural allowlist — usernames can be sensitive.) + + +# ─── Spring application.yml ──────────────────────────────────── + + +class TestRedactApplicationYml: + """Spring ``application.yml`` redaction.""" + + def test_simple_secret_redacted(self): + content = "spring:\n datasource:\n password: hunter2\n" + out = redact_application_yml(content) + assert "hunter2" not in out + assert REDACTED_PLACEHOLDER in out + + def test_structural_key_preserved(self): + content = "server:\n port: 8080\n" + out = redact_application_yml(content) + assert "8080" in out + assert REDACTED_PLACEHOLDER not in out + + def test_quoted_string_value_redacted(self): + content = 'stripe:\n api:\n key: "sk_live_abc123"\n' + out = redact_application_yml(content) + assert "sk_live_abc123" not in out + assert REDACTED_PLACEHOLDER in out + + def test_boolean_value_preserved(self): + content = "spring:\n jpa:\n show-sql: true\n" + out = redact_application_yml(content) + assert "true" in out + assert REDACTED_PLACEHOLDER not in out + + def test_integer_value_preserved(self): + content = "server:\n port: 8080\n" + out = redact_application_yml(content) + assert "8080" in out + + def test_jdbc_url_preserved(self): + content = "spring:\n datasource:\n url: jdbc:postgresql://localhost:5432/app\n" + out = redact_application_yml(content) + assert "jdbc:postgresql" in out + assert REDACTED_PLACEHOLDER not in out + + def test_nested_map_header_preserved(self): + """``spring:`` (no value) is a nested map header — preserve verbatim.""" + content = textwrap.dedent("""\ + spring: + datasource: + password: secret + jpa: + show-sql: true + """) + out = redact_application_yml(content) + # Map headers preserved + assert "spring:" in out + assert "datasource:" in out + assert "jpa:" in out + # Secret redacted, boolean preserved + assert "secret" not in out.replace(REDACTED_PLACEHOLDER, "") + assert "true" in out + + def test_list_items_redacted_if_secret_looking(self): + content = textwrap.dedent("""\ + allowed: + - 8080 + - sk_live_abc123def + - 9090 + """) + out = redact_application_yml(content) + # Numbers preserved, long alphanumeric redacted + assert "8080" in out + assert "9090" in out + assert "sk_live_abc123def" not in out + + def test_comments_preserved(self): + content = textwrap.dedent("""\ + # Server config + server: + port: 8080 # comment after value + """) + out = redact_application_yml(content) + # Note: the line-by-line approach may not preserve inline + # comments perfectly — that's acceptable for Phase 2. The + # leading comment line should be preserved. + assert "# Server config" in out + + def test_idempotent(self): + content = "spring:\n datasource:\n password: hunter2\nserver:\n port: 8080\n" + once = redact_application_yml(content) + twice = redact_application_yml(once) + assert once == twice + + def test_empty_content_returned_unchanged(self): + assert redact_application_yml("") == "" + + +# ─── Shopify {% schema %} ────────────────────────────────────── + + +class TestRedactShopifySchema: + """Shopify Liquid ``{% schema %}`` block redaction.""" + + def test_simple_default_redacted(self): + content = textwrap.dedent("""\ +

{{ section.settings.heading }}

+ {% schema %} + { + "settings": [ + { + "id": "api_key", + "type": "text", + "default": "sk_live_abc123", + "label": "API Key" + } + ] + } + {% endschema %} + """) + out = redact_shopify_schema(content) + assert "sk_live_abc123" not in out + assert REDACTED_PLACEHOLDER in out + # Non-default fields preserved + assert "api_key" in out + assert "API Key" in out + assert "text" in out + # Non-schema HTML preserved + assert "

{{ section.settings.heading }}

" in out + + def test_multiple_defaults_all_redacted(self): + content = textwrap.dedent("""\ + {% schema %} + { + "settings": [ + {"id": "key1", "default": "secret1"}, + {"id": "key2", "default": "secret2"}, + {"id": "key3", "default": "secret3"} + ] + } + {% endschema %} + """) + out = redact_shopify_schema(content) + assert "secret1" not in out + assert "secret2" not in out + assert "secret3" not in out + assert out.count(REDACTED_PLACEHOLDER) == 3 + + def test_numeric_default_preserved(self): + """Number defaults are never secrets — preserve them.""" + content = textwrap.dedent("""\ + {% schema %} + { + "settings": [ + {"id": "count", "type": "number", "default": 5} + ] + } + {% endschema %} + """) + out = redact_shopify_schema(content) + # Number default preserved (not redacted) + assert '"default": 5' in out or '"default":5' in out + assert REDACTED_PLACEHOLDER not in out + + def test_boolean_default_preserved(self): + content = textwrap.dedent("""\ + {% schema %} + { + "settings": [ + {"id": "enabled", "type": "checkbox", "default": true} + ] + } + {% endschema %} + """) + out = redact_shopify_schema(content) + assert "true" in out + assert REDACTED_PLACEHOLDER not in out + + def test_empty_string_default_preserved(self): + """Empty string defaults are not secrets — preserve (don't redact).""" + content = textwrap.dedent("""\ + {% schema %} + { + "settings": [ + {"id": "heading", "default": ""} + ] + } + {% endschema %} + """) + out = redact_shopify_schema(content) + assert REDACTED_PLACEHOLDER not in out + + def test_nested_blocks_redaction_recurses(self): + """``blocks`` array in Shopify schema has nested ``settings``.""" + content = textwrap.dedent("""\ + {% schema %} + { + "blocks": [ + { + "type": "item", + "settings": [ + {"id": "token", "default": "tok_xxx"} + ] + } + ] + } + {% endschema %} + """) + out = redact_shopify_schema(content) + assert "tok_xxx" not in out + assert REDACTED_PLACEHOLDER in out + + def test_malformed_json_replaced_with_comment(self): + """If schema JSON is malformed, replace block with a comment marker.""" + content = "{% schema %}{not valid json{% endschema %}" + out = redact_shopify_schema(content) + # Must NOT contain the raw (potentially secret-bearing) body. + assert "{not valid json" not in out + assert "schema redacted" in out.lower() or "malformed" in out.lower() + + def test_no_schema_block_returns_content_unchanged(self): + """A Liquid file without a schema block is untouched.""" + content = "

Hello

\n

{{ product.title }}

\n" + out = redact_shopify_schema(content) + assert out == content + + def test_multiple_schema_blocks_in_one_file(self): + """Multiple ``{% schema %}`` blocks (rare but valid) all redacted.""" + content = textwrap.dedent("""\ + {% schema %} + {"settings": [{"id": "a", "default": "secret_a"}]} + {% endschema %} +

middle

+ {% schema %} + {"settings": [{"id": "b", "default": "secret_b"}]} + {% endschema %} + """) + out = redact_shopify_schema(content) + assert "secret_a" not in out + assert "secret_b" not in out + assert "

middle

" in out + assert out.count(REDACTED_PLACEHOLDER) == 2 + + def test_case_insensitive_tag_matching(self): + """``{% SCHEMA %}`` and ``{% Schema %}`` should also match.""" + content = "{% SCHEMA %}{\"settings\": [{\"id\": \"a\", \"default\": \"x\"}]}{% ENDSChema %}" + out = redact_shopify_schema(content) + assert "x\"" not in out.replace(REDACTED_PLACEHOLDER, "") + assert REDACTED_PLACEHOLDER in out + + def test_idempotent(self): + content = textwrap.dedent("""\ + {% schema %} + {"settings": [{"id": "a", "default": "secret_a"}]} + {% endschema %} + """) + once = redact_shopify_schema(content) + twice = redact_shopify_schema(once) + assert once == twice + + def test_empty_content_returned_unchanged(self): + assert redact_shopify_schema("") == "" + + +# ─── redact_config_file (auto-detect) ────────────────────────── + + +class TestRedactConfigFile: + """Auto-detection dispatches to the correct per-format redactor.""" + + def test_properties_file_dispatches_to_spring(self, tmp_path): + f = tmp_path / "application.properties" + f.write_text("spring.datasource.password=hunter2\n", encoding="utf-8") + out = redact_config_file(str(f)) + assert "hunter2" not in out + assert REDACTED_PLACEHOLDER in out + + def test_yml_file_dispatches_to_spring_yml(self, tmp_path): + f = tmp_path / "application.yml" + f.write_text("spring:\n datasource:\n password: hunter2\n", encoding="utf-8") + out = redact_config_file(str(f)) + assert "hunter2" not in out + assert REDACTED_PLACEHOLDER in out + + def test_liquid_file_dispatches_to_shopify(self, tmp_path): + f = tmp_path / "theme.liquid" + f.write_text( + '{% schema %}\n{"settings": [{"id": "k", "default": "secret"}]}\n{% endschema %}\n', + encoding="utf-8", + ) + out = redact_config_file(str(f)) + assert "secret" not in out.replace(REDACTED_PLACEHOLDER, "") + assert REDACTED_PLACEHOLDER in out + + def test_non_config_file_returned_unchanged(self, tmp_path): + """A regular source file is NOT redacted.""" + f = tmp_path / "app.py" + original = "PASSWORD = 'hunter2'\n" + f.write_text(original, encoding="utf-8") + out = redact_config_file(str(f)) + assert out == original + + def test_content_arg_skips_read(self, tmp_path): + """When ``content`` is passed, the file is NOT read from disk.""" + f = tmp_path / "application.properties" + f.write_text("dummy_on_disk=hunter2\n", encoding="utf-8") + passed_content = "spring.datasource.password=from_memory\n" + out = redact_config_file(str(f), content=passed_content) + # Should redact the passed-in content, not the on-disk content. + assert "from_memory" not in out + assert "dummy_on_disk" not in out + assert REDACTED_PLACEHOLDER in out + + def test_profile_variant_dispatches_correctly(self, tmp_path): + f = tmp_path / "application-prod.properties" + f.write_text("stripe.api.key=sk_live_xxx\n", encoding="utf-8") + out = redact_config_file(str(f)) + assert "sk_live_xxx" not in out + + +# ─── safe_read_file_for_indexing (integration) ───────────────── + + +class TestSafeReadFileForIndexing: + """End-to-end: path-traversal + redaction in one call.""" + + def test_config_file_redacted(self, tmp_path): + proj = tmp_path / "proj" + proj.mkdir() + f = proj / "application.properties" + f.write_text("spring.datasource.password=hunter2\n", encoding="utf-8") + out = safe_read_file_for_indexing(str(f), str(proj)) + assert out is not None + assert "hunter2" not in out + assert REDACTED_PLACEHOLDER in out + + def test_non_config_file_returned_unredacted(self, tmp_path): + proj = tmp_path / "proj" + proj.mkdir() + f = proj / "app.py" + original = "PASSWORD = 'hunter2'\n" + f.write_text(original, encoding="utf-8") + out = safe_read_file_for_indexing(str(f), str(proj)) + # Non-config files are NOT redacted — that's the secrets_engine's + # job, not the config redactor's. We return content as-is. + assert out == original + + def test_path_traversal_refused(self, tmp_path): + """Phase 1 integration: path escape returns None.""" + proj = tmp_path / "proj" + proj.mkdir() + outside = tmp_path / "outside.txt" + outside.write_text("SECRET", encoding="utf-8") + candidate = str(proj / ".." / "outside.txt") + out = safe_read_file_for_indexing(candidate, str(proj)) + assert out is None + + def test_nonexistent_file_returns_none(self, tmp_path): + proj = tmp_path / "proj" + proj.mkdir() + candidate = str(proj / "application.properties") # doesn't exist + out = safe_read_file_for_indexing(candidate, str(proj)) + assert out is None