From 589d814578d46d9ae1118d28fec10602e92df277 Mon Sep 17 00:00:00 2001 From: Joey Shi Date: Mon, 13 Jul 2026 11:51:42 -0700 Subject: [PATCH 1/5] fix(sandbox): exclude home-dir dotfiles from capture_to artifacts --- src/coder_eval/sandbox.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 0804d979..0aa2b6ad 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -27,7 +27,25 @@ # sandbox-created bulk. `.claude` is the RW lean copy of ~/.claude (carries # .credentials.json). The rest are sandbox infra the standard artifacts path # also drops. Matched by basename at every level (shutil.ignore_patterns). -_WORKSPACE_CAPTURE_IGNORE = (".claude", ".venv", ".npm-prefix", "node_modules") +_WORKSPACE_CAPTURE_IGNORE = ( + # Credentials / Claude infra + ".claude", + # Python / JS build infra + ".venv", + ".npm-prefix", + "node_modules", + # Home-directory caches & config (created by uv, pip, npm, etc. when WORKDIR=HOME) + ".cache", + ".config", + ".npm", + ".local", + # Shell dotfiles pre-baked into the image + ".bashrc", + ".bash_history", + ".bash_logout", + ".profile", + ".wget-hsts", +) def _grant_read_traverse(root: Path) -> None: @@ -1080,8 +1098,10 @@ def capture_to(self, artifact_dir: Path) -> Path: :data:`_WORKSPACE_CAPTURE_IGNORE` -- most importantly ``.claude`` (the RW lean copy of the host ``~/.claude`` carries ``.credentials.json``; without this a ``/root`` WORKDIR would leak it into artifacts), plus - ``.venv``/``node_modules``/``.npm-prefix`` (sandbox-created bulk, already - stripped by the standard artifacts path's post-run cleanup). + ``.venv``/``node_modules``/``.npm-prefix`` (sandbox-created bulk), and + Linux home-directory noise (``.cache``, ``.config``, ``.npm``, + ``.local``, shell dotfiles) written by tools like uv/pip/npm when + HOME == WORKDIR. Returns the destination path; unlike preserve_to it does NOT repoint ``self.sandbox_dir`` -- the workspace persists in-container and is reaped From d45d4166ac6af1bacc593b73e3ef6d62c0d7437c Mon Sep 17 00:00:00 2001 From: Joey Shi Date: Mon, 13 Jul 2026 11:54:38 -0700 Subject: [PATCH 2/5] test(sandbox): cover home-dir dotfile exclusion in capture_to Extends test_capture_to_copies_and_tolerates_dangling_symlink to assert that .cache, .config, .npm, .local, .bashrc, .bash_history, .bash_logout, .profile, and .wget-hsts are excluded from captured artifacts. Co-Authored-By: Claude Sonnet 4.6 --- tests/test_sandbox.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 83176b8f..4b7567f0 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -1203,6 +1203,18 @@ def test_capture_to_copies_and_tolerates_dangling_symlink(tmp_path): (ws / ".venv" / "pyvenv.cfg").write_text("x", encoding="utf-8") (ws / "node_modules").mkdir() (ws / "node_modules" / "pkg.js").write_text("x", encoding="utf-8") + # Home-dir noise written by uv/pip/npm/shell when WORKDIR == HOME (/root). + (ws / ".cache").mkdir() + (ws / ".cache" / "uv").mkdir() + (ws / ".config").mkdir() + (ws / ".config" / "uv").mkdir() + (ws / ".npm").mkdir() + (ws / ".local").mkdir() + (ws / ".bashrc").write_text("# bash", encoding="utf-8") + (ws / ".bash_history").write_text("ls\n", encoding="utf-8") + (ws / ".bash_logout").write_text("# logout", encoding="utf-8") + (ws / ".profile").write_text("# profile", encoding="utf-8") + (ws / ".wget-hsts").write_text("", encoding="utf-8") artifacts = tmp_path / "artifacts" dest = sandbox.capture_to(artifacts) @@ -1215,6 +1227,16 @@ def test_capture_to_copies_and_tolerates_dangling_symlink(tmp_path): assert not (dest / ".claude").exists() assert not (dest / ".venv").exists() assert not (dest / "node_modules").exists() + # Home-dir noise excluded (written by tools when WORKDIR == HOME). + assert not (dest / ".cache").exists() + assert not (dest / ".config").exists() + assert not (dest / ".npm").exists() + assert not (dest / ".local").exists() + assert not (dest / ".bashrc").exists() + assert not (dest / ".bash_history").exists() + assert not (dest / ".bash_logout").exists() + assert not (dest / ".profile").exists() + assert not (dest / ".wget-hsts").exists() # Source workspace is COPIED, not moved (originals untouched). assert (ws / "real.txt").exists() assert (ws / ".claude" / ".credentials.json").exists() From e096b65aa12083eb4b6ad7876e208bda5c49927b Mon Sep 17 00:00:00 2001 From: Joey Shi Date: Mon, 13 Jul 2026 13:33:01 -0700 Subject: [PATCH 3/5] fix(deps): upgrade click to 8.4.2 to resolve PYSEC-2026-2132 click 8.3.1 has a command injection vulnerability in click.edit() (PYSEC-2026-2132, fixed in 8.3.3). Pin click>=8.3.3 explicitly so uv resolves to 8.4.2. Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 1 + uv.lock | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index de423a0e..0d25d99b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "pydantic-settings>=2.14.2", "pyyaml>=6.0.3", "typer>=0.24.1", + "click>=8.3.3", "rich>=14.3.3", "python-dotenv>=1.2.2", "anthropic>=0.86.0", diff --git a/uv.lock b/uv.lock index 3a5d429c..1c9baa98 100644 --- a/uv.lock +++ b/uv.lock @@ -431,14 +431,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.1" +version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] [[package]] @@ -450,6 +450,7 @@ dependencies = [ { name = "anyio" }, { name = "azure-monitor-opentelemetry-exporter" }, { name = "claude-agent-sdk" }, + { name = "click" }, { name = "jmespath" }, { name = "jsonschema" }, { name = "opentelemetry-sdk" }, @@ -495,6 +496,7 @@ requires-dist = [ { name = "azure-monitor-opentelemetry-exporter", specifier = ">=1.0.0b30,<1.1.0" }, { name = "bandit", extras = ["toml"], marker = "extra == 'dev'", specifier = ">=1.9.4" }, { name = "claude-agent-sdk", specifier = ">=0.2.82" }, + { name = "click", specifier = ">=8.3.3" }, { name = "google-antigravity", marker = "extra == 'antigravity'", specifier = "==0.1.5" }, { name = "jmespath", specifier = ">=1.1.0" }, { name = "jsonschema", specifier = ">=4.26.0" }, From 424632e0ee8d39f48bcc1029a272f08d6a461b27 Mon Sep 17 00:00:00 2001 From: Joey Shi Date: Mon, 13 Jul 2026 14:19:58 -0700 Subject: [PATCH 4/5] fix(sandbox): extend capture_to denylist with credential stores Adds .aws, .ssh, .gnupg, .docker, .azure, .netrc, .gitconfig to _WORKSPACE_CAPTURE_IGNORE as a security denylist (defense-in-depth: eval images don't bake credentials, but any image that does must not leak them into uploaded artifacts). Also restructures the comment to distinguish the two exclusion classes: security denylist vs. home-dir noise suppression. Co-Authored-By: Claude Sonnet 4.6 --- src/coder_eval/sandbox.py | 35 +++++++++++++++++++++++++---------- tests/test_sandbox.py | 27 +++++++++++++++++++++++---- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 0aa2b6ad..1ec60528 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -22,24 +22,39 @@ logger = logging.getLogger(__name__) -# Top-level entries excluded from Sandbox.capture_to (docker WORKDIR-alignment). -# The WORKDIR can be HOME or overlap framework mounts, so skip credentials and -# sandbox-created bulk. `.claude` is the RW lean copy of ~/.claude (carries -# .credentials.json). The rest are sandbox infra the standard artifacts path -# also drops. Matched by basename at every level (shutil.ignore_patterns). +# Entries excluded from Sandbox.capture_to (docker WORKDIR-alignment). +# Two classes of exclusion: +# +# 1. SECURITY denylist: credential files/dirs that must never leak into +# captured artifacts (which get uploaded). Defense-in-depth -- the eval +# images don't bake credentials, but any future image that does should +# not silently expose them. +# +# 2. NOISE suppression: sandbox-created bulk and home-dir infrastructure +# written by tools (uv, pip, npm, shell) when WORKDIR overlaps HOME +# (e.g. /root). These are never task deliverables. +# +# Matched by basename at every level via shutil.ignore_patterns. _WORKSPACE_CAPTURE_IGNORE = ( - # Credentials / Claude infra - ".claude", - # Python / JS build infra + # --- Security: credential stores --- + ".claude", # RW lean copy of host ~/.claude (carries .credentials.json) + ".aws", # AWS credentials / config + ".ssh", # SSH keys + ".gnupg", # GPG keys + ".docker", # Docker auth (config.json) + ".azure", # Azure CLI credentials + ".netrc", # FTP/curl/git credentials + ".gitconfig", # May embed PATs via credential.helper + # --- Noise: Python / JS build infra --- ".venv", ".npm-prefix", "node_modules", - # Home-directory caches & config (created by uv, pip, npm, etc. when WORKDIR=HOME) + # --- Noise: home-dir caches & config (uv, pip, npm, etc.) --- ".cache", ".config", ".npm", ".local", - # Shell dotfiles pre-baked into the image + # --- Noise: shell dotfiles pre-baked into the image --- ".bashrc", ".bash_history", ".bash_logout", diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 4b7567f0..c606fa92 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -1196,14 +1196,25 @@ def test_capture_to_copies_and_tolerates_dangling_symlink(tmp_path): (ws / "real.txt").write_text("hello", encoding="utf-8") # A dangling symlink is the exact failure the old `cp -a` bridge hit. (ws / "dangling").symlink_to(ws / "does_not_exist") - # Framework/sensitive entries that must NOT be captured (WORKDIR may be HOME). + # Security denylist: credential stores that must never leak into artifacts. (ws / ".claude").mkdir() (ws / ".claude" / ".credentials.json").write_text("SECRET", encoding="utf-8") + (ws / ".aws").mkdir() + (ws / ".aws" / "credentials").write_text("[default]\naws_access_key_id=FAKE", encoding="utf-8") + (ws / ".ssh").mkdir() + (ws / ".ssh" / "id_rsa").write_text("PRIVATE KEY", encoding="utf-8") + (ws / ".gnupg").mkdir() + (ws / ".docker").mkdir() + (ws / ".docker" / "config.json").write_text('{"auths":{}}', encoding="utf-8") + (ws / ".azure").mkdir() + (ws / ".netrc").write_text("machine github.com login user password TOKEN", encoding="utf-8") + (ws / ".gitconfig").write_text("[user]\n\tname = Test", encoding="utf-8") + # Noise: Python / JS build infra. (ws / ".venv").mkdir() (ws / ".venv" / "pyvenv.cfg").write_text("x", encoding="utf-8") (ws / "node_modules").mkdir() (ws / "node_modules" / "pkg.js").write_text("x", encoding="utf-8") - # Home-dir noise written by uv/pip/npm/shell when WORKDIR == HOME (/root). + # Noise: home-dir caches written by uv/pip/npm/shell when WORKDIR == HOME. (ws / ".cache").mkdir() (ws / ".cache" / "uv").mkdir() (ws / ".config").mkdir() @@ -1223,11 +1234,19 @@ def test_capture_to_copies_and_tolerates_dangling_symlink(tmp_path): assert (dest / "real.txt").read_text(encoding="utf-8") == "hello" # Dangling symlink skipped (ignore_dangling_symlinks) -> did not raise. assert not (dest / "dangling").exists() - # Credentials / sandbox bulk excluded from the captured artifacts. + # Security denylist: no credential stores in artifacts. assert not (dest / ".claude").exists() + assert not (dest / ".aws").exists() + assert not (dest / ".ssh").exists() + assert not (dest / ".gnupg").exists() + assert not (dest / ".docker").exists() + assert not (dest / ".azure").exists() + assert not (dest / ".netrc").exists() + assert not (dest / ".gitconfig").exists() + # Noise: build infra excluded. assert not (dest / ".venv").exists() assert not (dest / "node_modules").exists() - # Home-dir noise excluded (written by tools when WORKDIR == HOME). + # Noise: home-dir bulk excluded. assert not (dest / ".cache").exists() assert not (dest / ".config").exists() assert not (dest / ".npm").exists() From ebea7ec8bb3cfaba1927709b3d5aea2de3bb6482 Mon Sep 17 00:00:00 2001 From: Joey Shi Date: Mon, 13 Jul 2026 14:22:57 -0700 Subject: [PATCH 5/5] style: fix ruff formatting in _WORKSPACE_CAPTURE_IGNORE Co-Authored-By: Claude Sonnet 4.6 --- src/coder_eval/sandbox.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 1ec60528..07217061 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -37,14 +37,14 @@ # Matched by basename at every level via shutil.ignore_patterns. _WORKSPACE_CAPTURE_IGNORE = ( # --- Security: credential stores --- - ".claude", # RW lean copy of host ~/.claude (carries .credentials.json) - ".aws", # AWS credentials / config - ".ssh", # SSH keys - ".gnupg", # GPG keys - ".docker", # Docker auth (config.json) - ".azure", # Azure CLI credentials - ".netrc", # FTP/curl/git credentials - ".gitconfig", # May embed PATs via credential.helper + ".claude", # RW lean copy of host ~/.claude (carries .credentials.json) + ".aws", # AWS credentials / config + ".ssh", # SSH keys + ".gnupg", # GPG keys + ".docker", # Docker auth (config.json) + ".azure", # Azure CLI credentials + ".netrc", # FTP/curl/git credentials + ".gitconfig", # May embed PATs via credential.helper # --- Noise: Python / JS build infra --- ".venv", ".npm-prefix",