security.yml

name: Security Scan

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  schedule:
    - cron: "0 6 * * 1"

permissions:
  contents: read

concurrency:
  group: security-${{ github.ref }}
  cancel-in-progress: true

jobs:
  bandit:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.11"
      - name: Install Bandit
        run: python -m pip install --upgrade pip bandit
      - name: Run Bandit
        run: |
          bandit -r src sdk --ini .bandit --severity-level medium -f json -o /tmp/bandit-current.json || true
          python scripts/bandit_diff.py .bandit-baseline.json /tmp/bandit-current.json

  safety:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.11"
      - name: Resolve Safety dependency inputs
        run: |
          python - <<'PY'
          import os
          import subprocess
          import sys
          import tempfile
          from pathlib import Path
          import tomllib

          root = Path(".")
          output_dir = root / ".tmp-security"
          output_dir.mkdir(exist_ok=True)

          def load_project_dependencies(pyproject_path: Path) -> list[str]:
              if not pyproject_path.exists():
                  return []
              data = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))
              return list(data.get("project", {}).get("dependencies", []))

          def load_requirements(requirements_path: Path) -> list[str]:
              if not requirements_path.exists():
                  return []
              requirements: list[str] = []
              for line in requirements_path.read_text(encoding="utf-8").splitlines():
                  stripped = line.strip()
                  if not stripped or stripped.startswith("#"):
                      continue
                  requirements.append(stripped)
              return requirements

          def write_requirements(target: Path, entries: list[str]) -> None:
              deduped = list(dict.fromkeys(entries))
              target.write_text("\n".join(deduped) + "\n", encoding="utf-8")

          def resolve_requirements(name: str, entries: list[str], target: Path) -> int:
              temp_input = output_dir / f"{name}.in"
              write_requirements(temp_input, entries)

              with tempfile.TemporaryDirectory(prefix=f"safety-{name}-", dir=output_dir) as temp_dir:
                  venv_dir = Path(temp_dir) / "venv"
                  subprocess.run([sys.executable, "-m", "venv", str(venv_dir)], check=True)
                  scripts_dir = venv_dir / ("Scripts" if os.name == "nt" else "bin")
                  python = scripts_dir / ("python.exe" if os.name == "nt" else "python")
                  subprocess.run([str(python), "-m", "pip", "install", "--upgrade", "pip"], check=True)
                  subprocess.run([str(python), "-m", "pip", "install", "-r", str(temp_input)], check=True)
                  freeze = subprocess.run(
                      [str(python), "-m", "pip", "freeze"],
                      check=True,
                      capture_output=True,
                      text=True,
                  )

              resolved = [line for line in freeze.stdout.splitlines() if line.strip()]
              if any("==" not in line for line in resolved):
                  raise SystemExit(f"{name} requirements were not fully resolved")
              target.write_text("\n".join(resolved) + "\n", encoding="utf-8")
              print(f"Resolved {len(resolved)} packages for {name} -> {target}")
              return len(resolved)

          main_count = resolve_requirements(
              "main",
              load_project_dependencies(root / "pyproject.toml")
              + load_requirements(root / "requirements.txt"),
              output_dir / "requirements-main.txt",
          )
          sdk_count = resolve_requirements(
              "sdk",
              load_project_dependencies(root / "sdk" / "pyproject.toml"),
              output_dir / "requirements-sdk.txt",
          )
          # Drop intra-monorepo deps (agentflow-client / agentflow-runtime) — they
          # are not on PyPI yet during the v1.1.0 publish run and are scanned via
          # the "main" / "sdk" buckets above anyway.
          integrations_deps = [
              dep for dep in load_project_dependencies(root / "integrations" / "pyproject.toml")
              if not dep.lower().startswith(("agentflow-client", "agentflow-runtime"))
          ]
          integrations_count = resolve_requirements(
              "integrations",
              integrations_deps,
              output_dir / "requirements-integrations.txt",
          )

          def load_optional_dependencies(pyproject_path: Path, extra: str) -> list[str]:
              if not pyproject_path.exists():
                  return []
              data = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))
              return list(data.get("project", {}).get("optional-dependencies", {}).get(extra, []))

          # Audit P1-3: scan every published/deployed extra, not just the
          # unconditional core dependencies — cloud, postgres, the root
          # "integrations" extra (langchain/langgraph/llama-index; distinct
          # from the standalone integrations/pyproject.toml package resolved
          # above), load, and contract. Each extra resolves in its own
          # venv, same as main/sdk/integrations above, so unrelated extras
          # never have to share one dependency graph. Safety only reads the
          # frozen pins below, so the buckets do not need to be mutually
          # installable.
          extras_counts: dict[str, int] = {}
          for extra_name in ("cloud", "postgres", "integrations", "load", "contract"):
              extras_counts[extra_name] = resolve_requirements(
                  f"extra-{extra_name}",
                  load_optional_dependencies(root / "pyproject.toml", extra_name),
                  output_dir / f"requirements-extra-{extra_name}.txt",
              )

          # The Flink runtime is not an extra (its beam chain can never
          # co-install with core pyarrow>=17) — its manifest lives next to
          # the Flink image Dockerfile. Scan it as its own bucket.
          flink_count = resolve_requirements(
              "flink-runtime",
              load_requirements(
                  root / "src" / "processing" / "flink_jobs" / "requirements.txt"
              ),
              output_dir / "requirements-flink-runtime.txt",
          )

          summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
          if summary_path:
              summary = Path(summary_path)
              with summary.open("a", encoding="utf-8") as handle:
                  handle.write("## Safety dependency scope\n")
                  handle.write(
                      f"- Main app runtime: resolved install of `pyproject.toml` `[project.dependencies]` + `requirements.txt` ({main_count} packages)\n"
                  )
                  handle.write(
                      f"- SDK runtime: resolved install of `sdk/pyproject.toml` `[project.dependencies]` ({sdk_count} packages)\n"
                  )
                  handle.write(
                      f"- Integrations runtime: resolved install of `integrations/pyproject.toml` `[project.dependencies]` ({integrations_count} packages)\n"
                  )
                  for extra_name, extra_count in extras_counts.items():
                      handle.write(
                          f"- Extra `[{extra_name}]` (root `pyproject.toml`): resolved install ({extra_count} packages)\n"
                      )
                  handle.write(
                      f"- Flink runtime (`src/processing/flink_jobs/requirements.txt`): resolved install ({flink_count} packages)\n"
                  )
                  handle.write("- Exclusions: dev/CI/test extras, local tooling, and unrelated Docker image packages\n")
          PY
      - name: Install Safety
        run: python -m pip install --upgrade pip "safety<3"
      - name: Verify Safety fails on a known vulnerable pin
        run: |
          printf 'urllib3==1.24.1\n' > .tmp-security/requirements-regression.txt
          if safety check -r .tmp-security/requirements-regression.txt > .tmp-security/safety-regression.log 2>&1; then
            cat .tmp-security/safety-regression.log
            echo "Safety unexpectedly passed the vulnerable regression probe"
            exit 1
          fi
          if ! grep -q "urllib3" .tmp-security/safety-regression.log; then
            cat .tmp-security/safety-regression.log
            echo "Safety failed the regression probe for an unexpected reason"
            exit 1
          fi
      - name: Run Safety
        run: |
          # SFTY-20260217-93940 (CVE-2026-25087): use-after-free in Arrow
          # C++ 15.0.0-23.0.0, fixed in 23.0.1. Only the flink-runtime
          # bucket resolves an affected pyarrow (apache-flink's beam chain
          # caps pyarrow<17; the API image and every extra run >=23.0.1 via
          # uv.lock, gated by pip-audit). Why the risk is negligible here
          # (GHSA-rgxp-2hwp-jwgg, verified 2026-07-20):
          #   1. The vulnerable API is the C++ IPC *file* reader with
          #      pre-buffering enabled (RecordBatchFileReader::
          #      PreBufferMetadata, off by default). Per the advisory, "the
          #      functionality is not exposed in language bindings (Python,
          #      Ruby, C GLib), so these bindings are not vulnerable" — the
          #      pyarrow wheel this bucket installs cannot reach the bug.
          #   2. Our jobs are DataStream STRING/pickle only; PyFlink 2.3.0
          #      imports pyarrow lazily and solely for Table/pandas Arrow
          #      coders (fn_execution/coders.py), whose decode path is
          #      pa.ipc.open_stream — the IPC *stream* reader, which the
          #      advisory explicitly excludes. pyarrow is installed but
          #      never imported on our code path.
          # Blocked upstream, not by our pin: apache-flink 2.3.0 is the
          # newest PyPI release (checked 2026-07-20) and itself caps
          # pyarrow<21; beam accepts pyarrow<24 only from 2.75.0, which no
          # released apache-flink allows. Dependabot watches the Flink
          # manifest (src/processing/flink_jobs), so the next apache-flink
          # release opens a PR — re-check this ignore there. Remove when the
          # resolved flink-runtime bucket installs pyarrow>=23.0.1.
          # Do NOT retry "uninstall unused pyarrow from the image" — probed
          # in the real image 2026-07-21 and rejected: without pyarrow,
          # `import apache_beam` itself crashes (beam 2.61 io/__init__ does
          # `from apache_beam.io.parquetio import *`, and parquetio's class
          # bodies evaluate `pa.Table` annotations with pa=None ->
          # AttributeError). The dependency is load-bearing at import time
          # even though our jobs never use it.
          #
          # 88512 (langchain cross-ecosystem false positive) was ignored here
          # until 2026-07-20; PyUp corrected the entry (verified: safety
          # 2.3.5 reports 0 findings on the pinned langchain stack), so the
          # ignore is gone. If it ever resurfaces, it fails this job loudly.
          safety check \
            --ignore SFTY-20260217-93940 \
            -r .tmp-security/requirements-main.txt \
            -r .tmp-security/requirements-sdk.txt \
            -r .tmp-security/requirements-integrations.txt \
            -r .tmp-security/requirements-extra-cloud.txt \
            -r .tmp-security/requirements-extra-postgres.txt \
            -r .tmp-security/requirements-extra-integrations.txt \
            -r .tmp-security/requirements-flink-runtime.txt \
            -r .tmp-security/requirements-extra-load.txt \
            -r .tmp-security/requirements-extra-contract.txt

  # Audit P1-3: pip-audit against the hash-pinned export of uv.lock. The
  # unlocked run could not even finish resolving in three minutes; with the
  # complete pinned set (ci.yml lock-check proves completeness) there is
  # nothing to resolve — every pin is checked against the advisory DBs
  # directly.
  pip-audit:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.11"
      - name: Install pip-audit
        run: python -m pip install --upgrade pip "pip-audit>=2.7,<3"
      - name: Audit the locked production dependency set
        run: pip-audit --no-deps -r requirements-docker.lock

  npm-audit:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: "20"
      - name: Install TS SDK deps from lockfile
        working-directory: sdk-ts
        run: npm ci
      - name: npm audit
        working-directory: sdk-ts
        run: npm audit --audit-level=moderate

  trivy:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    permissions:
      contents: read
      security-events: write
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Build API image
        env:
          COMPOSE_PROJECT_NAME: agentflow-security
          # docker compose validates env vars across all services even when
          # building only one. agentflow-api itself does not consume these.
          CLICKHOUSE_USER: scan-only
          CLICKHOUSE_PASSWORD: scan-only
          GF_SECURITY_ADMIN_USER: scan-only
          GF_SECURITY_ADMIN_PASSWORD: scan-only
        run: |
          docker compose -f docker-compose.prod.yml build agentflow-api
          if (-not (docker image inspect agentflow-security-agentflow-api:latest 2>$null)) {
            throw "agentflow-security-agentflow-api:latest was not built"
          }
          docker tag agentflow-security-agentflow-api:latest agentflow-api:security-scan
        shell: pwsh
      - name: Generate CycloneDX SBOM
        # Pinned to release tag (audit p9 #3); update by bumping the
        # tag, never re-pin to @master (allows upstream to alter scanner).
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          image-ref: agentflow-api:security-scan
          format: cyclonedx
          output: agentflow-api.cdx.json
      - name: Upload CycloneDX SBOM
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: agentflow-api-sbom-cyclonedx
          path: agentflow-api.cdx.json
          if-no-files-found: error
      - name: Run Trivy scan
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          image-ref: agentflow-api:security-scan
          format: sarif
          output: trivy-results.sarif
          severity: HIGH,CRITICAL
          ignore-unfixed: true
          exit-code: "1"
          # Without this, trivy-action builds the SARIF "with all severities"
          # and drops BOTH filters above from the scan the exit code comes
          # from — an unfixable MEDIUM in the base image (e.g. liblzma5
          # CVE-2026-34743, no Debian fix available) fails the gate that
          # declares itself HIGH,CRITICAL-only. This makes the declared
          # filters real for the SARIF scan too.
          limit-severities-for-sarif: true
      - name: Upload Trivy scan results
        if: always()
        uses: github/codeql-action/upload-sarif@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1
        with:
          sarif_file: trivy-results.sarif

  iac:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      security-events: write
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Run Trivy IaC misconfiguration scan
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          scan-type: config
          scan-ref: infrastructure/terraform
          format: sarif
          output: trivy-iac.sarif
          severity: MEDIUM,HIGH,CRITICAL
          exit-code: "1"
          limit-severities-for-sarif: true
      - name: Upload Trivy IaC scan results
        if: always()
        uses: github/codeql-action/upload-sarif@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1
        with:
          sarif_file: trivy-iac.sarif
          # Distinct category so these alerts do not collide with the image
          # scan's SARIF upload in the trivy job.
          category: trivy-iac