diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c68b6e4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,76 @@ +name: CI + +on: + push: + pull_request: + +jobs: + # ------------------------------------------------------------------ + # Job 1: run pytest — every branch, every PR + # ------------------------------------------------------------------ + test: + name: pytest + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Run tests + run: pytest tests/ -v + + # ------------------------------------------------------------------ + # Job 2: build & push Docker image to GHCR + # Runs only on pushes to main or tag pushes; requires tests to pass. + # ------------------------------------------------------------------ + build-and-push: + name: Build and push Docker image + runs-on: ubuntu-latest + needs: test + if: | + github.event_name == 'push' && + (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) + + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Derive image tags + id: tags + run: | + REGISTRY="ghcr.io/${{ github.repository }}" + REF_SLUG="${GITHUB_REF_NAME//\//-}" # replace '/' with '-' for branch names + SHORT_SHA="${GITHUB_SHA::8}" + + TAGS="${REGISTRY}:${REF_SLUG},${REGISTRY}:${SHORT_SHA}" + + # Add :latest only for pushes to main + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + TAGS="${TAGS},${REGISTRY}:latest" + fi + + echo "tags=${TAGS}" >> "$GITHUB_OUTPUT" + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.tags.outputs.tags }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index ef29b4d..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,74 +0,0 @@ -# .gitlab-ci.yml - -# Define the stages for your CI/CD pipeline -stages: - - build_and_push_docker - -# Define the Docker service required for building and pushing images -# This uses the Docker-in-Docker (dind) service. -# The 'docker:latest' image is used for the runner environment itself. -image: docker:latest - -services: - - docker:dind - -# Before any script runs in the job, execute these commands. -# This section handles logging into the GitLab Container Registry. -before_script: - # Print a message indicating the login attempt - - echo "Attempting to log in to GitLab Container Registry..." - # Log in to the GitLab Container Registry using predefined CI/CD variables. - # CI_REGISTRY is the address of the registry (e.g., registry.gitlab.com). - # CI_REGISTRY_USER is 'gitlab-ci-token'. - # CI_REGISTRY_PASSWORD holds the unique token for the running job. - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" - -# Define the job for building and pushing the Docker image -build_and_push: - stage: build_and_push_docker - # Rules define when this job should run. - # This job will run for commits on the 'main' branch, or for any tags. - rules: - - if: '$CI_COMMIT_BRANCH == "main"' # Runs on 'main' branch - - if: '$CI_COMMIT_TAG' # Runs when a tag is pushed - - script: - # Define the image name using predefined CI/CD variables. - # CI_REGISTRY_IMAGE is the full path to your project's image repository - # in the GitLab Container Registry (e.g., registry.gitlab.com/your-group/your-project). - # CI_COMMIT_REF_SLUG is a slug-friendly version of the branch or tag name. - # CI_COMMIT_SHORT_SHA is the first 8 characters of the commit hash. - - IMAGE_TAG_BRANCH="$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" - - IMAGE_TAG_SHA="$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA" - - IMAGE_TAG_LATEST="$CI_REGISTRY_IMAGE:latest" - - # Build the Docker image. - # -t: Tags the image. We tag it with the branch/tag name and the short commit SHA. - # .: Indicates that the Dockerfile is in the current directory (root of the project). - # - echo "Building Docker image with tags: $IMAGE_TAG_BRANCH, $IMAGE_TAG_SHA' - - docker build -t "$IMAGE_TAG_BRANCH" -t "$IMAGE_TAG_SHA" . - - # Push the Docker image with the branch/tag name. - # - echo "Pushing image: $IMAGE_TAG_BRANCH" - - docker push "$IMAGE_TAG_BRANCH" - - # Push the Docker image with the short commit SHA. - # - echo "Pushing image: $IMAGE_TAG_SHA" - - docker push "$IMAGE_TAG_SHA" - - # Conditionally push the 'latest' tag. - # This will tag the image with 'latest' only if the commit is on the 'main' branch. - - | - if [ "$CI_COMMIT_BRANCH" == "main" ]; then - # echo "Tagging and pushing 'latest' for main branch: $IMAGE_TAG_LATEST"; - docker tag "$IMAGE_TAG_BRANCH" "$IMAGE_TAG_LATEST"; - docker push "$IMAGE_TAG_LATEST"; - fi - - # Cache Docker layers to speed up subsequent builds. - # This creates a cache specifically for the Docker build context. - cache: - key: "$CI_COMMIT_REF_SLUG" # Cache per branch/tag - paths: - - .docker_cache/ # Directory where Docker build cache will be stored - policy: pull-push diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2b418d4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: + - repo: local + hooks: + - id: pytest + name: pytest + entry: pytest tests/ + language: system + pass_filenames: false + always_run: true diff --git a/agents.py b/agents.py index 1815bc8..ef1339e 100644 --- a/agents.py +++ b/agents.py @@ -1,7 +1,9 @@ +from typing import Callable + from omegaconf import DictConfig from smolagents import CodeAgent, OpenAIServerModel, WikipediaSearchTool, tool, LogLevel -from tools import opencti_tools, osint_tools, todays_date +from tools import opencti_tools, opencti_write_tools, osint_tools, attack_tools, todays_date def create_llm_model(model_id: str, api_key: str, api_base: str = "https://openrouter.ai/api/v1") -> OpenAIServerModel: @@ -14,6 +16,19 @@ def create_llm_model(model_id: str, api_key: str, api_base: str = "https://openr max_tokens=10_000, ) + +# Registry: maps a tool-set name to a factory that returns a list of tool instances. +# Adding a new tool group = one dict entry. +_TOOL_REGISTRY: dict[str, Callable[[], list]] = { + "opencti": lambda: [tool(t) for t in opencti_tools], + "opencti_write": lambda: [tool(t) for t in opencti_write_tools], + "osint": lambda: [tool(t) for t in osint_tools], + "attack": lambda: [tool(t) for t in attack_tools], + "wikipedia": lambda: [WikipediaSearchTool()], + "todays_date": lambda: [tool(todays_date)], +} + + def get_agent_tools(tool_names: list[str] | str) -> list: """Gets the tool instances from their names.""" if isinstance(tool_names, str): @@ -21,61 +36,74 @@ def get_agent_tools(tool_names: list[str] | str) -> list: tools = [] for name in tool_names: - if name == "opencti": - tools.extend([tool(t) for t in opencti_tools]) - elif name == "osint": - tools.extend([tool(t) for t in osint_tools]) - elif name == "wikipedia": - tools.append(WikipediaSearchTool()) - elif name == "todays_date": - tools.append(tool(todays_date)) - else: + factory = _TOOL_REGISTRY.get(name) + if factory is None: raise ValueError(f"Unknown tool name: {name}") + tools.extend(factory()) return tools -def create_agent(cfg: DictConfig, model: OpenAIServerModel): - """Creates an agent instance based on the hydra configuration.""" - # Look up the agent configuration from the library using the agent_name - agent_cfg = cfg.agent_lib[cfg.agent_name] - agent_type = agent_cfg.type - if agent_type == "simple": - blueprint = cfg.agent_definitions[agent_cfg.blueprint] - return CodeAgent( +# --------------------------------------------------------------------------- +# Private builder functions — one per agent type +# --------------------------------------------------------------------------- + +def _build_simple(cfg: DictConfig, agent_cfg, model: OpenAIServerModel) -> CodeAgent: + blueprint = cfg.agent_definitions[agent_cfg.blueprint] + return CodeAgent( + model=model, + tools=get_agent_tools(blueprint.tools), + verbosity_level=LogLevel.DEBUG, + name=blueprint.name, + description=blueprint.description, + ) + + +def _build_all_in_one(cfg: DictConfig, agent_cfg, model: OpenAIServerModel) -> CodeAgent: + return CodeAgent( + model=model, + tools=get_agent_tools(["opencti", "osint", "wikipedia", "todays_date"]), + verbosity_level=LogLevel.DEBUG, + planning_interval=3, + name="all_task_agent", + ) + + +def _build_managing(cfg: DictConfig, agent_cfg, model: OpenAIServerModel) -> CodeAgent: + managed_agents = [] + for agent_key in agent_cfg.managed_agents: + blueprint = cfg.agent_definitions[agent_key] + managed_agent = CodeAgent( model=model, tools=get_agent_tools(blueprint.tools), verbosity_level=LogLevel.DEBUG, name=blueprint.name, description=blueprint.description, ) - elif agent_type == "all_in_one": - return CodeAgent( - model=model, - tools=get_agent_tools(["opencti", "osint", "wikipedia", "todays_date"]), - verbosity_level=LogLevel.DEBUG, - planning_interval=3, - name="all_task_agent", - ) - elif agent_type == "managing": - managed_agents = [] - for agent_key in agent_cfg.managed_agents: - blueprint = cfg.agent_definitions[agent_key] - managed_agent = CodeAgent( - model=model, - tools=get_agent_tools(blueprint.tools), - verbosity_level=LogLevel.DEBUG, - name=blueprint.name, - description=blueprint.description, - ) - managed_agents.append(managed_agent) - - return CodeAgent( - model=model, - tools=[tool(todays_date)], - verbosity_level=LogLevel.DEBUG, - planning_interval=3, - managed_agents=managed_agents, - name="managing_agent", - ) - else: - raise ValueError(f"Unknown agent type: {agent_type}") \ No newline at end of file + managed_agents.append(managed_agent) + + return CodeAgent( + model=model, + tools=[tool(todays_date)], + verbosity_level=LogLevel.DEBUG, + planning_interval=3, + managed_agents=managed_agents, + name="managing_agent", + ) + + +# Registry: adding a new agent type = one dict entry +_BUILDERS: dict[str, Callable] = { + "simple": _build_simple, + "all_in_one": _build_all_in_one, + "managing": _build_managing, +} + + +def create_agent(cfg: DictConfig, model: OpenAIServerModel) -> CodeAgent: + """Creates an agent instance based on the hydra configuration.""" + agent_cfg = cfg.agent_lib[cfg.agent_name] + builder = _BUILDERS.get(agent_cfg.type) + if builder is None: + raise ValueError(f"Unknown agent type: {agent_cfg.type}") + return builder(cfg, agent_cfg, model) + diff --git a/api.py b/api.py new file mode 100644 index 0000000..e2c9c32 --- /dev/null +++ b/api.py @@ -0,0 +1,176 @@ +""" +FastAPI REST interface for agentic-cti. + +Exposes the agent system over HTTP so SIEM/SOAR platforms and automation pipelines +can query threat intelligence without the Gradio UI. + +Endpoints: + POST /query — Submit a task; returns a job ID immediately (async) + GET /results/{id} — Poll for results by job ID + GET /health — Liveness check + +Usage: + uvicorn api:app --host 0.0.0.0 --port 8000 + +Environment variables required: + OPENROUTER_API_KEY, OPENCTI_URL, OPENCTI_TOKEN, ALIENVAULT_API_KEY +""" + +import asyncio +import logging +import os +import uuid +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException +from omegaconf import OmegaConf +from pydantic import BaseModel + +from agents import create_agent, create_llm_model + +load_dotenv() +log = logging.getLogger(__name__) + +app = FastAPI( + title="Agentic CTI API", + description="REST interface for the agentic threat intelligence platform.", + version="1.0.0", +) + +# In-memory job store — replace with Redis or a database for multi-process deployments +_jobs: Dict[str, Dict[str, Any]] = {} + +# Load the dashboard config once at startup so agents can be created per request +_cfg = OmegaConf.load(os.path.join(os.path.dirname(__file__), "conf", "dashboard.yaml")) +# Resolve env vars embedded in the config +OmegaConf.resolve(_cfg) + + +# --------------------------------------------------------------------------- +# Request / Response schemas +# --------------------------------------------------------------------------- + +class QueryRequest(BaseModel): + task: str + agent: str = "managing_agent" + model: str = "ms_ds_r1" + + +class QueryResponse(BaseModel): + id: str + status: str + message: str + + +class ResultResponse(BaseModel): + id: str + status: str + task: Optional[str] = None + agent: Optional[str] = None + model: Optional[str] = None + result: Optional[str] = None + submitted_at: Optional[str] = None + completed_at: Optional[str] = None + error: Optional[str] = None + + +# --------------------------------------------------------------------------- +# Background task runner +# --------------------------------------------------------------------------- + +async def _execute_job(job_id: str, request: QueryRequest) -> None: + """Runs the agent in a thread pool so it doesn't block the event loop.""" + try: + model_id = _cfg.model_lib[request.model].id + api_key = _cfg.api_key + + # Merge requested agent name into config for create_agent + job_cfg = OmegaConf.merge(_cfg, {"agent_name": request.agent}) + + def _run() -> str: + model = create_llm_model(model_id=model_id, api_key=api_key) + agent = create_agent(job_cfg, model) + return agent.run(request.task) + + loop = asyncio.get_event_loop() + result = await loop.run_in_executor(None, _run) + + _jobs[job_id].update({ + "status": "completed", + "result": result, + "completed_at": datetime.now(timezone.utc).isoformat(), + }) + log.info("Job %s completed.", job_id) + + except Exception as exc: + _jobs[job_id].update({ + "status": "failed", + "error": str(exc), + "completed_at": datetime.now(timezone.utc).isoformat(), + }) + log.exception("Job %s failed: %s", job_id, exc) + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + +@app.get("/health") +def health() -> Dict[str, str]: + """Liveness check.""" + return {"status": "ok"} + + +@app.post("/query", response_model=QueryResponse, status_code=202) +async def submit_query(request: QueryRequest) -> QueryResponse: + """ + Submit a threat intelligence task to the agent system. + + The task runs asynchronously. Poll GET /results/{id} to retrieve the output. + + - **task**: Natural-language question or instruction for the agent. + - **agent**: Agent configuration name from conf/dashboard.yaml (default: managing_agent). + - **model**: Model name from the model_lib in conf/dashboard.yaml (default: ms_ds_r1). + """ + if request.agent not in _cfg.agent_lib: + raise HTTPException(status_code=400, detail=f"Unknown agent '{request.agent}'. " + f"Valid agents: {list(_cfg.agent_lib.keys())}") + if request.model not in _cfg.model_lib: + raise HTTPException(status_code=400, detail=f"Unknown model '{request.model}'. " + f"Valid models: {list(_cfg.model_lib.keys())}") + + job_id = str(uuid.uuid4()) + _jobs[job_id] = { + "id": job_id, + "status": "running", + "task": request.task, + "agent": request.agent, + "model": request.model, + "result": None, + "error": None, + "submitted_at": datetime.now(timezone.utc).isoformat(), + "completed_at": None, + } + + asyncio.create_task(_execute_job(job_id, request)) + + return QueryResponse(id=job_id, status="running", + message=f"Job submitted. Poll GET /results/{job_id} for output.") + + +@app.get("/results/{job_id}", response_model=ResultResponse) +def get_result(job_id: str) -> ResultResponse: + """ + Retrieve the result of a previously submitted query. + + **status** values: + - `running` — agent is still executing + - `completed` — result is available in the `result` field + - `failed` — execution failed; see the `error` field + """ + job = _jobs.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found.") + return ResultResponse(**job) diff --git a/conf/dashboard.yaml b/conf/dashboard.yaml index 351f974..dab5f1a 100644 --- a/conf/dashboard.yaml +++ b/conf/dashboard.yaml @@ -27,10 +27,18 @@ agent_definitions: name: "OpenCTI_search" description: "Can search OpenCTI data for threat intelligence data. OpenCTI holds cyber threat intelligence knowledge and observables. It has been created in order to structure, store, organize and visualize technical and non-technical information about cyber threats." tools: "opencti" + opencti_writer: + name: "OpenCTI_writer" + description: "Can persist intelligence back into OpenCTI by creating Reports, Notes, Indicators (IOCs), and Relationships between objects. Use this agent to save synthesised threat intelligence so it is available to the broader team." + tools: "opencti_write" osint: name: "Open_Source_Threat_Intelligence_Agent" - description: "Can search various source for vulnerabilities, exploitations and critical vulnerabilities exploits against NIST CVEs, CISA Known exploits, and Open Threat Intelligence holdings" + description: "Can search various source for vulnerabilities, exploitations and critical vulnerabilities exploits against NIST CVEs, CISA Known exploits, and Open Threat Intelligence holdings. Can also cross-reference asset inventories against vulnerability databases for prioritised remediation lists." tools: "osint" + attack: + name: "ATT&CK_Navigator_Agent" + description: "Can generate MITRE ATT&CK Navigator layer JSON files for named threat actors or search terms by querying OpenCTI for associated Attack-Pattern objects. The output can be imported directly into the ATT&CK Navigator to visualise technique coverage and defensive gaps." + tools: "attack" wikipedia: name: "wikipedia_agent" description: "This agent can search wikipedia." @@ -47,7 +55,9 @@ agent_lib: # The user can modify this list to change the combination of managed agents managed_agents: - "opencti" + - "opencti_writer" - "osint" + - "attack" - "wikipedia" - "refine_task" all_task_agent: @@ -56,9 +66,15 @@ agent_lib: opencti: type: "simple" blueprint: "opencti" + opencti_writer: + type: "simple" + blueprint: "opencti_writer" osint: type: "simple" blueprint: "osint" + attack: + type: "simple" + blueprint: "attack" wikipedia: type: "simple" blueprint: "wikipedia" \ No newline at end of file diff --git a/conf/schedules.yaml b/conf/schedules.yaml new file mode 100644 index 0000000..dc33d99 --- /dev/null +++ b/conf/schedules.yaml @@ -0,0 +1,46 @@ +hydra: + run: + dir: ./logs/${now:%Y-%m-%d}/${hydra.job.name}/${now:%H-%M-%S} + +# API key from environment +api_key: ${oc.env:OPENROUTER_API_KEY} + +# Output directory for scheduled briefing markdown files +output_dir: reports/ + +# Scheduled briefing definitions +# cron format: "minute hour day_of_month month day_of_week" +schedules: + - name: weekly_threat_brief + cron: "0 8 * * MON" + model: ms_ds_r1 + agent: managing_agent + task: > + Generate a weekly threat intelligence briefing covering the most significant threats + from the past 7 days. Focus on active threat actors, critical vulnerabilities with + known exploitation, and key indicators of compromise. Provide an executive summary, + top 3-5 threats with severity assessment, a summary of relevant TTPs mapped to + MITRE ATT&CK, and prioritised defensive recommendations. Format as professional + markdown suitable for a security leadership audience. + + - name: daily_kev_digest + cron: "0 7 * * *" + model: ms_ds_r1 + agent: osint + task: > + Retrieve the latest CISA Known Exploited Vulnerabilities added in the past 24 hours. + For each new entry, provide the CVE ID, affected vendor and product, a brief description + of the vulnerability, CVSS score if available, and the remediation due date. + Format as a concise markdown table followed by brief analyst commentary on the + highest-severity entries. + + - name: monthly_threat_actor_review + cron: "0 9 1 * *" + model: qwen_235b + agent: managing_agent + task: > + Generate a monthly review of the top threat actors observed in the past 30 days. + For each actor, summarise recent campaigns, targeted sectors, primary TTPs, and + any shifts in tactics or targeting since last month. Include an ATT&CK Navigator + layer for the most active actor. Provide strategic recommendations for defensive + posture improvements. Format as a detailed markdown report. diff --git a/dspy_agent.py b/dspy_agent.py deleted file mode 100644 index 50d536a..0000000 --- a/dspy_agent.py +++ /dev/null @@ -1,43 +0,0 @@ -import dspy -import os -from tools.osint_tools import ( - search_cves, - get_cve, - get_all_exploited_vulnerabilities, - vulnerabilities_keyword_filter, - search_indicator -) - -import dotenv -dotenv.load_dotenv() - -class OSINTInvestigator(dspy.Signature): - """OSINT analyst investigating cybersecurity threats using available tools""" - - user_query: str = dspy.InputField() - investigation_result: str = dspy.OutputField( - desc="Comprehensive findings including CVEs, IOCs, and mitigation strategies" - ) - -osint_agent = dspy.ReAct( - OSINTInvestigator, - tools=[ - search_cves, - get_cve, - # get_all_exploited_vulnerabilities, - vulnerabilities_keyword_filter, - search_indicator - ] -) -lm = dspy.LM( - model="openrouter/google/gemma-3-27b-it:free", - api_base="https://openrouter.ai/api/v1", - api_key=os.getenv("OPENROUTER_API_KEY"), -) -dspy.configure(lm=lm) - -# task = input("Task: ") -task = "What are the top 10 known exploited vulnerabilities from the past 10 days?" -output = osint_agent(user_request=task) -print(output) - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7a66765 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,68 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.backends.legacy:build" + +# --------------------------------------------------------------------------- +# Project metadata +# --------------------------------------------------------------------------- +[project] +name = "agentic-cti" +version = "0.1.0" +description = "Agentic threat intelligence platform using smolagents and OpenCTI" +readme = "readme.md" +requires-python = ">=3.11" + +# Runtime dependencies (mirrors requirements.txt — requirements.txt is still +# used by Docker / pip install -r for compatibility) +dependencies = [ + "python-dotenv", + "pycti", + "smolagents[toolkit,openai,gradio]", + "wikipedia-api", + "OTXv2", + "nvdlib", + "hydra-core", + "hydra-joblib-launcher", + "fastapi", + "uvicorn[standard]", + "apscheduler", +] + +[project.optional-dependencies] +dev = [ + "pytest", + "pre-commit", +] + +# --------------------------------------------------------------------------- +# pytest +# --------------------------------------------------------------------------- +[tool.pytest.ini_options] +# Add the project root to sys.path so tests can import top-level modules +# (agents, tools, evaluations) without a manual conftest.py path hack. +pythonpath = ["."] +testpaths = ["tests"] +addopts = "-v" + +# --------------------------------------------------------------------------- +# ruff (linter + formatter) +# --------------------------------------------------------------------------- +[tool.ruff] +target-version = "py311" +line-length = 100 + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear +] +ignore = [ + "E501", # line too long — handled by formatter +] + +[tool.ruff.lint.isort] +known-first-party = ["tools", "agents", "evaluations", "api", "scheduler"] diff --git a/readme.md b/readme.md index 89df799..af1baaa 100644 --- a/readme.md +++ b/readme.md @@ -1,75 +1,186 @@ # Agentic CTI -An agentic threat intelligence system that orchestrates specialised sub-agents to answer complex CTI queries. A managing `CodeAgent` (powered by [smolagents](https://github.com/huggingface/smolagents)) coordinates three domain agents — OpenCTI, OSINT, and Wikipedia — selecting and delegating to whichever combination is needed to answer the task. +An agentic threat intelligence platform that orchestrates specialised sub-agents to answer complex CTI queries, enrich and persist intelligence, and deliver scheduled briefings. A managing `CodeAgent` (powered by [smolagents](https://github.com/huggingface/smolagents)) coordinates domain-specific agents — selecting and delegating to whichever combination is needed for the task. ``` Managing Agent -├── OpenCTI Agent — structured CTI: threat actors, TTPs, malware, reports -├── OSINT Agent — CVE/NVD, CISA KEV, AlienVault OTX -└── Wikipedia Agent — background context on threat actors and techniques +├── OpenCTI Agent — structured CTI: threat actors, TTPs, malware, reports +├── OpenCTI Writer Agent — persist reports, notes, indicators, and relationships back to OpenCTI +├── OSINT Agent — CVE/NVD, CISA KEV, AlienVault OTX, asset-correlated vuln prioritisation +├── ATT&CK Agent — MITRE ATT&CK Navigator layer generation from OpenCTI Attack-Patterns +├── Wikipedia Agent — background context on threat actors and techniques +└── Refine Task Agent — LLM-only task reformulation and query clarification ``` -Agent configurations and model selections are managed via [Hydra](https://hydra.cc), making it straightforward to sweep across models and architectures for evaluation. +Agent configurations, model selections, and tool composition are managed via [Hydra](https://hydra.cc), making it straightforward to swap models and sweep across architectures without changing code. + +--- + +## Supported Use Cases + +### Intelligence Retrieval +- **Threat actor and campaign profiling** — search OpenCTI Intrusion-Set and Campaign objects, enriched with Wikipedia background context +- **TTP and attack pattern discovery** — query OpenCTI Attack-Pattern and Course-Of-Action objects aligned to MITRE ATT&CK +- **Malware and tool analysis** — search OpenCTI Arsenal (Malware, Tool, Vulnerability objects) +- **Report and analysis retrieval** — fetch recent Notes and Reports from OpenCTI filtered by keyword and time window +- **IOC reputation lookup** — query AlienVault OTX for IP addresses, domains, file hashes, CVEs, and more + +### Vulnerability Intelligence +- **CVE research** — keyword and ID-based search across NIST NVD +- **Known exploited vulnerability triage** — retrieve and filter the CISA KEV list by product or keyword +- **Asset-correlated vulnerability prioritisation** — cross-reference a comma-separated list of system names or vendor/product keywords against CISA KEV and NVD; results sorted by KEV status and CVSS score + +### ATT&CK Integration +- **Navigator layer generation** — produce ATT&CK Navigator JSON for a named threat actor or keyword by querying OpenCTI Attack-Patterns; output is importable directly into the [ATT&CK Navigator](https://mitre-attack.github.io/attack-navigator/) + +### Intelligence Persistence +- **Write-back to OpenCTI** — create Reports, Notes, Indicators (IOCs), and STIX Relationships from agent-synthesised intelligence, marked TLP:AMBER by default + +### Reporting +- **On-demand threat briefs** — generate structured markdown reports for government, financial, or sector-specific audiences via the Gradio UI or REST API +- **Scheduled briefings** — automated cron-driven reports saved as markdown files (weekly threat brief, daily KEV digest, monthly threat actor review) + +### Programmatic Integration +- **REST API** — submit tasks and retrieve results over HTTP for SIEM/SOAR integration and pipeline automation + +--- + +## Architecture + +### Agent Types + +Three agent architectures are available, selectable via `conf/dashboard.yaml`: + +| Type | Description | +|---|---| +| `managing` | Orchestrating coordinator that dynamically delegates to specialised sub-agents. Recommended for complex, multi-source queries. | +| `all_in_one` | Single agent with access to all tools. Simpler reasoning path; lower overhead for straightforward tasks. | +| `simple` | Single domain-specific agent (e.g. `opencti`, `osint`, `attack`). Useful for focused queries. | + +### Models + +All models are accessed via [OpenRouter](https://openrouter.ai): + +| Key | Model | +|---|---| +| `ms_ds_r1` | Microsoft MAI-DS R1 *(default)* | +| `qwen_235b` | Qwen 3 235B | +| `qwen_2.5_coder` | Qwen 2.5 Coder 32B | +| `deepseek_r1` | DeepSeek R1 | + +### Tools + +| Module | Tool | Source | +|---|---|---| +| `opencti_tools` | `search_analyses` | OpenCTI Notes & Reports | +| | `search_threats` | OpenCTI Intrusion-Sets & Campaigns | +| | `search_techniques` | OpenCTI Attack-Patterns & Courses of Action | +| | `search_arsenal` | OpenCTI Malware, Tools & Vulnerabilities | +| | `get_systems` | OpenCTI System inventory | +| `opencti_write_tools` | `create_report` | OpenCTI write (TLP:AMBER default) | +| | `create_note` | OpenCTI write | +| | `create_indicator` | OpenCTI write | +| | `create_relationship` | OpenCTI write | +| `osint_tools` | `search_cves` | NIST NVD | +| | `get_cve` | NIST NVD | +| | `get_all_exploited_vulnerabilities` | CISA KEV | +| | `vulnerabilities_keyword_filter` | CISA KEV | +| | `search_indicator` | AlienVault OTX | +| | `correlate_systems_with_vulnerabilities` | CISA KEV + NIST NVD | +| `attack_tools` | `generate_navigator_layer` | OpenCTI → ATT&CK Navigator JSON | +| `general` | `todays_date` | System date | + +--- ## Building and Running the Project ### Prerequisites -- Docker -- Python 3.11 +- Python 3.11 (conda recommended) +- A running [OpenCTI](https://docs.opencti.io) instance +- API keys for OpenRouter and AlienVault OTX ### Environment Variables -Before running the project, you need to set the following environment variables. You can create a `.env` file in the root of the project to store these variables. - -- `OPENCTI_URL`: The URL of your OpenCTI instance. -- `OPENCTI_TOKEN`: The token for the OpenCTI API. -- `ALIENVAULT_API_KEY`: Your API key for AlienVault OTX. -- `OPENROUTER_API_KEY`: Your API key for OpenRouter (used to access LLMs). - -Example `.env` file: +Create a `.env` file in the project root: ``` OPENCTI_URL=http://localhost:8080 OPENCTI_TOKEN=your_opencti_token ALIENVAULT_API_KEY=your_alienvault_api_key -OPENROUTER_API_KEY=your_llm_api_key +OPENROUTER_API_KEY=your_openrouter_api_key +``` + +### Setting Up the Environment + +```bash +conda create -n agentic-cti python=3.11 +conda activate agentic-cti +pip install -r requirements.txt ``` +### Running the Gradio UI + +```bash +python dashboard.py +``` + +The interactive chat interface will be available at `http://localhost:7861`. The default configuration uses the Microsoft MAI-DS R1 model with the `managing_agent` architecture. + ### Running with Docker -The easiest way to run the project is with Docker. +```bash +docker build -t threat-intel-agent . +docker run -p 7861:7861 --env-file .env threat-intel-agent +``` -1. Build the Docker image: +### Running the REST API - ```bash - docker build -t threat-intel-agent . - ``` +The REST API exposes the agent system over HTTP for SIEM/SOAR integration and automation pipelines. -2. Run the Docker container: +```bash +uvicorn api:app --host 0.0.0.0 --port 8000 +``` + +**Endpoints:** + +| Method | Path | Description | +|---|---|---| +| `POST` | `/query` | Submit a task; returns a job ID immediately | +| `GET` | `/results/{id}` | Poll for results by job ID | +| `GET` | `/health` | Liveness check | + +**Example:** - ```bash - docker run -p 7861:7861 --env-file .env threat-intel-agent - ``` +```bash +# Submit a task +curl -X POST http://localhost:8000/query \ + -H "Content-Type: application/json" \ + -d '{"task": "What are the top 5 TTPs used by APT28?", "agent": "managing_agent", "model": "ms_ds_r1"}' -The Gradio UI will be available at `http://localhost:7861`. +# Poll for the result +curl http://localhost:8000/results/ +``` -### Running Locally +### Running the Scheduler -1. Install the required Python packages: +The scheduler runs automated threat briefings on a cron schedule, saving markdown reports to the `reports/` directory. - ```bash - pip install -r requirements.txt - ``` +```bash +python scheduler.py +``` -2. Run the dashboard: +Default schedules (configurable in `conf/schedules.yaml`): - ```bash - python dashboard.py - ``` +| Schedule | Cron | Description | +|---|---|---| +| `weekly_threat_brief` | Monday 08:00 UTC | Weekly threat intelligence summary | +| `daily_kev_digest` | Daily 07:00 UTC | New CISA KEV entries from the past 24 hours | +| `monthly_threat_actor_review` | 1st of month 09:00 UTC | Monthly review of active threat actors | -The Gradio UI will be available at `http://localhost:7861`. +To modify schedules or add new ones, edit `conf/schedules.yaml`. Each entry specifies a `cron` expression, `agent`, `model`, and `task` prompt. +--- ## Running Evaluations @@ -91,19 +202,96 @@ The sweep covers `{qwen_235b, qwen_2.5_coder, deepseek_r1}` × `{managing_agent, Each result includes a `scores` block with heuristic quality signals (markdown structure, recommendation content, threat actor mentions) to support comparison across runs. +--- + +## Synthetic Organisation Test Harness + +The project includes a reference synthetic organisation profile — the **Federal Financial Intelligence Unit (FFIU)** — for use in integration testing and evaluation against realistic, organisation-scoped scenarios. + +### Why a Synthetic Organisation? + +Generic evaluation tasks (`task.txt`) test general capability but cannot validate asset-specific queries like "which of our systems have known exploited vulnerabilities?" or "generate an ATT&CK layer for the threat actors targeting us." The FFIU profile fills this gap by providing a realistic but entirely fictional organisation with a defined asset inventory and threat actor context. + +### FFIU Profile + +``` +Name: Federal Financial Intelligence Unit (FFIU) +Sector: Government / Financial Regulation +Mission: Financial crime intelligence, interbank reporting, sanctions monitoring +Staff: 800 | Systems: 60 +``` + +**Representative system inventory** (subset used in harness tasks): + +| System | Vendor | Product | Criticality | +|---|---|---|---| +| Case Management Platform | Palantir | Gotham | Critical | +| Sanctions Screening Engine | Actimize | AML/Sanctions | Critical | +| Interbank Messaging Gateway | SWIFT | Alliance Gateway | Critical | +| SIEM | Splunk | Enterprise | High | +| Identity & Access Management | Okta | Workforce Identity | High | +| VPN / Remote Access | Palo Alto | GlobalProtect | High | +| Financial Reporting Portal | Microsoft | SharePoint | High | +| Email Gateway | Proofpoint | Email Protection | Medium | + +**Threat actors of concern:** APT28, APT41, FIN7, Lazarus Group, TA505 + +### Harness Tasks + +These tasks exercise the four priority use cases against the FFIU context: + +1. **Vulnerability prioritisation (US-03)** — Cross-reference CISA KEV and NVD against FFIU's system inventory and return a prioritised remediation list +2. **ATT&CK Navigator (US-05)** — Generate an ATT&CK Navigator JSON layer for APT28 and Lazarus Group targeting financial regulatory infrastructure +3. **Detection rule generation (US-06)** — Produce Sigma detection rules for the top 5 TTPs used by FIN7 against financial sector organisations +4. **Scheduled threat brief (US-04)** — Generate a weekly threat brief for FFIU scoped to APT28, APT41, and FIN7 activity + +To run FFIU-scoped tasks against the managing agent: + +```bash +# Example — asset-correlated vulnerability prioritisation +# Adjust the task text to match the FFIU system inventory +python evaluations.py agent_name=managing_agent model_name=ms_ds_r1 +``` + +For full harness automation with a seeded OpenCTI instance, see `tests/fixtures/` (seed scripts to be added). + +--- + ## Running Tests ```bash pytest tests/ ``` +--- + ## Project Structure -- `agents.py`: Agent factory — creates simple, all-in-one, and managing agent configurations. -- `dashboard.py`: Gradio UI entry point. -- `evaluations.py`: Evaluation harness; runs tasks against agent/model combinations and scores outputs. -- `Dockerfile`: Container build. -- `requirements.txt`: Python dependencies. -- `conf/`: Hydra configuration files (model library, agent library, sweep parameters). -- `tools/`: OpenCTI, OSINT, and utility tools available to agents. -- `tests/`: Unit tests for tools and agent logic. +``` +agentic-cti/ +├── agents.py # Agent factory — simple, all-in-one, and managing configurations +├── api.py # FastAPI REST interface (port 8000) +├── dashboard.py # Gradio UI entry point (port 7861) +├── scheduler.py # APScheduler cron runner for automated briefings +├── evaluations.py # Evaluation harness — sweeps model × agent combinations +├── task.txt # Evaluation task definitions (one per line) +├── Dockerfile # Container build +├── requirements.txt # Python dependencies +├── conf/ +│ ├── dashboard.yaml # Model library, agent library, default selections +│ ├── schedules.yaml # Scheduled briefing definitions (cron, agent, model, task) +│ └── validation.yaml # Evaluation sweep parameters +├── tools/ +│ ├── __init__.py # Package exports +│ ├── opencti_tools.py # Read tools: search threats, techniques, arsenal, analyses, systems +│ ├── opencti_write_tools.py # Write tools: create reports, notes, indicators, relationships +│ ├── osint_tools.py # OSINT tools: NVD, CISA KEV, AlienVault OTX, asset correlation +│ ├── attack_tools.py # ATT&CK Navigator layer generation +│ └── general.py # Utility tools (todays_date) +├── tests/ +│ ├── test_agents.py # Agent factory and scoring tests +│ └── test_tools.py # Tool functionality tests +├── experiments/ +│ └── dspy_agent.py # Experimental DSPy-based implementation (not integrated) +└── reports/ # Output directory for scheduled briefing markdown files +``` diff --git a/requirements.txt b/requirements.txt index 67ba2ee..4dbf7ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,13 @@ dotenv pycti -smolagents[toolkit,openai,gradio]@git+https://github.com/JakeBx/smolagents.git@gradio-reset-memory +smolagents[toolkit,openai,gradio] pytest +pre-commit wikipedia-api OTXv2 nvdlib hydra-core -hydra-joblib-launcher \ No newline at end of file +hydra-joblib-launcher +fastapi +uvicorn[standard] +apscheduler \ No newline at end of file diff --git a/scheduler.py b/scheduler.py new file mode 100644 index 0000000..398d62d --- /dev/null +++ b/scheduler.py @@ -0,0 +1,91 @@ +""" +Scheduled threat intelligence briefing runner. + +Reads schedule definitions from conf/schedules.yaml and runs each briefing +on its configured cron expression. Output is saved as markdown files to the +configured output_dir. + +Usage: + python scheduler.py + +Environment variables required (same as dashboard.py): + OPENROUTER_API_KEY, OPENCTI_URL, OPENCTI_TOKEN, ALIENVAULT_API_KEY +""" + +import logging +import os +from datetime import datetime, timezone +from pathlib import Path + +import hydra +from apscheduler.schedulers.blocking import BlockingScheduler +from omegaconf import DictConfig, OmegaConf + +from agents import create_agent, create_llm_model + +log = logging.getLogger(__name__) + + +def _run_briefing(cfg: DictConfig, schedule: DictConfig) -> None: + """Executes a single scheduled briefing and writes the result to a file.""" + name = schedule.name + log.info("Running scheduled briefing: %s", name) + + model_cfg = cfg.model_lib[schedule.model] + model = create_llm_model(model_id=model_cfg.id, api_key=cfg.api_key) + + # Temporarily override agent_name so create_agent picks the right config + agent_cfg = OmegaConf.merge(cfg, {"agent_name": schedule.agent}) + agent = create_agent(agent_cfg, model) + + result = agent.run(schedule.task) + + output_dir = Path(cfg.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + filename = output_dir / f"{timestamp}-{name}.md" + filename.write_text(result, encoding="utf-8") + log.info("Briefing saved to %s", filename) + + +@hydra.main(version_base=None, config_path="conf", config_name="schedules") +def main(cfg: DictConfig) -> None: + schedules = cfg.get("schedules", []) + if not schedules: + log.warning("No schedules defined in conf/schedules.yaml — nothing to run.") + return + + scheduler = BlockingScheduler(timezone="UTC") + + for schedule in schedules: + cron_parts = schedule.cron.split() + if len(cron_parts) != 5: + log.error("Invalid cron expression for '%s': %s", schedule.name, schedule.cron) + continue + + minute, hour, day, month, day_of_week = cron_parts + scheduler.add_job( + _run_briefing, + trigger="cron", + kwargs={"cfg": cfg, "schedule": schedule}, + minute=minute, + hour=hour, + day=day, + month=month, + day_of_week=day_of_week, + id=schedule.name, + name=schedule.name, + misfire_grace_time=300, + ) + log.info("Scheduled '%s' with cron: %s", schedule.name, schedule.cron) + + log.info("Scheduler started. %d job(s) registered. Press Ctrl+C to stop.", len(scheduler.get_jobs())) + try: + scheduler.start() + except (KeyboardInterrupt, SystemExit): + log.info("Scheduler stopped.") + + +if __name__ == "__main__": + main() diff --git a/tests/test_agents.py b/tests/test_agents.py index 8bb624a..687cfc4 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -1,11 +1,13 @@ import pytest from unittest.mock import MagicMock, patch -from agents import get_agent_tools, create_agent +from agents import get_agent_tools, create_agent, _BUILDERS from evaluations import score_output -# --- get_agent_tools --- +# --------------------------------------------------------------------------- +# get_agent_tools +# --------------------------------------------------------------------------- def test_get_agent_tools_none_returns_empty(): """'none' as a tool name should return an empty list without error.""" @@ -19,7 +21,16 @@ def test_get_agent_tools_unknown_raises(): get_agent_tools(["not_a_real_tool"]) -# --- create_agent --- +# --------------------------------------------------------------------------- +# create_agent — _BUILDERS registry +# --------------------------------------------------------------------------- + +def test_builders_registry_contains_all_types(): + """The _BUILDERS registry must contain all three expected agent types.""" + assert "simple" in _BUILDERS + assert "all_in_one" in _BUILDERS + assert "managing" in _BUILDERS + def test_create_agent_unknown_type_raises(): """An unrecognised agent type in config should raise ValueError.""" @@ -31,7 +42,9 @@ def test_create_agent_unknown_type_raises(): create_agent(cfg, MagicMock()) -# --- score_output --- +# --------------------------------------------------------------------------- +# score_output +# --------------------------------------------------------------------------- def test_score_output_well_formed_report(): """A markdown report with headers and recommendations should score positively.""" diff --git a/tests/test_tools.py b/tests/test_tools.py index 9b494a2..5caf7ac 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,39 +1,69 @@ import pytest -from unittest.mock import patch -from tools.opencti_tools import get_systems, search_analyses, search_techniques, search_threats, search_arsenal +from unittest.mock import MagicMock, patch + +from tools.opencti_tools import ( + _build_filter, + get_systems, + search_analyses, + search_techniques, + search_threats, + search_arsenal, +) + + +# --------------------------------------------------------------------------- +# _build_filter +# --------------------------------------------------------------------------- + +def test_build_filter_returns_well_formed_dict(): + """_build_filter should return a dict with the correct shape and the given entity types.""" + result = _build_filter(["Malware", "Tool"]) + + assert isinstance(result, dict) + assert result["mode"] == "and" + assert "filters" in result + assert "filterGroups" in result + + # Entity types must appear in the nested filterGroup + inner_filters = result["filterGroups"][0]["filters"] + assert inner_filters[0]["values"] == ["Malware", "Tool"] + + +# --------------------------------------------------------------------------- +# Search tools — parametrized fixture +# --------------------------------------------------------------------------- @pytest.fixture def mock_client(): - """Pytest fixture to mock the OpenCTIApiClient.""" - with patch('tools.client') as mock_client: - # Mock the call for most tools to return one item - mock_client.stix_core_object.list.return_value = [{'id': 'stix-core-object-1', 'name': 'mock_object'}] - # Mock the call for reports in search_analyses to return an empty list - # This ensures search_analyses returns exactly one result in the test - mock_client.report.list.return_value = [] - yield mock_client + """Pytest fixture: patches _get_client so no real OpenCTI connection is made.""" + client = MagicMock() + client.stix_core_object.list.return_value = [ + {'id': 'stix-core-object-1', 'name': 'mock_object', 'entity_type': 'Malware'} + ] + with patch('tools.opencti_tools._get_client', return_value=client): + yield client + search_functions = [search_analyses, search_threats, search_techniques, search_arsenal] -@pytest.mark.parametrize("tool", search_functions) -def test_search_tool_returns_one_result(tool, mock_client): - """ - Tests that each search tool returns a list with a single dictionary when max_results=1. - This test uses a mocked OpenCTI client to provide consistent results. - """ - result = tool(search_term="", max_results=1) - - assert isinstance(result, list), f"Tool {tool.__name__} did not return a list." - assert len(result) == 1, f"Tool {tool.__name__} did not return a list with one item." - assert isinstance(result[0], dict), f"Tool {tool.__name__} did not return a list of dictionaries." - - -def test_get_systems(mock_client): - """ - Tests that get_systems returns the correct system name. - """ - mock_client.stix_core_object.list.return_value = [{'name': 'SOHO router'}] + + +@pytest.mark.parametrize("search_fn", search_functions, ids=[f.__name__ for f in search_functions]) +def test_search_tool_returns_list_of_dicts(search_fn, mock_client): + """Each search function must return a non-empty list of dicts when the client returns one item.""" + result = search_fn(search_term="test", max_results=1) + + assert isinstance(result, list), f"{search_fn.__name__} did not return a list" + assert len(result) == 1, f"{search_fn.__name__} did not return exactly one item" + assert isinstance(result[0], dict), f"{search_fn.__name__} did not return a list of dicts" + + +def test_get_systems_returns_list_of_dicts(mock_client): + """get_systems must return a list of dicts with at least a 'name' key.""" + mock_client.stix_core_object.list.return_value = [{'name': 'SOHO router', 'entity_type': 'System'}] + result = get_systems() - + assert isinstance(result, list) assert len(result) == 1 - assert isinstance(result[0], dict) \ No newline at end of file + assert isinstance(result[0], dict) + assert result[0].get('name') == 'SOHO router' diff --git a/tools/__init__.py b/tools/__init__.py index 6254522..e58cd93 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -1,3 +1,5 @@ from .opencti_tools import opencti_tools +from .opencti_write_tools import opencti_write_tools from .osint_tools import osint_tools +from .attack_tools import attack_tools from .general import todays_date \ No newline at end of file diff --git a/tools/attack_tools.py b/tools/attack_tools.py new file mode 100644 index 0000000..785d3b3 --- /dev/null +++ b/tools/attack_tools.py @@ -0,0 +1,110 @@ +import json +from typing import Any, Dict, List + +from .opencti_tools import search_techniques + +# ATT&CK Navigator layer schema version +_NAVIGATOR_VERSION = "4.9.5" +_ATTACK_VERSION = "14" +_LAYER_VERSION = "4.5" + +# Highlight colour for techniques found in the layer +_TECHNIQUE_COLOUR = "#e97d37" + + +def _extract_mitre_id(external_references: Any) -> str | None: + """Extracts the ATT&CK technique ID (e.g. T1566) from an externalReferences list.""" + if not external_references: + return None + refs = external_references if isinstance(external_references, list) else [external_references] + for ref in refs: + if isinstance(ref, dict): + source = ref.get("sourceName", "") or ref.get("source_name", "") + ext_id = ref.get("externalId", "") or ref.get("external_id", "") + if "mitre-attack" in source.lower() and ext_id.startswith("T"): + return ext_id + return None + + +def generate_navigator_layer(threat_actor_name: str) -> str: + """ + Generates an ATT&CK Navigator layer JSON for a named threat actor or search term + by querying OpenCTI for associated Attack-Pattern objects. + + The returned JSON string can be saved as a .json file and imported directly into + the ATT&CK Navigator (https://mitre-attack.github.io/attack-navigator/). + + Args: + threat_actor_name (str): Name of the threat actor, campaign, or keyword to search + for in OpenCTI (e.g. 'APT28', 'Lazarus Group', 'ransomware'). + + Returns: + str: ATT&CK Navigator layer JSON as a formatted string. Contains the technique + IDs found in OpenCTI highlighted in orange, plus metadata about the layer. + Returns an error JSON string if no techniques are found. + """ + techniques_raw = search_techniques(threat_actor_name, max_results=200) + + technique_entries: List[Dict[str, Any]] = [] + seen_ids: set = set() + + for obj in techniques_raw: + if obj.get("entity_type") != "Attack-Pattern": + continue + mitre_id = _extract_mitre_id(obj.get("externalReferences")) + if not mitre_id or mitre_id in seen_ids: + continue + seen_ids.add(mitre_id) + technique_entries.append({ + "techniqueID": mitre_id, + "color": _TECHNIQUE_COLOUR, + "comment": obj.get("description", "")[:500], + "enabled": True, + "metadata": [], + "links": [], + "showSubtechniques": False, + }) + + if not technique_entries: + return json.dumps({ + "error": f"No ATT&CK techniques found in OpenCTI for query: '{threat_actor_name}'", + "hint": "Ensure OpenCTI contains Attack-Pattern objects with MITRE ATT&CK external references.", + }, indent=2) + + layer = { + "name": f"{threat_actor_name} — ATT&CK Layer", + "versions": { + "attack": _ATTACK_VERSION, + "navigator": _NAVIGATOR_VERSION, + "layer": _LAYER_VERSION, + }, + "domain": "enterprise-attack", + "description": ( + f"Techniques associated with '{threat_actor_name}' sourced from OpenCTI. " + f"Generated by agentic-cti. {len(technique_entries)} technique(s) mapped." + ), + "filters": {"platforms": ["Windows", "Linux", "macOS", "Network", "Cloud"]}, + "sorting": 0, + "layout": {"layout": "side", "aggregateFunction": "average", "showID": True, "showName": True}, + "hideDisabled": False, + "techniques": technique_entries, + "gradient": { + "colors": ["#ff6666ff", "#ffe766ff", "#8ec843ff"], + "minValue": 0, + "maxValue": 100, + }, + "legendItems": [ + {"label": f"Observed technique ({threat_actor_name})", "color": _TECHNIQUE_COLOUR} + ], + "metadata": [], + "links": [], + "showTacticRowBackground": False, + "tacticRowBackground": "#dddddd", + "selectTechniquesAcrossTactics": True, + "selectSubtechniquesWithParent": False, + } + + return json.dumps(layer, indent=2) + + +attack_tools = [generate_navigator_layer] diff --git a/tools/opencti_tools.py b/tools/opencti_tools.py index 545720a..1d425d0 100644 --- a/tools/opencti_tools.py +++ b/tools/opencti_tools.py @@ -2,16 +2,27 @@ from typing import List, Dict, Any from pycti import OpenCTIApiClient +_client = None -client = OpenCTIApiClient( - url=os.environ.get("OPENCTI_URL"), - token=os.environ.get("OPENCTI_TOKEN"), - custom_headers="Request-Timeout:600" -) +def _get_client() -> OpenCTIApiClient: + """Lazy factory: returns a module-level singleton OpenCTIApiClient. -def _filter_factory(entity_types: List[str]): - """Creates a filter for a list of entity types.""" + The client is created on first call, not at import time, so importing this + module never triggers a network connection or reads env-vars prematurely. + """ + global _client + if _client is None: + _client = OpenCTIApiClient( + url=os.environ.get("OPENCTI_URL"), + token=os.environ.get("OPENCTI_TOKEN"), + custom_headers="Request-Timeout:600", + ) + return _client + + +def _build_filter(entity_types: List[str]) -> Dict: + """Builds and returns a filter dict for a list of entity types.""" return { "mode": "and", "filters": [ @@ -60,9 +71,8 @@ def search_analyses(search_term: str, max_results: int = 50, n_days: int = 30) - Returns: List[dict]: A list of dictionaries with entity details, including entity_type. """ - # Get Notes - analyses_filters = _filter_factory(entity_types=["Note", "Report"]) - analyses_results = client.stix_core_object.list(search=search_term, first=max_results, filters=analyses_filters) + analyses_filters = _build_filter(entity_types=["Note", "Report"]) + analyses_results = _get_client().stix_core_object.list(search=search_term, first=max_results, filters=analyses_filters) analyses_keys = ['name', 'description', 'entity_type', 'externalReferences', 'last_seen', 'url'] @@ -82,8 +92,8 @@ def search_threats(search_term: str, max_results: int = 50) -> List[Dict[str, An """ limit = min(1_000, max_results) entity_types = ["Intrusion-Set", "Campaign"] - filters = _filter_factory(entity_types=entity_types) - results = client.stix_core_object.list(search=search_term, first=limit, filters=filters) + filters = _build_filter(entity_types=entity_types) + results = _get_client().stix_core_object.list(search=search_term, first=limit, filters=filters) keys = ['name', 'description', 'entity_type', 'externalReferences'] return _postprocess(results, keys) @@ -102,8 +112,8 @@ def search_techniques(search_term: str, max_results: int = 50) -> List[Dict[str, """ limit = min(1_000, max_results) entity_types = ["Attack-Pattern", "Course-Of-Action"] - filters = _filter_factory(entity_types=entity_types) - results = client.stix_core_object.list(search=search_term, first=limit, filters=filters) + filters = _build_filter(entity_types=entity_types) + results = _get_client().stix_core_object.list(search=search_term, first=limit, filters=filters) keys = ['name', 'description', 'entity_type', 'externalReferences'] return _postprocess(results, keys) @@ -122,8 +132,8 @@ def search_arsenal(search_term: str, max_results: int = 50) -> List[Dict[str, An """ limit = min(1_000, max_results) entity_types = ["Malware", "Tool", "Vulnerability"] - filters = _filter_factory(entity_types=entity_types) - results = client.stix_core_object.list(search=search_term, first=limit, filters=filters) + filters = _build_filter(entity_types=entity_types) + results = _get_client().stix_core_object.list(search=search_term, first=limit, filters=filters) keys = ['name', 'description', 'entity_type', 'externalReferences'] return _postprocess(results, keys) @@ -138,8 +148,8 @@ def get_systems() -> List[Dict[str, Any]]: Returns: List[Dict[str, Any]]: A list of dictionaries with system details. """ - filters = _filter_factory(entity_types=["System"]) - results = client.stix_core_object.list(search="", first=50, filters=filters) + filters = _build_filter(entity_types=["System"]) + results = _get_client().stix_core_object.list(search="", first=50, filters=filters) keys = ['name', 'description', 'entity_type', 'externalReferences'] return _postprocess(results, keys) @@ -149,5 +159,6 @@ def get_systems() -> List[Dict[str, Any]]: search_analyses, search_threats, search_techniques, + search_arsenal, get_systems, ] diff --git a/tools/opencti_write_tools.py b/tools/opencti_write_tools.py new file mode 100644 index 0000000..55d9005 --- /dev/null +++ b/tools/opencti_write_tools.py @@ -0,0 +1,171 @@ +import os +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +from pycti import OpenCTIApiClient + +from .opencti_tools import _get_client + +# Standard OpenCTI TLP marking definition IDs (STIX 2.1 spec) +_TLP_IDS = { + "TLP:CLEAR": "marking-definition--613f2e26-407d-48c7-9eca-b8e91ba519f9", + "TLP:GREEN": "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", + "TLP:AMBER": "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", + "TLP:RED": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", +} + +_DEFAULT_TLP = "TLP:AMBER" + + +def _tlp_id(tlp_level: str = _DEFAULT_TLP) -> str: + """Returns the OpenCTI marking definition ID for a TLP level.""" + tid = _TLP_IDS.get(tlp_level.upper()) + if tid is None: + raise ValueError(f"Unknown TLP level: {tlp_level}. Valid values: {list(_TLP_IDS)}") + return tid + + +def create_report( + name: str, + content: str, + tlp_level: str = _DEFAULT_TLP, + report_types: Optional[list] = None, +) -> Dict[str, Any]: + """ + Creates a Report object in OpenCTI from agent-synthesised intelligence. + + Args: + name (str): Title of the report. + content (str): The body / description of the report (markdown supported). + tlp_level (str): TLP marking for the report. Defaults to TLP:AMBER. + Valid values: 'TLP:CLEAR', 'TLP:GREEN', 'TLP:AMBER', 'TLP:RED'. + report_types (list, optional): OpenCTI report type tags. + Defaults to ["threat-report"]. + + Returns: + Dict[str, Any]: The created report object with its OpenCTI ID. + """ + if report_types is None: + report_types = ["threat-report"] + + published = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + result = _get_client().report.create( + name=name, + description=content, + report_types=report_types, + published=published, + objectMarkingIds=[_tlp_id(tlp_level)], + ) + return {"id": result.get("id"), "name": name, "tlp": tlp_level, "published": published} + + +def create_note( + content: str, + abstract: str = "", + tlp_level: str = _DEFAULT_TLP, + object_ids: Optional[list] = None, +) -> Dict[str, Any]: + """ + Creates a Note object in OpenCTI to capture analyst observations or + agent-generated commentary attached to existing CTI objects. + + Args: + content (str): The full body of the note (markdown supported). + abstract (str): A short summary / headline for the note. + tlp_level (str): TLP marking. Defaults to TLP:AMBER. + object_ids (list, optional): List of OpenCTI object IDs this note relates to. + + Returns: + Dict[str, Any]: The created note object with its OpenCTI ID. + """ + result = _get_client().note.create( + attribute_abstract=abstract or content[:120], + content=content, + objectMarkingIds=[_tlp_id(tlp_level)], + objects=object_ids or [], + ) + return {"id": result.get("id"), "abstract": abstract, "tlp": tlp_level} + + +def create_indicator( + name: str, + pattern: str, + pattern_type: str = "stix", + tlp_level: str = _DEFAULT_TLP, + description: str = "", +) -> Dict[str, Any]: + """ + Creates an Indicator (IOC) object in OpenCTI. + + Args: + name (str): Human-readable name for the indicator (e.g. 'Malicious IP 1.2.3.4'). + pattern (str): The detection pattern string. + For STIX patterns, use format: "[ipv4-addr:value = '1.2.3.4']" + For Sigma: provide the full YAML rule. + For YARA: provide the full YARA rule text. + pattern_type (str): Pattern language. One of: 'stix', 'sigma', 'yara', 'tanium-signal'. + Defaults to 'stix'. + tlp_level (str): TLP marking. Defaults to TLP:AMBER. + description (str): Optional context describing the indicator. + + Returns: + Dict[str, Any]: The created indicator with its OpenCTI ID. + """ + valid_types = {"stix", "sigma", "yara", "tanium-signal", "spl", "eql", "kql"} + if pattern_type not in valid_types: + raise ValueError(f"Invalid pattern_type '{pattern_type}'. Valid: {valid_types}") + + result = _get_client().indicator.create( + name=name, + description=description, + pattern=pattern, + pattern_type=pattern_type, + objectMarkingIds=[_tlp_id(tlp_level)], + ) + return {"id": result.get("id"), "name": name, "pattern_type": pattern_type, "tlp": tlp_level} + + +def create_relationship( + from_id: str, + to_id: str, + relationship_type: str, + description: str = "", + tlp_level: str = _DEFAULT_TLP, +) -> Dict[str, Any]: + """ + Creates a STIX relationship between two existing OpenCTI objects. + + Common relationship types: + - 'uses' : Threat actor/campaign → malware/tool/technique + - 'targets' : Threat actor/campaign → identity/sector/system + - 'attributed-to' : Campaign/intrusion-set → threat actor + - 'indicates' : Indicator → malware/tool/campaign + - 'mitigates' : Course-of-action → attack-pattern + + Args: + from_id (str): OpenCTI ID of the source object. + to_id (str): OpenCTI ID of the target object. + relationship_type (str): STIX relationship type (e.g. 'uses', 'targets'). + description (str): Optional description for the relationship. + tlp_level (str): TLP marking. Defaults to TLP:AMBER. + + Returns: + Dict[str, Any]: The created relationship with its OpenCTI ID. + """ + result = _get_client().stix_core_relationship.create( + fromId=from_id, + toId=to_id, + relationship_type=relationship_type, + description=description, + objectMarkingIds=[_tlp_id(tlp_level)], + ) + return { + "id": result.get("id"), + "from_id": from_id, + "to_id": to_id, + "relationship_type": relationship_type, + "tlp": tlp_level, + } + + +opencti_write_tools = [create_report, create_note, create_indicator, create_relationship] diff --git a/tools/osint_tools.py b/tools/osint_tools.py index 6730084..d1eab2f 100644 --- a/tools/osint_tools.py +++ b/tools/osint_tools.py @@ -107,4 +107,94 @@ def search_indicator(search_term: str, indicator_type: str) -> dict: return otx.get_indicator_details_full(indicator_type_enum, search_term) -osint_tools = [search_cves, get_cve, get_all_exploited_vulnerabilities, vulnerabilities_keyword_filter, search_indicator] +def correlate_systems_with_vulnerabilities(system_names: str) -> list: + """ + Cross-references a list of system names or vendor/product keywords against the + CISA Known Exploited Vulnerabilities list and NIST NVD to produce a prioritised + remediation list for a specific asset inventory. + + Each result includes KEV status, CVSS score, and remediation due date where available. + Results are sorted: KEV entries first (by due date), then by CVSS score descending. + + Args: + system_names (str): Comma-separated system names, vendor names, or product keywords + to search for (e.g. "Microsoft Windows, Splunk, Palo Alto GlobalProtect"). + + Returns: + list: Prioritised list of dicts, each with keys: + cveID, vulnerabilityName, product, vendorProject, shortDescription, + cvss_score, kev_status, kev_due_date, kev_ransomware_use. + """ + keywords = [k.strip() for k in system_names.split(",") if k.strip()] + + kev_list = get_all_exploited_vulnerabilities() + kev_by_cve = {v["cveID"]: v for v in kev_list} + + seen_cves: set = set() + results = [] + + for keyword in keywords: + # Search CISA KEV for this keyword + kev_matches = vulnerabilities_keyword_filter(keyword) + for vuln in kev_matches: + cve_id = vuln["cveID"] + if cve_id in seen_cves: + continue + seen_cves.add(cve_id) + nvd_data = nvdlib.searchCVE(cveId=cve_id) + cvss_score = None + if nvd_data: + cve_obj = nvd_data[0] + if hasattr(cve_obj, "score") and cve_obj.score: + cvss_score = cve_obj.score[1] + results.append({ + "cveID": cve_id, + "vulnerabilityName": vuln.get("vulnerabilityName", ""), + "product": vuln.get("product", ""), + "vendorProject": vuln.get("vendorProject", ""), + "shortDescription": vuln.get("shortDescription", ""), + "cvss_score": cvss_score, + "kev_status": "Known Exploited", + "kev_due_date": vuln.get("dueDate", ""), + "kev_ransomware_use": vuln.get("knownRansomwareCampaignUse", "Unknown"), + }) + + # Also search NVD for any additional CVEs not already in KEV results + nvd_matches = nvdlib.searchCVE(keywordSearch=keyword, limit=10) + for cve_obj in nvd_matches: + cve_id = cve_obj.id + if cve_id in seen_cves: + continue + seen_cves.add(cve_id) + cvss_score = None + if hasattr(cve_obj, "score") and cve_obj.score: + cvss_score = cve_obj.score[1] + kev_entry = kev_by_cve.get(cve_id) + results.append({ + "cveID": cve_id, + "vulnerabilityName": cve_obj.id, + "product": keyword, + "vendorProject": "", + "shortDescription": cve_obj.descriptions[0].value if cve_obj.descriptions else "", + "cvss_score": cvss_score, + "kev_status": "Known Exploited" if kev_entry else "Not in KEV", + "kev_due_date": kev_entry.get("dueDate", "") if kev_entry else "", + "kev_ransomware_use": kev_entry.get("knownRansomwareCampaignUse", "Unknown") if kev_entry else "Unknown", + }) + + # Sort: KEV entries first by due date, then non-KEV by CVSS descending + kev_entries = [r for r in results if r["kev_status"] == "Known Exploited"] + non_kev = [r for r in results if r["kev_status"] != "Known Exploited"] + kev_entries.sort(key=lambda x: x["kev_due_date"] or "") + non_kev.sort(key=lambda x: x["cvss_score"] or 0, reverse=True) + return kev_entries + non_kev + + +osint_tools = [ + search_cves, + get_cve, + get_all_exploited_vulnerabilities, + vulnerabilities_keyword_filter, + search_indicator, + correlate_systems_with_vulnerabilities, +]