From bf153e62251ca3e2185f682720990220ca23e99d Mon Sep 17 00:00:00 2001 From: L4stPL4Y3R Date: Mon, 30 Mar 2026 14:24:10 +0100 Subject: [PATCH] feat: integrate uncover multi-engine search, CDN/IP filtering, and consolidate Censys credentials - Add ProjectDiscovery uncover integration across pipeline, agent, and UI with configurable max results and per-engine API key management - Add CDN/Cloudflare/CloudFront IP filtering (ip_filter.py) to all OSINT enrichment modules (Shodan, Censys, FOFA, Netlas, OTX, VirusTotal, ZoomEye) - Harden Criminal IP enrichment with proper error classification, credit exhaustion detection, and single-warning-then-stop behavior - Consolidate 4 Censys credential fields into 2 (API Token + Org ID) and migrate censys_enrich.py from deprecated Search API v2 to Platform API v3 - Add 10 new API key fields in Global Settings for uncover engines (Quake, Hunter, PublicWWW, HunterHow, Google, Onyphe, Driftnet, Censys Platform) - Wire uncover as GROUP 2b target expansion phase in recon pipeline (before port scan) with Neo4j graph storage via update_graph_from_uncover mixin - Add UncoverToolManager to agentic layer with Docker-based execution - Add comprehensive unit tests (test_uncover_enrich.py) and update existing test suites for new key names and filtering behavior Made-with: Cursor --- agentic/orchestrator.py | 19 +- agentic/project_settings.py | 2 + agentic/prompts/tool_registry.py | 14 +- agentic/tools.py | 122 ++++- graph_db/mixins/osint_mixin.py | 91 ++++ recon/censys_enrich.py | 47 +- recon/criminalip_enrich.py | 124 ++++- recon/entrypoint.sh | 1 + recon/fofa_enrich.py | 6 + recon/ip_filter.py | 90 ++++ recon/main.py | 36 +- recon/netlas_enrich.py | 6 + recon/otx_enrich.py | 6 + recon/project_settings.py | 48 +- recon/shodan_enrich.py | 6 + recon/uncover_enrich.py | 462 ++++++++++++++++++ recon/virustotal_enrich.py | 6 + recon/zoomeye_enrich.py | 6 + recon_orchestrator/api.py | 4 +- recon_orchestrator/container_manager.py | 1 + tests/test_censys_enrich.py | 27 +- tests/test_criminalip_enrich.py | 108 +++- tests/test_graph_db_refactor.py | 3 +- tests/test_netlas_enrich.py | 8 +- tests/test_uncover_enrich.py | 248 ++++++++++ tests/test_virustotal_enrich.py | 2 +- .../migration.sql | 10 + webapp/prisma/schema.prisma | 14 +- .../src/app/api/users/[id]/settings/route.ts | 40 +- webapp/src/app/settings/page.tsx | 177 ++++++- .../sections/OsintEnrichmentSection.tsx | 39 +- 31 files changed, 1651 insertions(+), 122 deletions(-) create mode 100644 recon/ip_filter.py create mode 100644 recon/uncover_enrich.py create mode 100644 tests/test_uncover_enrich.py create mode 100644 webapp/prisma/migrations/20260329120000_consolidate_censys_keys/migration.sql diff --git a/agentic/orchestrator.py b/agentic/orchestrator.py index a79b29f0..9e1a1b13 100644 --- a/agentic/orchestrator.py +++ b/agentic/orchestrator.py @@ -35,6 +35,7 @@ VirusTotalToolManager, ZoomEyeToolManager, CriminalIpToolManager, + UncoverToolManager, PhaseAwareToolExecutor, ) from orchestrator_helpers import ( @@ -219,13 +220,14 @@ def _build_rotator(main_key: str, tool_name: str) -> KeyRotator: # OSINT tools — Censys, FOFA, OTX, Netlas, VirusTotal, ZoomEye, CriminalIP if hasattr(self, '_osint_managers') and self.tool_executor: _osint_key_map = { - 'censys': {'id_field': 'censysApiId', 'secret_field': 'censysApiSecret'}, + 'censys': {'token_field': 'censysApiToken', 'org_field': 'censysOrgId'}, 'fofa': {'key_field': 'fofaApiKey', 'rotation_name': 'fofa'}, 'otx': {'key_field': 'otxApiKey', 'rotation_name': 'otx'}, 'netlas': {'key_field': 'netlasApiKey', 'rotation_name': 'netlas'}, 'virustotal': {'key_field': 'virusTotalApiKey', 'rotation_name': 'virustotal'}, 'zoomeye': {'key_field': 'zoomEyeApiKey', 'rotation_name': 'zoomeye'}, 'criminalip': {'key_field': 'criminalIpApiKey', 'rotation_name': 'criminalip'}, + 'uncover': {'key_field': None}, } for tool_name, key_cfg in _osint_key_map.items(): mgr = self._osint_managers.get(tool_name) @@ -235,12 +237,14 @@ def _build_rotator(main_key: str, tool_name: str) -> KeyRotator: if not enabled: self.tool_executor.update_osint_tool(tool_name, None) continue - if tool_name == 'censys': - api_id = user_settings.get(key_cfg['id_field'], '') - api_secret = user_settings.get(key_cfg['secret_field'], '') - if api_id and api_secret and (mgr.api_id != api_id or mgr.api_secret != api_secret): - mgr.api_id = api_id - mgr.api_secret = api_secret + if tool_name == 'uncover': + self.tool_executor.update_osint_tool(tool_name, mgr.get_tool()) + elif tool_name == 'censys': + api_token = user_settings.get(key_cfg['token_field'], '') + org_id = user_settings.get(key_cfg['org_field'], '') + if api_token and org_id and (mgr.api_token != api_token or mgr.org_id != org_id): + mgr.api_token = api_token + mgr.org_id = org_id self.tool_executor.update_osint_tool(tool_name, mgr.get_tool()) logger.info(f"Updated {tool_name} tool with API credentials") else: @@ -321,6 +325,7 @@ async def _setup_tools(self) -> None: 'virustotal': VirusTotalToolManager(), 'zoomeye': ZoomEyeToolManager(), 'criminalip': CriminalIpToolManager(), + 'uncover': UncoverToolManager(), } osint_tools = { name: mgr.get_tool() diff --git a/agentic/project_settings.py b/agentic/project_settings.py index c95c4952..1395f98d 100644 --- a/agentic/project_settings.py +++ b/agentic/project_settings.py @@ -100,6 +100,7 @@ 'virustotal': ['informational', 'exploitation'], 'zoomeye': ['informational', 'exploitation'], 'criminalip': ['informational', 'exploitation'], + 'uncover': ['informational', 'exploitation'], }, # Kali Shell Library Installation @@ -128,6 +129,7 @@ 'VIRUSTOTAL_ENABLED': True, 'ZOOMEYE_ENABLED': True, 'CRIMINALIP_ENABLED': True, + 'UNCOVER_ENABLED': True, # Social Engineering Simulation 'PHISHING_SMTP_CONFIG': '', # Free-text SMTP config for phishing email delivery (optional) diff --git a/agentic/prompts/tool_registry.py b/agentic/prompts/tool_registry.py index c785fa87..494b4f78 100644 --- a/agentic/prompts/tool_registry.py +++ b/agentic/prompts/tool_registry.py @@ -193,7 +193,7 @@ '**censys** (Internet-wide host/service search)\n' ' - **action="search"** — Search hosts by query (e.g. "services.port=443 AND location.country=US")\n' ' - **action="host"** — Detailed IP info: services, TLS certs, OS, ASN\n' - ' - Paid API — requires Censys API ID + Secret' + ' - Paid API — requires Censys API Token + Organization ID' ), }, "fofa": { @@ -258,4 +258,16 @@ ' - **action="domain_report"** — Risk assessment, technologies, domain intel' ), }, + "uncover": { + "purpose": "Multi-engine internet search (Shodan, Censys, FOFA, ZoomEye, Netlas, etc.)", + "when_to_use": "Search multiple OSINT engines at once for exposed assets, or look up a specific IP across all engines", + "args_format": '"action": "search|ip", "query": "search query", "ip": "1.2.3.4"', + "description": ( + '**uncover** (Multi-engine internet search)\n' + ' - **action="search"** — Search across all configured engines simultaneously\n' + ' - **action="ip"** — Lookup a specific IP across all engines\n' + ' - Supports: Shodan, Censys, FOFA, ZoomEye, Netlas, CriminalIP, Quake, Hunter, and more\n' + ' - Returns IP, port, hostname, and source engine for each result' + ), + }, } diff --git a/agentic/tools.py b/agentic/tools.py index 6fd8b6f9..bd8e9663 100644 --- a/agentic/tools.py +++ b/agentic/tools.py @@ -1008,16 +1008,16 @@ def _osint_http_error(tool_name: str, e: 'httpx.HTTPStatusError') -> str: class CensysToolManager: - """Censys internet search — host/service discovery via certificate and banner data.""" + """Censys internet search — host/service discovery via Platform API v3.""" - API_BASE = "https://search.censys.io/api/v2" + API_BASE = "https://api.platform.censys.io/v3/global" - def __init__(self, api_id: str = '', api_secret: str = ''): - self.api_id = api_id - self.api_secret = api_secret + def __init__(self, api_token: str = '', org_id: str = ''): + self.api_token = api_token + self.org_id = org_id def get_tool(self) -> Optional[callable]: - if not self.api_id or not self.api_secret: + if not self.api_token or not self.org_id: logger.warning("Censys API credentials not configured - censys tool unavailable.") return None manager = self @@ -1039,20 +1039,25 @@ async def censys(action: str, query: str = "", ip: str = "") -> str: Returns: Formatted results from the Censys API """ - auth = (manager.api_id, manager.api_secret) + headers = { + "Authorization": f"Bearer {manager.api_token}", + "Accept": "application/json", + "Content-Type": "application/json", + } + params = {"organization_id": manager.org_id} try: - async with httpx.AsyncClient(timeout=30.0, auth=auth) as client: + async with httpx.AsyncClient(timeout=30.0, headers=headers, params=params) as client: if action == "search": if not query: return "Error: 'query' required for action='search'" - resp = await client.get( - f"{manager.API_BASE}/hosts/search", - params={"q": query, "per_page": 25}, + resp = await client.post( + f"{manager.API_BASE}/search/query", + json={"query": query, "page_size": 25}, ) elif action == "host": if not ip: return "Error: 'ip' required for action='host'" - resp = await client.get(f"{manager.API_BASE}/hosts/{ip}") + resp = await client.get(f"{manager.API_BASE}/asset/host/{ip}") else: return f"Error: Unknown action '{action}'. Valid: search, host" @@ -1060,11 +1065,12 @@ async def censys(action: str, query: str = "", ip: str = "") -> str: data = resp.json() if action == "search": - hits = data.get("result", {}).get("hits", []) - total = data.get("result", {}).get("total", 0) + result = data.get("result", {}) + hits = result.get("hits", []) + total = result.get("total", 0) if not hits: return f"No Censys results for: {query}" - lines = [f"Censys search: {total} hosts (showing {len(hits)})"] + lines = [f"Censys search: {total} hits (showing {len(hits)})"] for i, h in enumerate(hits[:25], 1): ip_addr = h.get("ip", "?") services = h.get("services", []) @@ -1748,7 +1754,17 @@ async def criminalip(action: str, ip: str = "", domain: str = "") -> str: return "\n".join(lines) except httpx.HTTPStatusError as e: - return _osint_http_error("Criminal IP", e) + status = e.response.status_code + if status in (401, 403): + return "Criminal IP API error: API key is invalid or expired. Check Global Settings." + if status == 402: + return "Criminal IP API error: Credit/quota exhausted. Check your plan." + body = e.response.text[:300].lower() + if any(kw in body for kw in ("credit", "quota", "exceeded", "insufficient")): + return "Criminal IP API error: Credit/quota exhausted. Check your plan." + if status == 429: + return "Criminal IP API error: Rate limit exceeded. Try again later." + return f"Criminal IP API error: HTTP {status}" except Exception as e: logger.error(f"Criminal IP {action} failed: {e}") return f"Criminal IP error: {str(e)}" @@ -1757,6 +1773,80 @@ async def criminalip(action: str, ip: str = "", domain: str = "") -> str: return criminalip +class UncoverToolManager: + """ProjectDiscovery Uncover — multi-engine internet search.""" + + def __init__(self, api_key: str = ''): + self.api_key = api_key + self.key_rotator = None + + def get_tool(self) -> Optional[callable]: + manager = self + + @tool + async def uncover(action: str, query: str = '', ip: str = '') -> str: + """ + Uncover multi-engine internet search for exposed assets. + + Searches across Shodan, Censys, FOFA, ZoomEye, Netlas, CriminalIP, + and other engines simultaneously. + + Args: + action: "search" to search by query, "ip" to lookup a specific IP + query: Search query (e.g. 'ssl:"Example Inc."', 'hostname:example.com') + ip: IP address for IP-specific lookup + + Returns: + Discovered hosts with IP, port, and source engine + """ + if action == "ip" and ip: + query = ip + elif action == "search" and not query: + return "Error: 'query' is required for search action" + elif not query: + return "Error: provide action='search' with query, or action='ip' with ip" + + try: + import subprocess + import json as _json + cmd = ["docker", "run", "--rm", + "projectdiscovery/uncover:latest", + "-q", query, "-json", "-silent", "-l", "25"] + result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) + if result.returncode != 0: + return f"Uncover error: {result.stderr[:200]}" + + lines = [] + for line in result.stdout.strip().split('\n'): + if not line.strip(): + continue + try: + entry = _json.loads(line) + ip_val = entry.get('ip', '') + port_val = entry.get('port', '') + host_val = entry.get('host', '') + source = entry.get('source', '') + parts = [f"{ip_val}:{port_val}"] + if host_val and host_val != ip_val: + parts.append(f"host={host_val}") + if source: + parts.append(f"[{source}]") + lines.append(" ".join(parts)) + except _json.JSONDecodeError: + continue + + if not lines: + return f"Uncover: no results for '{query}'" + return f"Uncover results ({len(lines)}):\n" + "\n".join(lines[:25]) + except subprocess.TimeoutExpired: + return "Uncover: search timed out" + except Exception as e: + return f"Uncover error: {str(e)}" + + logger.info("Uncover multi-engine search tool configured") + return uncover + + # ============================================================================= # PHASE-AWARE TOOL EXECUTOR # ============================================================================= diff --git a/graph_db/mixins/osint_mixin.py b/graph_db/mixins/osint_mixin.py index 14887e71..dcf5b406 100644 --- a/graph_db/mixins/osint_mixin.py +++ b/graph_db/mixins/osint_mixin.py @@ -2109,4 +2109,95 @@ def update_graph_from_criminalip(self, recon_data: dict, user_id: str, project_i print(f"[graph-db] update_graph_from_criminalip complete: {stats}") return stats + def update_graph_from_uncover(self, recon_data: dict, user_id: str, project_id: str) -> dict: + """Update Neo4j graph with uncover target expansion results. + Creates Subdomain and IP nodes for newly discovered assets. + Uses ON CREATE SET to avoid overwriting richer data from other tools. + """ + stats = { + "subdomains_created": 0, "ips_created": 0, + "relationships_created": 0, "errors": [], + } + domain = recon_data.get("domain", "") or "" + try: + uncover = recon_data.get("uncover") or {} + hosts = uncover.get("hosts") or [] + ips = uncover.get("ips") or [] + ip_ports = uncover.get("ip_ports") or {} + + if not hosts and not ips: + return stats + + with self.driver.session() as session: + for hostname in hosts: + if not hostname: + continue + try: + session.run( + """ + MERGE (s:Subdomain {name: $name, user_id: $user_id, project_id: $project_id}) + ON CREATE SET s.discovered_at = datetime(), s.updated_at = datetime(), + s.source = 'uncover', s.status = 'unverified' + """, + name=hostname, user_id=user_id, project_id=project_id, + ) + stats["subdomains_created"] += 1 + if domain: + session.run( + """ + MATCH (s:Subdomain {name: $name, user_id: $user_id, project_id: $project_id}) + MATCH (d:Domain {name: $domain, user_id: $user_id, project_id: $project_id}) + MERGE (s)-[:BELONGS_TO]->(d) + MERGE (d)-[:HAS_SUBDOMAIN]->(s) + """, + name=hostname, domain=domain, + user_id=user_id, project_id=project_id, + ) + stats["relationships_created"] += 2 + except Exception as e: + stats["errors"].append(f"Uncover subdomain {hostname}: {e}") + + for ip in ips: + if not ip: + continue + try: + session.run( + """ + MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) + ON CREATE SET i.updated_at = datetime(), i.uncover_discovered = true + SET i.uncover_enriched = true, i.updated_at = datetime() + """, + address=ip, user_id=user_id, project_id=project_id, + ) + stats["ips_created"] += 1 + + ports = ip_ports.get(ip, []) + for port_num in ports: + if not port_num or port_num <= 0: + continue + session.run( + """ + MERGE (p:Port {number: $port, protocol: 'tcp', ip_address: $ip, + user_id: $user_id, project_id: $project_id}) + ON CREATE SET p.state = 'open', p.source = 'uncover', + p.updated_at = datetime() + MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + MERGE (i)-[:HAS_PORT]->(p) + """, + port=int(port_num), ip=ip, + user_id=user_id, project_id=project_id, + ) + stats["relationships_created"] += 1 + except Exception as e: + stats["errors"].append(f"Uncover IP {ip}: {e}") + + except Exception as e: + stats["errors"].append(f"update_graph_from_uncover: {e}") + + print(f"[+][graph-db] Uncover Graph Update: " + f"{stats['subdomains_created']} subdomains, " + f"{stats['ips_created']} IPs, " + f"{stats['relationships_created']} relationships") + print(f"[graph-db] update_graph_from_uncover complete") + return stats diff --git a/recon/censys_enrich.py b/recon/censys_enrich.py index e29d0341..c8e09e1b 100644 --- a/recon/censys_enrich.py +++ b/recon/censys_enrich.py @@ -1,12 +1,12 @@ """ Censys Pipeline Enrichment Module -Passive OSINT enrichment using the Censys Search API v2 (hosts). +Passive OSINT enrichment using the Censys Platform API v3. Queries host records for discovered IPv4 addresses: services, geo location, autonomous system, and operating system metadata. -Requires CENSYS_API_ID and CENSYS_API_SECRET (HTTP Basic Auth). No key rotation -(pair credentials). +Requires CENSYS_API_TOKEN (Personal Access Token) and CENSYS_ORG_ID +(Organization ID). Uses Bearer-token auth against api.platform.censys.io. """ from __future__ import annotations @@ -14,11 +14,16 @@ import logging from typing import Any +try: + from recon.ip_filter import filter_ips_for_enrichment +except ImportError: + from ip_filter import filter_ips_for_enrichment + import requests logger = logging.getLogger(__name__) -CENSYS_API_BASE = "https://search.censys.io/api/v2" +CENSYS_API_BASE = "https://api.platform.censys.io/v3/global" def _extract_ips_from_recon(combined_result: dict) -> list[str]: @@ -70,18 +75,19 @@ def _censys_normalize_software(svc: dict) -> list: return out -def _censys_get_host(ip: str, api_id: str, api_secret: str) -> tuple[dict | None, bool]: - """GET /v2/hosts/{ip} with Basic auth. +def _censys_get_host(ip: str, api_token: str, org_id: str) -> tuple[dict | None, bool]: + """GET /v3/global/asset/host/{ip} with Bearer token auth. Returns (result_or_none, rate_limited). If rate_limited, caller should stop. """ - url = f"{CENSYS_API_BASE}/hosts/{ip}" + url = f"{CENSYS_API_BASE}/asset/host/{ip}" + headers = { + "Authorization": f"Bearer {api_token}", + "Accept": "application/json", + } + params = {"organization_id": org_id} try: - resp = requests.get( - url, - auth=(api_id, api_secret), - timeout=30, - ) + resp = requests.get(url, headers=headers, params=params, timeout=30) if resp.status_code == 200: body = resp.json() result = body.get("result") @@ -96,6 +102,10 @@ def _censys_get_host(ip: str, api_id: str, api_secret: str) -> tuple[dict | None logger.warning("Censys rate limit (429) — stopping host fetches for this run") print("[!][Censys] Rate limit hit — skipping remaining hosts") return None, True + if resp.status_code in (401, 403): + logger.warning(f"Censys {resp.status_code} — auth failed (check token/org-id)") + print(f"[!][Censys] Auth error {resp.status_code} — verify API Token and Organization ID") + return None, True logger.warning(f"Censys {resp.status_code} for {ip}: {resp.text[:200]}") return None, False except requests.RequestException as e: @@ -263,16 +273,17 @@ def run_censys_enrichment(combined_result: dict, settings: dict[str, Any]) -> di if not settings.get("CENSYS_ENABLED", False): return combined_result - api_id = settings.get("CENSYS_API_ID", "") or "" - api_secret = settings.get("CENSYS_API_SECRET", "") or "" - if not api_id or not api_secret: - logger.warning("Censys API ID or secret missing — skipping enrichment") - print("[!][Censys] CENSYS_API_ID / CENSYS_API_SECRET not configured — skipping") + api_token = settings.get("CENSYS_API_TOKEN", "") or "" + org_id = settings.get("CENSYS_ORG_ID", "") or "" + if not api_token or not org_id: + logger.warning("Censys API Token or Organization ID missing — skipping enrichment") + print("[!][Censys] CENSYS_API_TOKEN / CENSYS_ORG_ID not configured — skipping") return combined_result print(f"[*][Censys] Starting OSINT enrichment") ips = _extract_ips_from_recon(combined_result) + ips = filter_ips_for_enrichment(ips, combined_result, "Censys") print(f"[+][Censys] Extracted {len(ips)} unique IPs for enrichment") censys_data: dict[str, Any] = {"hosts": []} @@ -283,7 +294,7 @@ def run_censys_enrichment(combined_result: dict, settings: dict[str, Any]) -> di else: print(f"[*][Censys] Querying host view for {len(ips)} IPs...") for ip in ips: - result, rate_limited = _censys_get_host(ip, api_id, api_secret) + result, rate_limited = _censys_get_host(ip, api_token, org_id) if rate_limited: break if result is None: diff --git a/recon/criminalip_enrich.py b/recon/criminalip_enrich.py index 647dbe11..4c4a1808 100644 --- a/recon/criminalip_enrich.py +++ b/recon/criminalip_enrich.py @@ -10,6 +10,11 @@ import requests +try: + from recon.ip_filter import filter_ips_for_enrichment +except ImportError: + from ip_filter import filter_ips_for_enrichment + logger = logging.getLogger(__name__) CRIMINALIP_API_BASE = "https://api.criminalip.io/v1/" @@ -44,17 +49,40 @@ def _effective_key(api_key: str, key_rotator) -> str: return (api_key or "").strip() +STOP_AUTH = "auth" +STOP_CREDIT = "credit" +STOP_RATE = "rate" + + +def _classify_stop_reason(status: int, body_text: str) -> str | None: + """Return a stop reason if the response indicates further requests are futile.""" + if status in (401, 403): + return STOP_AUTH + if status == 402: + return STOP_CREDIT + lower = body_text.lower() + if any(kw in lower for kw in ("credit", "quota", "exceeded", "limit reached", "insufficient")): + return STOP_CREDIT + if "unauthorized" in lower or "invalid api key" in lower or "invalid key" in lower: + return STOP_AUTH + return None + + def _cip_get( path: str, api_key: str, key_rotator, params: dict | None = None, timeout: int = 30, -) -> dict | None: - """GET Criminal IP v1 with 429 retry once.""" +) -> tuple[dict | None, str | None]: + """GET Criminal IP v1 with 429 retry once. + + Returns (body_or_none, stop_reason). stop_reason is non-None when further + requests should be skipped (auth failure, credit exhaustion, rate limit). + """ eff = _effective_key(api_key, key_rotator) if not eff: - return None + return None, STOP_AUTH url = f"{CRIMINALIP_API_BASE.rstrip('/')}/{path.lstrip('/')}" headers = {"x-api-key": eff} merged = dict(params or {}) @@ -66,27 +94,36 @@ def _cip_get( key_rotator.tick() if resp.status_code == 200: try: - return resp.json() + return resp.json(), None except ValueError: logger.warning(f"CriminalIP invalid JSON for {path}") - return None + return None, None if resp.status_code == 404: logger.debug(f"CriminalIP 404 for {path}") - return None + return None, None if resp.status_code == 429: logger.warning("CriminalIP rate limit (429), sleeping and retrying once") if attempt == 0: time.sleep(2) continue - return None + return None, STOP_RATE + + body_text = resp.text[:300] + stop = _classify_stop_reason(resp.status_code, body_text) + if stop: + logger.warning( + f"CriminalIP {resp.status_code} for {path}: {body_text}" + ) + return None, stop + logger.warning( - f"CriminalIP {resp.status_code} for {path}: {resp.text[:200]}" + f"CriminalIP {resp.status_code} for {path}: {body_text[:200]}" ) - return None + return None, None except requests.RequestException as e: logger.warning(f"CriminalIP request failed for {path}: {e}") - return None - return None + return None, None + return None, STOP_RATE def _parse_ip_report(ip: str, body: dict | None) -> dict | None: @@ -267,10 +304,22 @@ def _parse_domain_report(domain: str, body: dict | None) -> dict | None: return out +_STOP_MESSAGES = { + STOP_AUTH: "API key is invalid or expired — skipping remaining Criminal IP requests", + STOP_CREDIT: "API credit/quota exhausted — skipping remaining Criminal IP requests", + STOP_RATE: "Rate limit exceeded — skipping remaining Criminal IP requests", +} + +_MAX_CONSECUTIVE_FAILURES = 3 + + def run_criminalip_enrichment(combined_result: dict, settings: dict) -> dict: """ Run Criminal IP enrichment: domain report (domain mode) and per-IP data. + Stops early on auth/credit errors (single message) or after + ``_MAX_CONSECUTIVE_FAILURES`` consecutive data failures. + Mutates combined_result in place with key ``criminalip``. """ if not settings.get("CRIMINALIP_ENABLED", False): @@ -280,14 +329,15 @@ def run_criminalip_enrichment(combined_result: dict, settings: dict) -> dict: key_rotator = settings.get("CRIMINALIP_KEY_ROTATOR") if not _effective_key(api_key, key_rotator): - print(f"[!][CriminalIP] No API key configured — skipping") + print("[!][CriminalIP] No API key configured — skipping") return combined_result domain = combined_result.get("domain", "") is_ip_mode = combined_result.get("metadata", {}).get("ip_mode", False) ips = _extract_ips_from_recon(combined_result) + ips = filter_ips_for_enrichment(ips, combined_result, "CriminalIP") - print(f"[*][CriminalIP] Starting OSINT enrichment") + print("[*][CriminalIP] Starting OSINT enrichment") print(f"[+][CriminalIP] Extracted {len(ips)} unique IPs for enrichment") cip_data: dict = { @@ -295,40 +345,74 @@ def run_criminalip_enrichment(combined_result: dict, settings: dict) -> dict: "domain_report": None, } + def _handle_stop(reason: str) -> None: + msg = _STOP_MESSAGES.get(reason, f"Stopping Criminal IP requests ({reason})") + print(f"[!][CriminalIP] {msg}") + try: need_sleep = False + stopped = False + if domain and not is_ip_mode: print(f"[*][CriminalIP] Fetching domain report for {domain}...") - raw = _cip_get( + raw, stop = _cip_get( "domain/report", api_key, key_rotator, params={"query": domain}, ) - cip_data["domain_report"] = _parse_domain_report(domain, raw) - if cip_data["domain_report"]: - print(f"[+][CriminalIP] Domain report retrieved for {domain}") + if stop: + _handle_stop(stop) + stopped = True else: - print(f"[!][CriminalIP] No domain report data for {domain}") + cip_data["domain_report"] = _parse_domain_report(domain, raw) + if cip_data["domain_report"]: + print(f"[+][CriminalIP] Domain report retrieved for {domain}") + else: + print(f"[!][CriminalIP] No domain report data for {domain}") need_sleep = True + consecutive_fails = 0 + ips_attempted = 0 for ip in ips: + if stopped: + break if need_sleep: time.sleep(1) need_sleep = True + ips_attempted += 1 print(f"[*][CriminalIP] Fetching IP data for {ip}...") - raw = _cip_get("ip/data", api_key, key_rotator, params={"ip": ip, "full": "true"}) + raw, stop = _cip_get("ip/data", api_key, key_rotator, params={"ip": ip, "full": "true"}) + + if stop: + _handle_stop(stop) + stopped = True + break + report = _parse_ip_report(ip, raw) if report: cip_data["ip_reports"].append(report) + consecutive_fails = 0 vuln_count = len(report.get("vulnerabilities") or []) print( f"[+][CriminalIP] IP data retrieved for {ip} " f"(ports={len(report['ports'])}, vulns={vuln_count})" ) else: + consecutive_fails += 1 logger.warning(f"CriminalIP: no data for {ip}") - + if consecutive_fails >= _MAX_CONSECUTIVE_FAILURES: + print( + f"[!][CriminalIP] {consecutive_fails} consecutive failures " + f"— skipping remaining IPs" + ) + stopped = True + break + + if stopped: + skipped = len(ips) - ips_attempted + if skipped > 0: + print(f"[!][CriminalIP] Skipped {skipped} remaining IP(s)") print( f"[+][CriminalIP] Enrichment complete: " f"{len(cip_data['ip_reports'])} IP report(s), " diff --git a/recon/entrypoint.sh b/recon/entrypoint.sh index 07f1fda5..b140e01c 100644 --- a/recon/entrypoint.sh +++ b/recon/entrypoint.sh @@ -140,6 +140,7 @@ IMAGES=( "caffix/amass:latest" "frost19k/puredns:latest" "jauderho/hakrawler:latest" + "projectdiscovery/uncover:latest" ) for IMAGE in "${IMAGES[@]}"; do diff --git a/recon/fofa_enrich.py b/recon/fofa_enrich.py index d97a61a3..d4134a71 100644 --- a/recon/fofa_enrich.py +++ b/recon/fofa_enrich.py @@ -14,6 +14,11 @@ import requests +try: + from recon.ip_filter import filter_ips_for_enrichment +except ImportError: + from ip_filter import filter_ips_for_enrichment + logger = logging.getLogger(__name__) FOFA_API_URL = "https://fofa.info/api/v1/search/all" @@ -174,6 +179,7 @@ def run_fofa_enrichment(combined_result: dict, settings: dict[str, Any]) -> dict domain = combined_result.get("domain", "") or "" is_ip_mode = combined_result.get("metadata", {}).get("ip_mode", False) ips = _extract_ips_from_recon(combined_result) + ips = filter_ips_for_enrichment(ips, combined_result, "FOFA") print(f"[*][FOFA] Starting OSINT enrichment") diff --git a/recon/ip_filter.py b/recon/ip_filter.py new file mode 100644 index 00000000..bd35fc27 --- /dev/null +++ b/recon/ip_filter.py @@ -0,0 +1,90 @@ +""" +RedAmon - IP Filtering Helpers +=============================== +Classify and filter IPs before OSINT enrichment to avoid wasting API +credits on non-routable, reserved, or CDN addresses. +""" +from __future__ import annotations + +import ipaddress +import logging +from typing import Set + +logger = logging.getLogger(__name__) + +CGNAT_NETWORK = ipaddress.ip_network("100.64.0.0/10") + + +def is_non_routable_ip(ip_str: str) -> bool: + """Return True if *ip_str* should NOT be sent to external OSINT APIs. + + Covers RFC 1918 private, loopback, link-local, CGNAT (100.64.0.0/10), + and IETF reserved ranges. + """ + try: + addr = ipaddress.ip_address(ip_str) + except ValueError: + return True + return ( + addr.is_private + or addr.is_loopback + or addr.is_link_local + or addr.is_reserved + or addr.is_multicast + or addr.is_unspecified + or addr in CGNAT_NETWORK + ) + + +def collect_cdn_ips(combined_result: dict) -> Set[str]: + """Gather IPs flagged as CDN by Naabu/httpx from port_scan and http_probe data.""" + cdn_ips: Set[str] = set() + + port_scan = combined_result.get("port_scan") or {} + for ip, info in (port_scan.get("by_ip") or {}).items(): + if isinstance(info, dict) and info.get("is_cdn"): + cdn_ips.add(ip) + + http_probe = combined_result.get("http_probe") or {} + for _url, info in (http_probe.get("by_url") or {}).items(): + if isinstance(info, dict) and info.get("is_cdn"): + ip = info.get("ip") + if ip: + cdn_ips.add(ip) + + return cdn_ips + + +def filter_ips_for_enrichment( + ips: list[str], + combined_result: dict, + module_name: str = "OSINT", +) -> list[str]: + """Filter an IP list, removing non-routable and CDN IPs. + + Logs a summary of skipped IPs once (not per-IP) to keep output clean. + """ + cdn_ips = collect_cdn_ips(combined_result) + + kept: list[str] = [] + skipped_private = 0 + skipped_cdn = 0 + + for ip in ips: + if is_non_routable_ip(ip): + skipped_private += 1 + continue + if ip in cdn_ips: + skipped_cdn += 1 + continue + kept.append(ip) + + parts: list[str] = [] + if skipped_private: + parts.append(f"{skipped_private} non-routable/reserved") + if skipped_cdn: + parts.append(f"{skipped_cdn} CDN") + if parts: + print(f"[*][{module_name}] Skipped {', '.join(parts)} IP(s) from enrichment") + + return kept diff --git a/recon/main.py b/recon/main.py index f2aee49d..04565cdc 100644 --- a/recon/main.py +++ b/recon/main.py @@ -678,6 +678,22 @@ def run_ip_recon(target_ips: list, settings: dict) -> dict: # Background graph update: IP recon _graph_update_bg("update_graph_from_ip_recon", combined_result, USER_ID, PROJECT_ID) + # ===================================================================== + # GROUP 2b — Uncover Target Expansion (before port scan / OSINT) + # ===================================================================== + if settings.get('UNCOVER_ENABLED', False): + try: + from recon.uncover_enrich import run_uncover_expansion, merge_uncover_into_pipeline + uncover_data = run_uncover_expansion(combined_result, settings) + if uncover_data: + combined_result["uncover"] = uncover_data + merge_uncover_into_pipeline(combined_result, uncover_data, combined_result.get('domain', '')) + combined_result["metadata"]["modules_executed"].append("uncover_expansion") + save_recon_file(combined_result, output_file) + _graph_update_bg("update_graph_from_uncover", combined_result, USER_ID, PROJECT_ID) + except Exception as e: + print(f"[!][Uncover] Expansion failed: {e}") + # ===================================================================== # Shodan + Port Scan (parallel fan-out) — same pattern as domain recon # ===================================================================== @@ -760,7 +776,7 @@ def run_ip_recon(target_ips: list, settings: dict) -> dict: if settings.get(cfg[0], False) and ( settings.get(f'{name.upper()}_API_KEY', '') - or (name == 'censys' and settings.get('CENSYS_API_ID', '')) + or (name == 'censys' and settings.get('CENSYS_API_TOKEN', '')) or name == 'otx' # OTX supports anonymous requests without an API key ) } @@ -1050,6 +1066,22 @@ def run_domain_recon(target: str, anonymous: bool = False, bruteforce: bool = Fa if "urlscan" in combined_result: _graph_update_bg("update_graph_from_urlscan_discovery", combined_result, USER_ID, PROJECT_ID) + # ===================================================================== + # GROUP 2b — Uncover Target Expansion (before port scan / OSINT) + # ===================================================================== + if _settings.get('UNCOVER_ENABLED', False): + try: + from recon.uncover_enrich import run_uncover_expansion, merge_uncover_into_pipeline + uncover_data = run_uncover_expansion(combined_result, _settings) + if uncover_data: + combined_result["uncover"] = uncover_data + merge_uncover_into_pipeline(combined_result, uncover_data, TARGET_DOMAIN) + combined_result["metadata"]["modules_executed"].append("uncover_expansion") + save_recon_file(combined_result, output_file) + _graph_update_bg("update_graph_from_uncover", combined_result, USER_ID, PROJECT_ID) + except Exception as e: + print(f"[!][Uncover] Expansion failed: {e}") + # ===================================================================== # GROUP 3 — Fan-Out: Shodan + Port Scan (parallel) # Both need IPs/hostnames from DNS. Independent of each other. @@ -1168,7 +1200,7 @@ def run_domain_recon(target: str, anonymous: bool = False, bruteforce: bool = Fa if _settings.get(cfg[0], False) and ( _settings.get(f'{name.upper()}_API_KEY', '') - or (name == 'censys' and _settings.get('CENSYS_API_ID', '')) + or (name == 'censys' and _settings.get('CENSYS_API_TOKEN', '')) or name == 'otx' # OTX supports anonymous requests without an API key ) } diff --git a/recon/netlas_enrich.py b/recon/netlas_enrich.py index 7929cc77..690e546a 100644 --- a/recon/netlas_enrich.py +++ b/recon/netlas_enrich.py @@ -13,6 +13,11 @@ import requests +try: + from recon.ip_filter import filter_ips_for_enrichment +except ImportError: + from ip_filter import filter_ips_for_enrichment + logger = logging.getLogger(__name__) NETLAS_API_BASE = "https://app.netlas.io/api" @@ -210,6 +215,7 @@ def run_netlas_enrichment(combined_result: dict, settings: dict[str, Any]) -> di domain = combined_result.get("domain", "") or "" is_ip_mode = combined_result.get("metadata", {}).get("ip_mode", False) ips = _extract_ips_from_recon(combined_result) + ips = filter_ips_for_enrichment(ips, combined_result, "Netlas") print(f"[*][Netlas] Starting OSINT enrichment") diff --git a/recon/otx_enrich.py b/recon/otx_enrich.py index 12b3b6f3..a5fcf195 100644 --- a/recon/otx_enrich.py +++ b/recon/otx_enrich.py @@ -23,6 +23,11 @@ import requests +try: + from recon.ip_filter import filter_ips_for_enrichment +except ImportError: + from ip_filter import filter_ips_for_enrichment + logger = logging.getLogger(__name__) OTX_API_BASE = "https://otx.alienvault.com/api/v1/indicators" @@ -345,6 +350,7 @@ def run_otx_enrichment(combined_result: dict, settings: dict[str, Any]) -> dict: domain = combined_result.get("domain", "") or "" is_ip_mode = combined_result.get("metadata", {}).get("ip_mode", False) ips = _extract_ips_from_recon(combined_result) + ips = filter_ips_for_enrichment(ips, combined_result, "OTX") print(f"[+][OTX] Extracted {len(ips)} unique IPs") diff --git a/recon/project_settings.py b/recon/project_settings.py index f9d7662d..75227d94 100644 --- a/recon/project_settings.py +++ b/recon/project_settings.py @@ -383,8 +383,8 @@ # OSINT & Threat Intelligence Enrichment 'CENSYS_ENABLED': False, - 'CENSYS_API_ID': '', - 'CENSYS_API_SECRET': '', + 'CENSYS_API_TOKEN': '', + 'CENSYS_ORG_ID': '', 'FOFA_ENABLED': False, 'FOFA_MAX_RESULTS': 1000, 'FOFA_API_KEY': '', @@ -403,6 +403,18 @@ 'CRIMINALIP_ENABLED': False, 'CRIMINALIP_API_KEY': '', + # Uncover (ProjectDiscovery multi-engine search) + 'UNCOVER_ENABLED': False, + 'UNCOVER_MAX_RESULTS': 500, + 'UNCOVER_QUAKE_API_KEY': '', + 'UNCOVER_HUNTER_API_KEY': '', + 'UNCOVER_PUBLICWWW_API_KEY': '', + 'UNCOVER_HUNTERHOW_API_KEY': '', + 'UNCOVER_GOOGLE_API_KEY': '', + 'UNCOVER_GOOGLE_API_CX': '', + 'UNCOVER_ONYPHE_API_KEY': '', + 'UNCOVER_DRIFTNET_API_KEY': '', + # Subdomain Discovery Tool Toggles 'CRTSH_ENABLED': True, 'CRTSH_MAX_RESULTS': 5000, @@ -810,6 +822,8 @@ def fetch_project_settings(project_id: str, webapp_url: str) -> dict[str, Any]: settings['ZOOMEYE_ENABLED'] = project.get('zoomEyeEnabled', DEFAULT_SETTINGS['ZOOMEYE_ENABLED']) settings['ZOOMEYE_MAX_RESULTS'] = project.get('zoomEyeMaxResults', DEFAULT_SETTINGS['ZOOMEYE_MAX_RESULTS']) settings['CRIMINALIP_ENABLED'] = project.get('criminalIpEnabled', DEFAULT_SETTINGS['CRIMINALIP_ENABLED']) + settings['UNCOVER_ENABLED'] = project.get('uncoverEnabled', DEFAULT_SETTINGS['UNCOVER_ENABLED']) + settings['UNCOVER_MAX_RESULTS'] = int(project.get('uncoverMaxResults', DEFAULT_SETTINGS['UNCOVER_MAX_RESULTS']) or DEFAULT_SETTINGS['UNCOVER_MAX_RESULTS']) # Subdomain Discovery Tool Toggles settings['CRTSH_ENABLED'] = project.get('crtshEnabled', DEFAULT_SETTINGS['CRTSH_ENABLED']) @@ -883,8 +897,8 @@ def _build_rotator(main_key: str, tool_name: str) -> 'KeyRotator': # OSINT & Threat Intelligence keys if settings.get('CENSYS_ENABLED'): - settings['CENSYS_API_ID'] = user_global.get('censysApiId', '') - settings['CENSYS_API_SECRET'] = user_global.get('censysApiSecret', '') + settings['CENSYS_API_TOKEN'] = user_global.get('censysApiToken', '') + settings['CENSYS_ORG_ID'] = user_global.get('censysOrgId', '') if settings.get('FOFA_ENABLED'): fofa_key = user_global.get('fofaApiKey', '') @@ -916,6 +930,32 @@ def _build_rotator(main_key: str, tool_name: str) -> 'KeyRotator': settings['CRIMINALIP_API_KEY'] = cip_key settings['CRIMINALIP_KEY_ROTATOR'] = _build_rotator(cip_key, 'criminalip') + # Uncover keys — always load shared OSINT keys so uncover can use + # engines even when the per-tool enrichment toggles are off. + if settings.get('UNCOVER_ENABLED'): + if not settings.get('SHODAN_API_KEY'): + settings['SHODAN_API_KEY'] = user_global.get('shodanApiKey', '') + if not settings.get('FOFA_API_KEY'): + settings['FOFA_API_KEY'] = user_global.get('fofaApiKey', '') + if not settings.get('ZOOMEYE_API_KEY'): + settings['ZOOMEYE_API_KEY'] = user_global.get('zoomEyeApiKey', '') + if not settings.get('NETLAS_API_KEY'): + settings['NETLAS_API_KEY'] = user_global.get('netlasApiKey', '') + if not settings.get('CRIMINALIP_API_KEY'): + settings['CRIMINALIP_API_KEY'] = user_global.get('criminalIpApiKey', '') + if not settings.get('CENSYS_API_TOKEN'): + settings['CENSYS_API_TOKEN'] = user_global.get('censysApiToken', '') + if not settings.get('CENSYS_ORG_ID'): + settings['CENSYS_ORG_ID'] = user_global.get('censysOrgId', '') + settings['UNCOVER_QUAKE_API_KEY'] = user_global.get('quakeApiKey', '') + settings['UNCOVER_HUNTER_API_KEY'] = user_global.get('hunterApiKey', '') + settings['UNCOVER_PUBLICWWW_API_KEY'] = user_global.get('publicWwwApiKey', '') + settings['UNCOVER_HUNTERHOW_API_KEY'] = user_global.get('hunterHowApiKey', '') + settings['UNCOVER_GOOGLE_API_KEY'] = user_global.get('googleApiKey', '') + settings['UNCOVER_GOOGLE_API_CX'] = user_global.get('googleApiCx', '') + settings['UNCOVER_ONYPHE_API_KEY'] = user_global.get('onypheApiKey', '') + settings['UNCOVER_DRIFTNET_API_KEY'] = user_global.get('driftnetApiKey', '') + # Rules of Engagement settings['ROE_ENABLED'] = project.get('roeEnabled', DEFAULT_SETTINGS['ROE_ENABLED']) settings['ROE_EXCLUDED_HOSTS'] = project.get('roeExcludedHosts', DEFAULT_SETTINGS['ROE_EXCLUDED_HOSTS']) diff --git a/recon/shodan_enrich.py b/recon/shodan_enrich.py index 3aa80062..67587acc 100644 --- a/recon/shodan_enrich.py +++ b/recon/shodan_enrich.py @@ -18,6 +18,11 @@ import requests +try: + from recon.ip_filter import filter_ips_for_enrichment +except ImportError: + from ip_filter import filter_ips_for_enrichment + logger = logging.getLogger(__name__) SHODAN_API_BASE = "https://api.shodan.io" @@ -346,6 +351,7 @@ def run_shodan_enrichment(combined_result: dict, settings: dict[str, Any]) -> di print("-" * 40) ips = _extract_ips_from_recon(combined_result) + ips = filter_ips_for_enrichment(ips, combined_result, "Shodan") domain = combined_result.get("domain", "") is_ip_mode = combined_result.get("metadata", {}).get("ip_mode", False) diff --git a/recon/uncover_enrich.py b/recon/uncover_enrich.py new file mode 100644 index 00000000..9d6c021b --- /dev/null +++ b/recon/uncover_enrich.py @@ -0,0 +1,462 @@ +""" +Uncover Pipeline Enrichment Module +==================================== +Target expansion using ProjectDiscovery's uncover tool. +Queries multiple search engines (Shodan, Censys, FOFA, ZoomEye, Netlas, +CriminalIP, Quake, Hunter, PublicWWW, HunterHow, Google, Onyphe, Driftnet) +to discover exposed hosts associated with the target domain/org. + +Runs as a Docker container (projectdiscovery/uncover) with a dynamically +generated provider-config.yaml containing only engines that have API keys. + +This module is a DISCOVERY tool (finds new IPs/hosts), not an enrichment +tool. Its output is merged back into the pipeline's DNS structures so +downstream modules (Shodan, port scan, OSINT enrichment, HTTP probe) can +process the newly discovered assets. +""" +from __future__ import annotations + +import json +import logging +import os +import subprocess +from typing import Any, Dict, Set + +try: + from recon.ip_filter import filter_ips_for_enrichment, is_non_routable_ip +except ImportError: + from ip_filter import filter_ips_for_enrichment, is_non_routable_ip + +logger = logging.getLogger(__name__) + +UNCOVER_DOCKER_IMAGE = "projectdiscovery/uncover:latest" +UNCOVER_TIMEOUT = 600 # 10 minutes max + + +def _build_provider_config(settings: dict) -> tuple[dict, list[str]]: + """Build uncover provider-config.yaml content and matching engine list. + + Only includes engines that have valid credentials configured. + Reuses existing OSINT keys (Shodan, FOFA, ZoomEye, Netlas, CriminalIP) + alongside uncover-specific keys (Quake, Hunter, etc.). + + Returns (yaml_dict, enabled_engines). + """ + config: Dict[str, list] = {} + engines: list[str] = [] + + # Shodan (reuses existing pipeline key) + shodan_key = settings.get('SHODAN_API_KEY', '') + if shodan_key: + config['shodan'] = [shodan_key] + engines.append('shodan') + + # Censys (platform token + org ID) + censys_token = settings.get('CENSYS_API_TOKEN', '') + censys_org = settings.get('CENSYS_ORG_ID', '') + if censys_token and censys_org: + config['censys'] = [f"{censys_token}:{censys_org}"] + engines.append('censys') + + # FOFA (reuses existing key, format: email:key or key-only) + fofa_key = settings.get('FOFA_API_KEY', '') + if fofa_key: + config['fofa'] = [fofa_key] + engines.append('fofa') + + # ZoomEye (reuses existing key) + ze_key = settings.get('ZOOMEYE_API_KEY', '') + if ze_key: + config['zoomeye'] = [ze_key] + engines.append('zoomeye') + + # Netlas (reuses existing key) + netlas_key = settings.get('NETLAS_API_KEY', '') + if netlas_key: + config['netlas'] = [netlas_key] + engines.append('netlas') + + # CriminalIP (reuses existing key) + cip_key = settings.get('CRIMINALIP_API_KEY', '') + if cip_key: + config['criminalip'] = [cip_key] + engines.append('criminalip') + + # Uncover-only engines + quake_key = settings.get('UNCOVER_QUAKE_API_KEY', '') + if quake_key: + config['quake'] = [quake_key] + engines.append('quake') + + hunter_key = settings.get('UNCOVER_HUNTER_API_KEY', '') + if hunter_key: + config['hunter'] = [hunter_key] + engines.append('hunter') + + publicwww_key = settings.get('UNCOVER_PUBLICWWW_API_KEY', '') + if publicwww_key: + config['publicwww'] = [publicwww_key] + engines.append('publicwww') + + hunterhow_key = settings.get('UNCOVER_HUNTERHOW_API_KEY', '') + if hunterhow_key: + config['hunterhow'] = [hunterhow_key] + engines.append('hunterhow') + + google_key = settings.get('UNCOVER_GOOGLE_API_KEY', '') + google_cx = settings.get('UNCOVER_GOOGLE_API_CX', '') + if google_key and google_cx: + config['google'] = [f"{google_key}:{google_cx}"] + engines.append('google') + + onyphe_key = settings.get('UNCOVER_ONYPHE_API_KEY', '') + if onyphe_key: + config['onyphe'] = [onyphe_key] + engines.append('onyphe') + + driftnet_key = settings.get('UNCOVER_DRIFTNET_API_KEY', '') + if driftnet_key: + config['driftnet'] = [driftnet_key] + engines.append('driftnet') + + # shodan-idb works without keys, always include for IP lookups + if 'shodan' not in engines: + engines.append('shodan-idb') + + return config, engines + + +def _build_queries(domain: str, settings: dict) -> list[str]: + """Build search queries for the target domain. + + Uses hostname and SSL certificate org searches to maximize coverage. + """ + queries = [domain] + + whois_org = settings.get('_WHOIS_ORG', '') + if whois_org and whois_org.lower() not in ('n/a', 'unknown', '', 'none'): + queries.append(f'ssl:"{whois_org}"') + + return queries + + +def _run_uncover_docker( + queries: list[str], + engines: list[str], + config_path: str, + max_results: int, + temp_dir: str, +) -> list[dict]: + """Run uncover via Docker and parse JSON output. + + Returns list of parsed JSON result dicts. + """ + output_file = os.path.join(temp_dir, "uncover_output.jsonl") + + cmd = [ + "docker", "run", "--rm", + "-v", f"{temp_dir}:/config:ro", + "-v", f"{temp_dir}:/output", + UNCOVER_DOCKER_IMAGE, + "-pc", "/config/provider-config.yaml", + "-e", ",".join(engines), + "-json", + "-silent", + "-l", str(max_results), + "-timeout", "60", + "-o", "/output/uncover_output.jsonl", + ] + + for q in queries: + cmd.extend(["-q", q]) + + logger.info(f"Running uncover: engines={engines}, queries={queries}") + + result = None + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=UNCOVER_TIMEOUT, + ) + + if result.returncode != 0: + stderr = (result.stderr or '').strip() + if stderr: + for line in stderr.split('\n')[:5]: + line = line.strip() + if line and not line.startswith('[WRN]'): + logger.warning(f"Uncover stderr: {line}") + + except subprocess.TimeoutExpired: + print("[!][Uncover] Timed out — partial results may be available") + except FileNotFoundError: + print("[!][Uncover] Docker not found — skipping") + return [] + except Exception as e: + print(f"[!][Uncover] Error: {e}") + return [] + + results = [] + if os.path.isfile(output_file): + with open(output_file, 'r') as f: + for line in f: + line = line.strip() + if not line: + continue + try: + results.append(json.loads(line)) + except json.JSONDecodeError: + continue + + if not results and result is not None and result.stdout: + for line in result.stdout.strip().split('\n'): + line = line.strip() + if not line: + continue + try: + results.append(json.loads(line)) + except json.JSONDecodeError: + continue + + return results + + +def _deduplicate_results(results: list[dict]) -> list[dict]: + """Deduplicate by (ip, port) keeping the first occurrence per source.""" + seen: Set[tuple] = set() + unique = [] + for r in results: + ip = r.get('ip', '') + if not ip: + continue + try: + port = int(r.get('port', 0) or 0) + except (ValueError, TypeError): + port = 0 + key = (ip, port) + if key not in seen: + seen.add(key) + unique.append(r) + return unique + + +def _extract_hosts_and_ips( + results: list[dict], + domain: str, + combined_result: dict, +) -> tuple[list[str], list[str], dict]: + """Extract unique IPs, hostnames, and per-IP port data from uncover results. + + Filters out non-routable and CDN IPs. + Returns (new_ips, new_hostnames, ip_ports_map). + """ + all_ips: Set[str] = set() + all_hosts: Set[str] = set() + ip_ports: Dict[str, Set[int]] = {} + + for r in results: + ip = r.get('ip', '') + host = r.get('host', '') + + try: + port = int(r.get('port', 0) or 0) + except (ValueError, TypeError): + port = 0 + + if ip: + all_ips.add(ip) + if port > 0: + ip_ports.setdefault(ip, set()).add(port) + + if host and host != ip: + h = host.lower().strip().rstrip('.') + if h and (h == domain or h.endswith('.' + domain)): + all_hosts.add(h) + + # Filter IPs + filtered_ips = filter_ips_for_enrichment( + sorted(all_ips), combined_result, "Uncover" + ) + + # Build ip_ports map for filtered IPs only + filtered_ip_ports = {} + for ip in filtered_ips: + if ip in ip_ports: + filtered_ip_ports[ip] = sorted(ip_ports[ip]) + + return filtered_ips, sorted(all_hosts), filtered_ip_ports + + +def run_uncover_expansion( + combined_result: dict, + settings: dict[str, Any], +) -> dict: + """Run uncover target expansion and merge new hosts/IPs into combined_result. + + Designed to run BEFORE Shodan + port scan (GROUP 3) so newly discovered + assets are processed by all downstream modules. + + Args: + combined_result: Pipeline's combined result dict (mutated in place) + settings: Project settings dict + + Returns: + Uncover data dict with discovered hosts, or empty dict. + """ + if not settings.get('UNCOVER_ENABLED', False): + return {} + + print("\n[*][Uncover] Starting multi-engine target expansion") + print("-" * 40) + + config, engines = _build_provider_config(settings) + if not engines or (len(engines) == 1 and engines[0] == 'shodan-idb'): + print("[!][Uncover] No API keys configured for any search engine — skipping") + return {} + + keyed_engines = [e for e in engines if e != 'shodan-idb'] + print(f"[+][Uncover] Engines: {', '.join(keyed_engines)}") + + domain = combined_result.get('domain', '') + if not domain: + print("[!][Uncover] No target domain — skipping") + return {} + + # Extract org name from whois if available + whois_data = combined_result.get('whois', {}) + if isinstance(whois_data, dict): + org = whois_data.get('registrant_org', '') or whois_data.get('org', '') + if org: + settings['_WHOIS_ORG'] = org + + queries = _build_queries(domain, settings) + max_results = int(settings.get('UNCOVER_MAX_RESULTS', 500)) + + print(f"[*][Uncover] Queries: {queries}") + print(f"[*][Uncover] Max results: {max_results}") + + temp_dir = "/tmp/redamon/.uncover_temp" + os.makedirs(temp_dir, exist_ok=True) + + try: + # Write provider config (no yaml dependency -- write manually) + config_path = os.path.join(temp_dir, "provider-config.yaml") + with open(config_path, 'w') as f: + for engine_name, keys in config.items(): + f.write(f"{engine_name}:\n") + for k in keys: + f.write(f" - {k}\n") + + raw_results = _run_uncover_docker( + queries, engines, config_path, max_results, temp_dir, + ) + + if not raw_results: + print("[*][Uncover] No results found") + return {} + + deduped = _deduplicate_results(raw_results) + print(f"[+][Uncover] Raw: {len(raw_results)} results, deduplicated: {len(deduped)}") + + # Collect source stats + source_counts: Dict[str, int] = {} + for r in deduped: + src = r.get('source', 'unknown') + source_counts[src] = source_counts.get(src, 0) + 1 + + new_ips, new_hosts, ip_ports = _extract_hosts_and_ips( + deduped, domain, combined_result, + ) + + print(f"[+][Uncover] Discovered: {len(new_ips)} unique IPs, {len(new_hosts)} subdomains") + for src, cnt in sorted(source_counts.items()): + print(f" [{src}] {cnt} results") + + uncover_data = { + "hosts": new_hosts, + "ips": new_ips, + "ip_ports": ip_ports, + "sources": list(source_counts.keys()), + "source_counts": source_counts, + "total_raw": len(raw_results), + "total_deduped": len(deduped), + } + + return uncover_data + + except Exception as e: + logger.error(f"Uncover expansion failed: {e}") + print(f"[!][Uncover] Error: {e}") + return {} + finally: + # Cleanup temp files + for fname in ("provider-config.yaml", "uncover_output.jsonl"): + fpath = os.path.join(temp_dir, fname) + if os.path.isfile(fpath): + try: + os.remove(fpath) + except OSError: + pass + + +def merge_uncover_into_pipeline( + combined_result: dict, + uncover_data: dict, + domain: str, +) -> int: + """Merge uncover discoveries into the pipeline's DNS/subdomain structures. + + New subdomains go into dns.subdomains so downstream modules + (port scan, HTTP probe, OSINT enrichment) process them. + + Returns count of new assets merged. + """ + if not uncover_data: + return 0 + + dns = combined_result.setdefault("dns", {}) + subdomains = dns.setdefault("subdomains", {}) + merged = 0 + + # Merge new hostnames as subdomains with their IPs + for host in uncover_data.get("hosts", []): + if host not in subdomains: + subdomains[host] = { + "ips": {"ipv4": [], "ipv6": []}, + "source": "uncover", + } + merged += 1 + + # For IPs discovered without a hostname, track them + # so OSINT enrichment modules can process them + ip_ports = uncover_data.get("ip_ports", {}) + existing_ips = set() + domain_dns = dns.get("domain", {}) + for ip in domain_dns.get("ips", {}).get("ipv4", []): + existing_ips.add(ip) + for _sub, info in subdomains.items(): + for ip in info.get("ips", {}).get("ipv4", []): + existing_ips.add(ip) + + new_ip_count = 0 + for ip in uncover_data.get("ips", []): + if ip not in existing_ips: + new_ip_count += 1 + + # Store the uncover IP data for OSINT enrichment modules to consume + if uncover_data.get("ips"): + expanded = combined_result.get("metadata", {}).get("expanded_ips", []) + if isinstance(expanded, list): + existing_expanded = set(expanded) + for ip in uncover_data["ips"]: + if ip not in existing_expanded: + expanded.append(ip) + combined_result.setdefault("metadata", {})["expanded_ips"] = expanded + + total = merged + new_ip_count + if total: + print(f"[+][Uncover] Merged: {merged} new subdomains, {new_ip_count} new IPs into pipeline") + + return total diff --git a/recon/virustotal_enrich.py b/recon/virustotal_enrich.py index da4c0370..504e1eac 100644 --- a/recon/virustotal_enrich.py +++ b/recon/virustotal_enrich.py @@ -11,6 +11,11 @@ import requests +try: + from recon.ip_filter import filter_ips_for_enrichment +except ImportError: + from ip_filter import filter_ips_for_enrichment + logger = logging.getLogger(__name__) VIRUSTOTAL_API_BASE = "https://www.virustotal.com/api/v3/" @@ -155,6 +160,7 @@ def run_virustotal_enrichment(combined_result: dict, settings: dict) -> dict: domain = combined_result.get("domain", "") is_ip_mode = combined_result.get("metadata", {}).get("ip_mode", False) ips = _extract_ips_from_recon(combined_result) + ips = filter_ips_for_enrichment(ips, combined_result, "VirusTotal") ip_slice = ips[:max_targets] if max_targets else [] print(f"[*][VirusTotal] Starting OSINT enrichment") diff --git a/recon/zoomeye_enrich.py b/recon/zoomeye_enrich.py index 40e5b059..e4ecdfbc 100644 --- a/recon/zoomeye_enrich.py +++ b/recon/zoomeye_enrich.py @@ -10,6 +10,11 @@ import requests +try: + from recon.ip_filter import filter_ips_for_enrichment +except ImportError: + from ip_filter import filter_ips_for_enrichment + logger = logging.getLogger(__name__) ZOOMEYE_API_BASE = "https://api.zoomeye.ai/" @@ -231,6 +236,7 @@ def run_zoomeye_enrichment(combined_result: dict, settings: dict) -> dict: domain = combined_result.get("domain", "") is_ip_mode = combined_result.get("metadata", {}).get("ip_mode", False) ips = _extract_ips_from_recon(combined_result) + ips = filter_ips_for_enrichment(ips, combined_result, "ZoomEye") print(f"[*][ZoomEye] Starting OSINT enrichment") diff --git a/recon_orchestrator/api.py b/recon_orchestrator/api.py index 11b53e93..a43a5e43 100644 --- a/recon_orchestrator/api.py +++ b/recon_orchestrator/api.py @@ -175,8 +175,8 @@ async def get_defaults(): # API keys fetched at runtime from user's global settings (not stored per-project) 'SHODAN_API_KEY', 'URLSCAN_API_KEY', - 'CENSYS_API_ID', - 'CENSYS_API_SECRET', + 'CENSYS_API_TOKEN', + 'CENSYS_ORG_ID', 'OTX_API_KEY', 'NETLAS_API_KEY', 'VIRUSTOTAL_API_KEY', diff --git a/recon_orchestrator/container_manager.py b/recon_orchestrator/container_manager.py index b492f6a3..63896769 100644 --- a/recon_orchestrator/container_manager.py +++ b/recon_orchestrator/container_manager.py @@ -31,6 +31,7 @@ "projectdiscovery/httpx", "projectdiscovery/katana", "projectdiscovery/nuclei", + "projectdiscovery/uncover", "sxcurity/gau", "frost19k/puredns", ] diff --git a/tests/test_censys_enrich.py b/tests/test_censys_enrich.py index 1fe619c9..f484ae7c 100644 --- a/tests/test_censys_enrich.py +++ b/tests/test_censys_enrich.py @@ -1,8 +1,8 @@ """ Unit tests for Censys OSINT enrichment (recon/censys_enrich.py). -Mocks requests.get for https://search.censys.io/api/v2/hosts/{ip}. -Censys uses HTTP Basic auth and has no KEY_ROTATOR (.tick() N/A). +Mocks requests.get for the Censys Platform API v3. +Censys uses Bearer-token auth (Personal Access Token + Organization ID). """ from __future__ import annotations @@ -56,8 +56,8 @@ class TestCensysEnrich(unittest.TestCase): def _settings(self, **overrides) -> dict: base = { "CENSYS_ENABLED": True, - "CENSYS_API_ID": "id-test", - "CENSYS_API_SECRET": "secret-test", + "CENSYS_API_TOKEN": "token-test", + "CENSYS_ORG_ID": "org-test", } base.update(overrides) return base @@ -83,15 +83,15 @@ def test_enrichment_success(self, mock_get, _sleep): mock_get.assert_called() args, kwargs = mock_get.call_args - self.assertTrue(str(args[0]).startswith("https://search.censys.io/api/v2/hosts/")) - self.assertEqual(kwargs.get("auth"), ("id-test", "secret-test")) + self.assertIn("api.platform.censys.io/v3/global/asset/host/", str(args[0])) + self.assertIn("Bearer token-test", kwargs.get("headers", {}).get("Authorization", "")) @patch("censys_enrich.requests.get") def test_missing_api_key(self, mock_get): cr = _combined_result() for settings in ( - self._settings(CENSYS_API_ID=""), - self._settings(CENSYS_API_SECRET=""), + self._settings(CENSYS_API_TOKEN=""), + self._settings(CENSYS_ORG_ID=""), ): out = run_censys_enrichment(cr, settings) self.assertNotIn("censys", out) @@ -100,7 +100,7 @@ def test_missing_api_key(self, mock_get): @patch("censys_enrich.time.sleep") @patch("censys_enrich.requests.get") def test_http_error(self, mock_get, _sleep): - for code in (401, 500): + for code in (500, 502): with self.subTest(code=code): mock_get.reset_mock() mock_get.return_value = _mock_response(code, {}, text="err") @@ -108,6 +108,15 @@ def test_http_error(self, mock_get, _sleep): out = run_censys_enrichment(cr, self._settings()) self.assertEqual(out["censys"]["hosts"], []) + @patch("censys_enrich.time.sleep") + @patch("censys_enrich.requests.get") + def test_auth_error_stops(self, mock_get, _sleep): + """401/403 should stop all further fetches (treated like rate limit).""" + mock_get.return_value = _mock_response(401, {}, text="unauthorized") + cr = _combined_result() + out = run_censys_enrichment(cr, self._settings()) + self.assertEqual(out["censys"]["hosts"], []) + @patch("censys_enrich.time.sleep") @patch("censys_enrich.requests.get") def test_rate_limit(self, mock_get, _sleep): diff --git a/tests/test_criminalip_enrich.py b/tests/test_criminalip_enrich.py index e16cbe9c..0a70af58 100644 --- a/tests/test_criminalip_enrich.py +++ b/tests/test_criminalip_enrich.py @@ -211,8 +211,19 @@ def test_missing_api_key(self, mock_get): @patch("criminalip_enrich.time.sleep") @patch("criminalip_enrich.requests.get") - def test_http_error(self, mock_get, _sleep): - for code in (401, 500): + def test_http_error_500(self, mock_get, _sleep): + """Non-auth errors (500) return None without triggering a hard stop.""" + mock_get.return_value = _mock_response(500, {}, text="err") + cr = _combined_result() + out = run_criminalip_enrichment(cr, self._settings()) + self.assertIsNone(out["criminalip"]["domain_report"]) + self.assertEqual(out["criminalip"]["ip_reports"], []) + + @patch("criminalip_enrich.time.sleep") + @patch("criminalip_enrich.requests.get") + def test_auth_error_stops_early(self, mock_get, _sleep): + """401/403 on the first request stops all further requests.""" + for code in (401, 403): with self.subTest(code=code): mock_get.reset_mock() mock_get.return_value = _mock_response(code, {}, text="err") @@ -220,6 +231,7 @@ def test_http_error(self, mock_get, _sleep): out = run_criminalip_enrichment(cr, self._settings()) self.assertIsNone(out["criminalip"]["domain_report"]) self.assertEqual(out["criminalip"]["ip_reports"], []) + self.assertEqual(mock_get.call_count, 1, "Should stop after first auth failure") @patch("criminalip_enrich.time.sleep") @patch("criminalip_enrich.requests.get") @@ -339,5 +351,97 @@ def side_effect(url, **_kwargs): self.assertEqual(rep["ports"][0]["port"], 80) + @patch("criminalip_enrich.time.sleep") + @patch("criminalip_enrich.requests.get") + def test_credit_exhaustion_stops_early(self, mock_get, _sleep): + """Response body mentioning 'credit' or 'quota' triggers early stop.""" + mock_get.return_value = _mock_response( + 400, {}, text='{"status":400,"message":"credit exceeded"}' + ) + cr = _combined_result() + out = run_criminalip_enrichment(cr, self._settings()) + self.assertIsNone(out["criminalip"]["domain_report"]) + self.assertEqual(out["criminalip"]["ip_reports"], []) + self.assertEqual(mock_get.call_count, 1) + + @patch("criminalip_enrich.time.sleep") + @patch("criminalip_enrich.requests.get") + def test_consecutive_data_failures_stop(self, mock_get, _sleep): + """After 3 consecutive IPs returning no data, remaining IPs are skipped.""" + cr = { + "domain": "", + "metadata": {"ip_mode": True, "expanded_ips": [ + "1.2.3.1", "1.2.3.2", "1.2.3.3", "1.2.3.4", "1.2.3.5", + ]}, + "dns": {"domain": {}, "subdomains": {}}, + } + mock_get.return_value = _mock_response( + 400, {}, text='{"status":400,"message":"Invalid IP Address","data":{}}' + ) + out = run_criminalip_enrichment(cr, self._settings()) + self.assertEqual(out["criminalip"]["ip_reports"], []) + self.assertEqual(mock_get.call_count, 3, "Should stop after 3 consecutive failures") + + @patch("criminalip_enrich.time.sleep") + @patch("criminalip_enrich.requests.get") + def test_consecutive_counter_resets_on_success(self, mock_get, _sleep): + """A successful response resets the consecutive failure counter.""" + call_count = {"n": 0} + + def side_effect(url, **_kwargs): + call_count["n"] += 1 + if call_count["n"] == 2: + return _mock_response(200, _cip_ip_body()) + return _mock_response( + 400, {}, text='{"status":400,"message":"Invalid IP Address"}' + ) + + mock_get.side_effect = side_effect + cr = { + "domain": "", + "metadata": {"ip_mode": True, "expanded_ips": [ + "1.2.3.1", "1.2.3.2", "1.2.3.3", "1.2.3.4", "1.2.3.5", + "1.2.3.6", "1.2.3.7", + ]}, + "dns": {"domain": {}, "subdomains": {}}, + } + out = run_criminalip_enrichment(cr, self._settings()) + self.assertEqual(len(out["criminalip"]["ip_reports"]), 1) + self.assertEqual(mock_get.call_count, 5, + "1 fail, 1 success (reset), 3 more fails then stop") + + @patch("criminalip_enrich.time.sleep") + @patch("criminalip_enrich.requests.get") + def test_rate_limit_stops_ip_loop(self, mock_get, mock_sleep): + """429 on an IP request stops all further IP queries.""" + def side_effect(url, **_kwargs): + if "domain/report" in url: + return _mock_response(200, _cip_domain_body()) + return _mock_response(429, {}, text="rate limited") + + mock_get.side_effect = side_effect + cr = { + "domain": "example.com", + "metadata": {"ip_mode": False}, + "dns": {"domain": {"ips": {"ipv4": ["1.2.3.1", "1.2.3.2", "1.2.3.3"]}}, "subdomains": {}}, + } + out = run_criminalip_enrichment(cr, self._settings()) + self.assertIsNotNone(out["criminalip"]["domain_report"]) + self.assertEqual(out["criminalip"]["ip_reports"], []) + ip_calls = [c for c in mock_get.call_args_list if "ip/data" in str(c)] + self.assertEqual(len(ip_calls), 2, + "One attempt + one retry for the first IP, then stop") + + @patch("criminalip_enrich.time.sleep") + @patch("criminalip_enrich.requests.get") + def test_402_credit_stops(self, mock_get, _sleep): + """HTTP 402 is treated as credit exhaustion and stops immediately.""" + mock_get.return_value = _mock_response(402, {}, text="payment required") + cr = _combined_result() + out = run_criminalip_enrichment(cr, self._settings()) + self.assertIsNone(out["criminalip"]["domain_report"]) + self.assertEqual(mock_get.call_count, 1) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_graph_db_refactor.py b/tests/test_graph_db_refactor.py index badd255e..416e9333 100644 --- a/tests/test_graph_db_refactor.py +++ b/tests/test_graph_db_refactor.py @@ -157,7 +157,7 @@ def test_all_original_public_methods_preserved(self): new_methods |= self._methods(path) missing = original - new_methods - extra = new_methods - original - {"__init__", "__enter__", "__exit__"} + extra = new_methods - original - {"__init__", "__enter__", "__exit__", "update_graph_from_uncover"} self.assertEqual(missing, set(), f"Methods missing from refactored code: {missing}") self.assertEqual(extra, set(), f"Unexpected extra methods: {extra}") @@ -186,6 +186,7 @@ def test_expected_methods_per_mixin(self): "update_graph_from_censys", "update_graph_from_fofa", "update_graph_from_otx", "update_graph_from_netlas", "update_graph_from_virustotal", "update_graph_from_zoomeye", "update_graph_from_criminalip", + "update_graph_from_uncover", }, } for fpath, required in checks.items(): diff --git a/tests/test_netlas_enrich.py b/tests/test_netlas_enrich.py index f11ce0d7..582814d3 100644 --- a/tests/test_netlas_enrich.py +++ b/tests/test_netlas_enrich.py @@ -433,14 +433,14 @@ def test_ip_mode_queries_per_ip(self, mock_get, _sleep): mock_get.return_value = _mock_response(200, _netlas_body()) cr = { "domain": "", - "metadata": {"ip_mode": True, "expanded_ips": ["10.0.0.1", "10.0.0.2"]}, + "metadata": {"ip_mode": True, "expanded_ips": ["93.184.1.1", "93.184.1.2"]}, "dns": {}, } run_netlas_enrichment(cr, self._settings()) self.assertEqual(mock_get.call_count, 2) calls = [mock_get.call_args_list[i][1]["params"]["q"] for i in range(2)] - self.assertIn("host:10.0.0.1", calls) - self.assertIn("host:10.0.0.2", calls) + self.assertIn("host:93.184.1.1", calls) + self.assertIn("host:93.184.1.2", calls) @patch("netlas_enrich.time.sleep") @patch("netlas_enrich.requests.get") @@ -451,7 +451,7 @@ def test_ip_mode_stops_on_rate_limit(self, mock_get, _sleep): ] cr = { "domain": "", - "metadata": {"ip_mode": True, "expanded_ips": ["10.0.0.1", "10.0.0.2", "10.0.0.3"]}, + "metadata": {"ip_mode": True, "expanded_ips": ["93.184.1.1", "93.184.1.2", "93.184.1.3"]}, "dns": {}, } run_netlas_enrichment(cr, self._settings()) diff --git a/tests/test_uncover_enrich.py b/tests/test_uncover_enrich.py new file mode 100644 index 00000000..c2f562e2 --- /dev/null +++ b/tests/test_uncover_enrich.py @@ -0,0 +1,248 @@ +"""Tests for recon/uncover_enrich.py""" +import json +import os +import sys +import unittest +from unittest.mock import patch, MagicMock + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'recon')) + +from uncover_enrich import ( + _build_provider_config, + _deduplicate_results, + _extract_hosts_and_ips, + _build_queries, + merge_uncover_into_pipeline, + run_uncover_expansion, +) + + +class TestBuildProviderConfig(unittest.TestCase): + + def test_empty_settings_returns_shodan_idb_only(self): + config, engines = _build_provider_config({}) + self.assertEqual(config, {}) + self.assertEqual(engines, ['shodan-idb']) + + def test_shodan_key_adds_engine(self): + config, engines = _build_provider_config({'SHODAN_API_KEY': 'key123'}) + self.assertIn('shodan', config) + self.assertIn('shodan', engines) + self.assertNotIn('shodan-idb', engines) + + def test_censys_needs_both_token_and_org(self): + config, engines = _build_provider_config({'CENSYS_API_TOKEN': 'tok'}) + self.assertNotIn('censys', engines) + config, engines = _build_provider_config({ + 'CENSYS_API_TOKEN': 'tok', + 'CENSYS_ORG_ID': 'org', + }) + self.assertIn('censys', engines) + self.assertEqual(config['censys'], ['tok:org']) + + def test_google_needs_both_key_and_cx(self): + config, engines = _build_provider_config({'UNCOVER_GOOGLE_API_KEY': 'gkey'}) + self.assertNotIn('google', engines) + config, engines = _build_provider_config({ + 'UNCOVER_GOOGLE_API_KEY': 'gkey', + 'UNCOVER_GOOGLE_API_CX': 'gcx', + }) + self.assertIn('google', engines) + self.assertEqual(config['google'], ['gkey:gcx']) + + def test_all_engines_configured(self): + settings = { + 'SHODAN_API_KEY': 's1', + 'CENSYS_API_TOKEN': 'ct', 'CENSYS_ORG_ID': 'co', + 'FOFA_API_KEY': 'f1', + 'ZOOMEYE_API_KEY': 'z1', + 'NETLAS_API_KEY': 'n1', + 'CRIMINALIP_API_KEY': 'c1', + 'UNCOVER_QUAKE_API_KEY': 'q1', + 'UNCOVER_HUNTER_API_KEY': 'h1', + 'UNCOVER_PUBLICWWW_API_KEY': 'pw1', + 'UNCOVER_HUNTERHOW_API_KEY': 'hh1', + 'UNCOVER_GOOGLE_API_KEY': 'gk', 'UNCOVER_GOOGLE_API_CX': 'gc', + 'UNCOVER_ONYPHE_API_KEY': 'o1', + 'UNCOVER_DRIFTNET_API_KEY': 'd1', + } + config, engines = _build_provider_config(settings) + expected = [ + 'shodan', 'censys', 'fofa', 'zoomeye', 'netlas', + 'criminalip', 'quake', 'hunter', 'publicwww', 'hunterhow', + 'google', 'onyphe', 'driftnet', + ] + for e in expected: + self.assertIn(e, engines, f"{e} missing from engines") + self.assertNotIn('shodan-idb', engines) + + +class TestDeduplicateResults(unittest.TestCase): + + def test_dedup_by_ip_port(self): + results = [ + {'ip': '1.2.3.4', 'port': 80, 'source': 'shodan'}, + {'ip': '1.2.3.4', 'port': 80, 'source': 'censys'}, + {'ip': '1.2.3.4', 'port': 443, 'source': 'shodan'}, + {'ip': '5.6.7.8', 'port': 80, 'source': 'fofa'}, + ] + deduped = _deduplicate_results(results) + self.assertEqual(len(deduped), 3) + self.assertEqual(deduped[0]['source'], 'shodan') + + def test_skips_empty_ip(self): + results = [ + {'ip': '', 'port': 80}, + {'ip': '1.2.3.4', 'port': 80}, + ] + deduped = _deduplicate_results(results) + self.assertEqual(len(deduped), 1) + + +class TestExtractHostsAndIps(unittest.TestCase): + + def test_filters_non_routable(self): + results = [ + {'ip': '10.0.0.1', 'port': 80, 'host': 'internal.example.com'}, + {'ip': '93.184.216.34', 'port': 443, 'host': 'www.example.com'}, + {'ip': '100.64.1.5', 'port': 8080, 'host': 'cgnat.example.com'}, + ] + ips, hosts, ip_ports = _extract_hosts_and_ips( + results, 'example.com', {} + ) + self.assertIn('93.184.216.34', ips) + self.assertNotIn('10.0.0.1', ips) + self.assertNotIn('100.64.1.5', ips) + + def test_extracts_in_scope_hostnames(self): + results = [ + {'ip': '93.184.216.34', 'port': 443, 'host': 'sub.example.com'}, + {'ip': '1.2.3.4', 'port': 80, 'host': 'other.net'}, + ] + ips, hosts, ip_ports = _extract_hosts_and_ips( + results, 'example.com', {} + ) + self.assertIn('sub.example.com', hosts) + self.assertNotIn('other.net', hosts) + + def test_collects_ports_per_ip(self): + results = [ + {'ip': '93.184.216.34', 'port': 80}, + {'ip': '93.184.216.34', 'port': 443}, + ] + ips, hosts, ip_ports = _extract_hosts_and_ips( + results, 'example.com', {} + ) + self.assertEqual(sorted(ip_ports.get('93.184.216.34', [])), [80, 443]) + + +class TestBuildQueries(unittest.TestCase): + + def test_basic_domain(self): + queries = _build_queries('example.com', {}) + self.assertEqual(queries, ['example.com']) + + def test_with_whois_org(self): + queries = _build_queries('example.com', {'_WHOIS_ORG': 'Example Inc.'}) + self.assertEqual(len(queries), 2) + self.assertIn('ssl:"Example Inc."', queries) + + def test_skips_na_org(self): + queries = _build_queries('example.com', {'_WHOIS_ORG': 'N/A'}) + self.assertEqual(len(queries), 1) + + +class TestMergeIntoPipeline(unittest.TestCase): + + def test_merge_new_subdomains(self): + combined = {"dns": {"subdomains": {}}, "domain": "example.com"} + uncover_data = { + "hosts": ["new.example.com", "api.example.com"], + "ips": ["1.2.3.4"], + "ip_ports": {"1.2.3.4": [80, 443]}, + } + count = merge_uncover_into_pipeline(combined, uncover_data, "example.com") + self.assertGreater(count, 0) + self.assertIn("new.example.com", combined["dns"]["subdomains"]) + self.assertIn("api.example.com", combined["dns"]["subdomains"]) + self.assertEqual( + combined["dns"]["subdomains"]["new.example.com"]["source"], + "uncover", + ) + + def test_no_duplicate_subdomains(self): + combined = { + "dns": {"subdomains": {"existing.example.com": {"ips": {"ipv4": []}}}}, + "domain": "example.com", + } + uncover_data = { + "hosts": ["existing.example.com", "new.example.com"], + "ips": [], + "ip_ports": {}, + } + count = merge_uncover_into_pipeline(combined, uncover_data, "example.com") + self.assertGreater(count, 0) + + def test_empty_data(self): + combined = {"dns": {"subdomains": {}}} + count = merge_uncover_into_pipeline(combined, {}, "example.com") + self.assertEqual(count, 0) + + +class TestRunUncoverExpansion(unittest.TestCase): + + def test_disabled_returns_empty(self): + result = run_uncover_expansion({}, {'UNCOVER_ENABLED': False}) + self.assertEqual(result, {}) + + def test_no_keys_returns_empty(self): + result = run_uncover_expansion( + {"domain": "example.com"}, + {'UNCOVER_ENABLED': True}, + ) + self.assertEqual(result, {}) + + @patch('uncover_enrich.subprocess.run') + def test_parses_json_output(self, mock_run): + output_lines = [ + json.dumps({"ip": "93.184.216.34", "port": 443, "host": "www.example.com", "source": "shodan"}), + json.dumps({"ip": "93.184.216.35", "port": 80, "host": "api.example.com", "source": "censys"}), + ] + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = "\n".join(output_lines) + mock_result.stderr = "" + mock_run.return_value = mock_result + + settings = { + 'UNCOVER_ENABLED': True, + 'SHODAN_API_KEY': 'test_key', + 'UNCOVER_MAX_RESULTS': 100, + } + combined = { + "domain": "example.com", + "metadata": {"modules_executed": []}, + } + result = run_uncover_expansion(combined, settings) + self.assertIn("ips", result) + self.assertIn("hosts", result) + + @patch('uncover_enrich.subprocess.run') + def test_timeout_returns_partial(self, mock_run): + import subprocess + mock_run.side_effect = subprocess.TimeoutExpired(cmd="docker", timeout=600) + settings = { + 'UNCOVER_ENABLED': True, + 'SHODAN_API_KEY': 'test_key', + 'UNCOVER_MAX_RESULTS': 100, + } + combined = { + "domain": "example.com", + "metadata": {"modules_executed": []}, + } + result = run_uncover_expansion(combined, settings) + self.assertIsInstance(result, dict) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_virustotal_enrich.py b/tests/test_virustotal_enrich.py index 0c2ab4df..87ed276b 100644 --- a/tests/test_virustotal_enrich.py +++ b/tests/test_virustotal_enrich.py @@ -490,7 +490,7 @@ def side(url, **_kw): @patch("virustotal_enrich.requests.get") def test_max_targets_limits_ip_requests(self, mock_get, _sleep): mock_get.return_value = _mock_response(200, _vt_ip_body()) - ips = [f"10.0.0.{i}" for i in range(1, 11)] # 10 IPs + ips = [f"93.184.{i}.1" for i in range(1, 11)] # 10 IPs cr = _combined_result(ips=ips) cr["domain"] = "" # skip domain request out = run_virustotal_enrichment(cr, _default_settings(VIRUSTOTAL_MAX_TARGETS=3)) diff --git a/webapp/prisma/migrations/20260329120000_consolidate_censys_keys/migration.sql b/webapp/prisma/migrations/20260329120000_consolidate_censys_keys/migration.sql new file mode 100644 index 00000000..d1793a52 --- /dev/null +++ b/webapp/prisma/migrations/20260329120000_consolidate_censys_keys/migration.sql @@ -0,0 +1,10 @@ +-- Consolidate Censys credentials from 4 fields to 2. +-- The legacy Search API v2 (censys_api_id / censys_api_secret) is replaced +-- by the Platform API v3 (censys_api_token / censys_org_id). +-- +-- If a user already stored a platform token in censys_api_token, keep it. +-- Otherwise, no data migration is needed since the old API ID/Secret pair +-- is fundamentally different credentials (Basic Auth vs Bearer PAT). + +ALTER TABLE "user_settings" DROP COLUMN IF EXISTS "censys_api_id"; +ALTER TABLE "user_settings" DROP COLUMN IF EXISTS "censys_api_secret"; diff --git a/webapp/prisma/schema.prisma b/webapp/prisma/schema.prisma index 74508a77..ac2c284a 100644 --- a/webapp/prisma/schema.prisma +++ b/webapp/prisma/schema.prisma @@ -70,14 +70,22 @@ model UserSettings { urlscanApiKey String @default("") @map("urlscan_api_key") // OSINT & Threat Intelligence API Keys - censysApiId String @default("") @map("censys_api_id") - censysApiSecret String @default("") @map("censys_api_secret") + censysApiToken String @default("") @map("censys_api_token") + censysOrgId String @default("") @map("censys_org_id") fofaApiKey String @default("") @map("fofa_api_key") otxApiKey String @default("") @map("otx_api_key") netlasApiKey String @default("") @map("netlas_api_key") virusTotalApiKey String @default("") @map("virustotal_api_key") zoomEyeApiKey String @default("") @map("zoomeye_api_key") criminalIpApiKey String @default("") @map("criminalip_api_key") + quakeApiKey String @default("") @map("quake_api_key") + hunterApiKey String @default("") @map("hunter_api_key") + publicWwwApiKey String @default("") @map("publicwww_api_key") + hunterHowApiKey String @default("") @map("hunterhow_api_key") + googleApiKey String @default("") @map("google_api_key") + googleApiCx String @default("") @map("google_api_cx") + onypheApiKey String @default("") @map("onyphe_api_key") + driftnetApiKey String @default("") @map("driftnet_api_key") // Tunneling & Infrastructure ngrokAuthtoken String @default("") @map("ngrok_authtoken") @@ -492,6 +500,8 @@ model Project { zoomEyeEnabled Boolean @default(false) @map("zoomeye_enabled") zoomEyeMaxResults Int @default(1000) @map("zoomeye_max_results") criminalIpEnabled Boolean @default(false) @map("criminalip_enabled") + uncoverEnabled Boolean @default(false) @map("uncover_enabled") + uncoverMaxResults Int @default(500) @map("uncover_max_results") // Subdomain Discovery Tool Toggles crtshEnabled Boolean @default(true) @map("crtsh_enabled") diff --git a/webapp/src/app/api/users/[id]/settings/route.ts b/webapp/src/app/api/users/[id]/settings/route.ts index 1fb961d3..ae5407e2 100644 --- a/webapp/src/app/api/users/[id]/settings/route.ts +++ b/webapp/src/app/api/users/[id]/settings/route.ts @@ -12,7 +12,7 @@ function maskSecret(value: string): string { } const TUNNEL_FIELDS = ['ngrokAuthtoken', 'chiselServerUrl', 'chiselAuth'] as const -const TOOL_NAMES = ['tavily', 'shodan', 'serp', 'nvd', 'vulners', 'urlscan', 'censys', 'fofa', 'otx', 'netlas', 'virustotal', 'zoomeye', 'criminalip'] as const +const TOOL_NAMES = ['tavily', 'shodan', 'serp', 'nvd', 'vulners', 'urlscan', 'censys', 'fofa', 'otx', 'netlas', 'virustotal', 'zoomeye', 'criminalip', 'quake', 'hunter', 'publicwww', 'hunterhow', 'onyphe', 'driftnet'] as const // GET /api/users/[id]/settings export async function GET(request: NextRequest, { params }: RouteParams) { @@ -49,14 +49,22 @@ export async function GET(request: NextRequest, { params }: RouteParams) { nvdApiKey: '', vulnersApiKey: '', urlscanApiKey: '', - censysApiId: '', - censysApiSecret: '', + censysApiToken: '', + censysOrgId: '', fofaApiKey: '', otxApiKey: '', netlasApiKey: '', virusTotalApiKey: '', zoomEyeApiKey: '', criminalIpApiKey: '', + quakeApiKey: '', + hunterApiKey: '', + publicWwwApiKey: '', + hunterHowApiKey: '', + googleApiKey: '', + googleApiCx: '', + onypheApiKey: '', + driftnetApiKey: '', ngrokAuthtoken: '', chiselServerUrl: '', chiselAuth: '', @@ -74,14 +82,22 @@ export async function GET(request: NextRequest, { params }: RouteParams) { nvdApiKey: maskSecret(settings.nvdApiKey), vulnersApiKey: maskSecret(settings.vulnersApiKey), urlscanApiKey: maskSecret(settings.urlscanApiKey), - censysApiId: maskSecret(settings.censysApiId), - censysApiSecret: maskSecret(settings.censysApiSecret), + censysApiToken: maskSecret(settings.censysApiToken), + censysOrgId: maskSecret(settings.censysOrgId), fofaApiKey: maskSecret(settings.fofaApiKey), otxApiKey: maskSecret(settings.otxApiKey), netlasApiKey: maskSecret(settings.netlasApiKey), virusTotalApiKey: maskSecret(settings.virusTotalApiKey), zoomEyeApiKey: maskSecret(settings.zoomEyeApiKey), criminalIpApiKey: maskSecret(settings.criminalIpApiKey), + quakeApiKey: maskSecret(settings.quakeApiKey), + hunterApiKey: maskSecret(settings.hunterApiKey), + publicWwwApiKey: maskSecret(settings.publicWwwApiKey), + hunterHowApiKey: maskSecret(settings.hunterHowApiKey), + googleApiKey: maskSecret(settings.googleApiKey), + googleApiCx: maskSecret(settings.googleApiCx), + onypheApiKey: maskSecret(settings.onypheApiKey), + driftnetApiKey: maskSecret(settings.driftnetApiKey), ngrokAuthtoken: maskSecret(settings.ngrokAuthtoken), chiselAuth: maskSecret(settings.chiselAuth), } @@ -109,7 +125,7 @@ export async function PUT(request: NextRequest, { params }: RouteParams) { }) const data: Record = {} - const fields = ['githubAccessToken', 'tavilyApiKey', 'shodanApiKey', 'serpApiKey', 'nvdApiKey', 'vulnersApiKey', 'urlscanApiKey', 'censysApiId', 'censysApiSecret', 'fofaApiKey', 'otxApiKey', 'netlasApiKey', 'virusTotalApiKey', 'zoomEyeApiKey', 'criminalIpApiKey', 'ngrokAuthtoken', 'chiselServerUrl', 'chiselAuth'] as const + const fields = ['githubAccessToken', 'tavilyApiKey', 'shodanApiKey', 'serpApiKey', 'nvdApiKey', 'vulnersApiKey', 'urlscanApiKey', 'censysApiToken', 'censysOrgId', 'fofaApiKey', 'otxApiKey', 'netlasApiKey', 'virusTotalApiKey', 'zoomEyeApiKey', 'criminalIpApiKey', 'quakeApiKey', 'hunterApiKey', 'publicWwwApiKey', 'hunterHowApiKey', 'googleApiKey', 'googleApiCx', 'onypheApiKey', 'driftnetApiKey', 'ngrokAuthtoken', 'chiselServerUrl', 'chiselAuth'] as const for (const field of fields) { if (field in body) { @@ -219,14 +235,22 @@ export async function PUT(request: NextRequest, { params }: RouteParams) { nvdApiKey: maskSecret(settings.nvdApiKey), vulnersApiKey: maskSecret(settings.vulnersApiKey), urlscanApiKey: maskSecret(settings.urlscanApiKey), - censysApiId: maskSecret(settings.censysApiId), - censysApiSecret: maskSecret(settings.censysApiSecret), + censysApiToken: maskSecret(settings.censysApiToken), + censysOrgId: maskSecret(settings.censysOrgId), fofaApiKey: maskSecret(settings.fofaApiKey), otxApiKey: maskSecret(settings.otxApiKey), netlasApiKey: maskSecret(settings.netlasApiKey), virusTotalApiKey: maskSecret(settings.virusTotalApiKey), zoomEyeApiKey: maskSecret(settings.zoomEyeApiKey), criminalIpApiKey: maskSecret(settings.criminalIpApiKey), + quakeApiKey: maskSecret(settings.quakeApiKey), + hunterApiKey: maskSecret(settings.hunterApiKey), + publicWwwApiKey: maskSecret(settings.publicWwwApiKey), + hunterHowApiKey: maskSecret(settings.hunterHowApiKey), + googleApiKey: maskSecret(settings.googleApiKey), + googleApiCx: maskSecret(settings.googleApiCx), + onypheApiKey: maskSecret(settings.onypheApiKey), + driftnetApiKey: maskSecret(settings.driftnetApiKey), ngrokAuthtoken: maskSecret(settings.ngrokAuthtoken), chiselAuth: maskSecret(settings.chiselAuth), rotationConfigs, diff --git a/webapp/src/app/settings/page.tsx b/webapp/src/app/settings/page.tsx index 6552cb81..d6667d7a 100644 --- a/webapp/src/app/settings/page.tsx +++ b/webapp/src/app/settings/page.tsx @@ -19,14 +19,22 @@ interface UserSettings { nvdApiKey: string vulnersApiKey: string urlscanApiKey: string - censysApiId: string - censysApiSecret: string + censysApiToken: string + censysOrgId: string fofaApiKey: string otxApiKey: string netlasApiKey: string virusTotalApiKey: string zoomEyeApiKey: string criminalIpApiKey: string + quakeApiKey: string + hunterApiKey: string + publicWwwApiKey: string + hunterHowApiKey: string + googleApiKey: string + googleApiCx: string + onypheApiKey: string + driftnetApiKey: string ngrokAuthtoken: string chiselServerUrl: string chiselAuth: string @@ -40,14 +48,22 @@ const EMPTY_SETTINGS: UserSettings = { nvdApiKey: '', vulnersApiKey: '', urlscanApiKey: '', - censysApiId: '', - censysApiSecret: '', + censysApiToken: '', + censysOrgId: '', fofaApiKey: '', otxApiKey: '', netlasApiKey: '', virusTotalApiKey: '', zoomEyeApiKey: '', criminalIpApiKey: '', + quakeApiKey: '', + hunterApiKey: '', + publicWwwApiKey: '', + hunterHowApiKey: '', + googleApiKey: '', + googleApiCx: '', + onypheApiKey: '', + driftnetApiKey: '', ngrokAuthtoken: '', chiselServerUrl: '', chiselAuth: '', @@ -72,6 +88,12 @@ const TOOL_NAME_MAP: Record = { virusTotalApiKey: 'virustotal', zoomEyeApiKey: 'zoomeye', criminalIpApiKey: 'criminalip', + quakeApiKey: 'quake', + hunterApiKey: 'hunter', + publicWwwApiKey: 'publicwww', + hunterHowApiKey: 'hunterhow', + onypheApiKey: 'onyphe', + driftnetApiKey: 'driftnet', } function getProviderIcon(providerType: string): string { @@ -273,14 +295,22 @@ export default function SettingsPage() { nvdApiKey: data.nvdApiKey || '', vulnersApiKey: data.vulnersApiKey || '', urlscanApiKey: data.urlscanApiKey || '', - censysApiId: data.censysApiId || '', - censysApiSecret: data.censysApiSecret || '', + censysApiToken: data.censysApiToken || '', + censysOrgId: data.censysOrgId || '', fofaApiKey: data.fofaApiKey || '', otxApiKey: data.otxApiKey || '', netlasApiKey: data.netlasApiKey || '', virusTotalApiKey: data.virusTotalApiKey || '', zoomEyeApiKey: data.zoomEyeApiKey || '', criminalIpApiKey: data.criminalIpApiKey || '', + quakeApiKey: data.quakeApiKey || '', + hunterApiKey: data.hunterApiKey || '', + publicWwwApiKey: data.publicWwwApiKey || '', + hunterHowApiKey: data.hunterHowApiKey || '', + googleApiKey: data.googleApiKey || '', + googleApiCx: data.googleApiCx || '', + onypheApiKey: data.onypheApiKey || '', + driftnetApiKey: data.driftnetApiKey || '', ngrokAuthtoken: data.ngrokAuthtoken || '', chiselServerUrl: data.chiselServerUrl || '', chiselAuth: data.chiselAuth || '', @@ -352,14 +382,22 @@ export default function SettingsPage() { nvdApiKey: data.nvdApiKey || '', vulnersApiKey: data.vulnersApiKey || '', urlscanApiKey: data.urlscanApiKey || '', - censysApiId: data.censysApiId || '', - censysApiSecret: data.censysApiSecret || '', + censysApiToken: data.censysApiToken || '', + censysOrgId: data.censysOrgId || '', fofaApiKey: data.fofaApiKey || '', otxApiKey: data.otxApiKey || '', netlasApiKey: data.netlasApiKey || '', virusTotalApiKey: data.virusTotalApiKey || '', zoomEyeApiKey: data.zoomEyeApiKey || '', criminalIpApiKey: data.criminalIpApiKey || '', + quakeApiKey: data.quakeApiKey || '', + hunterApiKey: data.hunterApiKey || '', + publicWwwApiKey: data.publicWwwApiKey || '', + hunterHowApiKey: data.hunterHowApiKey || '', + googleApiKey: data.googleApiKey || '', + googleApiCx: data.googleApiCx || '', + onypheApiKey: data.onypheApiKey || '', + driftnetApiKey: data.driftnetApiKey || '', ngrokAuthtoken: data.ngrokAuthtoken || '', chiselServerUrl: data.chiselServerUrl || '', chiselAuth: data.chiselAuth || '', @@ -692,23 +730,24 @@ export default function SettingsPage() { /> toggleFieldVisibility('censysApiId')} - onChange={v => updateSetting('censysApiId', v)} + badges={['AI Agent', 'Recon Pipeline', 'Uncover']} + value={settings.censysApiToken} + visible={!!visibleFields.censysApiToken} + onToggle={() => toggleFieldVisibility('censysApiToken')} + onChange={v => updateSetting('censysApiToken', v)} /> toggleFieldVisibility('censysApiSecret')} - onChange={v => updateSetting('censysApiSecret', v)} + label="Censys Organization ID" + hint="Censys Organization ID — paired with API Token above. Found on your Censys account page" + signupUrl="https://accounts.censys.io/settings/personal-access-tokens" + badges={['AI Agent', 'Recon Pipeline', 'Uncover']} + value={settings.censysOrgId} + visible={!!visibleFields.censysOrgId} + onToggle={() => toggleFieldVisibility('censysOrgId')} + onChange={v => updateSetting('censysOrgId', v)} /> openRotationModal('criminalIpApiKey')} rotationInfo={rotationConfigs.criminalip || null} /> + toggleFieldVisibility('quakeApiKey')} + onChange={v => updateSetting('quakeApiKey', v)} + onConfigureRotation={() => openRotationModal('quakeApiKey')} + rotationInfo={rotationConfigs.quake || null} + /> + toggleFieldVisibility('hunterApiKey')} + onChange={v => updateSetting('hunterApiKey', v)} + onConfigureRotation={() => openRotationModal('hunterApiKey')} + rotationInfo={rotationConfigs.hunter || null} + /> + toggleFieldVisibility('publicWwwApiKey')} + onChange={v => updateSetting('publicWwwApiKey', v)} + onConfigureRotation={() => openRotationModal('publicWwwApiKey')} + rotationInfo={rotationConfigs.publicwww || null} + /> + toggleFieldVisibility('hunterHowApiKey')} + onChange={v => updateSetting('hunterHowApiKey', v)} + onConfigureRotation={() => openRotationModal('hunterHowApiKey')} + rotationInfo={rotationConfigs.hunterhow || null} + /> + toggleFieldVisibility('googleApiKey')} + onChange={v => updateSetting('googleApiKey', v)} + /> + toggleFieldVisibility('googleApiCx')} + onChange={v => updateSetting('googleApiCx', v)} + /> + toggleFieldVisibility('onypheApiKey')} + onChange={v => updateSetting('onypheApiKey', v)} + onConfigureRotation={() => openRotationModal('onypheApiKey')} + rotationInfo={rotationConfigs.onyphe || null} + /> + toggleFieldVisibility('driftnetApiKey')} + onChange={v => updateSetting('driftnetApiKey', v)} + onConfigureRotation={() => openRotationModal('driftnetApiKey')} + rotationInfo={rotationConfigs.driftnet || null} + /> )} @@ -1280,7 +1411,7 @@ function SecretField({ onChange(e.target.value)} placeholder={`Enter ${label.toLowerCase()}`} /> diff --git a/webapp/src/components/projects/ProjectForm/sections/OsintEnrichmentSection.tsx b/webapp/src/components/projects/ProjectForm/sections/OsintEnrichmentSection.tsx index 739ef444..33e5768a 100644 --- a/webapp/src/components/projects/ProjectForm/sections/OsintEnrichmentSection.tsx +++ b/webapp/src/components/projects/ProjectForm/sections/OsintEnrichmentSection.tsx @@ -37,7 +37,7 @@ export function OsintEnrichmentSection({ data, updateField }: OsintEnrichmentSec .then(settings => { if (settings) { setKeyStatus({ - censys: !!(settings.censysApiId && settings.censysApiSecret), + censys: !!(settings.censysApiToken && settings.censysOrgId), fofa: !!settings.fofaApiKey, otx: !!settings.otxApiKey, netlas: !!settings.netlasApiKey, @@ -90,7 +90,7 @@ export function OsintEnrichmentSection({ data, updateField }: OsintEnrichmentSec {noKey('censys') && (
- No Censys API credentials — add ID & Secret in Global Settings to enable. + No Censys API credentials — add API Token & Organization ID in Global Settings to enable.
)} @@ -272,6 +272,41 @@ export function OsintEnrichmentSection({ data, updateField }: OsintEnrichmentSec + {/* Uncover */} +
+
+
+ Uncover (Multi-Engine Search) +

+ ProjectDiscovery Uncover — searches Shodan, Censys, FOFA, ZoomEye, Netlas, + CriminalIP, Quake, Hunter, and more simultaneously for target expansion. + Discovers additional IPs, subdomains, and open ports before port scanning. + Configure API keys for each engine in Global Settings. +

+
+ updateField('uncoverEnabled', checked)} + /> +
+ {data.uncoverEnabled && ( +
+
+ + updateField('uncoverMaxResults', parseInt(e.target.value) || 500)} + min={1} + max={10000} + /> + Maximum total results across all engines (1–10 000) +
+
+ )} +
+ )}