From 317decdc19a0ce5c7873defcbb4a6845e5444559 Mon Sep 17 00:00:00 2001 From: "David J. Bianco" Date: Thu, 14 May 2026 10:16:01 -0400 Subject: [PATCH 1/3] feat: improve source identity metadata coherence --- .../references/config-apps-processes.md | 7 +- .../eforge/references/config-dns-network.md | 8 + .../config/activity/application_catalog.yaml | 36 +++++ .../config/activity/tls_realism.yaml | 47 ++++++ src/evidenceforge/config/schemas.py | 150 ++++++++++++++++++ .../generation/activity/generator.py | 48 +++++- .../generation/activity/tls_realism.py | 53 +++++++ tests/unit/test_activity.py | 58 +++++++ tests/unit/test_dhcp_and_certs.py | 74 +++++++++ 9 files changed, 472 insertions(+), 9 deletions(-) diff --git a/commands/eforge/references/config-apps-processes.md b/commands/eforge/references/config-apps-processes.md index 1a17be87..8a7bf64f 100644 --- a/commands/eforge/references/config-apps-processes.md +++ b/commands/eforge/references/config-apps-processes.md @@ -80,7 +80,7 @@ applications: ### Loaded Module Fields (Windows only) -DLLs characteristically loaded by this process, used for Sysmon Event 7 (ImageLoaded) generation. All fields except `path` have defaults — only specify what differs. +DLLs characteristically loaded by this process, used for Sysmon Event 7 (ImageLoaded) generation. Microsoft OS loader DLLs can rely on defaults. Third-party modules should set source-native signer metadata, and known vendor modules should carry PE metadata so rendered `Company`, `Product`, `Description`, and `FileVersion` do not fall back to Microsoft or blank values. | Field | Type | Default | Description | |-------|------|---------|-------------| @@ -88,8 +88,9 @@ DLLs characteristically loaded by this process, used for Sysmon Event 7 (ImageLo | `signed` | bool | `true` | Whether the DLL is digitally signed | | `signature` | string | `"Microsoft Windows"` | Signer name (e.g., `"Google LLC"`, `"Mozilla Corporation"`) | | `signature_status` | string | `"Valid"` | One of: `Valid`, `Expired`, `Revoked`, `Unavailable` | +| `pe_metadata` | object | inherited or blank | Optional DLL-specific PE fields: `file_version`, `description`, `product`, `company`, `original_filename` | -Every Windows process also receives the common OS loader DLLs (ntdll.dll, kernel32.dll, etc.) defined in `system_processes.yaml` under `common_loaded_modules.windows` — you don't need to repeat those in per-app profiles. +Application DLLs inherit the owning app's PE version/product/company if `pe_metadata` is omitted. Every Windows process also receives the common OS loader DLLs (ntdll.dll, kernel32.dll, etc.) defined in `system_processes.yaml` under `common_loaded_modules.windows` — you don't need to repeat those in per-app profiles. ### Valid Categories @@ -369,7 +370,7 @@ Provides file path, registry key, and DLL pools for probabilistic background eve - `registry_keys_hklm:` — `[key, value_name, details]` triples for HKLM writes (Run, Defender, WDigest, Firewall) - `dll_pool:` — System32 and application DLL paths for module load events -Overlay replaces entire sections (section-replace merge). Details values use Sysmon format: `"DWORD (0x00000001)"` for REG_DWORD, string for REG_SZ. Registry and DLL entries may use `{user}`, `{rand}`, `{hex}`, `{guid}`, `{mru}`, `{doc}`, `{package}`, and `{version}` placeholders; these are materialized per emitted event to avoid repetitive TargetObject paths. +Overlay replaces entire sections (section-replace merge). Details values use Sysmon format: `"DWORD (0x00000001)"` for REG_DWORD, string for REG_SZ. Registry and DLL entries may use `{user}`, `{rand}`, `{hex}`, `{guid}`, `{mru}`, `{doc}`, `{package}`, and `{version}` placeholders; these are materialized per emitted event to avoid repetitive TargetObject paths. DHCP interface registry values are additionally controlled by `endpoint_noise.yaml`, which reserves them for actual DHCP lease/reconfigure activity unless explicitly relaxed. --- diff --git a/commands/eforge/references/config-dns-network.md b/commands/eforge/references/config-dns-network.md index f289c826..bd155483 100644 --- a/commands/eforge/references/config-dns-network.md +++ b/commands/eforge/references/config-dns-network.md @@ -459,6 +459,12 @@ ocsp: certificate_chains: include_intermediate_probability: 0.86 include_second_intermediate_probability: 0.08 + subject_key_profiles: + - subject_patterns: ["CN=R3, O=Let's Encrypt, C=US"] + issuer_family: rsa_public_ca + key_type: rsa + key_length: 2048 + child_signature_algorithms: ["sha256WithRSAEncryption"] templates: - name: lets_encrypt issuer_patterns: ["*Let's Encrypt*"] @@ -484,6 +490,8 @@ warns when an OCSP responder host is missing from the registry. `ocsp.suppress_revoked_suffixes` prevents routine mainstream browsing certificates from being marked revoked while still allowing rare revoked statuses for uncategorized or intentionally suspicious certificate identities. +`certificate_chains.subject_key_profiles` declares the issuer-side key family used when signing child certificates. The `certificate.sig_alg` rendered in Zeek `x509.log` follows the issuer key and one of the profile's compatible `child_signature_algorithms`, so RSA and ECDSA public CAs do not produce impossible mixed chains. Run `eforge validate-config` after changing this section; it rejects empty pattern/algorithm lists and RSA/ECDSA signature mismatches. + `destinations.profiles` keeps TLS volume heavy-tailed without collapsing all hosts onto the same few SNI values. Profiles can list explicit `domains`, pull from `dns_registry.yaml` through `dns_tags`, limit by `os`, `personas`, `system_types`, or `purpose_tags`, and add `os_overrides` for OS-specific update/package endpoints. When an OS override provides domains or DNS tags, that override replaces the profile's generic pool for that OS so Windows update traffic does not drift into Linux package mirrors, and vice versa. Overlays merge nested dicts and extend lists, so project-local profiles can add domains without replacing the default pool. ## smb_file_transfers.yaml diff --git a/src/evidenceforge/config/activity/application_catalog.yaml b/src/evidenceforge/config/activity/application_catalog.yaml index 8e5f2c18..0aab0266 100644 --- a/src/evidenceforge/config/activity/application_catalog.yaml +++ b/src/evidenceforge/config/activity/application_catalog.yaml @@ -41,10 +41,28 @@ applications: loaded_modules: - path: 'C:\Program Files\Google\Chrome\Application\chrome_elf.dll' signature: "Google LLC" + pe_metadata: + file_version: "120.0.6099.225" + description: "Google Chrome ELF" + product: "Google Chrome" + company: "Google LLC" + original_filename: "chrome_elf.dll" - path: 'C:\Program Files\Google\Chrome\Application\libEGL.dll' signature: "Google LLC" + pe_metadata: + file_version: "120.0.6099.225" + description: "ANGLE libEGL" + product: "Google Chrome" + company: "Google LLC" + original_filename: "libEGL.dll" - path: 'C:\Program Files\Google\Chrome\Application\libGLESv2.dll' signature: "Google LLC" + pe_metadata: + file_version: "120.0.6099.225" + description: "ANGLE libGLESv2" + product: "Google Chrome" + company: "Google LLC" + original_filename: "libGLESv2.dll" - path: 'C:\Windows\System32\user32.dll' - path: 'C:\Windows\System32\gdi32.dll' - path: 'C:\Windows\System32\ws2_32.dll' @@ -77,10 +95,28 @@ applications: loaded_modules: - path: 'C:\Program Files\Mozilla Firefox\mozglue.dll' signature: "Mozilla Corporation" + pe_metadata: + file_version: "121.0" + description: "Mozilla Glue Library" + product: "Firefox" + company: "Mozilla Corporation" + original_filename: "mozglue.dll" - path: 'C:\Program Files\Mozilla Firefox\nss3.dll' signature: "Mozilla Corporation" + pe_metadata: + file_version: "121.0" + description: "Network Security Services" + product: "Firefox" + company: "Mozilla Corporation" + original_filename: "nss3.dll" - path: 'C:\Program Files\Mozilla Firefox\lgpllibs.dll' signature: "Mozilla Corporation" + pe_metadata: + file_version: "121.0" + description: "Mozilla LGPL Libraries" + product: "Firefox" + company: "Mozilla Corporation" + original_filename: "lgpllibs.dll" - path: 'C:\Windows\System32\user32.dll' - path: 'C:\Windows\System32\gdi32.dll' - path: 'C:\Windows\System32\ws2_32.dll' diff --git a/src/evidenceforge/config/activity/tls_realism.yaml b/src/evidenceforge/config/activity/tls_realism.yaml index b14118fd..148e5de3 100644 --- a/src/evidenceforge/config/activity/tls_realism.yaml +++ b/src/evidenceforge/config/activity/tls_realism.yaml @@ -91,6 +91,53 @@ certificate_chains: - {type: "rsa", length: 2048, weight: 55} - {type: "rsa", length: 4096, weight: 20} - {type: "ecdsa", length: 256, weight: 25} + subject_key_profiles: + - subject_patterns: + - "CN=R3, O=Let's Encrypt, C=US" + - "CN=ISRG Root X1, O=Internet Security Research Group, C=US" + - "CN=DigiCert*" + - "CN=Amazon RSA*" + - "CN=Amazon Root CA 1*" + - "CN=Microsoft RSA Root Certificate Authority 2017*" + - "CN=Baltimore CyberTrust Root*" + - "CN=GlobalSign Root CA*" + - "CN=GlobalSign Root R3*" + - "CN=USERTrust RSA Certification Authority*" + - "CN=Enterprise Root CA*" + issuer_family: rsa_public_ca + key_type: rsa + key_length: 2048 + child_signature_algorithms: ["sha256WithRSAEncryption"] + - subject_patterns: + - "CN=E1, O=Let's Encrypt, C=US" + - "CN=ISRG Root X2, O=Internet Security Research Group, C=US" + - "CN=Cloudflare Inc ECC*" + - "CN=Amazon Root CA 3*" + - "CN=Apple Root CA - G3*" + - "CN=Apple Public EV Server ECC*" + issuer_family: ecdsa_public_ca + key_type: ecdsa + key_length: 256 + child_signature_algorithms: ["ecdsa-with-SHA256"] + - subject_patterns: + - "CN=GTS CA 1C3, O=Google Trust Services LLC, C=US" + - "CN=GTS Root R1, O=Google Trust Services LLC, C=US" + issuer_family: google_trust_services_rsa + key_type: rsa + key_length: 2048 + child_signature_algorithms: ["sha256WithRSAEncryption"] + - subject_patterns: + - "CN=GTS Root R4, O=Google Trust Services LLC, C=US" + issuer_family: google_trust_services_ecdsa + key_type: ecdsa + key_length: 384 + child_signature_algorithms: ["ecdsa-with-SHA384"] + - subject_patterns: + - "CN=Sectigo Public Server Authentication Root R46*" + issuer_family: sectigo_rsa + key_type: rsa + key_length: 4096 + child_signature_algorithms: ["sha384WithRSAEncryption"] templates: - name: lets_encrypt issuer_patterns: ["*Let's Encrypt*"] diff --git a/src/evidenceforge/config/schemas.py b/src/evidenceforge/config/schemas.py index 707d1d38..d6d6ad50 100644 --- a/src/evidenceforge/config/schemas.py +++ b/src/evidenceforge/config/schemas.py @@ -53,6 +53,40 @@ class LoadedModuleEntry(BaseModel, extra="forbid"): signature_status: str = "Valid" pe_metadata: dict[str, str] | None = None + @model_validator(mode="after") + def known_vendor_modules_have_native_identity(self) -> Self: + """Require explicit source-native identity for known third-party DLL families.""" + known_vendors = { + "google\\chrome": ("Google LLC",), + "mozilla firefox": ("Mozilla Corporation",), + "7-zip": ("Igor Pavlov", "-"), + "vmware": ("VMware, Inc.",), + "dell": ("Dell Inc.",), + "cisco": ("Cisco Systems, Inc.",), + } + path_lower = self.path.replace("/", "\\").lower() + for path_fragment, allowed_signatures in known_vendors.items(): + if path_fragment not in path_lower: + continue + if self.signature not in allowed_signatures: + raise ValueError(f"known third-party module {self.path!r} must use a native signer") + if not self.pe_metadata: + raise ValueError(f"known third-party module {self.path!r} must define pe_metadata") + required_fields = { + "file_version", + "description", + "product", + "company", + "original_filename", + } + missing = sorted(field for field in required_fields if not self.pe_metadata.get(field)) + if missing: + raise ValueError( + f"known third-party module {self.path!r} missing pe_metadata fields: " + f"{', '.join(missing)}" + ) + return self + class PlatformConfig(BaseModel, extra="forbid"): """Per-OS platform config within an application entry.""" @@ -259,6 +293,61 @@ def non_empty_list(cls, v: list[str]) -> list[str]: return v +class TlsSubjectKeyProfile(BaseModel, extra="forbid"): + """A CA subject-name to public-key profile mapping in tls_realism.yaml.""" + + subject_patterns: list[str] + issuer_family: str + key_type: Literal["rsa", "ecdsa"] + key_length: int + child_signature_algorithms: list[ + Literal[ + "sha256WithRSAEncryption", + "sha384WithRSAEncryption", + "ecdsa-with-SHA256", + "ecdsa-with-SHA384", + ] + ] + + @field_validator("subject_patterns") + @classmethod + def patterns_non_empty(cls, v: list[str]) -> list[str]: + if not v: + raise ValueError("subject_patterns must not be empty") + if any(not pattern for pattern in v): + raise ValueError("subject_patterns entries must be non-empty") + return v + + @field_validator("key_length") + @classmethod + def key_length_valid(cls, v: int) -> int: + if v <= 0: + raise ValueError("key_length must be positive") + return v + + @field_validator("child_signature_algorithms") + @classmethod + def child_signature_algorithms_non_empty(cls, v: list[str]) -> list[str]: + if not v: + raise ValueError("child_signature_algorithms must not be empty") + return v + + @model_validator(mode="after") + def child_algorithms_match_key_type(self) -> Self: + """Reject child signature algorithms incompatible with the issuer key.""" + has_ecdsa_alg = any( + algorithm.startswith("ecdsa-") for algorithm in self.child_signature_algorithms + ) + has_rsa_alg = any( + algorithm.endswith("RSAEncryption") for algorithm in self.child_signature_algorithms + ) + if self.key_type == "rsa" and has_ecdsa_alg: + raise ValueError("rsa issuer profiles cannot use ecdsa child signature algorithms") + if self.key_type == "ecdsa" and has_rsa_alg: + raise ValueError("ecdsa issuer profiles cannot use RSA child signature algorithms") + return self + + class TlsCertificateChainConfig(BaseModel, extra="forbid"): """Certificate-chain behavior settings in tls_realism.yaml.""" @@ -268,6 +357,7 @@ class TlsCertificateChainConfig(BaseModel, extra="forbid"): intermediate_validity_days_max: int intermediate_not_before_max_days: int key_types: list[TlsKeyType] + subject_key_profiles: list[TlsSubjectKeyProfile] = Field(default_factory=list) templates: list[TlsChainTemplate] @field_validator( @@ -1028,6 +1118,66 @@ def has_matchers_and_paths(self) -> Self: return self +# --- Endpoint Noise --- + + +class WindowsScheduledProcessNoiseConfig(BaseModel, extra="forbid"): + """Windows scheduled/background process timing policy.""" + + count_min: int = Field(ge=0) + count_max: int = Field(ge=0) + trigger_window_start_seconds: int = Field(ge=0, le=3599) + trigger_window_end_seconds: int = Field(ge=0, le=3599) + slot_spacing_seconds: int = Field(gt=0, le=3600) + host_phase_window_seconds: int = Field(gt=0, le=3600) + jitter_seconds_min: int + jitter_seconds_max: int + skip_probability: float = Field(ge=0.0, le=1.0) + + @model_validator(mode="after") + def bounds_are_ordered(self) -> Self: + """Reject timing windows that would reintroduce boundary clamping.""" + if self.count_min > self.count_max: + raise ValueError("count_min must be <= count_max") + if self.trigger_window_start_seconds >= self.trigger_window_end_seconds: + raise ValueError("trigger_window_start_seconds must be < trigger_window_end_seconds") + if self.jitter_seconds_min > self.jitter_seconds_max: + raise ValueError("jitter_seconds_min must be <= jitter_seconds_max") + return self + + +class DhcpInterfaceRegistryNoiseConfig(BaseModel, extra="forbid"): + """Policy for DHCP-related interface registry values.""" + + value_names: list[str] + require_dhcp_state: bool = True + emit_on_lease_events: bool = True + suppress_system_types: list[str] = Field(default_factory=list) + suppress_roles: list[str] = Field(default_factory=list) + + @field_validator("value_names") + @classmethod + def value_names_non_empty(cls, v: list[str]) -> list[str]: + if not v: + raise ValueError("value_names must not be empty") + if any(not name for name in v): + raise ValueError("value_names entries must be non-empty") + return v + + +class RegistryNoiseConfig(BaseModel, extra="forbid"): + """Ambient endpoint registry-noise policy.""" + + dhcp_interface_values: DhcpInterfaceRegistryNoiseConfig + + +class EndpointNoiseConfig(BaseModel, extra="forbid"): + """Root schema for endpoint_noise.yaml.""" + + windows_scheduled_processes: WindowsScheduledProcessNoiseConfig + registry_noise: RegistryNoiseConfig + + # --- CreateRemoteThread Patterns --- diff --git a/src/evidenceforge/generation/activity/generator.py b/src/evidenceforge/generation/activity/generator.py index a766423c..d87f5ad8 100644 --- a/src/evidenceforge/generation/activity/generator.py +++ b/src/evidenceforge/generation/activity/generator.py @@ -1571,6 +1571,22 @@ def _tls_key_for_certificate_name( return key_type, key_length +def _tls_signature_algorithm_for_issuer( + issuer_name: str, + *, + fallback_key_type: str = "rsa", + fallback_key_length: int = 2048, +) -> str: + """Return the certificate signature algorithm implied by the issuer key.""" + from evidenceforge.generation.activity.tls_realism import signature_algorithm_for_issuer + + return signature_algorithm_for_issuer( + issuer_name, + fallback_type=fallback_key_type, + fallback_length=fallback_key_length, + ) + + class ActivityGenerator: """Generates specific activity events using StateManager and emitters. @@ -2493,7 +2509,11 @@ def _attach_ssl_context( certificate_not_valid_before=validity[0], certificate_not_valid_after=validity[1], certificate_key_alg="id-ecPublicKey" if is_ecdsa else "rsaEncryption", - certificate_sig_alg="ecdsa-with-SHA256" if is_ecdsa else "sha256WithRSAEncryption", + certificate_sig_alg=_tls_signature_algorithm_for_issuer( + issuer_cfg["name"], + fallback_key_type=key_type, + fallback_key_length=key_length, + ), certificate_key_type=key_type, certificate_key_length=key_length, certificate_exponent="65537" if not is_ecdsa else "", @@ -2764,7 +2784,9 @@ def _build_tls_certificate_chain( from evidenceforge.events.contexts import X509Context from evidenceforge.generation.activity.tls_realism import ( certificate_chain_config, + certificate_subject_key_profile, chain_template_for_issuer, + signature_algorithm_for_issuer, ) chain = [leaf] @@ -2828,6 +2850,11 @@ def _build_tls_certificate_chain( selected_key = profile_rng.choices(key_types, weights=weights, k=1)[0] key_type = str(selected_key.get("type", "rsa")) key_length = int(selected_key.get("length", 2048)) + key_type, key_length = certificate_subject_key_profile( + subject, + fallback_type=key_type, + fallback_length=key_length, + ) key_type, key_length = _tls_key_for_certificate_name(subject, key_type, key_length) serial_seed = "|".join( [ @@ -2870,6 +2897,11 @@ def _build_tls_certificate_chain( key_type = str(profile["certificate_key_type"]) key_length = int(profile["certificate_key_length"]) is_ecdsa = key_type == "ecdsa" + signature_alg = signature_algorithm_for_issuer( + str(profile["certificate_issuer"]), + fallback_type=key_type, + fallback_length=key_length, + ) chain.append( X509Context( fuid=generate_stable_zeek_uid( @@ -2884,9 +2916,7 @@ def _build_tls_certificate_chain( certificate_not_valid_before=int(profile["certificate_not_valid_before"]), certificate_not_valid_after=int(profile["certificate_not_valid_after"]), certificate_key_alg="id-ecPublicKey" if is_ecdsa else "rsaEncryption", - certificate_sig_alg="ecdsa-with-SHA256" - if is_ecdsa - else "sha256WithRSAEncryption", + certificate_sig_alg=signature_alg, certificate_key_type=key_type, certificate_key_length=key_length, certificate_exponent="65537" if not is_ecdsa else "", @@ -4564,7 +4594,8 @@ def generate_process( from evidenceforge.generation.activity.dll_load_profiles import get_dlls_for_process dll_profiles = get_dlls_for_process(_exe_lower) - dll_path = rng.choice(dll_profiles)["path"] if dll_profiles else "" + dll_profile = rng.choice(dll_profiles) if dll_profiles else {} + dll_path = dll_profile.get("path", "") module_delay_ms = rng.randint(120, 1500) process_start = running_proc.start_time if running_proc is not None else None if dll_path and self._mark_loaded_module( @@ -4588,7 +4619,12 @@ def generate_process( logon_id=process_logon_id, start_time=process_start, ), - image_load=ImageLoadContext(image_loaded=dll_path), + image_load=ImageLoadContext( + image_loaded=dll_path, + signed=bool(dll_profile.get("signed", True)), + signature=str(dll_profile.get("signature", "Microsoft Windows")), + signature_status=str(dll_profile.get("signature_status", "Valid")), + ), edr=EdrContext(object_id=str(uuid.uuid4()), actor_id=proc_obj_id), storyline_origin=from_storyline, ) diff --git a/src/evidenceforge/generation/activity/tls_realism.py b/src/evidenceforge/generation/activity/tls_realism.py index 2c64dc2f..b8601132 100644 --- a/src/evidenceforge/generation/activity/tls_realism.py +++ b/src/evidenceforge/generation/activity/tls_realism.py @@ -90,6 +90,59 @@ def certificate_chain_config() -> dict[str, Any]: return load_tls_realism().get("certificate_chains", {}) +def _subject_key_profile(subject_name: str) -> dict[str, Any] | None: + """Return the configured CA key profile matching a subject/issuer name.""" + for profile in certificate_chain_config().get("subject_key_profiles", []): + if not isinstance(profile, dict): + continue + patterns = [str(pattern) for pattern in profile.get("subject_patterns", [])] + if any(fnmatch.fnmatch(subject_name, pattern) for pattern in patterns): + return profile + return None + + +def certificate_subject_key_profile( + subject_name: str, + fallback_type: str = "rsa", + fallback_length: int = 2048, +) -> tuple[str, int]: + """Return the configured key profile for a CA subject or issuer name. + + X.509 ``certificate.sig_alg`` describes the issuer's signing key, not the + child certificate's own public key. These profiles let chain construction + choose that issuer key from source-owned CA metadata instead of inferring it + from the child row. + """ + key_type = fallback_type + key_length = fallback_length + profile = _subject_key_profile(subject_name) + if profile is not None: + key_type = str(profile.get("key_type", key_type)) + key_length = int(profile.get("key_length", key_length)) + return key_type, key_length + + +def signature_algorithm_for_issuer( + issuer_name: str, + fallback_type: str = "rsa", + fallback_length: int = 2048, +) -> str: + """Return a Zeek x509 ``certificate.sig_alg`` value for an issuer key.""" + profile = _subject_key_profile(issuer_name) + if profile is not None: + algorithms = [str(algorithm) for algorithm in profile.get("child_signature_algorithms", [])] + if algorithms: + return algorithms[0] + issuer_key_type, _issuer_key_length = certificate_subject_key_profile( + issuer_name, + fallback_type=fallback_type, + fallback_length=fallback_length, + ) + if issuer_key_type == "ecdsa" and _issuer_key_length >= 384: + return "ecdsa-with-SHA384" + return "ecdsa-with-SHA256" if issuer_key_type == "ecdsa" else "sha256WithRSAEncryption" + + def certificate_analyzer_delay_ms( *, zeek_uid: str, diff --git a/tests/unit/test_activity.py b/tests/unit/test_activity.py index 4f2009cf..431be9da 100644 --- a/tests/unit/test_activity.py +++ b/tests/unit/test_activity.py @@ -1788,6 +1788,64 @@ def getrandbits(self, bits): ] assert registry_events == [] + def test_process_module_load_preserves_profile_signature_metadata( + self, + activity_gen, + test_user, + test_system, + state_manager, + mock_emitters, + monkeypatch, + ): + """Probabilistic process ImageLoad events should carry DLL profile signer fields.""" + + class ModuleLoadRandom(random.Random): + def __init__(self): + super().__init__(7) + self._random_values = iter([0.99, 0.01]) + + def random(self): + return next(self._random_values, 0.99) + + import evidenceforge.generation.activity.dll_load_profiles as dll_profiles + + timestamp = datetime(2024, 1, 15, 10, 0, 0, tzinfo=UTC) + state_manager.set_current_time(timestamp) + logon_id = activity_gen.generate_logon(test_user, test_system, timestamp) + monkeypatch.setattr(generator_module, "_get_rng", ModuleLoadRandom) + monkeypatch.setattr( + dll_profiles, + "get_dlls_for_process", + lambda _exe: [ + { + "path": r"C:\Program Files\Mozilla Firefox\mozglue.dll", + "signed": True, + "signature": "Mozilla Corporation", + "signature_status": "Valid", + } + ], + ) + + activity_gen.generate_process( + test_user, + test_system, + timestamp + timedelta(seconds=5), + logon_id, + r"C:\Program Files\Mozilla Firefox\firefox.exe", + r'"C:\Program Files\Mozilla Firefox\firefox.exe"', + parent_pid=4, + ) + + image_load_events = [ + call.args[0] + for call in mock_emitters["windows_event_security"].emit.call_args_list + if call.args[0].event_type == "image_load" + ] + assert image_load_events + assert image_load_events[-1].image_load.image_loaded.endswith("mozglue.dll") + assert image_load_events[-1].image_load.signature == "Mozilla Corporation" + assert image_load_events[-1].image_load.signature_status == "Valid" + def test_image_load_is_clamped_after_process_start( self, activity_gen, test_user, test_system, state_manager, mock_emitters ): diff --git a/tests/unit/test_dhcp_and_certs.py b/tests/unit/test_dhcp_and_certs.py index a13aac11..9a722128 100644 --- a/tests/unit/test_dhcp_and_certs.py +++ b/tests/unit/test_dhcp_and_certs.py @@ -26,12 +26,14 @@ ) from evidenceforge.generation.activity.tls_realism import ( certificate_chain_config, + certificate_subject_key_profile, chain_template_for_issuer, multi_label_public_suffixes, ocsp_config, pick_ocsp_responder, pick_tls_destination, reset_tls_realism_cache, + signature_algorithm_for_issuer, tls_destination_config, ) from evidenceforge.generation.state_manager import StateManager @@ -817,6 +819,78 @@ def test_intermediate_ca_profile_is_stable_across_leaf_certificates(self): == second_intermediate.certificate_not_valid_after ) + def test_intermediate_signature_algorithm_follows_issuer_key(self): + """Intermediate certificate signatures should be signed by the issuer key.""" + generator = ActivityGenerator(StateManager(), {}) + issuer_name = "CN=E1, O=Let's Encrypt, C=US" + intermediate = None + for seed in range(1, 50): + chain = generator._build_tls_certificate_chain( + leaf=X509Context( + fuid="FLeaf", + certificate_subject="CN=leaf.example", + certificate_issuer=issuer_name, + ), + cert_name=f"leaf-{seed}.example", + issuer_name=issuer_name, + event_time=datetime(2024, 10, 14, 12, 0, tzinfo=UTC), + connection_uid=f"CLeE1{seed}", + rng=random.Random(seed), + ) + candidate = chain[1] + if ( + certificate_subject_key_profile(candidate.certificate_subject)[0] + != certificate_subject_key_profile(candidate.certificate_issuer)[0] + ): + intermediate = candidate + break + + assert intermediate is not None + + assert intermediate.certificate_subject == issuer_name + assert intermediate.certificate_issuer != intermediate.certificate_subject + expected = signature_algorithm_for_issuer(intermediate.certificate_issuer) + assert intermediate.certificate_sig_alg == expected + + def test_leaf_signature_algorithm_follows_issuer_not_leaf_key(self): + """An ECDSA leaf signed by an RSA CA should render an RSA signature algorithm.""" + state_manager = StateManager() + state_manager.set_current_time(datetime(2024, 10, 14, 12, 0, tzinfo=UTC)) + generator = ActivityGenerator(state_manager, {}) + generator._emit_ocsp_http_response = lambda *args, **kwargs: None + event = None + + for seed in range(1, 100): + candidate = SecurityEvent( + timestamp=datetime(2024, 10, 14, 12, 0, tzinfo=UTC), + event_type="connection", + network=NetworkContext( + src_ip="10.30.40.101", + src_port=50123 + seed, + dst_ip="142.250.190.99", + dst_port=443, + protocol="tcp", + service="ssl", + zeek_uid=f"CGtsLeafSignature{seed}", + ), + ) + generator._attach_ssl_context( + candidate, + hostname=f"asset-{seed}.google.com", + dns=None, + dst_ip="142.250.190.99", + rng=random.Random(seed), + allow_failure=False, + ) + if candidate.x509 is not None: + event = candidate + break + + assert event is not None and event.x509 is not None + assert event.x509.certificate_issuer == "CN=GTS CA 1C3, O=Google Trust Services LLC, C=US" + expected = signature_algorithm_for_issuer(event.x509.certificate_issuer) + assert event.x509.certificate_sig_alg == expected + class TestDnsRtt: """Tests for resolver-aware DNS timing realism.""" From 5931c8a9ceee912b27799683baffc91d04f89c23 Mon Sep 17 00:00:00 2001 From: "David J. Bianco" Date: Thu, 14 May 2026 10:16:19 -0400 Subject: [PATCH 2/3] feat: tune endpoint baseline noise policy --- TODO.md | 1 + commands/eforge/config.md | 4 +- .../references/config-dependency-graph.md | 12 +- .../eforge/references/config-host-activity.md | 38 +++- .../eforge/references/config-validation.md | 2 + docs/reference/CUSTOMIZING_CONFIG.md | 9 +- src/evidenceforge/cli/validate_config.py | 10 +- src/evidenceforge/config/activity/README.md | 3 +- .../config/activity/endpoint_noise.yaml | 37 ++++ .../generation/activity/endpoint_noise.py | 49 +++++ .../generation/engine/baseline.py | 208 +++++++++++++++++- tests/unit/test_baseline_canonical.py | 84 ++++++- tests/unit/test_validate_config.py | 100 +++++++++ 13 files changed, 539 insertions(+), 18 deletions(-) create mode 100644 src/evidenceforge/config/activity/endpoint_noise.yaml create mode 100644 src/evidenceforge/generation/activity/endpoint_noise.py diff --git a/TODO.md b/TODO.md index a090be20..4b111d9a 100644 --- a/TODO.md +++ b/TODO.md @@ -240,6 +240,7 @@ Replaced manual per-emitter field coordination with SecurityEvent intermediate r - [x] **P2** Scenario skill anti-curation guidance follow-up — Revised the dev scenario skill so attacker-controlled domains, service accounts, scheduled tasks, files, and process names blend into ordinary naming conventions without becoming semantic breadcrumbs that reveal the attack narrative. Verification: `uv run pytest tests/unit/test_install_skills.py -q --no-cov` passed (`30 passed`); the same focused test file also passed under the default coverage run, but that command failed the whole-repo coverage threshold because it intentionally ran only one test module. - [x] **P1** Web application response/session realism follow-up — Added data-driven inbound `web_server` visitor profiles so human visitors consume `traffic_rates.web` as top-level actions, then fan out into required page assets/API calls through `site_maps.yaml`; crawler, health-check, API-client, and opportunistic-probe traffic now uses source-native configured request/status/User-Agent profiles. Static resource sizes are stable per host/path, human navigation and render fanout timing use `timing_profiles.yaml`, and docs/skill references now explain the budget and config ownership. Verification passed: focused web/timing/baseline tests (`107 passed, 1 skipped`), config-related tests (`64 passed`), `uv run eforge validate-config`, repo-wide Ruff checks/format checks, full normal `uv run pytest -q` (`3012 passed, 15 skipped`), and `git diff --check`. - [x] **P1** Well-synced network sensor timing follow-up — Replaced hardcoded multi-sensor Zeek +/-400ms skew plus broad path delay with a validated `network_sensor_observation` timing profile. The default `well_synced` profile keeps stable per-sensor clock skew within +/-1.5ms and per-flow capture/path delay within 50-2000us while preserving canonical packet/byte truth unless source-native observation variance is explicitly enabled. Verification passed with focused Zeek/timing tests, `uv run eforge validate-config`, repo-wide Ruff checks/format checks, full normal `uv run pytest -q` (`3012 passed, 15 skipped`), and `git diff --check`. +- [x] **P1** Source identity and endpoint baseline realism sprint — completed TLS/X.509 issuer-compatible chain signatures, Sysmon Event 7 native third-party module identity, config-driven Windows scheduled-process timing, and DHCP registry emission policy tied to lease activity. Verified with `uv run eforge validate-config`, focused regressions, Ruff, normal pytest, and slow-inclusive pytest. - [ ] **DEFERRED with observation/source coverage architecture** **P2** Endpoint/eCAR baseline variance follow-up — Loop 96 found workstation eCAR category volumes and Linux process lifecycle evidence too uniform and complete. Defer with the broader observation/profile sprint so host/persona-specific variance, long-lived process state, benign unmatched artifacts, and realistic endpoint observation gaps are modeled coherently rather than as eCAR-only omissions. - [ ] **Later architectural sprint: imperfect observation and source coverage** — defer the broad "too-complete telemetry" problem until after the sharper defects are gone. Model source-specific drop rates, ingestion delay, audit-policy gaps, endpoint coverage variance, and asymmetric Security/Sysmon/eCAR/Zeek visibility as a coherent observation/profile layer rather than one-off omissions. Bundle the related deferred items into this sprint: endpoint/eCAR baseline variance, source-specific process lifecycle completeness modeling, configurable cross-source evidence disagreement, per-host/source log coverage, and the host/activity profile items for per-entity artifact and volume variance. - [x] Full slow-suite regression cleanup after loop-65 merge — explicit-proxy storyline beacons now preserve authored hostname+destination IP pairs only when the storyline marks that pair as intentional, normal proxy-origin DNS resolution remains intact, and the parallel-generation LogonID assertion treats Type 7 unlock reuse as valid slice-of-time Windows behavior. Verified with targeted proxy/parallel tests, `uv run ruff check .`, `uv run ruff format --check .`, and `uv run pytest -v --include-slow` (`2875 passed, 23 skipped`). diff --git a/commands/eforge/config.md b/commands/eforge/config.md index e3248735..d4eedb37 100644 --- a/commands/eforge/config.md +++ b/commands/eforge/config.md @@ -66,8 +66,10 @@ When writing to the overlay, files are partial — they contain ONLY the user's | Modify CallTrace patterns | `calltrace_patterns.yaml` | (standalone — Event 10 ProcessAccess call chain templates) | | Modify ProcessAccess masks | `process_access_patterns.yaml` | (standalone — Event 10 baseline source/target pairs and GrantedAccess masks) | | Modify CreateRemoteThread pairs | `create_remote_thread_patterns.yaml` | (standalone — Event 8 baseline source/target pairs) | +| Modify TLS chain/OCSP/SNI realism | `tls_realism.yaml` | `dns_registry.yaml` for OCSP responder hosts and domains selected by `dns_tags` | | Modify Windows auth realism | `windows_auth_realism.yaml` | (standalone — Security log auth timing and failed-logon profile knobs) | | Modify baseline auth noise | `auth_noise.yaml` | (standalone — stale scheduled-credential accounts and irregular recurrence timing) | +| Modify endpoint background noise | `endpoint_noise.yaml` | (standalone — scheduled-process timing and DHCP registry emission policy) | | Modify causal/source timing | `timing_profiles.yaml` | (standalone — causal prerequisite, source latency, teardown, and Windows/Sysmon collision-spacing knobs) | | ~~Format definitions~~ | Not user-customizable | Engine internals — requires code changes | | ~~Evaluation rules~~ | Not user-customizable | Must match format definitions — requires code changes | @@ -88,7 +90,7 @@ Also read the relevant reference doc for field schemas and conventions: | Applications, spawn rules, processes | `references/config-apps-processes.md` | | Sysmon filters, EDR pools, CallTrace, ProcessAccess masks, CreateRemoteThread pairs | `references/config-apps-processes.md` (Sysmon sections) | | Persona file structure | `references/config-personas.md` | -| Host activity (bash, systemd, syslog) | `references/config-host-activity.md` | +| Host activity (bash, systemd, syslog, endpoint noise) | `references/config-host-activity.md` | | Timing profiles | `references/config-host-activity.md` | | Format definitions | `references/config-formats.md` (read-only reference — not user-customizable) | | Evaluation rules | `references/config-evaluation.md` (read-only reference — not user-customizable) | diff --git a/commands/eforge/references/config-dependency-graph.md b/commands/eforge/references/config-dependency-graph.md index 4f121947..c8840b7a 100644 --- a/commands/eforge/references/config-dependency-graph.md +++ b/commands/eforge/references/config-dependency-graph.md @@ -151,6 +151,13 @@ Each row is a file; columns show what it depends on and what depends on it. | depends on | nothing | Standalone authentication-noise profile data | | **depended on by** | Engine (runtime) | Drives stale scheduled-credential account pools, recurrence timing, jitter, skips, and backoff | +### endpoint_noise.yaml +| Direction | File | Relationship | +|-----------|------|-------------| +| depends on | nothing | Standalone endpoint background timing and registry-emission policy data | +| **depended on by** | Engine (runtime) | Drives Windows scheduled-process trigger windows, host drift, skips, and DHCP interface registry write policy | +| validated by | `eforge validate-config` | Enforces coherent timing bounds, probability ranges, and non-empty DHCP registry value lists | + ### network_params.yaml | Direction | File | Relationship | |-----------|------|-------------| @@ -166,8 +173,9 @@ Each row is a file; columns show what it depends on and what depends on it. ### tls_realism.yaml | Direction | File | Relationship | |-----------|------|-------------| -| depends on | tls_issuers.yaml, dns_registry.yaml | Chain templates match issuer names/patterns selected from issuer config; OCSP responder hosts must exist in dns_registry; destination profiles can pull domains by DNS tag | -| **depended on by** | Engine (runtime) | Drives Zeek TLS SAN, x509 chain depth, OCSP cache/status behavior, and profiled TLS SNI/destination selection | +| depends on | tls_issuers.yaml, dns_registry.yaml | Chain templates and subject-key profiles match issuer names/patterns selected from issuer config; OCSP responder hosts must exist in dns_registry; destination profiles can pull domains by DNS tag | +| **depended on by** | Engine (runtime) | Drives Zeek TLS SAN, x509 chain depth, issuer-compatible certificate signature algorithms, OCSP cache/status behavior, and profiled TLS SNI/destination selection | +| validated by | `eforge validate-config` | Enforces coherent chain profile structure, non-empty subject-key patterns, and RSA/ECDSA child signature compatibility | ### smb_file_transfers.yaml | Direction | File | Relationship | diff --git a/commands/eforge/references/config-host-activity.md b/commands/eforge/references/config-host-activity.md index 4ee80dbf..9abefe53 100644 --- a/commands/eforge/references/config-host-activity.md +++ b/commands/eforge/references/config-host-activity.md @@ -12,8 +12,11 @@ Schema documentation for host-level activity config files. User customizations g 2. [systemd_schedules.yaml](#systemd_schedulesyaml) 3. [extra_syslog_messages.yaml](#extra_syslog_messagesyaml) 4. [kerberos_realism.yaml](#kerberos_realismyaml) -5. [timing_profiles.yaml](#timing_profilesyaml) -6. [Domain Controller Baseline Activity](#domain-controller-baseline-activity) +5. [windows_auth_realism.yaml](#windows_auth_realismyaml) +6. [auth_noise.yaml](#auth-noise-auth_noiseyaml) +7. [endpoint_noise.yaml](#endpoint-noise-endpoint_noiseyaml) +8. [timing_profiles.yaml](#timing_profilesyaml) +9. [Domain Controller Baseline Activity](#domain-controller-baseline-activity) --- @@ -315,6 +318,37 @@ scheduled_stale_credentials: --- +## Endpoint Noise (`endpoint_noise.yaml`) + +Controls endpoint background timing and registry-emission policies that are too source-specific for scenario YAML. Use it to tune routine Windows scheduled-process spacing and whether DHCP interface registry values appear as ambient Sysmon/EDR noise. + +```yaml +windows_scheduled_processes: + count_min: 2 + count_max: 5 + trigger_window_start_seconds: 90 + trigger_window_end_seconds: 3510 + slot_spacing_seconds: 300 + host_phase_window_seconds: 900 + jitter_seconds_min: -42 + jitter_seconds_max: 73 + skip_probability: 0.08 + +registry_noise: + dhcp_interface_values: + value_names: [DhcpIPAddress, DhcpNameServer] + require_dhcp_state: true + emit_on_lease_events: true + suppress_system_types: [server, domain_controller] + suppress_roles: [domain_controller, dns_server, file_server, web_server] +``` + +`windows_scheduled_processes` replaces hour-end clamping with profile-driven trigger windows, per-host phase offsets, jitter, and skips. Keep `trigger_window_end_seconds` comfortably below 3599 to avoid synthetic `xx:59:59` clusters. + +`registry_noise.dhcp_interface_values` reserves DHCP interface registry writes for actual DHCP lease/reconfigure activity. Static infrastructure roles should stay in `suppress_system_types` or `suppress_roles` so they do not repeatedly rewrite DHCP values as ambient registry noise. Run `eforge validate-config` after overlay changes; it rejects inverted ranges, empty value-name lists, and invalid probabilities. + +--- + ## timing_profiles.yaml Data-driven timing windows for causal relationships, source-native latency, teardown margins, and Windows/Sysmon same-timestamp collision spacing. Use this when tuning realism of correlated event gaps without changing scenario YAML. diff --git a/commands/eforge/references/config-validation.md b/commands/eforge/references/config-validation.md index 3e5400ad..a5895b57 100644 --- a/commands/eforge/references/config-validation.md +++ b/commands/eforge/references/config-validation.md @@ -83,6 +83,8 @@ Run `eforge info ` to get specific values (e.g., `eforge info paths.activ | 36 | kerberos_realism.yaml structure | ERROR | Invalid Kerberos 4768 pre-auth/ticket/encryption distribution, unsupported hex values, PKINIT without certificate profile, non-PKINIT with certificate fields, excessive no-preauth/PKINIT/RC4 weights, or malformed certificate profile fields | | 37 | web_session_profiles.yaml structure | ERROR | Invalid inbound web visitor class, missing User-Agent pool, malformed configured request, or invalid request-count range | | 38 | auth_noise.yaml structure | ERROR | Invalid stale scheduled-credential account pool, host-count range, recurrence interval range, jitter range, skip probability, or backoff bounds | +| 39 | endpoint_noise.yaml structure | ERROR | Invalid Windows scheduled-process timing bounds, skip probability, or DHCP registry emission policy | +| 40 | tls_realism.yaml chain metadata | ERROR | Invalid TLS subject-key profile fields or RSA/ECDSA child signature algorithm mismatch | ## Scenario Validation: traffic_rates diff --git a/docs/reference/CUSTOMIZING_CONFIG.md b/docs/reference/CUSTOMIZING_CONFIG.md index 8e6f5c52..b8fb7906 100644 --- a/docs/reference/CUSTOMIZING_CONFIG.md +++ b/docs/reference/CUSTOMIZING_CONFIG.md @@ -143,7 +143,7 @@ This is a **partial overlay** — it adds `nurse` to Chrome's and Outlook's pers eforge info personas # Should include "nurse" eforge info dns_tags # Should include your new tags -# Run full validation (27 cross-reference checks) +# Run full validation across merged package + overlay config eforge validate-config ``` @@ -157,11 +157,12 @@ Configuration files are interconnected. When you add an entry to one file, other | Certificate/update/telemetry proxy behavior | `proxy_uri_templates.yaml` (`domain_class`, infra-specific paths/content types, and `referrer_policy: none`; non-browser classes are excluded from site-map browsing sessions) | | New proxy User-Agent behavior | `proxy_user_agents.yaml` (workstation/server UA pools, package-manager host bindings, domain-specific update/cert/telemetry overrides) | | Inbound web visitor mix | `web_session_profiles.yaml` (visitor classes, configured tool/API requests, and User-Agent pools). Human visitor sessions use `site_maps.yaml`; timing lives in `timing_profiles.yaml`; `traffic_rates.yaml` `web` counts top-level actions only. | -| New TLS issuer behavior | `tls_issuers.yaml` (issuer validity, key-type weights, and domain CA overrides). RSA-branded issuer names should only advertise RSA key types unless the chain/signature model is also updated to distinguish issuer signature algorithm from leaf public-key algorithm. | -| New TLS OCSP responder behavior | `tls_realism.yaml` (`ocsp.responders`) plus `dns_registry.yaml` for each responder hostname | +| New TLS issuer behavior | `tls_issuers.yaml` (issuer validity, key-type weights, and domain CA overrides). RSA-branded issuer names should only advertise RSA key types unless matching `tls_realism.yaml` subject-key profiles distinguish issuer signature algorithm from leaf public-key algorithm. | +| New TLS OCSP responder or chain behavior | `tls_realism.yaml` (`ocsp.responders`, `certificate_chains.templates`, and `certificate_chains.subject_key_profiles`) plus `dns_registry.yaml` for each responder hostname. Subject key profiles must include issuer family, key type/size, and compatible child signature algorithms. | | Kerberos TGT pre-auth realism | `kerberos_realism.yaml` (`tgt_success.pre_auth_types`, ticket options, encryption types, and PKINIT certificate profiles). Run `eforge validate-config`; PKINIT (`PreAuthType: 15`) requires populated certificate profile support. | | Windows auth realism | `windows_auth_realism.yaml` (`workstation_lock.min_unlock_gap_seconds`, failed-logon local/network profiles, and optional companion network connection rates) | | Baseline auth noise | `auth_noise.yaml` (stale scheduled-credential account pools, host counts, recurrence intervals, jitter, skips, and backoff) | +| Endpoint background noise | `endpoint_noise.yaml` (Windows scheduled-process trigger windows, host drift, skip probability, and DHCP registry emission policy) | | Causal/source-native timing | `timing_profiles.yaml` (`relationships` for causal prerequisites, source latency, teardown margins, Zeek analyzer offsets and TLS duration floors, plus Windows/Sysmon collision spacing) | | Public NTP fallback servers and DNS tunnel timing | `network_params.yaml` (`public_ntp_servers`, `dns_tunnel_rtt`; scenario-defined internal/domain NTP servers still take precedence) | | A new application | `spawn_rules.yaml` (process tree), `process_network_map.yaml` (if it generates traffic) | @@ -203,4 +204,4 @@ For full field schemas and conventions, see the reference docs installed with th | Persona file structure | `/eforge:references:config-personas` | | Host activity (bash, systemd, syslog) | `/eforge:references:config-host-activity` | | Cross-file dependency map | `/eforge:references:config-dependency-graph` | -| Validation checks (27) | `/eforge:references:config-validation` | +| Validation checks | `/eforge:references:config-validation` | diff --git a/src/evidenceforge/cli/validate_config.py b/src/evidenceforge/cli/validate_config.py index f3cbf5c3..51e9e488 100644 --- a/src/evidenceforge/cli/validate_config.py +++ b/src/evidenceforge/cli/validate_config.py @@ -114,7 +114,7 @@ def _safe_load_yaml(path: Path) -> tuple[Any, str | None]: def validate_config() -> ValidationResult: - """Run all 27 validation checks across config files. + """Run validation checks across config files. Uses the same loader paths the engine uses (including overlay merges). """ @@ -230,6 +230,9 @@ def validate_config() -> ValidationResult: "dll_pool", }, }, + "activity/endpoint_noise.yaml": { + "dict_fields": {"windows_scheduled_processes", "registry_noise"}, + }, "activity/ids_signatures.yaml": { "list_fields": {"signatures": None}, }, @@ -445,6 +448,7 @@ def validate_config() -> ValidationResult: load_create_remote_thread_patterns, ) from evidenceforge.generation.activity.dns_registry import load_dns_registry + from evidenceforge.generation.activity.endpoint_noise import load_endpoint_noise from evidenceforge.generation.activity.ids_signatures import load_ids_signatures from evidenceforge.generation.activity.process_access_patterns import ( load_process_access_patterns, @@ -475,6 +479,7 @@ def validate_config() -> ValidationResult: proxy_ua_data = load_proxy_user_agents() site_data = load_site_maps() sys_proc_data = load_system_processes() + endpoint_noise_data = load_endpoint_noise() tls_realism_data = load_tls_realism() windows_auth_data = load_windows_auth_realism() timing_profiles_data = load_timing_profiles() @@ -1689,6 +1694,7 @@ def _record_ids_rule_identity( DnsTunnelRttConfig, DnsTunnelTtlEntry, EdrFileSideEffectProfile, + EndpointNoiseConfig, KerberosRealismConfig, OuiEntry, PersonaEntry, @@ -1815,6 +1821,8 @@ def _record_ids_rule_identity( "edr_pools.yaml (file_side_effect_profiles)", ) ) + if endpoint_noise_data: + _SCHEMA_CHECKS.append(([endpoint_noise_data], EndpointNoiseConfig, "endpoint_noise.yaml")) # traffic_profiles.yaml: connection entries all_traffic_connection_entries = [] diff --git a/src/evidenceforge/config/activity/README.md b/src/evidenceforge/config/activity/README.md index fb8221f8..a10543cd 100644 --- a/src/evidenceforge/config/activity/README.md +++ b/src/evidenceforge/config/activity/README.md @@ -18,10 +18,11 @@ caches data after first load. Two files (`network_params.yaml`, | `bash_commands.yaml` | `bash_commands.py` | Per-role bash command pools (sysadmin, dba, developer, generic) with `{placeholder}` templates. | | `system_processes.yaml` | `system_processes.py` | Baseline Windows scheduled tasks and system services (svchost, MpCmdRun, etc.). | | `tls_issuers.yaml` | `tls_issuers.py` | Certificate issuer configs (Let's Encrypt, DigiCert, etc.) with validity periods and key types. RSA-named issuers should not include ECDSA key types under the current simplified x509 model. | -| `tls_realism.yaml` | `tls_realism.py` | TLS SAN, OCSP, certificate-chain, and destination-profile settings with overlay support. | +| `tls_realism.yaml` | `tls_realism.py` | TLS SAN, OCSP, certificate-chain, CA key/signature metadata, and destination-profile settings with overlay support. | | `kerberos_realism.yaml` | `kerberos_realism.py` | Kerberos 4768 TGT PreAuthType, TicketOptions, encryption, and PKINIT certificate field distributions with overlay support. | | `windows_auth_realism.yaml` | `windows_auth_realism.py` | Windows Security authentication realism knobs such as minimum 4800→4801 lock/unlock gap, failed-logon validation paths, companion network evidence, and 4672 privilege profiles. | | `auth_noise.yaml` | `auth_noise.py` | Baseline authentication-noise profiles such as stale scheduled-credential account pools and irregular recurrence timing. | +| `endpoint_noise.yaml` | `endpoint_noise.py` | Endpoint background timing and registry-emission policies for Windows scheduled processes and DHCP interface registry writes. | | `proxy_uri_templates.yaml` | `proxy_uri.py` | Per-domain URI path templates for proxy logs (Windows Update, CRL, OCSP, Azure AD, etc.). | | `network_params.yaml` | `network_params.py`, `engine/emitter_setup.py` | MAC address OUI prefixes, public NTP fallback servers, and DNS tunnel RTT bounds. | | `systemd_schedules.yaml` | `engine/baseline.py` | Systemd timer and cron job schedules (logrotate, fstrim, apt-daily, etc.). | diff --git a/src/evidenceforge/config/activity/endpoint_noise.yaml b/src/evidenceforge/config/activity/endpoint_noise.yaml new file mode 100644 index 00000000..204d9142 --- /dev/null +++ b/src/evidenceforge/config/activity/endpoint_noise.yaml @@ -0,0 +1,37 @@ +# Endpoint baseline noise policy for Windows scheduled/background process and +# ambient registry telemetry. +# +# User customizations go in: +# .eforge/config/activity/endpoint_noise.yaml +# +# Overlay behavior: nested dicts merge and lists extend. + +windows_scheduled_processes: + count_min: 2 + count_max: 5 + trigger_window_start_seconds: 90 + trigger_window_end_seconds: 3510 + slot_spacing_seconds: 300 + host_phase_window_seconds: 900 + jitter_seconds_min: -42 + jitter_seconds_max: 73 + skip_probability: 0.08 + +registry_noise: + dhcp_interface_values: + value_names: + - DhcpIPAddress + - DhcpNameServer + require_dhcp_state: true + emit_on_lease_events: true + suppress_system_types: + - server + - domain_controller + suppress_roles: + - domain_controller + - dns_server + - file_server + - web_server + - forward_proxy + - app_server + - database diff --git a/src/evidenceforge/generation/activity/endpoint_noise.py b/src/evidenceforge/generation/activity/endpoint_noise.py new file mode 100644 index 00000000..2bcff1d5 --- /dev/null +++ b/src/evidenceforge/generation/activity/endpoint_noise.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: MIT + +"""Endpoint baseline noise policy loader.""" + +from __future__ import annotations + +from typing import Any + +from evidenceforge.config import get_activity_directory +from evidenceforge.config.overlay import deep_merge_dict, load_with_overlay + +_CONFIG_PATH = get_activity_directory() / "endpoint_noise.yaml" +_CACHED_DATA: dict[str, Any] | None = None + + +def _merge_endpoint_noise(default: dict, overlay: dict) -> dict: + """Merge endpoint noise overlay with package defaults.""" + return deep_merge_dict(default, overlay) + + +def load_endpoint_noise() -> dict[str, Any]: + """Load endpoint noise config from YAML, merged with overlay. Cached after first call.""" + global _CACHED_DATA + if _CACHED_DATA is not None: + return _CACHED_DATA + + _CACHED_DATA = load_with_overlay( + _CONFIG_PATH, + "activity/endpoint_noise.yaml", + _merge_endpoint_noise, + ) + return _CACHED_DATA + + +def reset_endpoint_noise_cache() -> None: + """Clear cached endpoint noise config. Intended for tests.""" + global _CACHED_DATA + _CACHED_DATA = None + + +def windows_scheduled_process_config() -> dict[str, Any]: + """Return Windows scheduled/background process timing policy.""" + return load_endpoint_noise().get("windows_scheduled_processes", {}) + + +def registry_noise_config() -> dict[str, Any]: + """Return ambient endpoint registry-noise policy.""" + return load_endpoint_noise().get("registry_noise", {}) diff --git a/src/evidenceforge/generation/engine/baseline.py b/src/evidenceforge/generation/engine/baseline.py index 7dc9e7d9..14a3b784 100644 --- a/src/evidenceforge/generation/engine/baseline.py +++ b/src/evidenceforge/generation/engine/baseline.py @@ -473,6 +473,94 @@ def _materialize_registry_value_for_time( return prior_time.strftime("%Y-%m-%dT%H:%M:%S") +def _is_dhcp_managed_registry_value( + key: str, + value_name: str, + policy: dict[str, Any] | None = None, +) -> bool: + """Return whether a registry value belongs to DHCP lease state.""" + if policy is None: + from evidenceforge.generation.activity.endpoint_noise import registry_noise_config + + policy = registry_noise_config().get("dhcp_interface_values", {}) + key_lower = key.lower() + if r"services\tcpip\parameters" not in key_lower: + return False + managed_names = {str(name).lower() for name in policy.get("value_names", [])} + return value_name.lower() in managed_names + + +def _system_suppresses_dhcp_registry_noise(system: Any, policy: dict[str, Any]) -> bool: + """Return whether DHCP registry noise should be suppressed for this static host.""" + system_type = str(getattr(system, "type", "") or "").lower() + roles = {str(role).lower() for role in (getattr(system, "roles", []) or [])} + suppressed_types = {str(value).lower() for value in policy.get("suppress_system_types", [])} + suppressed_roles = {str(value).lower() for value in policy.get("suppress_roles", [])} + return system_type in suppressed_types or bool(roles.intersection(suppressed_roles)) + + +def _ambient_registry_entry_allowed( + system: Any, + key: str, + value_name: str, + dhcp_state: dict[str, Any] | None, + registry_cfg: dict[str, Any] | None = None, +) -> bool: + """Return whether an ambient registry pool entry can emit for this host.""" + if registry_cfg is None: + from evidenceforge.generation.activity.endpoint_noise import registry_noise_config + + registry_cfg = registry_noise_config() + policy = registry_cfg.get("dhcp_interface_values", {}) + if not _is_dhcp_managed_registry_value(key, value_name, policy): + return True + if policy.get("emit_on_lease_events", True): + return False + if _system_suppresses_dhcp_registry_noise(system, policy): + return False + return bool(dhcp_state) if policy.get("require_dhcp_state", True) else True + + +def _windows_scheduled_task_offsets( + current_hour: datetime, + system: Any, + rng: random.Random, +) -> list[float]: + """Return config-driven Windows scheduled/background task offsets for this hour.""" + from evidenceforge.generation.activity.endpoint_noise import windows_scheduled_process_config + + cfg = windows_scheduled_process_config() + count_min = max(0, int(cfg.get("count_min", 2))) + count_max = max(count_min, int(cfg.get("count_max", 5))) + start = max(0, min(3599, int(cfg.get("trigger_window_start_seconds", 90)))) + end = max(start + 1, min(3599, int(cfg.get("trigger_window_end_seconds", 3510)))) + spacing = max(1, int(cfg.get("slot_spacing_seconds", 300))) + phase_window = max(1, int(cfg.get("host_phase_window_seconds", 900))) + jitter_min = float(cfg.get("jitter_seconds_min", -42)) + jitter_max = float(cfg.get("jitter_seconds_max", 73)) + if jitter_min > jitter_max: + jitter_min, jitter_max = jitter_max, jitter_min + skip_probability = max(0.0, min(1.0, float(cfg.get("skip_probability", 0.08)))) + window_len = max(1, end - start) + candidate_slots = list(range(0, max(1, window_len), spacing)) or [0] + num_tasks = rng.randint(count_min, count_max) if count_max > 0 else 0 + num_tasks = min(num_tasks, len(candidate_slots)) + if num_tasks <= 0: + return [] + + host_phase = _stable_seed( + f"task_phase:{system.hostname}:{current_hour.date().isoformat()}" + ) % min(phase_window, window_len) + selected_slots = sorted(rng.sample(candidate_slots, num_tasks)) + offsets: list[float] = [] + for slot in selected_slots: + if rng.random() < skip_probability: + continue + offset = start + ((slot + host_phase) % window_len) + rng.uniform(jitter_min, jitter_max) + offsets.append(max(float(start), min(float(end), offset))) + return sorted(offsets) + + # Synthetic SYSTEM user for baseline Event 8/10 generation _SYSTEM_USER = User( username="SYSTEM", @@ -638,6 +726,101 @@ def _resolve_traffic_rate(self, traffic_type: str) -> tuple[int, int]: rate = defaults[traffic_type] return (rate[0], rate[1]) + def _emit_dhcp_registry_side_effect( + self, + *, + system: Any, + time: datetime, + rng: random.Random, + sys_pids: dict[str, int], + dhcp_state: dict[str, Any] | None, + ) -> None: + """Emit DHCP interface registry writes coupled to a lease/renewal event.""" + if _get_os_category(system.os) != "windows" or "windows_event_sysmon" not in self.emitters: + return + + from evidenceforge.events.base import SecurityEvent + from evidenceforge.events.contexts import AuthContext, ProcessContext, RegistryContext + from evidenceforge.generation.activity.edr_pools import ( + get_registry_keys_hklm, + materialize_edr_template_group, + ) + from evidenceforge.generation.activity.endpoint_noise import registry_noise_config + + registry_cfg = registry_noise_config() + policy = registry_cfg.get("dhcp_interface_values", {}) + if not policy.get("emit_on_lease_events", True): + return + if policy.get("require_dhcp_state", True) and not dhcp_state: + return + if _system_suppresses_dhcp_registry_noise(system, policy): + return + + dhcp_entries = [ + (key, value_name, details) + for key, value_name, details in get_registry_keys_hklm() + if _is_dhcp_managed_registry_value(key, value_name, policy) + ] + if not dhcp_entries: + return + + _host_ctx = self.activity_generator._build_host_context(system) + count = min(len(dhcp_entries), rng.randint(1, min(2, len(dhcp_entries)))) + for key_tmpl, value_tmpl, details_tmpl in rng.sample(dhcp_entries, count): + reg_ts = time + timedelta(milliseconds=rng.randint(45, 900)) + key, value_name, details = materialize_edr_template_group( + (key_tmpl, value_tmpl, details_tmpl), + rng, + system.assigned_user or "SYSTEM", + host_ip=system.ip, + host_key=system.hostname, + host_os=system.os, + ) + writer_candidates = _registry_writer_candidates( + key, + sys_pids, + system.assigned_user, + ) + if writer_candidates: + reg_pid, reg_image, reg_user = rng.choice(writer_candidates) + else: + reg_pid = sys_pids.get("svchost_netsvcs", sys_pids.get("services", 4)) + reg_image = r"C:\Windows\System32\svchost.exe" + reg_user = "NETWORK SERVICE" + reg_proc = self.state_manager.get_process(system.hostname, reg_pid) + if reg_proc is not None: + reg_image = reg_proc.image + if reg_proc.start_time and reg_ts <= reg_proc.start_time: + reg_ts = reg_proc.start_time + timedelta(milliseconds=1) + target = f"{key}\\{value_name}" + self.activity_generator.dispatcher.dispatch( + SecurityEvent( + timestamp=reg_ts, + event_type="registry_modify", + src_host=_host_ctx, + auth=AuthContext( + username=reg_user, + user_sid=self.activity_generator._get_sid(reg_user), + logon_id=reg_proc.logon_id if reg_proc is not None else "", + ), + process=ProcessContext( + pid=reg_pid, + parent_pid=reg_proc.parent_pid if reg_proc is not None else 0, + image=reg_image, + command_line=reg_proc.command_line if reg_proc is not None else "", + username=reg_proc.username if reg_proc is not None else reg_user, + logon_id=reg_proc.logon_id if reg_proc is not None else "", + start_time=reg_proc.start_time if reg_proc is not None else None, + ), + registry=RegistryContext( + key=target, + value=_materialize_registry_value_for_time(target, details, reg_ts, rng), + action="modify", + pid=reg_pid, + ), + ) + ) + def _generate_scheduled_tasks( self, current_hour: datetime, @@ -3860,6 +4043,13 @@ def _svc_pid(*keys: str, _pids: dict = sys_pids) -> int: # noqa: B006 uid=generate_zeek_uid("C"), msg_types=["REQUEST", "ACK"], # Renewal, not discovery ) + self._emit_dhcp_registry_side_effect( + system=dhcp_state["system"], + time=renewal_ts, + rng=rng, + sys_pids=sys_pids, + dhcp_state=dhcp_state, + ) dhcp_state["last_renewal"] = next_renewal # SMB browsing: Windows workstations to DCs (SYSVOL/GPO) and file servers @@ -4053,12 +4243,15 @@ def _svc_pid(*keys: str, _pids: dict = sys_pids) -> int: # noqa: B006 get_registry_keys_hklm, materialize_edr_template, ) + from evidenceforge.generation.activity.endpoint_noise import registry_noise_config _REG_KEYS_HKCU = get_registry_keys_hkcu() _REG_KEYS_HKLM = get_registry_keys_hklm() _reg_count = rng.randint(18, 42) _svc_pid = sys_pids.get("svchost_netsvcs", sys_pids.get("services", 4)) _host_ctx = self.activity_generator._build_host_context(system) + _registry_cfg = registry_noise_config() + _dhcp_state = getattr(self, "_dhcp_lease_state", {}).get(system.hostname) # Only emit HKCU on workstations with a logged-in user; # servers and DCs run services, not user desktops. _has_desktop = getattr( @@ -4100,6 +4293,14 @@ def _svc_pid(*keys: str, _pids: dict = sys_pids) -> int: # noqa: B006 else: pool = rare_static_hklm _key, _vname, _details = rng.choice(pool or _REG_KEYS_HKLM) + if not _ambient_registry_entry_allowed( + system, + _key, + _vname, + _dhcp_state, + _registry_cfg, + ): + continue _template_user = system.assigned_user or "SYSTEM" _key = materialize_edr_template( _key, @@ -4187,12 +4388,7 @@ def _svc_pid(*keys: str, _pids: dict = sys_pids) -> int: # noqa: B006 pick_scheduled_task, ) - host_seed = _stable_seed(f"task_phase_{system.hostname}") % 900 - num_tasks = rng.randint(2, 5) - slot_bases = sorted(rng.sample(range(0, 3600, 300), min(num_tasks, 12))) - for slot_base in slot_bases: - offset = slot_base + host_seed + rng.gauss(0, 30) + rng.uniform(0, 10) - offset = max(0, min(3599, offset)) + for offset in _windows_scheduled_task_offsets(current_hour, system, rng): ts = current_hour + timedelta(seconds=offset) self.state_manager.set_current_time(ts) task_image, task_cmd, task_parent_key = pick_scheduled_task(rng) diff --git a/tests/unit/test_baseline_canonical.py b/tests/unit/test_baseline_canonical.py index 72725d69..2cdbe72e 100644 --- a/tests/unit/test_baseline_canonical.py +++ b/tests/unit/test_baseline_canonical.py @@ -35,7 +35,11 @@ from evidenceforge.events.contexts import HttpContext, IdsContext from evidenceforge.generation.activity import ActivityGenerator -from evidenceforge.generation.engine.baseline import _materialize_registry_value_for_time +from evidenceforge.generation.engine.baseline import ( + _ambient_registry_entry_allowed, + _materialize_registry_value_for_time, + _windows_scheduled_task_offsets, +) from evidenceforge.generation.state_manager import StateManager from evidenceforge.models import System, User @@ -877,6 +881,84 @@ def test_registry_noise_prefers_dynamic_pools_and_filters_repeated_tells(self): assert "Services\\\\EventLog\\\\Application" in source assert "driverdesc" in source + def test_ambient_registry_noise_suppresses_dhcp_values_for_static_hosts(self): + """Static infrastructure should not emit DHCP registry churn as ambient noise.""" + dc = System( + hostname="DC-01", + ip="10.10.2.10", + os="Windows Server 2022", + type="domain_controller", + roles=["domain_controller", "dns_server"], + ) + workstation = System( + hostname="WS-01", + ip="10.10.2.55", + os="Windows 11", + type="workstation", + ) + cfg = { + "dhcp_interface_values": { + "value_names": ["DhcpIPAddress"], + "require_dhcp_state": True, + "emit_on_lease_events": False, + "suppress_system_types": ["server", "domain_controller"], + "suppress_roles": ["domain_controller", "dns_server"], + } + } + key = r"HKLM\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces\{GUID}" + + assert not _ambient_registry_entry_allowed(dc, key, "DhcpIPAddress", {}, cfg) + assert not _ambient_registry_entry_allowed(workstation, key, "DhcpIPAddress", None, cfg) + assert _ambient_registry_entry_allowed( + workstation, + key, + "DhcpIPAddress", + {"lease_time": 3600}, + cfg, + ) + + def test_dhcp_registry_values_are_reserved_for_lease_side_effects(self): + """Default DHCP registry policy should keep lease-owned values out of random pools.""" + workstation = System( + hostname="WS-01", + ip="10.10.2.55", + os="Windows 11", + type="workstation", + ) + cfg = { + "dhcp_interface_values": { + "value_names": ["DhcpIPAddress"], + "require_dhcp_state": True, + "emit_on_lease_events": True, + "suppress_system_types": ["server", "domain_controller"], + "suppress_roles": [], + } + } + key = r"HKLM\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces\{GUID}" + + assert not _ambient_registry_entry_allowed( + workstation, + key, + "DhcpIPAddress", + {"lease_time": 3600}, + cfg, + ) + + +class TestWindowsScheduledProcessNoise: + """Regression tests for Windows scheduled/background process timing.""" + + def test_scheduled_task_offsets_avoid_hour_boundaries_and_vary(self): + system = System(hostname="WS-01", ip="10.10.2.55", os="Windows 11", type="workstation") + current_hour = datetime(2024, 3, 18, 12, 0, tzinfo=UTC) + + offsets = _windows_scheduled_task_offsets(current_hour, system, random.Random(3)) + + assert offsets + assert all(90 <= offset <= 3510 for offset in offsets) + assert not any(int(offset) == 3599 for offset in offsets) + assert len({round(offset, 3) for offset in offsets}) == len(offsets) + class TestSensorStartup: """Sensor startup events dispatch through canonical path.""" diff --git a/tests/unit/test_validate_config.py b/tests/unit/test_validate_config.py index 2b7833a6..38ebb755 100644 --- a/tests/unit/test_validate_config.py +++ b/tests/unit/test_validate_config.py @@ -42,6 +42,106 @@ def load_invalid_web_scan_presets(): for issue in result.issues ) + def test_validate_config_rejects_invalid_endpoint_noise_bounds(self, monkeypatch): + from evidenceforge.generation.activity import endpoint_noise + + def load_invalid_endpoint_noise(): + return { + "windows_scheduled_processes": { + "count_min": 5, + "count_max": 2, + "trigger_window_start_seconds": 3510, + "trigger_window_end_seconds": 90, + "slot_spacing_seconds": 300, + "host_phase_window_seconds": 900, + "jitter_seconds_min": 20, + "jitter_seconds_max": -20, + "skip_probability": 0.05, + }, + "registry_noise": { + "dhcp_interface_values": { + "value_names": ["DhcpIPAddress"], + "require_dhcp_state": True, + "emit_on_lease_events": True, + "suppress_system_types": ["server", "domain_controller"], + "suppress_roles": ["domain_controller"], + } + }, + } + + monkeypatch.setattr(endpoint_noise, "load_endpoint_noise", load_invalid_endpoint_noise) + + result = validate_config() + + assert any( + issue.severity == "ERROR" + and issue.file == "endpoint_noise.yaml" + and "count_min must be <= count_max" in issue.message + for issue in result.issues + ) + + def test_validate_config_rejects_third_party_module_with_microsoft_identity(self, monkeypatch): + from evidenceforge.generation.activity import application_catalog + + real_catalog_loader = application_catalog.load_catalog + + def load_invalid_catalog(): + data = real_catalog_loader() + apps = [dict(app) for app in data.get("applications", [])] + windows = dict(apps[0]["platforms"]["windows"]) + windows["loaded_modules"] = [ + { + "path": r"C:\Program Files\Google\Chrome\Application\chrome_elf.dll", + "signature": "Microsoft Windows", + } + ] + apps[0] = { + **apps[0], + "platforms": {**apps[0]["platforms"], "windows": windows}, + } + return {**data, "applications": apps} + + monkeypatch.setattr(application_catalog, "load_catalog", load_invalid_catalog) + + result = validate_config() + + assert any( + issue.severity == "ERROR" + and issue.file == "application_catalog.yaml" + and "must use a native signer" in issue.message + for issue in result.issues + ) + + def test_validate_config_rejects_incompatible_tls_subject_key_profile(self, monkeypatch): + from evidenceforge.generation.activity import tls_realism + + real_tls_loader = tls_realism.load_tls_realism + + def load_invalid_tls_realism(): + data = real_tls_loader() + certificate_chains = dict(data.get("certificate_chains", {})) + certificate_chains["subject_key_profiles"] = [ + { + "subject_patterns": ["CN=Invalid ECDSA CA*"], + "issuer_family": "invalid_ecdsa", + "key_type": "ecdsa", + "key_length": 256, + "child_signature_algorithms": ["sha256WithRSAEncryption"], + } + ] + return {**data, "certificate_chains": certificate_chains} + + monkeypatch.setattr(tls_realism, "load_tls_realism", load_invalid_tls_realism) + + result = validate_config() + + assert any( + issue.severity == "ERROR" + and issue.file == "tls_realism.yaml" + and "ecdsa issuer profiles cannot use RSA child signature algorithms" in issue.message + for issue in result.issues + ) + def test_validate_config_warns_for_unknown_ocsp_responder(self, monkeypatch): from evidenceforge.generation.activity import dns_registry, tls_realism From 0ed18dff7b8093ff085125235c79192e981f2cec Mon Sep 17 00:00:00 2001 From: "David J. Bianco" Date: Thu, 14 May 2026 12:45:28 -0400 Subject: [PATCH 3/3] feat: add observation profiles for source coverage --- TODO.md | 10 +- commands/eforge/config.md | 1 + .../references/config-dependency-graph.md | 7 + .../eforge/references/config-host-activity.md | 39 ++- .../eforge/references/config-validation.md | 3 +- .../eforge/references/scenario-reference.md | 15 + commands/eforge/scenario.md | 5 + docs/reference/CUSTOMIZING_CONFIG.md | 1 + docs/reference/scenario-reference.md | 15 + src/evidenceforge/cli/validate_config.py | 7 + src/evidenceforge/config/activity/README.md | 1 + .../config/activity/observation_profiles.yaml | 140 ++++++++++ .../config/observation_profiles.py | 49 ++++ src/evidenceforge/config/schemas.py | 99 +++++++ src/evidenceforge/events/dispatcher.py | 66 ++++- src/evidenceforge/events/observation.py | 264 ++++++++++++++++++ src/evidenceforge/generation/engine/core.py | 4 + .../generation/engine/storyline.py | 4 +- src/evidenceforge/generation/ground_truth.py | 37 +++ src/evidenceforge/models/scenario.py | 15 + src/evidenceforge/validation/schema.py | 20 ++ tests/unit/test_dispatcher.py | 129 ++++++++- tests/unit/test_ground_truth.py | 25 ++ tests/unit/test_validate_config.py | 33 +++ tests/unit/test_validation.py | 15 + 25 files changed, 988 insertions(+), 16 deletions(-) create mode 100644 src/evidenceforge/config/activity/observation_profiles.yaml create mode 100644 src/evidenceforge/config/observation_profiles.py create mode 100644 src/evidenceforge/events/observation.py diff --git a/TODO.md b/TODO.md index 4b111d9a..663a96e6 100644 --- a/TODO.md +++ b/TODO.md @@ -241,8 +241,8 @@ Replaced manual per-emitter field coordination with SecurityEvent intermediate r - [x] **P1** Web application response/session realism follow-up — Added data-driven inbound `web_server` visitor profiles so human visitors consume `traffic_rates.web` as top-level actions, then fan out into required page assets/API calls through `site_maps.yaml`; crawler, health-check, API-client, and opportunistic-probe traffic now uses source-native configured request/status/User-Agent profiles. Static resource sizes are stable per host/path, human navigation and render fanout timing use `timing_profiles.yaml`, and docs/skill references now explain the budget and config ownership. Verification passed: focused web/timing/baseline tests (`107 passed, 1 skipped`), config-related tests (`64 passed`), `uv run eforge validate-config`, repo-wide Ruff checks/format checks, full normal `uv run pytest -q` (`3012 passed, 15 skipped`), and `git diff --check`. - [x] **P1** Well-synced network sensor timing follow-up — Replaced hardcoded multi-sensor Zeek +/-400ms skew plus broad path delay with a validated `network_sensor_observation` timing profile. The default `well_synced` profile keeps stable per-sensor clock skew within +/-1.5ms and per-flow capture/path delay within 50-2000us while preserving canonical packet/byte truth unless source-native observation variance is explicitly enabled. Verification passed with focused Zeek/timing tests, `uv run eforge validate-config`, repo-wide Ruff checks/format checks, full normal `uv run pytest -q` (`3012 passed, 15 skipped`), and `git diff --check`. - [x] **P1** Source identity and endpoint baseline realism sprint — completed TLS/X.509 issuer-compatible chain signatures, Sysmon Event 7 native third-party module identity, config-driven Windows scheduled-process timing, and DHCP registry emission policy tied to lease activity. Verified with `uv run eforge validate-config`, focused regressions, Ruff, normal pytest, and slow-inclusive pytest. -- [ ] **DEFERRED with observation/source coverage architecture** **P2** Endpoint/eCAR baseline variance follow-up — Loop 96 found workstation eCAR category volumes and Linux process lifecycle evidence too uniform and complete. Defer with the broader observation/profile sprint so host/persona-specific variance, long-lived process state, benign unmatched artifacts, and realistic endpoint observation gaps are modeled coherently rather than as eCAR-only omissions. -- [ ] **Later architectural sprint: imperfect observation and source coverage** — defer the broad "too-complete telemetry" problem until after the sharper defects are gone. Model source-specific drop rates, ingestion delay, audit-policy gaps, endpoint coverage variance, and asymmetric Security/Sysmon/eCAR/Zeek visibility as a coherent observation/profile layer rather than one-off omissions. Bundle the related deferred items into this sprint: endpoint/eCAR baseline variance, source-specific process lifecycle completeness modeling, configurable cross-source evidence disagreement, per-host/source log coverage, and the host/activity profile items for per-entity artifact and volume variance. +- [ ] **P2** Endpoint/eCAR baseline variance follow-up — Loop 96 found workstation eCAR category volumes and Linux process lifecycle evidence too uniform and complete. The realistic endpoint observation-gap portion is now handled by named observation profiles; remaining work should focus on host/persona-specific volume variance, long-lived process state, and benign unmatched endpoint artifacts. +- [x] **Later architectural sprint: imperfect observation and source coverage** — implemented a training-friendly `complete` default plus overlay-compatible named observation profiles that apply deterministic source-level drop/delay/coverage semantics without modeling contradictions. The policy covers endpoint, network, proxy/web, firewall, IDS, Windows, Sysmon, Zeek, syslog, bash history, and eCAR source families, while ground truth preserves canonical truth and records source evidence status. Verification passed: focused observation/config/ground-truth tests, `uv run eforge validate-config`, Ruff checks/format checks, full normal `uv run pytest -v` (`3036 passed, 15 skipped`), and slow-inclusive `uv run pytest -v --include-slow` (`3050 passed, 1 skipped`). - [x] Full slow-suite regression cleanup after loop-65 merge — explicit-proxy storyline beacons now preserve authored hostname+destination IP pairs only when the storyline marks that pair as intentional, normal proxy-origin DNS resolution remains intact, and the parallel-generation LogonID assertion treats Type 7 unlock reuse as valid slice-of-time Windows behavior. Verified with targeted proxy/parallel tests, `uv run ruff check .`, `uv run ruff format --check .`, and `uv run pytest -v --include-slow` (`2875 passed, 23 skipped`). Detection Engineer blind review completed for the regenerated Loop 61 dataset at `scenarios/iteration-test/data`; reviewer verdict: Synthetic, 63/100 confidence. Main findings: one PROXY-01 sshd accepted-login lifecycle gap/self-source artifact and Windows 4648 explicit-credential caller PID/image provenance ambiguity around `WS-MCHEN-01`. @@ -581,7 +581,7 @@ Data works but experienced analysts spot tells. Grouped by format for efficient **Cisco ASA:** - [x] Security: bound threat-detection deny timestamp tracking window to prevent unbounded memory/CPU growth -- [ ] ASA imperfect-observation realism — deferred to a general solution for configurable evidence gaps. Built/Teardown counts are currently perfectly balanced, while real logs can have orphans from rotation boundaries, packet loss, sensor downtime, or collection windows. Keep exact pairing as the training-friendly default unless a realism profile enables dropped/partial firewall evidence. +- [x] ASA imperfect-observation realism — addressed by the general observation profile layer. `complete` preserves paired training-friendly firewall evidence, while non-default profiles can apply deterministic ASA source-family gaps that create realistic missing/partial firewall evidence without rewriting canonical truth. - [ ] ASA message type diversity limited to 106023/302013-16/305011-12 — missing 111008, 113004, 733100, 106001, 725001, 304001 - [ ] ASA deny baseline burstiness/profile variance — defer to a general per-source activity profile rather than a one-off ASA fix. Current deny events are uniformly spaced (3-7s); real scans should have configurable burst/quiet periods, campaign-level cadence, and source-specific variance. - [ ] ASA deny metadata diversity — defer to a general field-distribution realism layer. Current deny events use `[0x0, 0x0]` hash values uniformly; a later profile should model when hashes remain zero vs vary by platform/message/context. @@ -596,12 +596,12 @@ Data works but experienced analysts spot tells. Grouped by format for efficient - [x] Template variable leak — literal `{psql_db}` appearing in eCAR output; stale audit finding: Linux query placeholders are handled by `_parameterize_command()`, with `tests/unit/test_activity_helpers.py` covering `{psql_db}` replacement. **Cross-Source / General:** -- [ ] Configurable cross-source evidence disagreement — deferred by design. Perfect cross-source correlation is useful for training/huntability and should remain the default feature unless a scenario/evaluation profile asks for realism gaps. Later design a deterministic setting for dropped/partial/ambiguous corroborating evidence across Zeek, web, proxy, firewall, IDS, Windows, Sysmon, and eCAR without breaking ground-truth traceability. Include broader sensor-observation timing realism beyond the current per-event jitter: sensor clock skew/drift, NTP corrections, capture-path latency, log buffering, occasional source-specific missing/late records, and policy differences between proxy access and Zeek HTTP. +- [x] Configurable cross-source evidence disagreement — implemented as named observation profiles with `complete` as the default. Non-default profiles can introduce deterministic dropped/delayed/filtered/out-of-window evidence across Zeek, web, proxy, firewall, IDS, Windows, Sysmon, syslog, bash history, and eCAR without contradictions or ambiguous rewrites; ground truth retains source evidence status for traceability. - [x] Cross-sensor timestamp precision identical to 15+ decimal places — microsecond jitter added in snort.py, windows.py, and storyline.py - [ ] **P2** Per-host-type event rate multiplier — Domain controllers generate ~50 events/hr but real DCs running AD/DNS/DFS/GPO produce thousands/hr. `system.type` is used for routing but never for volume scaling. Need `event_rate_multiplier` on System model (or implicit per-type defaults) applied in `_calculate_events_for_hour()` and `_generate_system_traffic()`. DCs should be 3-5x workstation baseline; file servers and web servers similarly elevated. - [ ] Configurable per-entity artifact variation — deferred to the general host/activity profile layer. Encoded PowerShell baseline noise is currently identical across hosts (same Get-Service blob); later profiles should derive stable per-host command variants, encoded payloads, tool versions, and operator habits. - [ ] Configurable per-host volume variance — deferred to the general host/activity profile layer. Workstation connection counts are suspiciously uniform (808-1068 range); later profiles should widen variance by role, persona, weekday, installed apps, and stable host-specific multipliers. -- [ ] Configurable per-host/source log coverage — deferred to the general imperfect-observation/profile layer. Uniform log file sets across all hosts can be useful for training, but a later setting should allow host-specific telemetry coverage differences, disabled sensors, partial deployment, and collection gaps. +- [ ] Configurable per-host/source log deployment coverage — observation profiles now support source-family gaps and host-scoped missingness multipliers, but explicit per-host source enablement/disablement remains future work. A later setting should model named host groups, disabled sensors, partial deployments, and collection windows when users need topology-level telemetry coverage differences rather than event-level missingness. - [x] DNS IP pool reuse causes cross-provider resolution (CloudFront→Microsoft IPs, etc.) — domain-first selection ensures consistent domain→IP mapping via FORWARD_DNS - [x] AWS region mismatch between DNS PTR and SSL SNI for same IP — AWS hostname/PTR generation now derives a stable per-IP region/edge identity and PTR generation respects known forward hostname context. - [x] TLS volume clustering design — added data-driven TLS destination profiles with overlay support and `eforge validate-config` schema/tag checks. Auto-generated external TLS now uses weighted enterprise, certificate-infra, package-update, developer-tool, and long-tail browsing profiles with stable per-host preferences. Smoke output had 28,544 TLS SNI rows, 116 distinct names, top SNI share 5.5%, and top-5 share 18.0%. diff --git a/commands/eforge/config.md b/commands/eforge/config.md index d4eedb37..b2d8b88b 100644 --- a/commands/eforge/config.md +++ b/commands/eforge/config.md @@ -70,6 +70,7 @@ When writing to the overlay, files are partial — they contain ONLY the user's | Modify Windows auth realism | `windows_auth_realism.yaml` | (standalone — Security log auth timing and failed-logon profile knobs) | | Modify baseline auth noise | `auth_noise.yaml` | (standalone — stale scheduled-credential accounts and irregular recurrence timing) | | Modify endpoint background noise | `endpoint_noise.yaml` | (standalone — scheduled-process timing and DHCP registry emission policy) | +| Modify source observation coverage | `observation_profiles.yaml` | Scenario `observation_profile` selects the named profile; keep `complete` as the default training profile | | Modify causal/source timing | `timing_profiles.yaml` | (standalone — causal prerequisite, source latency, teardown, and Windows/Sysmon collision-spacing knobs) | | ~~Format definitions~~ | Not user-customizable | Engine internals — requires code changes | | ~~Evaluation rules~~ | Not user-customizable | Must match format definitions — requires code changes | diff --git a/commands/eforge/references/config-dependency-graph.md b/commands/eforge/references/config-dependency-graph.md index c8840b7a..95a720b2 100644 --- a/commands/eforge/references/config-dependency-graph.md +++ b/commands/eforge/references/config-dependency-graph.md @@ -158,6 +158,13 @@ Each row is a file; columns show what it depends on and what depends on it. | **depended on by** | Engine (runtime) | Drives Windows scheduled-process trigger windows, host drift, skips, and DHCP interface registry write policy | | validated by | `eforge validate-config` | Enforces coherent timing bounds, probability ranges, and non-empty DHCP registry value lists | +### observation_profiles.yaml +| Direction | File | Relationship | +|-----------|------|-------------| +| depends on | scenario `observation_profile` | The scenario selects a named profile; the profile file owns source-level missingness/delay values | +| **depended on by** | Event dispatcher, GROUND_TRUTH.md | Applies deterministic source-observation drops/delays after canonical state updates and reports source evidence status | +| validated by | `eforge validate-config` and `eforge validate` | Config validation checks source-family names/ranges; scenario validation checks that the named profile exists | + ### network_params.yaml | Direction | File | Relationship | |-----------|------|-------------| diff --git a/commands/eforge/references/config-host-activity.md b/commands/eforge/references/config-host-activity.md index 9abefe53..fae076df 100644 --- a/commands/eforge/references/config-host-activity.md +++ b/commands/eforge/references/config-host-activity.md @@ -15,8 +15,9 @@ Schema documentation for host-level activity config files. User customizations g 5. [windows_auth_realism.yaml](#windows_auth_realismyaml) 6. [auth_noise.yaml](#auth-noise-auth_noiseyaml) 7. [endpoint_noise.yaml](#endpoint-noise-endpoint_noiseyaml) -8. [timing_profiles.yaml](#timing_profilesyaml) -9. [Domain Controller Baseline Activity](#domain-controller-baseline-activity) +8. [observation_profiles.yaml](#observation-profiles-observation_profilesyaml) +9. [timing_profiles.yaml](#timing_profilesyaml) +10. [Domain Controller Baseline Activity](#domain-controller-baseline-activity) --- @@ -349,6 +350,40 @@ registry_noise: --- +## Observation Profiles (`observation_profiles.yaml`) + +Defines named source-observation profiles selected by scenario `observation_profile`. Keep `complete` as the default for training-friendly perfect source coverage and correlation. Use non-default profiles only when a scenario intentionally needs realistic source gaps or ingestion delays. + +```yaml +profiles: + complete: + description: Perfect source coverage for training-friendly datasets. + default: + missingness: 0.0 + delay_ms: {min_ms: 0, max_ms: 0} + host_missingness_multiplier: {min: 1.0, max: 1.0} + sources: {} + + enterprise_standard: + default: + missingness: 0.0 + delay_ms: {min_ms: 0, max_ms: 0} + host_missingness_multiplier: {min: 0.85, max: 1.15} + sources: + zeek: + missingness: 0.002 + delay_ms: {min_ms: 0, max_ms: 3} + sysmon: + missingness: 0.005 + delay_ms: {min_ms: 5, max_ms: 250} +``` + +Profiles are intentionally source-level, not event-type matrices. Scenario authors select a named profile; code owns safe source-native application semantics so new event types inherit their source-family default. Non-complete profiles may make evidence `visible`, `delayed`, `dropped`, `filtered`, or `out_of_window`, but must not create contradictory identifiers or field values across sources. + +Valid source families are `windows_security`, `sysmon`, `ecar`, `syslog`, `bash_history`, `zeek`, `proxy`, `web`, `asa`, and `ids`. Run `eforge validate-config` after overlay changes; it rejects unknown source-family names, invalid probabilities, and inverted ranges. Run `eforge validate` on scenarios that use a non-default profile so unknown profile names are caught before generation. + +--- + ## timing_profiles.yaml Data-driven timing windows for causal relationships, source-native latency, teardown margins, and Windows/Sysmon same-timestamp collision spacing. Use this when tuning realism of correlated event gaps without changing scenario YAML. diff --git a/commands/eforge/references/config-validation.md b/commands/eforge/references/config-validation.md index a5895b57..a0aa6ac9 100644 --- a/commands/eforge/references/config-validation.md +++ b/commands/eforge/references/config-validation.md @@ -84,7 +84,8 @@ Run `eforge info ` to get specific values (e.g., `eforge info paths.activ | 37 | web_session_profiles.yaml structure | ERROR | Invalid inbound web visitor class, missing User-Agent pool, malformed configured request, or invalid request-count range | | 38 | auth_noise.yaml structure | ERROR | Invalid stale scheduled-credential account pool, host-count range, recurrence interval range, jitter range, skip probability, or backoff bounds | | 39 | endpoint_noise.yaml structure | ERROR | Invalid Windows scheduled-process timing bounds, skip probability, or DHCP registry emission policy | -| 40 | tls_realism.yaml chain metadata | ERROR | Invalid TLS subject-key profile fields or RSA/ECDSA child signature algorithm mismatch | +| 40 | observation_profiles.yaml structure | ERROR | Invalid source-family name, missing `complete` profile, invalid missingness probability, or inverted delay/host multiplier range | +| 41 | tls_realism.yaml chain metadata | ERROR | Invalid TLS subject-key profile fields or RSA/ECDSA child signature algorithm mismatch | ## Scenario Validation: traffic_rates diff --git a/commands/eforge/references/scenario-reference.md b/commands/eforge/references/scenario-reference.md index 67eae45c..0820e334 100644 --- a/commands/eforge/references/scenario-reference.md +++ b/commands/eforge/references/scenario-reference.md @@ -22,6 +22,7 @@ personas: [...] # Optional time_window: ... baseline_activity: ... logon_grace_period: "30m" # Optional (default: "30m") — suppresses "no prior logon" warnings within this duration of time_window.start +observation_profile: complete # Optional (default: complete) — named source-observation profile storyline: [...] # Optional red_herrings: [...] # Optional: suspicious-but-benign events for analyst training output: ... @@ -392,6 +393,20 @@ baseline_activity: Intensity mapping: low=5, medium=15, high=40 events/user/hour. +## Observation Profile + +```yaml +observation_profile: complete # complete | enterprise_standard | messy_collection +``` + +`observation_profile` selects a named source-observation profile from +`config/activity/observation_profiles.yaml`. The default `complete` profile preserves +training-friendly perfect source coverage and correlation. Non-default profiles may introduce +deterministic source-level missingness and source-native delays while preserving canonical truth: +they can make evidence `visible`, `delayed`, `dropped`, `filtered`, or `out_of_window`, but they +must not create contradictory users, PIDs, ports, hashes, UIDs, or session identifiers across +sources. `GROUND_TRUTH.md` records source evidence status when a non-complete profile is used. + ## Storyline Storyline events define specific actions at specific times. Each entry declares what happened (`activity`, for documentation/GROUND_TRUTH.md) and what events to generate (`events` list with typed, validated fields). diff --git a/commands/eforge/scenario.md b/commands/eforge/scenario.md index 5e1f1044..3e56991e 100644 --- a/commands/eforge/scenario.md +++ b/commands/eforge/scenario.md @@ -66,6 +66,8 @@ Inbound traffic respects network topology: DMZ-placed `web_server` hosts attract **Traffic volume** — For scenarios that output server-side logs (especially `web_access`), the `intensity` setting controls how many top-level visitor actions web servers receive (low: ~20/hr, medium: ~1000/hr, high: ~5000/hr). Human page views automatically fan out into required page assets (JS, CSS, images, fonts, same-origin API calls) without consuming additional `web` budget. If the scenario focuses on server-side analysis (web scanners, access log anomalies), you likely need `intensity: high` or explicit `traffic_rates: {web: [5000, 12000]}` overrides to ensure attackers are buried in realistic background noise. Ask about expected noise-to-signal ratios for server-focused scenarios. +**Observation profile** — Default to `observation_profile: complete`. This preserves training-friendly perfect source coverage and correlation. Only choose another named profile such as `enterprise_standard` or `messy_collection` when the user explicitly wants source-native gaps, ingestion delays, or blind-review realism; do not invent per-source rates in scenario YAML. + **Stale accounts** — Does the organization have any disabled or inactive accounts that haven't been fully cleaned up? Former employees, decommissioned service accounts, or un-revoked contractor access are common in real environments. Add 2-4 stale accounts to `environment.stale_accounts` with `username`, `last_active` (ISO date), and `reason`. The engine automatically generates background noise from these: failed logons, Kerberos pre-auth failures on DCs, scheduled task failures, and service startup failures — creating realistic "why is this disabled account still here?" ambiguity for analysts. **Attacker realism / messiness** — How polished is the attacker? Real attacks are messy — even skilled operators make mistakes, hit dead ends, and waste time on paths that go nowhere. Ask the user how much "fumbling" they want in the storyline. This ranges from a near-perfect surgical strike (rare, but appropriate for APT scenarios) to a sloppy novice who tries multiple approaches before succeeding. See the "Attacker Fumbles and Dead Ends" section below for implementation details. @@ -258,6 +260,9 @@ baseline_activity: logon_grace_period: "30m" # Optional (default "30m") — suppresses "no prior logon" # warnings for events within this duration of time_window.start +observation_profile: complete # Optional (default complete). Use complete unless the user + # explicitly wants realistic source gaps/delays. + storyline: # The attack events to bury in the data - id: evt-recon-whoami # Required: unique event ID. Use descriptive labels # (e.g., "evt-lateral-ssh", "evt-c2-beacon-day2") or diff --git a/docs/reference/CUSTOMIZING_CONFIG.md b/docs/reference/CUSTOMIZING_CONFIG.md index b8fb7906..c2d0a76d 100644 --- a/docs/reference/CUSTOMIZING_CONFIG.md +++ b/docs/reference/CUSTOMIZING_CONFIG.md @@ -163,6 +163,7 @@ Configuration files are interconnected. When you add an entry to one file, other | Windows auth realism | `windows_auth_realism.yaml` (`workstation_lock.min_unlock_gap_seconds`, failed-logon local/network profiles, and optional companion network connection rates) | | Baseline auth noise | `auth_noise.yaml` (stale scheduled-credential account pools, host counts, recurrence intervals, jitter, skips, and backoff) | | Endpoint background noise | `endpoint_noise.yaml` (Windows scheduled-process trigger windows, host drift, skip probability, and DHCP registry emission policy) | +| Observation/source coverage | `observation_profiles.yaml` (named source-level missingness/delay profiles selected by scenario `observation_profile`; default `complete` keeps perfect coverage) | | Causal/source-native timing | `timing_profiles.yaml` (`relationships` for causal prerequisites, source latency, teardown margins, Zeek analyzer offsets and TLS duration floors, plus Windows/Sysmon collision spacing) | | Public NTP fallback servers and DNS tunnel timing | `network_params.yaml` (`public_ntp_servers`, `dns_tunnel_rtt`; scenario-defined internal/domain NTP servers still take precedence) | | A new application | `spawn_rules.yaml` (process tree), `process_network_map.yaml` (if it generates traffic) | diff --git a/docs/reference/scenario-reference.md b/docs/reference/scenario-reference.md index f74e98f6..118fa2bd 100644 --- a/docs/reference/scenario-reference.md +++ b/docs/reference/scenario-reference.md @@ -22,6 +22,7 @@ personas: [...] # Optional time_window: ... baseline_activity: ... logon_grace_period: "30m" # Optional (default: "30m") — suppresses "no prior logon" warnings within this duration of time_window.start +observation_profile: complete # Optional (default: complete) — named source-observation profile storyline: [...] # Optional red_herrings: [...] # Optional: suspicious-but-benign events for analyst training output: ... @@ -392,6 +393,20 @@ baseline_activity: Intensity mapping: low=5, medium=15, high=40 events/user/hour. +## Observation Profile + +```yaml +observation_profile: complete # complete | enterprise_standard | messy_collection +``` + +`observation_profile` selects a named source-observation profile from +`config/activity/observation_profiles.yaml`. The default `complete` profile preserves +training-friendly perfect source coverage and correlation. Non-default profiles may introduce +deterministic source-level missingness and source-native delays while preserving canonical truth: +they can make evidence `visible`, `delayed`, `dropped`, `filtered`, or `out_of_window`, but they +must not create contradictory users, PIDs, ports, hashes, UIDs, or session identifiers across +sources. `GROUND_TRUTH.md` records source evidence status when a non-complete profile is used. + ## Storyline Storyline events define specific actions at specific times. Each entry declares what happened (`activity`, for documentation/GROUND_TRUTH.md) and what events to generate (`events` list with typed, validated fields). diff --git a/src/evidenceforge/cli/validate_config.py b/src/evidenceforge/cli/validate_config.py index 51e9e488..80ac0aaf 100644 --- a/src/evidenceforge/cli/validate_config.py +++ b/src/evidenceforge/cli/validate_config.py @@ -441,6 +441,7 @@ def validate_config() -> ValidationResult: # Load all data through overlay-aware loaders for consistency. # Every config file should be loaded via its loader (not raw yaml.safe_load) # so that overlay customizations are visible to validation. + from evidenceforge.config.observation_profiles import load_observation_profiles from evidenceforge.generation.activity.application_catalog import load_catalog from evidenceforge.generation.activity.auth_noise import load_auth_noise_config from evidenceforge.generation.activity.create_remote_thread_patterns import ( @@ -480,6 +481,7 @@ def validate_config() -> ValidationResult: site_data = load_site_maps() sys_proc_data = load_system_processes() endpoint_noise_data = load_endpoint_noise() + observation_profiles_data = load_observation_profiles() tls_realism_data = load_tls_realism() windows_auth_data = load_windows_auth_realism() timing_profiles_data = load_timing_profiles() @@ -1696,6 +1698,7 @@ def _record_ids_rule_identity( EdrFileSideEffectProfile, EndpointNoiseConfig, KerberosRealismConfig, + ObservationProfilesConfig, OuiEntry, PersonaEntry, ProcessAccessPatternEntry, @@ -1823,6 +1826,10 @@ def _record_ids_rule_identity( ) if endpoint_noise_data: _SCHEMA_CHECKS.append(([endpoint_noise_data], EndpointNoiseConfig, "endpoint_noise.yaml")) + if observation_profiles_data: + _SCHEMA_CHECKS.append( + ([observation_profiles_data], ObservationProfilesConfig, "observation_profiles.yaml") + ) # traffic_profiles.yaml: connection entries all_traffic_connection_entries = [] diff --git a/src/evidenceforge/config/activity/README.md b/src/evidenceforge/config/activity/README.md index a10543cd..84f8050b 100644 --- a/src/evidenceforge/config/activity/README.md +++ b/src/evidenceforge/config/activity/README.md @@ -23,6 +23,7 @@ caches data after first load. Two files (`network_params.yaml`, | `windows_auth_realism.yaml` | `windows_auth_realism.py` | Windows Security authentication realism knobs such as minimum 4800→4801 lock/unlock gap, failed-logon validation paths, companion network evidence, and 4672 privilege profiles. | | `auth_noise.yaml` | `auth_noise.py` | Baseline authentication-noise profiles such as stale scheduled-credential account pools and irregular recurrence timing. | | `endpoint_noise.yaml` | `endpoint_noise.py` | Endpoint background timing and registry-emission policies for Windows scheduled processes and DHCP interface registry writes. | +| `observation_profiles.yaml` | `config/observation_profiles.py` | Named source-observation profiles for optional source-level missingness and delays. Scenario `observation_profile` defaults to `complete`. | | `proxy_uri_templates.yaml` | `proxy_uri.py` | Per-domain URI path templates for proxy logs (Windows Update, CRL, OCSP, Azure AD, etc.). | | `network_params.yaml` | `network_params.py`, `engine/emitter_setup.py` | MAC address OUI prefixes, public NTP fallback servers, and DNS tunnel RTT bounds. | | `systemd_schedules.yaml` | `engine/baseline.py` | Systemd timer and cron job schedules (logrotate, fstrim, apt-daily, etc.). | diff --git a/src/evidenceforge/config/activity/observation_profiles.yaml b/src/evidenceforge/config/activity/observation_profiles.yaml new file mode 100644 index 00000000..ac65e22e --- /dev/null +++ b/src/evidenceforge/config/activity/observation_profiles.yaml @@ -0,0 +1,140 @@ +# Source-observation profiles control optional realism gaps after canonical +# events are planned. The default complete profile preserves training-friendly +# perfect coverage and correlation. + +profiles: + complete: + description: Perfect source coverage for training-friendly datasets. + default: + missingness: 0.0 + delay_ms: + min_ms: 0 + max_ms: 0 + host_missingness_multiplier: + min: 1.0 + max: 1.0 + sources: {} + + enterprise_standard: + description: Mild source-native gaps for realistic enterprise collection. + default: + missingness: 0.0 + delay_ms: + min_ms: 0 + max_ms: 0 + host_missingness_multiplier: + min: 0.85 + max: 1.15 + sources: + windows_security: + missingness: 0.001 + delay_ms: + min_ms: 0 + max_ms: 100 + sysmon: + missingness: 0.005 + delay_ms: + min_ms: 5 + max_ms: 250 + ecar: + missingness: 0.01 + delay_ms: + min_ms: 10 + max_ms: 500 + syslog: + missingness: 0.003 + delay_ms: + min_ms: 0 + max_ms: 250 + bash_history: + missingness: 0.002 + delay_ms: + min_ms: 0 + max_ms: 0 + zeek: + missingness: 0.002 + delay_ms: + min_ms: 0 + max_ms: 3 + proxy: + missingness: 0.002 + delay_ms: + min_ms: 0 + max_ms: 750 + web: + missingness: 0.003 + delay_ms: + min_ms: 0 + max_ms: 500 + asa: + missingness: 0.002 + delay_ms: + min_ms: 0 + max_ms: 100 + ids: + missingness: 0.005 + delay_ms: + min_ms: 0 + max_ms: 50 + + messy_collection: + description: More visibly incomplete source coverage for blind realism evaluation. + default: + missingness: 0.0 + delay_ms: + min_ms: 0 + max_ms: 0 + host_missingness_multiplier: + min: 0.65 + max: 1.45 + sources: + windows_security: + missingness: 0.003 + delay_ms: + min_ms: 0 + max_ms: 300 + sysmon: + missingness: 0.015 + delay_ms: + min_ms: 10 + max_ms: 1500 + ecar: + missingness: 0.025 + delay_ms: + min_ms: 25 + max_ms: 2500 + syslog: + missingness: 0.01 + delay_ms: + min_ms: 0 + max_ms: 1000 + bash_history: + missingness: 0.006 + delay_ms: + min_ms: 0 + max_ms: 0 + zeek: + missingness: 0.006 + delay_ms: + min_ms: 0 + max_ms: 8 + proxy: + missingness: 0.007 + delay_ms: + min_ms: 0 + max_ms: 3000 + web: + missingness: 0.01 + delay_ms: + min_ms: 0 + max_ms: 1500 + asa: + missingness: 0.006 + delay_ms: + min_ms: 0 + max_ms: 500 + ids: + missingness: 0.02 + delay_ms: + min_ms: 0 + max_ms: 250 diff --git a/src/evidenceforge/config/observation_profiles.py b/src/evidenceforge/config/observation_profiles.py new file mode 100644 index 00000000..29274a02 --- /dev/null +++ b/src/evidenceforge/config/observation_profiles.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: MIT + +"""Observation profile config loader.""" + +from __future__ import annotations + +from typing import Any + +from evidenceforge.config import get_activity_directory +from evidenceforge.config.overlay import deep_merge_dict, load_with_overlay + +_CONFIG_PATH = get_activity_directory() / "observation_profiles.yaml" +_CACHED_DATA: dict[str, Any] | None = None + + +def load_observation_profiles() -> dict[str, Any]: + """Load source-observation profiles, merged with project-local overlay.""" + global _CACHED_DATA + if _CACHED_DATA is None: + _CACHED_DATA = load_with_overlay( + _CONFIG_PATH, + "activity/observation_profiles.yaml", + deep_merge_dict, + ) + return _CACHED_DATA + + +def reset_observation_profiles_cache() -> None: + """Clear cached observation profile config. Intended for tests.""" + global _CACHED_DATA + _CACHED_DATA = None + + +def observation_profile_names() -> set[str]: + """Return configured observation profile names.""" + profiles = load_observation_profiles().get("profiles", {}) + if not isinstance(profiles, dict): + return set() + return set(profiles) + + +def get_observation_profile(name: str) -> dict[str, Any]: + """Return a named observation profile config.""" + profiles = load_observation_profiles().get("profiles", {}) + if not isinstance(profiles, dict): + return {} + profile = profiles.get(name, {}) + return profile if isinstance(profile, dict) else {} diff --git a/src/evidenceforge/config/schemas.py b/src/evidenceforge/config/schemas.py index d6d6ad50..99862ea6 100644 --- a/src/evidenceforge/config/schemas.py +++ b/src/evidenceforge/config/schemas.py @@ -1178,6 +1178,105 @@ class EndpointNoiseConfig(BaseModel, extra="forbid"): registry_noise: RegistryNoiseConfig +# --- Observation Profiles --- + + +class ObservationDelayRange(BaseModel, extra="forbid"): + """Source-observation delay bounds in milliseconds.""" + + min_ms: int = Field(ge=0, le=3_600_000) + max_ms: int = Field(ge=0, le=3_600_000) + + @model_validator(mode="after") + def bounds_are_ordered(self) -> Self: + """Reject inverted delay ranges.""" + if self.min_ms > self.max_ms: + raise ValueError("min_ms must be <= max_ms") + return self + + +class ObservationMultiplierRange(BaseModel, extra="forbid"): + """Deterministic per-host multiplier bounds for source missingness.""" + + min: float = Field(ge=0.0, le=10.0) + max: float = Field(ge=0.0, le=10.0) + + @model_validator(mode="after") + def bounds_are_ordered(self) -> Self: + """Reject inverted multiplier ranges.""" + if self.min > self.max: + raise ValueError("min must be <= max") + return self + + +class ObservationSourceProfile(BaseModel, extra="forbid"): + """Source-level observation behavior for a profile.""" + + missingness: float = Field(default=0.0, ge=0.0, le=1.0) + delay_ms: ObservationDelayRange = Field( + default_factory=lambda: ObservationDelayRange(min_ms=0, max_ms=0) + ) + host_missingness_multiplier: ObservationMultiplierRange = Field( + default_factory=lambda: ObservationMultiplierRange(min=1.0, max=1.0) + ) + + +class ObservationProfileEntry(BaseModel, extra="forbid"): + """A named source-observation profile.""" + + VALID_SOURCE_FAMILIES: ClassVar[set[str]] = { + "windows_security", + "sysmon", + "ecar", + "syslog", + "bash_history", + "zeek", + "proxy", + "web", + "asa", + "ids", + } + + description: str = "" + default: ObservationSourceProfile = Field(default_factory=ObservationSourceProfile) + sources: dict[str, ObservationSourceProfile] = Field(default_factory=dict) + + @model_validator(mode="after") + def source_names_are_known(self) -> Self: + """Reject source-family typos.""" + unknown = sorted(set(self.sources) - self.VALID_SOURCE_FAMILIES) + if unknown: + raise ValueError(f"unknown observation source families: {', '.join(unknown)}") + return self + + +class ObservationProfilesConfig(BaseModel, extra="forbid"): + """Root schema for observation_profiles.yaml.""" + + profiles: dict[str, ObservationProfileEntry] + + @field_validator("profiles") + @classmethod + def profile_names_are_simple( + cls, v: dict[str, ObservationProfileEntry] + ) -> dict[str, ObservationProfileEntry]: + if not v: + raise ValueError("profiles must not be empty") + invalid = sorted( + name for name in v if not name or not name.replace("_", "").replace("-", "").isalnum() + ) + if invalid: + raise ValueError(f"invalid observation profile names: {', '.join(invalid)}") + return v + + @model_validator(mode="after") + def complete_profile_exists(self) -> Self: + """The complete profile is the stable training-friendly default.""" + if "complete" not in self.profiles: + raise ValueError('profiles must include "complete"') + return self + + # --- CreateRemoteThread Patterns --- diff --git a/src/evidenceforge/events/dispatcher.py b/src/evidenceforge/events/dispatcher.py index efeed01f..0a73719b 100644 --- a/src/evidenceforge/events/dispatcher.py +++ b/src/evidenceforge/events/dispatcher.py @@ -30,10 +30,17 @@ from __future__ import annotations import logging +from dataclasses import replace from datetime import datetime from typing import TYPE_CHECKING from evidenceforge.events.base import RawLogEntry, SecurityEvent +from evidenceforge.events.observation import ( + ObservationPolicy, + ObservationStatus, + ObservationSummary, + source_family_for_format, +) if TYPE_CHECKING: from evidenceforge.generation.emitters.base import LogEmitter @@ -90,13 +97,28 @@ def __init__( emitters: dict[str, LogEmitter], visibility_engine: NetworkVisibilityEngine | None = None, output_start_time: datetime | None = None, + observation_policy: ObservationPolicy | None = None, ) -> None: self.state_manager = state_manager self.emitters = emitters self.visibility_engine = visibility_engine self.output_start_time = output_start_time + self.observation_policy = observation_policy or ObservationPolicy("complete") + self._source_evidence_status: dict[str, dict[str, ObservationSummary]] = {} self.storyline_cluster_id: str | None = None + @property + def source_evidence_status(self) -> dict[str, dict[str, dict[str, int]]]: + """Return source evidence status summaries for ground truth generation.""" + return { + cluster_id: { + source: summary.as_dict() + for source, summary in sorted(source_summaries.items()) + if summary.as_dict() + } + for cluster_id, source_summaries in sorted(self._source_evidence_status.items()) + } + def _is_suppressed(self, timestamp: datetime) -> bool: """Return True if the event falls before the output window (warm-up period).""" if self.output_start_time is None: @@ -120,12 +142,23 @@ def dispatch(self, event: SecurityEvent) -> None: event.storyline_cluster_id = self.storyline_cluster_id self.state_manager.apply(event) if self._is_suppressed(event.timestamp): + self._record_observation(event, "all", "out_of_window") return - for emitter in self._get_matching_emitters(event): + for format_name, emitter in self._get_matching_emitters(event): + decision = self.observation_policy.decide(format_name, event) + if decision.status == "dropped": + self._record_observation(event, format_name, "dropped") + continue + event_to_emit = event + status: ObservationStatus = "visible" + if decision.delay.total_seconds() > 0: + event_to_emit = replace(event, timestamp=event.timestamp + decision.delay) + status = "delayed" + self._record_observation(event, format_name, status) if event.raw is not None: - emitter.emit_raw(event.raw.fields) + emitter.emit_raw(event_to_emit.raw.fields) else: - emitter.emit(event) + emitter.emit(event_to_emit) def dispatch_raw(self, entry: RawLogEntry) -> None: """Route a raw log entry directly to a specific emitter (escape hatch). @@ -137,9 +170,12 @@ def dispatch_raw(self, entry: RawLogEntry) -> None: emitter = self.emitters.get(entry.target_emitter) if emitter is None: raise KeyError(f"Unknown emitter: {entry.target_emitter!r}") + decision = self.observation_policy.decide_raw(entry) + if decision.status == "dropped": + return emitter.emit_raw(entry.data) - def _get_matching_emitters(self, event: SecurityEvent) -> list[LogEmitter]: + def _get_matching_emitters(self, event: SecurityEvent) -> list[tuple[str, LogEmitter]]: """Two-layer filtering: format eligibility + network visibility.""" # Raw event routing: target a single specific emitter if event.raw is not None: @@ -148,8 +184,9 @@ def _get_matching_emitters(self, event: SecurityEvent) -> list[LogEmitter]: logger.warning(f"Raw event targets unknown emitter: {event.raw.target_format!r}") return [] if event.local_only and event.raw.target_format in _NETWORK_FORMATS: + self._record_observation(event, event.raw.target_format, "filtered") return [] - return [emitter] + return [(event.raw.target_format, emitter)] # For network events, determine which formats can see this traffic # and annotate the event with observing sensor hostnames @@ -246,10 +283,27 @@ def _get_matching_emitters(self, event: SecurityEvent) -> list[LogEmitter]: continue # Host-local events (same src/dst IP) are invisible to network sensors if event.local_only and format_name in _NETWORK_FORMATS: + self._record_observation(event, format_name, "filtered") continue # Network visibility filter: only applies to network-format emitters if visible_formats is not None and format_name in _NETWORK_FORMATS: if format_name not in visible_formats: + self._record_observation(event, format_name, "filtered") continue - matched.append(emitter) + matched.append((format_name, emitter)) return matched + + def _record_observation( + self, + event: SecurityEvent, + format_name: str, + status: ObservationStatus, + ) -> None: + """Record source evidence status for storyline/red-herring ground truth.""" + cluster_id = event.storyline_cluster_id + if not cluster_id: + return + source = source_family_for_format(format_name) + cluster = self._source_evidence_status.setdefault(cluster_id, {}) + source_counts = cluster.setdefault(source, ObservationSummary()) + source_counts.record(status) diff --git a/src/evidenceforge/events/observation.py b/src/evidenceforge/events/observation.py new file mode 100644 index 00000000..ff03ee07 --- /dev/null +++ b/src/evidenceforge/events/observation.py @@ -0,0 +1,264 @@ +# Copyright (c) 2026 Cisco Systems, Inc. and its affiliates +# SPDX-License-Identifier: MIT + +"""Source-observation policy for optional collection gaps and delays.""" + +from __future__ import annotations + +import random +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, Literal + +from evidenceforge.config.observation_profiles import get_observation_profile +from evidenceforge.events.base import RawLogEntry, SecurityEvent +from evidenceforge.utils.rng import _stable_seed + +ObservationStatus = Literal["visible", "delayed", "dropped", "filtered", "out_of_window"] + +SOURCE_FAMILIES: frozenset[str] = frozenset( + { + "windows_security", + "sysmon", + "ecar", + "syslog", + "bash_history", + "zeek", + "proxy", + "web", + "asa", + "ids", + } +) + +_FORMAT_TO_SOURCE: dict[str, str] = { + "windows_event_security": "windows_security", + "windows_event_sysmon": "sysmon", + "ecar": "ecar", + "syslog": "syslog", + "bash_history": "bash_history", + "proxy_access": "proxy", + "web_access": "web", + "cisco_asa": "asa", + "snort_alert": "ids", +} + + +@dataclass(frozen=True, slots=True) +class ObservationDecision: + """Decision for one source rendering attempt.""" + + status: ObservationStatus + delay: timedelta = timedelta(0) + + +@dataclass(slots=True) +class ObservationSummary: + """Aggregated source evidence status for a storyline/red-herring cluster.""" + + visible: int = 0 + delayed: int = 0 + dropped: int = 0 + filtered: int = 0 + out_of_window: int = 0 + + def record(self, status: ObservationStatus) -> None: + """Increment the counter for an observation status.""" + setattr(self, status, getattr(self, status) + 1) + + def as_dict(self) -> dict[str, int]: + """Return non-zero status counts.""" + return { + status: count + for status, count in { + "visible": self.visible, + "delayed": self.delayed, + "dropped": self.dropped, + "filtered": self.filtered, + "out_of_window": self.out_of_window, + }.items() + if count + } + + +def source_family_for_format(format_name: str) -> str: + """Return the observation source family for an emitter format name.""" + if format_name.startswith("zeek_"): + return "zeek" + return _FORMAT_TO_SOURCE.get(format_name, format_name) + + +class ObservationPolicy: + """Applies a named observation profile to rendered source evidence.""" + + def __init__(self, profile_name: str = "complete") -> None: + self.profile_name = profile_name or "complete" + self.profile = get_observation_profile(self.profile_name) + if not self.profile: + raise ValueError(f"Unknown observation_profile: {self.profile_name}") + self.default = self.profile.get("default", {}) + self.sources = self.profile.get("sources", {}) + + @property + def is_complete(self) -> bool: + """Return True when the profile preserves perfect source coverage.""" + return self.profile_name == "complete" + + def decide(self, format_name: str, event: SecurityEvent) -> ObservationDecision: + """Return the source-observation decision for an event/emitter pair.""" + source = source_family_for_format(format_name) + settings = self._settings_for_source(source) + missingness = self._effective_missingness(source, event, settings) + identity = self._event_identity(source, format_name, event) + drop_rng = random.Random(_stable_seed(f"observation.drop|{self.profile_name}|{identity}")) + if missingness > 0 and drop_rng.random() < missingness: + return ObservationDecision(status="dropped") + + delay = self._sample_delay(source, event, settings, identity) + if delay > timedelta(0): + return ObservationDecision(status="delayed", delay=delay) + return ObservationDecision(status="visible") + + def decide_raw(self, entry: RawLogEntry) -> ObservationDecision: + """Return the source-observation decision for a direct raw entry.""" + source = source_family_for_format(entry.target_emitter) + settings = self._settings_for_source(source) + missingness = self._effective_missingness_for_host(source, "", settings) + identity = self._raw_identity(source, entry) + drop_rng = random.Random(_stable_seed(f"observation.drop|{self.profile_name}|{identity}")) + if missingness > 0 and drop_rng.random() < missingness: + return ObservationDecision(status="dropped") + return ObservationDecision(status="visible") + + def _settings_for_source(self, source: str) -> dict[str, Any]: + settings = self.sources.get(source, {}) + if not isinstance(settings, dict): + settings = {} + if not isinstance(self.default, dict): + return settings + merged = dict(self.default) + merged.update(settings) + return merged + + def _effective_missingness( + self, source: str, event: SecurityEvent, settings: dict[str, Any] + ) -> float: + host = self._host_key_for_event(event) + return self._effective_missingness_for_host(source, host, settings) + + def _effective_missingness_for_host( + self, source: str, host: str, settings: dict[str, Any] + ) -> float: + base = _safe_probability(settings.get("missingness", 0.0)) + multiplier_range = settings.get("host_missingness_multiplier", {}) + if not isinstance(multiplier_range, dict): + multiplier_range = {} + min_mult = _safe_float(multiplier_range.get("min", 1.0), 1.0, minimum=0.0, maximum=10.0) + max_mult = _safe_float(multiplier_range.get("max", 1.0), 1.0, minimum=0.0, maximum=10.0) + if max_mult < min_mult: + min_mult, max_mult = 1.0, 1.0 + if min_mult == max_mult: + multiplier = min_mult + else: + seed = _stable_seed(f"observation.host-mult|{self.profile_name}|{source}|{host}") + multiplier = random.Random(seed).uniform(min_mult, max_mult) + return max(0.0, min(base * multiplier, 1.0)) + + def _sample_delay( + self, + source: str, + event: SecurityEvent, + settings: dict[str, Any], + identity: str, + ) -> timedelta: + if event.raw is not None: + return timedelta(0) + delay = settings.get("delay_ms", {}) + if not isinstance(delay, dict): + return timedelta(0) + min_ms = _safe_int(delay.get("min_ms", 0), 0, minimum=0, maximum=3_600_000) + max_ms = _safe_int(delay.get("max_ms", 0), 0, minimum=0, maximum=3_600_000) + if max_ms <= 0 or max_ms < min_ms: + return timedelta(0) + seed = _stable_seed(f"observation.delay|{self.profile_name}|{source}|{identity}") + delay_ms = random.Random(seed).randint(min_ms, max_ms) + return timedelta(milliseconds=delay_ms) + + def _event_identity(self, source: str, format_name: str, event: SecurityEvent) -> str: + group = self._coherent_group_key(source, event) + host = self._host_key_for_event(event) + timestamp = int(event.timestamp.timestamp() * 1_000_000) + return "|".join( + [ + source, + format_name, + event.event_type, + host, + group, + str(timestamp), + ] + ) + + def _raw_identity(self, source: str, entry: RawLogEntry) -> str: + timestamp = int(entry.timestamp.timestamp() * 1_000_000) + return "|".join( + [ + source, + entry.target_emitter, + str(timestamp), + str(sorted(entry.data.items()))[:500], + ] + ) + + def _coherent_group_key(self, source: str, event: SecurityEvent) -> str: + if event.network: + uid = getattr(event.network, "uid", "") or getattr(event.network, "zeek_uid", "") + if uid: + return f"uid:{uid}" + if source == "zeek" and event.dns: + src_ip = event.network.src_ip if event.network else "" + return f"dns:{event.dns.query}:{event.dns.query_type}:{src_ip}" + if event.process: + pid = event.process.pid if event.process.pid is not None else "" + guid = getattr(event.process, "process_guid", "") or "" + image = event.process.image.rsplit("\\", 1)[-1].rsplit("/", 1)[-1] + return f"process:{event.process.username}:{pid}:{guid}:{image}" + if event.auth and event.auth.logon_id: + return f"session:{event.auth.username}:{event.auth.logon_id}" + if event.registry: + return f"registry:{event.registry.key}:{event.registry.value}" + if event.file: + return f"file:{event.file.path}:{event.file.action}" + if event.ids: + return f"ids:{event.ids.sid}:{event.ids.message}" + return "event" + + def _host_key_for_event(self, event: SecurityEvent) -> str: + host = event.dst_host or event.src_host + if host: + return host.hostname or host.ip + if event.process and event.process.hostname: + return event.process.hostname + if event.network: + return event.network.src_ip + return "" + + +def _safe_probability(value: Any) -> float: + return _safe_float(value, 0.0, minimum=0.0, maximum=1.0) + + +def _safe_float(value: Any, fallback: float, *, minimum: float, maximum: float) -> float: + try: + parsed = float(value) + except (TypeError, ValueError): + parsed = fallback + return max(minimum, min(parsed, maximum)) + + +def _safe_int(value: Any, fallback: int, *, minimum: int, maximum: int) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + parsed = fallback + return max(minimum, min(parsed, maximum)) diff --git a/src/evidenceforge/generation/engine/core.py b/src/evidenceforge/generation/engine/core.py index 078b589d..87a42e2c 100644 --- a/src/evidenceforge/generation/engine/core.py +++ b/src/evidenceforge/generation/engine/core.py @@ -266,11 +266,14 @@ def _initialize(self) -> None: } # Initialize event dispatcher and activity generator + from evidenceforge.events.observation import ObservationPolicy + self.dispatcher = EventDispatcher( state_manager=self.state_manager, emitters=self.emitters, visibility_engine=visibility_engine, output_start_time=self.start_time, + observation_policy=ObservationPolicy(self.scenario.observation_profile), ) self.activity_generator = ActivityGenerator( state_manager=self.state_manager, @@ -469,6 +472,7 @@ def _generate_ground_truth(self) -> None: scenario=self.scenario, malicious_events=self.malicious_events, red_herring_events=self.red_herring_events, + source_evidence_status=self.dispatcher.source_evidence_status, ) generator.generate(output_path) diff --git a/src/evidenceforge/generation/engine/storyline.py b/src/evidenceforge/generation/engine/storyline.py index 72df7c32..0ac7df80 100644 --- a/src/evidenceforge/generation/engine/storyline.py +++ b/src/evidenceforge/generation/engine/storyline.py @@ -1147,18 +1147,20 @@ def _execute_typed_event( Returns a malicious_event dict for GROUND_TRUTH.md. """ rng = _get_rng() + dispatcher = getattr(self, "dispatcher", None) malicious_event = { "time": time, "actor": actor.username, "system": system.hostname, "activity": activity, "type": spec.type, + "storyline_cluster_id": getattr(dispatcher, "storyline_cluster_id", None), } def _ground_truth_uid(uid: str, src_ip: str, dst_ip: str) -> str: if not uid: return "(filtered by sensor placement)" - visibility = getattr(self.dispatcher, "visibility_engine", None) + visibility = getattr(dispatcher, "visibility_engine", None) if visibility is None: return uid from evidenceforge.events.dispatcher import expand_formats diff --git a/src/evidenceforge/generation/ground_truth.py b/src/evidenceforge/generation/ground_truth.py index c60fb074..d7cfb3f7 100644 --- a/src/evidenceforge/generation/ground_truth.py +++ b/src/evidenceforge/generation/ground_truth.py @@ -54,6 +54,7 @@ def __init__( scenario: Scenario, malicious_events: list[dict], red_herring_events: list[dict] | None = None, + source_evidence_status: dict[str, dict[str, dict[str, int]]] | None = None, ): """Initialize ground truth generator. @@ -65,6 +66,7 @@ def __init__( self.scenario = scenario self.malicious_events = malicious_events self.red_herring_events = red_herring_events or [] + self.source_evidence_status = source_evidence_status or {} def generate(self, output_path: Path) -> None: """Generate GROUND_TRUTH.md file. @@ -89,6 +91,11 @@ def generate(self, output_path: Path) -> None: content.append("\n## Timeline\n") content.append(self._create_timeline()) + # 3. Source evidence status for profiles with imperfect observation. + if self._include_source_evidence_status(): + content.append("\n## Source Evidence Status\n") + content.append(self._create_source_evidence_status_section()) + # 3. Indicators of Compromise content.append("\n## Indicators of Compromise (IOCs)\n") iocs = self._extract_iocs() @@ -299,6 +306,36 @@ def _format_event_details(self, event: dict) -> str: else: return event.get("activity", "N/A") + def _include_source_evidence_status(self) -> bool: + """Return True when ground truth should show source observation status.""" + if not self.source_evidence_status: + return False + if self.scenario.observation_profile != "complete": + return True + for source_status in self.source_evidence_status.values(): + for counts in source_status.values(): + if any(status != "visible" and count for status, count in counts.items()): + return True + return False + + def _create_source_evidence_status_section(self) -> str: + """Create a compact per-storyline source evidence status table.""" + lines = [ + "Canonical ground truth remains authoritative. Source rows may be " + "`visible`, `delayed`, `dropped`, `filtered`, or `out_of_window` depending on " + "the selected observation profile and sensor placement.\n", + "| Storyline ID | Source | Status Counts |", + "|--------------|--------|---------------|", + ] + for cluster_id, source_status in sorted(self.source_evidence_status.items()): + for source, counts in sorted(source_status.items()): + rendered_counts = ", ".join( + f"{status}: {count}" for status, count in sorted(counts.items()) if count + ) + if rendered_counts: + lines.append(f"| {cluster_id} | {source} | {rendered_counts} |") + return "\n".join(lines) + "\n" + def _extract_iocs(self) -> dict[str, set]: """Extract indicators of compromise from malicious events. diff --git a/src/evidenceforge/models/scenario.py b/src/evidenceforge/models/scenario.py index f4a69aa5..7912759e 100644 --- a/src/evidenceforge/models/scenario.py +++ b/src/evidenceforge/models/scenario.py @@ -1444,6 +1444,13 @@ class Scenario(BaseModel): personas: list[Persona] | None = Field(default_factory=list) time_window: TimeWindow baseline_activity: BaselineActivity + observation_profile: str = Field( + default="complete", + description=( + "Named source-observation profile. Defaults to complete for " + "training-friendly perfect source coverage." + ), + ) storyline: list[StorylineEvent] | None = Field(default_factory=list) red_herrings: list[RedHerringEvent] = Field( default_factory=list, @@ -1467,3 +1474,11 @@ def validate_logon_grace_period(cls, v: str) -> str: if not re.match(r"^(\d+(ms|[hdms]))+$", v): raise ValueError("logon_grace_period must be a duration like '30m', '1h', '2h30m'") return v + + @field_validator("observation_profile") + @classmethod + def validate_observation_profile_name(cls, v: str) -> str: + """Validate observation profile names are simple config keys.""" + if not re.match(r"^[a-zA-Z0-9_-]+$", v): + raise ValueError("observation_profile must be a simple profile name") + return v diff --git a/src/evidenceforge/validation/schema.py b/src/evidenceforge/validation/schema.py index 399cbfd1..2548f29d 100644 --- a/src/evidenceforge/validation/schema.py +++ b/src/evidenceforge/validation/schema.py @@ -207,6 +207,7 @@ def validate(self) -> list[ValidationIssue]: self._validate_expansion_redundancy() self._validate_process_network_pairing() self._validate_firewall_config() + self._validate_observation_profile() self._sort_issues() return self.issues @@ -218,6 +219,25 @@ def has_errors(self) -> bool: """ return any(issue.severity == "error" for issue in self.issues) + def _validate_observation_profile(self) -> None: + """Validate that the scenario references a configured observation profile.""" + from evidenceforge.config.observation_profiles import observation_profile_names + + available = observation_profile_names() + profile = self.scenario.observation_profile + if profile not in available: + self.issues.append( + ValidationIssue( + severity="error", + field_path="observation_profile", + message=f"Unknown observation_profile: {profile}", + suggestion=( + "Use one of the configured observation profiles: " + f"{', '.join(sorted(available))}" + ), + ) + ) + def _validate_user_persona_references(self) -> None: """Check that user persona references exist in personas list.""" for idx, user in enumerate(self.scenario.environment.users): diff --git a/tests/unit/test_dispatcher.py b/tests/unit/test_dispatcher.py index b0bcf4a9..20ebe794 100644 --- a/tests/unit/test_dispatcher.py +++ b/tests/unit/test_dispatcher.py @@ -22,7 +22,7 @@ """Tests for EventDispatcher routing, visibility filtering, and StateManager.apply().""" -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from unittest.mock import MagicMock import pytest @@ -37,6 +37,11 @@ ) from evidenceforge.events.contexts import SyslogContext from evidenceforge.events.dispatcher import FORMAT_GROUPS, EventDispatcher +from evidenceforge.events.observation import ( + SOURCE_FAMILIES, + ObservationPolicy, + source_family_for_format, +) from evidenceforge.generation.state_manager import StateManager @@ -108,6 +113,128 @@ def test_dispatch_applies_storyline_cluster_provenance_only(self): assert event.storyline_origin is False +class TestObservationProfiles: + """Tests for optional source-observation policy in dispatcher.""" + + def test_complete_profile_preserves_visible_emission(self): + """The default complete profile keeps current perfect-coverage behavior.""" + sm = MagicMock(spec=StateManager) + emitter = _make_mock_emitter("sysmon", handles=True) + dispatcher = EventDispatcher( + state_manager=sm, + emitters={"windows_event_sysmon": emitter}, + ) + dispatcher.storyline_cluster_id = "story-001" + + event = SecurityEvent(timestamp=_make_ts(), event_type="process_create") + dispatcher.dispatch(event) + + emitter.emit.assert_called_once_with(event) + assert dispatcher.source_evidence_status["story-001"]["sysmon"] == {"visible": 1} + + def test_source_missingness_drops_rendering_without_skipping_state(self, monkeypatch): + """Non-complete profiles can drop source rows without corrupting canonical state.""" + monkeypatch.setattr( + "evidenceforge.events.observation.get_observation_profile", + lambda _name: { + "default": { + "missingness": 0.0, + "delay_ms": {"min_ms": 0, "max_ms": 0}, + "host_missingness_multiplier": {"min": 1.0, "max": 1.0}, + }, + "sources": { + "sysmon": { + "missingness": 1.0, + "delay_ms": {"min_ms": 0, "max_ms": 0}, + } + }, + }, + ) + sm = MagicMock(spec=StateManager) + emitter = _make_mock_emitter("sysmon", handles=True) + dispatcher = EventDispatcher( + state_manager=sm, + emitters={"windows_event_sysmon": emitter}, + observation_policy=ObservationPolicy("messy_test"), + ) + dispatcher.storyline_cluster_id = "story-001" + + event = SecurityEvent(timestamp=_make_ts(), event_type="process_create") + dispatcher.dispatch(event) + + sm.apply.assert_called_once_with(event) + emitter.emit.assert_not_called() + assert dispatcher.source_evidence_status["story-001"]["sysmon"] == {"dropped": 1} + + def test_source_delay_uses_copy_and_preserves_canonical_state(self, monkeypatch): + """Source delays render a timestamp-adjusted copy while state sees canonical time.""" + monkeypatch.setattr( + "evidenceforge.events.observation.get_observation_profile", + lambda _name: { + "default": { + "missingness": 0.0, + "delay_ms": {"min_ms": 0, "max_ms": 0}, + "host_missingness_multiplier": {"min": 1.0, "max": 1.0}, + }, + "sources": { + "sysmon": { + "missingness": 0.0, + "delay_ms": {"min_ms": 17, "max_ms": 17}, + } + }, + }, + ) + sm = MagicMock(spec=StateManager) + emitter = _make_mock_emitter("sysmon", handles=True) + dispatcher = EventDispatcher( + state_manager=sm, + emitters={"windows_event_sysmon": emitter}, + observation_policy=ObservationPolicy("delay_test"), + ) + dispatcher.storyline_cluster_id = "story-001" + + event = SecurityEvent(timestamp=_make_ts(), event_type="process_create") + dispatcher.dispatch(event) + + sm.apply.assert_called_once_with(event) + emitted_event = emitter.emit.call_args.args[0] + assert emitted_event is not event + assert emitted_event.timestamp == event.timestamp + timedelta(milliseconds=17) + assert event.timestamp == _make_ts() + assert dispatcher.source_evidence_status["story-001"]["sysmon"] == {"delayed": 1} + + def test_network_visibility_records_filtered_source_status(self): + """Network visibility filtering is reflected in source evidence status.""" + sm = MagicMock(spec=StateManager) + zeek = _make_mock_emitter("zeek_conn", handles=True) + dispatcher = EventDispatcher(state_manager=sm, emitters={"zeek_conn": zeek}) + dispatcher.storyline_cluster_id = "story-001" + + event = SecurityEvent( + timestamp=_make_ts(), + event_type="connection", + network=NetworkContext( + src_ip="10.0.1.50", + src_port=54321, + dst_ip="10.0.1.50", + dst_port=443, + protocol="tcp", + ), + local_only=True, + ) + dispatcher.dispatch(event) + + zeek.emit.assert_not_called() + assert dispatcher.source_evidence_status["story-001"]["zeek"] == {"filtered": 1} + + def test_all_emitter_formats_map_to_source_families(self): + """Every current emitter belongs to a source-observation family.""" + from evidenceforge.generation.engine.emitter_setup import _build_emitter_classes + + for format_name in _build_emitter_classes(): + assert source_family_for_format(format_name) in SOURCE_FAMILIES + + class TestNetworkVisibilityFiltering: """Tests for network visibility integration in dispatcher.""" diff --git a/tests/unit/test_ground_truth.py b/tests/unit/test_ground_truth.py index f2bb261f..8c9b704e 100644 --- a/tests/unit/test_ground_truth.py +++ b/tests/unit/test_ground_truth.py @@ -177,6 +177,31 @@ def test_create_timeline_with_events(self, minimal_scenario, malicious_events): assert "| 2024-01-15 10:30:00 UTC | attacker | TEST-01 | Process |" in timeline assert "| 2024-01-15 10:35:00 UTC | attacker | TEST-01 | Connection |" in timeline + def test_source_evidence_status_section_for_non_complete_profile( + self, minimal_scenario, malicious_events, tmp_path + ): + """Ground truth documents source evidence status when observation is imperfect.""" + minimal_scenario.observation_profile = "enterprise_standard" + malicious_events[0]["storyline_cluster_id"] = "evt-test-1" + output_path = tmp_path / "GROUND_TRUTH.md" + generator = GroundTruthGenerator( + minimal_scenario, + malicious_events, + source_evidence_status={ + "evt-test-1": { + "sysmon": {"visible": 2, "delayed": 1}, + "ecar": {"dropped": 1}, + } + }, + ) + + generator.generate(output_path) + content = output_path.read_text() + + assert "## Source Evidence Status" in content + assert "| evt-test-1 | ecar | dropped: 1 |" in content + assert "| evt-test-1 | sysmon | delayed: 1, visible: 2 |" in content + def test_create_timeline_sorted_by_time(self, minimal_scenario): """_create_timeline() should sort events chronologically.""" # Create events out of order diff --git a/tests/unit/test_validate_config.py b/tests/unit/test_validate_config.py index 38ebb755..6728f400 100644 --- a/tests/unit/test_validate_config.py +++ b/tests/unit/test_validate_config.py @@ -80,6 +80,39 @@ def load_invalid_endpoint_noise(): for issue in result.issues ) + def test_validate_config_rejects_invalid_observation_profile_source(self, monkeypatch): + from evidenceforge.config import observation_profiles + + def load_invalid_observation_profiles(): + return { + "profiles": { + "complete": { + "description": "bad", + "default": { + "missingness": 0.0, + "delay_ms": {"min_ms": 0, "max_ms": 0}, + "host_missingness_multiplier": {"min": 1.0, "max": 1.0}, + }, + "sources": {"zeek_http": {"missingness": 0.1}}, + } + } + } + + monkeypatch.setattr( + observation_profiles, + "load_observation_profiles", + load_invalid_observation_profiles, + ) + + result = validate_config() + + assert any( + issue.severity == "ERROR" + and issue.file == "observation_profiles.yaml" + and "unknown observation source families" in issue.message + for issue in result.issues + ) + def test_validate_config_rejects_third_party_module_with_microsoft_identity(self, monkeypatch): from evidenceforge.generation.activity import application_catalog diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py index b0814fa3..71b46a91 100644 --- a/tests/unit/test_validation.py +++ b/tests/unit/test_validation.py @@ -62,6 +62,21 @@ def test_valid_scenario_no_issues(self, scenarios_dir): assert len(issues) == 0 assert not validator.has_errors() + def test_unknown_observation_profile_errors(self, scenarios_dir): + """Scenario observation_profile must refer to a configured profile.""" + scenario_data = load_yaml(scenarios_dir / "minimal.yaml") + scenario_data["observation_profile"] = "does_not_exist" + scenario = Scenario(**scenario_data) + + issues = ScenarioValidator(scenario).validate() + + assert any( + issue.severity == "error" + and issue.field_path == "observation_profile" + and "Unknown observation_profile" in issue.message + for issue in issues + ) + def test_invalid_persona_reference(self): """User referencing non-existent persona should error.""" scenario = Scenario(