[codex] improve assessment-loop realism

commands/eforge/references/config-host-activity.md

@@ -112,6 +112,8 @@ schedules:
     jitter_minutes: 60
     distro: all
     role: web_server
+    services_any: [nginx, apache2]
+    slot_skip_probability: 0.08
     cron_user: root
     cron_commands:
       debian: "/usr/bin/certbot renew --quiet --deploy-hook 'systemctl reload nginx'"
@@ -130,6 +132,12 @@ schedules:
 | `jitter_minutes` | int | yes | Max jitter offset (per-host deterministic) |
 | `distro` | string | yes | `all`, `debian`, or `rhel` |
 | `role` | string | no | Host role filter (e.g., `web_server`) |
+| `roles` | list[string] | no | Host role filter where any role may match |
+| `exclude_roles` | list[string] | no | Host roles that suppress this schedule |
+| `services_any` | list[string] | no | Required host service/package signals where any service may match |
+| `host_probability` | float | no | Deterministic per-host enable probability between `0.0` and `1.0` |
+| `slot_skip_probability` | float | no | Deterministic per-slot skip probability for frequent timers |
+| `slot_jitter_seconds` | int | no | Extra runtime jitter for frequent timer slots |
 | `process_path` | string | no | Path to service binary for process create events |
 
 **Systemd timer additional fields:**
@@ -322,7 +330,7 @@ scheduled_stale_credentials:
 
 ## Endpoint Noise (`endpoint_noise.yaml`)
 
-Controls endpoint background timing and registry-emission policies that are too source-specific for scenario YAML. Use it to tune routine Windows scheduled-process spacing and whether DHCP interface registry values appear as ambient Sysmon/EDR noise.
+Controls endpoint background timing, registry-emission, and EDR attribution policies that are too source-specific for scenario YAML. Use it to tune routine Windows scheduled-process spacing, whether DHCP interface registry values appear as ambient Sysmon/EDR noise, and how often eCAR FLOW rows expose process/user principal context.
 
 ```yaml
 windows_scheduled_processes:
@@ -343,11 +351,19 @@ registry_noise:
     emit_on_lease_events: true
     suppress_system_types: [server, domain_controller]
     suppress_roles: [domain_controller, dns_server, file_server, web_server]
+
+ecar_flow_identity:
+  user_process_probability: 0.88
+  service_process_probability: 0.48
+  root_process_probability: 0.42
+  inbound_listener_probability: 0.36
 ```
 
 `windows_scheduled_processes` replaces hour-end clamping with profile-driven trigger windows, per-host phase offsets, jitter, and skips. Keep `trigger_window_end_seconds` comfortably below 3599 to avoid synthetic `xx:59:59` clusters.
 
-`registry_noise.dhcp_interface_values` reserves DHCP interface registry writes for actual DHCP lease/reconfigure activity. Static infrastructure roles should stay in `suppress_system_types` or `suppress_roles` so they do not repeatedly rewrite DHCP values as ambient registry noise. Run `eforge validate-config` after overlay changes; it rejects inverted ranges, empty value-name lists, and invalid probabilities.
+`registry_noise.dhcp_interface_values` reserves DHCP interface registry writes for actual DHCP lease/reconfigure activity. Static infrastructure roles should stay in `suppress_system_types` or `suppress_roles` so they do not repeatedly rewrite DHCP values as ambient registry noise.
+
+`ecar_flow_identity` controls mixed FLOW principal attribution. User-owned process flows usually carry `principal`, service/root flows carry it less often, inbound listener flows carry it occasionally, and unknown or rejected flows remain unattributed. Run `eforge validate-config` after overlay changes; it rejects inverted ranges, empty value-name lists, and invalid probabilities.
 
 ---
 

src/evidenceforge/cli/validate_config.py

@@ -197,6 +197,9 @@ def validate_config() -> ValidationResult:
         "activity/tls_realism.yaml": {
             "dict_fields": {"san", "serial_numbers", "ocsp", "certificate_chains", "destinations"},
         },
+        "activity/public_dns_profiles.yaml": {
+            "list_fields": {"nameserver_profiles": "name", "mail_profiles": "name"},
+        },
         "activity/smb_file_transfers.yaml": {
             "list_fields": {"mime_types": None, "analyzer_sets": None},
         },
@@ -231,7 +234,11 @@ def validate_config() -> ValidationResult:
             },
         },
         "activity/endpoint_noise.yaml": {
-            "dict_fields": {"windows_scheduled_processes", "registry_noise"},
+            "dict_fields": {
+                "windows_scheduled_processes",
+                "registry_noise",
+                "ecar_flow_identity",
+            },
         },
         "activity/host_activity_profiles.yaml": {
             "dict_fields": {
@@ -470,6 +477,7 @@ def validate_config() -> ValidationResult:
     from evidenceforge.generation.activity.process_network import load_process_network_map
     from evidenceforge.generation.activity.proxy_uri import load_proxy_uri_templates
     from evidenceforge.generation.activity.proxy_user_agents import load_proxy_user_agents
+    from evidenceforge.generation.activity.public_dns_profiles import load_public_dns_profiles
     from evidenceforge.generation.activity.site_maps import load_site_maps
     from evidenceforge.generation.activity.spawn_rules import load_spawn_rules
     from evidenceforge.generation.activity.system_processes import load_system_processes
@@ -480,6 +488,7 @@ def validate_config() -> ValidationResult:
     from evidenceforge.generation.activity.windows_auth_realism import load_windows_auth_realism
 
     dns_data = load_dns_registry()
+    public_dns_profiles_data = load_public_dns_profiles()
     ids_data = load_ids_signatures()
     catalog_data = load_catalog()
     traffic_data = load_traffic_profiles()
@@ -1719,6 +1728,7 @@ def _record_ids_rule_identity(
         ProcessAccessPatternEntry,
         ProcessNetworkEntry,
         ProxyUserAgentOverrideEntry,
+        PublicDnsProfilesConfig,
         PublicNtpServerEntry,
         RemoteThreadStartLocationEntry,
         ScheduledTaskEntry,
@@ -1892,6 +1902,12 @@ def _record_ids_rule_identity(
     if tls_realism_data:
         _SCHEMA_CHECKS.append(([tls_realism_data], TlsRealismConfig, "tls_realism.yaml"))
 
+    # public_dns_profiles.yaml
+    if public_dns_profiles_data:
+        _SCHEMA_CHECKS.append(
+            ([public_dns_profiles_data], PublicDnsProfilesConfig, "public_dns_profiles.yaml")
+        )
+
     # kerberos_realism.yaml
     from evidenceforge.generation.activity.kerberos_realism import load_kerberos_realism
 
@@ -1920,6 +1936,19 @@ def _record_ids_rule_identity(
             if not isinstance(entry, dict):
                 continue
             app = str(entry.get("app") or "<unknown>")
+            _VALID_SYSLOG_SYSTEM_TYPES = {"workstation", "server", "domain_controller"}
+            for system_type in entry.get("system_types", []):
+                if system_type not in _VALID_SYSLOG_SYSTEM_TYPES:
+                    result.issues.append(
+                        Issue(
+                            "ERROR",
+                            "extra_syslog_messages.yaml",
+                            (
+                                f'App "{app}" has invalid system_type "{system_type}" '
+                                f"(valid: {sorted(_VALID_SYSLOG_SYSTEM_TYPES)})"
+                            ),
+                        )
+                    )
             for message in entry.get("messages", []):
                 if not isinstance(message, str):
                     continue

src/evidenceforge/config/activity/README.md

@@ -22,7 +22,7 @@ caches data after first load. Two files (`network_params.yaml`,
 | `kerberos_realism.yaml` | `kerberos_realism.py` | Kerberos 4768 TGT PreAuthType, TicketOptions, encryption, and PKINIT certificate field distributions with overlay support. |
 | `windows_auth_realism.yaml` | `windows_auth_realism.py` | Windows Security authentication realism knobs such as minimum 4800→4801 lock/unlock gap, failed-logon validation paths, companion network evidence, and 4672 privilege profiles. |
 | `auth_noise.yaml` | `auth_noise.py` | Baseline authentication-noise profiles such as stale scheduled-credential account pools and irregular recurrence timing. |
-| `endpoint_noise.yaml` | `endpoint_noise.py` | Endpoint background timing and registry-emission policies for Windows scheduled processes and DHCP interface registry writes. |
+| `endpoint_noise.yaml` | `endpoint_noise.py` | Endpoint background timing, registry-emission, and EDR attribution policies for Windows scheduled processes, DHCP interface registry writes, and eCAR FLOW principal context. |
 | `host_activity_profiles.yaml` | `host_activity_profiles.py` | Coarse host/persona/role rate multipliers for baseline volume, endpoint noise, firewall deny bursts, and data-driven artifact variation. |
 | `observation_profiles.yaml` | `config/observation_profiles.py` | Named source-observation profiles for optional source-level missingness and delays. Scenario `observation_profile` defaults to `complete`; generation records status in `OBSERVATION_MANIFEST.json` for eval. |
 | `proxy_uri_templates.yaml` | `proxy_uri.py` | Per-domain URI path templates for proxy logs (Windows Update, CRL, OCSP, Azure AD, etc.). |

src/evidenceforge/config/activity/application_catalog.yaml

@@ -541,7 +541,7 @@ applications:
       linux:
         image_path: "/usr/bin/curl"
         command_templates:
-          - "curl -s https://api.example.com/status"
+          - "curl -s {external_api_url}"
           - "curl -sS -o /dev/null -w '%{http_code}' {internal_url}"
           - "curl -X GET {internal_url} -H 'Accept: application/json'"
     categories: [user_app]
@@ -802,6 +802,7 @@ applications:
 
   - id: dsquery
     display_name: "Directory Service Query"
+    selection_weight: 2
     platforms:
       windows:
         image_path: "C:\\Windows\\System32\\dsquery.exe"
@@ -820,6 +821,7 @@ applications:
           - 'dsquery.exe group -samid "*admin*" -limit {ad_limit}'
     categories: [query]
     personas: [sysadmin, help_desk]
+    system_types: [domain_controller]
 
   - id: ldapsearch
     display_name: "LDAP Search"
@@ -868,6 +870,7 @@ applications:
 
   - id: ntdsutil
     display_name: "NTDS Utility"
+    selection_weight: 1
     platforms:
       windows:
         image_path: 'C:\Windows\System32\ntdsutil.exe'

src/evidenceforge/config/activity/bash_commands.yaml

@@ -27,29 +27,76 @@ common:
   - "w"
   - "whoami"
   - "uname -a"
+  - "uname -sr"
+  - "uname -mrs"
+  - "cat /proc/version | cut -d' ' -f1-3"
   - "uptime"
   - "df -h"
+  - "df -h /"
+  - "df -h /var"
+  - "df -h /tmp"
   - "free -m"
+  - "free -h"
   - "ps aux"
+  - "ps -ef"
+  - "ps -ef | head"
+  - "ps aux --sort=-%mem | head"
   - "ps aux | grep {service}"
   - "clear"
   - "hostname -f"
   - "hostname"
   - "date"
+  - "date -u"
   - "history"
   - "cat /etc/hostname"
   - "cat /etc/os-release"
+  - "cat /etc/issue"
+  - "cat /etc/passwd | head"
   - "ll"
   - "exit"
   - "cat /etc/resolv.conf"
   - "cat /proc/cpuinfo | grep 'model name' | head -1"
+  - "grep -m1 'model name' /proc/cpuinfo"
   - "echo $SHELL"
+  - "locale"
+  - "umask"
+  - "ulimit -n"
   - "env | head -20"
+  - "env | sort | head"
   - "which python3"
+  - "command -v python3"
   - "file /usr/bin/ls"
+  - "stat /etc/passwd"
+  - "getent hosts localhost"
+  - "getent passwd $(whoami)"
+  - "groups"
+  - "users"
+  - "last -5"
+  - "who -a"
+  - "ip -br addr"
+  - "ip route"
+  - "ip route get 8.8.8.8"
+  - "ss -tan | head"
+  - "ss -s"
   - "ls -ltr /var/log/ | tail -10"
+  - "ls -lt /var/log | head"
+  - "ls -ltr /var/log | tail"
+  - "ls -ld /var/log"
+  - "ls -lah /tmp | head"
+  - "find /tmp -maxdepth 1 -type f | head"
+  - "du -sh /var/log"
+  - "du -sh /home/* 2>/dev/null | head"
   - "journalctl --no-pager -n 5"
   - "journalctl -p err --no-pager -n 10"
+  - "journalctl --since '10 min ago' --no-pager -n 20"
+  - "journalctl -xe --no-pager | tail -20"
+  - "systemctl --failed --no-pager"
+  - "tail -20 /var/log/syslog"
+  - "tail -50 /var/log/auth.log"
+  - "grep -i error /var/log/syslog | tail"
+  - "grep -i failed /var/log/auth.log | tail"
+  - "lsmod | head"
+  - "dmesg --ctime | tail -20"
 
 sysadmin:
   - "systemctl status sshd"

src/evidenceforge/config/activity/dns_registry.yaml

@@ -186,7 +186,7 @@ domains:
     tags: [saas, background, outlook, teams, onedrive]
   - domain: packages.microsoft.com
     ips: ["13.107.246.52", "13.107.246.53"]
-    tags: [background, windows, linux]
+    tags: [background, linux]
   - domain: res.cdn.office.net
     ips: ["13.107.6.171", "13.107.9.171"]
     tags: [cdn, outlook, teams, onedrive]

src/evidenceforge/config/activity/endpoint_noise.yaml

@@ -35,3 +35,9 @@ registry_noise:
       - forward_proxy
       - app_server
       - database
+
+ecar_flow_identity:
+  user_process_probability: 0.88
+  service_process_probability: 0.48
+  root_process_probability: 0.42
+  inbound_listener_probability: 0.36