From e9f1f1907fca88eb0482199c7de6e392ba12406d Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Sat, 21 Feb 2026 01:42:40 -0500 Subject: [PATCH 01/24] First draft --- .../alerts-cases/alerts/alerting-setup.md | 4 +- .../alerts-cases/alerts/rule-types.md | 2 +- get-started/_snippets/security-overview.md | 4 +- .../logs-data-stream-configure.md | 2 +- .../logs-data-stream-integrations.md | 2 +- .../data-streams/logs-data-stream.md | 2 +- redirects.yml | 14 + reference/glossary/index.md | 4 +- .../fields-and-object-schemas/alert-schema.md | 4 +- .../anomaly-detection.md | 4 +- .../entity-risk-scoring.md | 2 +- .../machine-learning-job-rule-requirements.md | 2 +- .../cloud/d4c/get-started-with-d4c.md | 2 +- solutions/security/detect-and-alert.md | 111 +-- .../about-building-block-rules.md | 3 +- .../detect-and-alert/about-detection-rules.md | 95 -- .../add-detection-alerts-to-cases.md | 1 + .../detect-and-alert/add-manage-exceptions.md | 11 +- .../advanced-data-source-configuration.md | 22 + ...tection-alerts.md => alert-suppression.md} | 14 +- .../detect-and-alert/before-you-begin.md | 28 + .../choose-the-right-rule-type.md | 123 +++ .../create-a-detection-rule.md | 35 + .../detect-and-alert/create-detection-rule.md | 880 ------------------ .../create-manage-shared-exception-lists.md | 1 + .../create-manage-value-lists.md | 4 +- .../cross-cluster-search-detection-rules.md | 7 +- .../security/detect-and-alert/custom-query.md | 107 +++ .../detect-and-alert/detections-privileges.md | 2 +- .../detect-and-alert/detections-reference.md | 20 + solutions/security/detect-and-alert/eql.md | 120 +++ solutions/security/detect-and-alert/esql.md | 118 +++ ...-cold-frozen-data-from-individual-rules.md | 57 -- .../detect-and-alert/indicator-match.md | 110 +++ ...es.md => install-manage-prebuilt-rules.md} | 19 +- ...unch-timeline-from-investigation-guides.md | 140 --- .../detect-and-alert/machine-learning.md | 103 ++ .../manage-detection-alerts.md | 7 +- .../manage-detection-rules.md | 15 +- .../detect-and-alert/mitre-attack-coverage.md | 84 ++ .../mitre-attandckr-coverage.md | 76 -- .../monitor-rule-executions.md | 1 + .../security/detect-and-alert/new-terms.md | 100 ++ .../detect-and-alert/prebuilt-rules.md | 23 + .../detect-and-alert/query-alert-indices.md | 1 + .../reduce-noise-and-false-positives.md | 103 ++ ...irements.md => requirements-privileges.md} | 10 +- .../detect-and-alert/rule-exceptions.md | 1 + .../rule-settings-reference.md | 303 ++++++ .../security/detect-and-alert/rule-types.md | 26 + .../detect-and-alert/set-rule-data-sources.md | 91 ++ ...tions-alerts.md => snooze-rule-actions.md} | 7 +- .../security/detect-and-alert/threshold.md | 103 ++ .../detect-and-alert/tune-detection-rules.md | 5 +- ...unmodified.md => update-prebuilt-rules.md} | 17 +- ...logsdb-index-mode-with-elastic-security.md | 1 + .../detect-and-alert/using-the-api.md | 45 + .../using-the-rule-builder.md | 61 ++ .../validate-and-test-rules.md | 16 + .../view-detection-alert-details.md | 11 +- .../visualize-detection-alerts.md | 1 + .../write-investigation-guides.md | 242 +++++ .../configure-third-party-response-actions.md | 6 +- .../endpoint-response-actions/isolate-host.md | 2 +- solutions/security/esql-for-security.md | 2 +- .../get-started/automatic-migration.md | 2 +- .../configure-advanced-settings.md | 6 +- .../elastic-security-requirements.md | 2 +- .../get-started/elastic-security-ui.md | 2 +- .../get-started/get-started-cloud-security.md | 2 +- .../get-started-detect-with-siem.md | 4 +- .../get-started-endpoint-security.md | 2 +- .../add-osquery-response-actions.md | 2 +- .../investigate/cases-requirements.md | 2 +- .../investigate/timeline-templates.md | 2 +- .../endpoint-protection-rules.md | 4 +- solutions/toc.yml | 68 +- troubleshoot/security/detection-rules.md | 8 +- 78 files changed, 2180 insertions(+), 1465 deletions(-) delete mode 100644 solutions/security/detect-and-alert/about-detection-rules.md create mode 100644 solutions/security/detect-and-alert/advanced-data-source-configuration.md rename solutions/security/detect-and-alert/{suppress-detection-alerts.md => alert-suppression.md} (90%) create mode 100644 solutions/security/detect-and-alert/before-you-begin.md create mode 100644 solutions/security/detect-and-alert/choose-the-right-rule-type.md create mode 100644 solutions/security/detect-and-alert/create-a-detection-rule.md delete mode 100644 solutions/security/detect-and-alert/create-detection-rule.md create mode 100644 solutions/security/detect-and-alert/custom-query.md create mode 100644 solutions/security/detect-and-alert/detections-reference.md create mode 100644 solutions/security/detect-and-alert/eql.md create mode 100644 solutions/security/detect-and-alert/esql.md delete mode 100644 solutions/security/detect-and-alert/exclude-cold-frozen-data-from-individual-rules.md create mode 100644 solutions/security/detect-and-alert/indicator-match.md rename solutions/security/detect-and-alert/{install-manage-elastic-prebuilt-rules.md => install-manage-prebuilt-rules.md} (90%) delete mode 100644 solutions/security/detect-and-alert/launch-timeline-from-investigation-guides.md create mode 100644 solutions/security/detect-and-alert/machine-learning.md create mode 100644 solutions/security/detect-and-alert/mitre-attack-coverage.md delete mode 100644 solutions/security/detect-and-alert/mitre-attandckr-coverage.md create mode 100644 solutions/security/detect-and-alert/new-terms.md create mode 100644 solutions/security/detect-and-alert/prebuilt-rules.md create mode 100644 solutions/security/detect-and-alert/reduce-noise-and-false-positives.md rename solutions/security/detect-and-alert/{detections-requirements.md => requirements-privileges.md} (85%) create mode 100644 solutions/security/detect-and-alert/rule-settings-reference.md create mode 100644 solutions/security/detect-and-alert/rule-types.md create mode 100644 solutions/security/detect-and-alert/set-rule-data-sources.md rename solutions/security/detect-and-alert/{reduce-notifications-alerts.md => snooze-rule-actions.md} (70%) create mode 100644 solutions/security/detect-and-alert/threshold.md rename solutions/security/detect-and-alert/{prebuilt-rules-update-modified-unmodified.md => update-prebuilt-rules.md} (85%) create mode 100644 solutions/security/detect-and-alert/using-the-api.md create mode 100644 solutions/security/detect-and-alert/using-the-rule-builder.md create mode 100644 solutions/security/detect-and-alert/validate-and-test-rules.md create mode 100644 solutions/security/detect-and-alert/write-investigation-guides.md diff --git a/explore-analyze/alerts-cases/alerts/alerting-setup.md b/explore-analyze/alerts-cases/alerts/alerting-setup.md index a66c7f9e40..2b336ed55a 100644 --- a/explore-analyze/alerts-cases/alerts/alerting-setup.md +++ b/explore-analyze/alerts-cases/alerts/alerting-setup.md @@ -55,7 +55,7 @@ The rule type also affects the privileges that are required to create and edit r * For {{ml}} rules, you must have `all` privileges for the **Analytics > {{ml-app}}** feature. * For {{stack-monitor-app}} rules, you must have the `monitoring_user` role. * For most {{observability}} rules, you must have `all` privileges for the appropriate {{observability}} features. However, for a custom threshold rule, you only need the `stack alerts` privilege. -* For Security rules, refer to [](../../../solutions/security/detect-and-alert/detections-requirements.md). +* For Security rules, refer to [](../../../solutions/security/detect-and-alert/requirements-privileges.md). :::: @@ -68,7 +68,7 @@ The rule type also affects the privileges that are required to create and edit r * `Read` for the **Management > {{connectors-feature}}** feature. ::::{note} -The rule type also affects the privileges that are required. For example, to view {{ml}} rules, you must have `read` privileges for the **Analytics > {{ml-app}}** feature. For {{stack-monitor-app}} rules, you must have the `monitoring_user` role. For {{observability}} rules, you must have `read` privileges for the appropriate {{observability}} features. For Security rules, refer to [](../../../solutions/security/detect-and-alert/detections-requirements.md). +The rule type also affects the privileges that are required. For example, to view {{ml}} rules, you must have `read` privileges for the **Analytics > {{ml-app}}** feature. For {{stack-monitor-app}} rules, you must have the `monitoring_user` role. For {{observability}} rules, you must have `read` privileges for the appropriate {{observability}} features. For Security rules, refer to [](../../../solutions/security/detect-and-alert/requirements-privileges.md). :::: diff --git a/explore-analyze/alerts-cases/alerts/rule-types.md b/explore-analyze/alerts-cases/alerts/rule-types.md index 6696da2983..1c1ade8ad1 100644 --- a/explore-analyze/alerts-cases/alerts/rule-types.md +++ b/explore-analyze/alerts-cases/alerts/rule-types.md @@ -10,7 +10,7 @@ products: # Rule types [rule-types] -A rule is a set of [conditions](../alerts.md#rules-conditions), [schedules](../alerts.md#rules-schedule), and [actions](../alerts.md#rules-actions ) that enable notifications. {{kib}} provides rules built into the {{stack}} and rules registered by one of the {{kib}} apps. You can create most rules types in [{{stack-manage-app}} > {{rules-ui}}](create-manage-rules.md). Security rules must be defined in the Security app. For more information, refer to the documentation about [creating a detection rule](../../../solutions/security/detect-and-alert/create-detection-rule.md). +A rule is a set of [conditions](../alerts.md#rules-conditions), [schedules](../alerts.md#rules-schedule), and [actions](../alerts.md#rules-actions ) that enable notifications. {{kib}} provides rules built into the {{stack}} and rules registered by one of the {{kib}} apps. You can create most rules types in [{{stack-manage-app}} > {{rules-ui}}](create-manage-rules.md). Security rules must be defined in the Security app. For more information, refer to the documentation about [creating a detection rule](../../../solutions/security/detect-and-alert/using-the-rule-builder.md). ::::{note} Some rule types are subscription features, while others are free features. For a comparison of the Elastic subscription levels, see [the subscription page](https://www.elastic.co/subscriptions). diff --git a/get-started/_snippets/security-overview.md b/get-started/_snippets/security-overview.md index 8948a44208..28b58d887f 100644 --- a/get-started/_snippets/security-overview.md +++ b/get-started/_snippets/security-overview.md @@ -13,7 +13,7 @@ Use {{elastic-sec}} to protect your systems from security threats. * [**SIEM:**](https://www.elastic.co/security/siem): {{elastic-sec}}'s modern SIEM provides a centralized platform for ingesting, analyzing, and managing security data from various sources. * [**Third-party integration support**](/solutions/security/get-started/ingest-data-to-elastic-security.md): Ingest data from a various tools and data sources so you can centralize your security data. -* [**Threat detection and analytics:**](/solutions/security/detect-and-alert.md): Identify threats by using [prebuilt rules](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md) with the ability to customize or create custom detection rules, automatically detect anomalous activity with built-in machine learning jobs, or proactively search for threats using our powerful [threat hunting and interactive visualization tools](/solutions/security/investigate.md). +* [**Threat detection and analytics:**](/solutions/security/detect-and-alert.md): Identify threats by using [prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md) with the ability to customize or create custom detection rules, automatically detect anomalous activity with built-in machine learning jobs, or proactively search for threats using our powerful [threat hunting and interactive visualization tools](/solutions/security/investigate.md). * [**Automatic migration**](/solutions/security/get-started/automatic-migration.md): Migrate SIEM rules from other platforms to {{elastic-sec}}. * [**Endpoint protection and threat prevention**](/solutions/security/configure-elastic-defend.md): Automatically stop cybersecurity attacks—such as malware and ransomware—before damage and loss can occur. * [**AI-powered features**](/solutions/security/ai.md): Leverage generative AI to help enhance threat detection, assist with incident response, and improve day-to-day security operations. @@ -37,7 +37,7 @@ Before diving into setup and configuration, familiarize yourself with the founda * [**{{elastic-defend}}:**](/solutions/security/configure-elastic-defend/install-elastic-defend.md) {{elastic-sec}}'s Endpoint Detection and Response (EDR) tool that protects endpoints from malicious activity. {{elastic-defend}} uses a combination of techniques like machine learning, behavioral analysis, and prebuilt rules to detect, prevent, and respond to threats in real-time. * [**{{elastic-endpoint}}:**](/solutions/security/manage-elastic-defend/elastic-endpoint-self-protection-features.md) The security component, enabled by {{agent}}, that performs {{elastic-defend}}'s threat monitoring and prevention capabilities. * [**Detection engine:**](/solutions/security/detect-and-alert.md) The framework that detects threats by using rules to search for suspicious events in your data, and generates alerts when events meet a rule's criteria. -* [**Detection rules:**](/solutions/security/detect-and-alert/about-detection-rules.md) Sets of conditions that identify potential threats and malicious activities. Rules analyze various data sources, including logs and network traffic, to detect anomalies, suspicious behaviors, or known attack patterns. {{elastic-sec}} ships out-of-the-box prebuilt rules, and you can create your own custom rules. +* [**Detection rules:**](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) Sets of conditions that identify potential threats and malicious activities. Rules analyze various data sources, including logs and network traffic, to detect anomalies, suspicious behaviors, or known attack patterns. {{elastic-sec}} ships out-of-the-box prebuilt rules, and you can create your own custom rules. * [**Alerts:**](/solutions/security/detect-and-alert/manage-detection-alerts.md) Notifications that are generated when rule conditions are met. Alerts include a wide range of information about potential threats, including host, user, network, and other contextual data to assist your investigation. * [**Machine learning and anomaly detection:**](/solutions/security/advanced-entity-analytics/anomaly-detection.md) Anomaly detection jobs identify anomalous events or patterns in your data. Use these with machine learning detection rules to generate alerts when behavior deviates from normal activity. * [**Entity analytics:**](/solutions/security/advanced-entity-analytics/overview.md) A threat detection feature that combines the power of Elastic’s detection engine and machine learning capabilities to identify unusual behavior for hosts, users, and services. diff --git a/manage-data/data-store/data-streams/logs-data-stream-configure.md b/manage-data/data-store/data-streams/logs-data-stream-configure.md index bfb6675fda..a6abae7030 100644 --- a/manage-data/data-store/data-streams/logs-data-stream-configure.md +++ b/manage-data/data-store/data-streams/logs-data-stream-configure.md @@ -147,4 +147,4 @@ By default, logs data streams use the following settings: - [](logs-data-stream-integrations.md) - [](/manage-data/data-store/templates.md) - [](/solutions/observability/logs/logs-index-template-defaults.md) -- [](/solutions/security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md) \ No newline at end of file +- [](/solutions/security/detect-and-alert/advanced-data-source-configuration.md) \ No newline at end of file diff --git a/manage-data/data-store/data-streams/logs-data-stream-integrations.md b/manage-data/data-store/data-streams/logs-data-stream-integrations.md index dcb58ca003..9a983058a0 100644 --- a/manage-data/data-store/data-streams/logs-data-stream-integrations.md +++ b/manage-data/data-store/data-streams/logs-data-stream-integrations.md @@ -173,4 +173,4 @@ If your cluster is already near capacity, stability issues can occur if you enab - Review the documentation for [](logs-data-stream.md), [](/manage-data/data-store/templates.md), and the [](/solutions/observability/logs/logs-index-template-defaults.md) - [](logs-data-stream-configure.md) -- [](/solutions/security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md) +- [](/solutions/security/detect-and-alert/advanced-data-source-configuration.md) diff --git a/manage-data/data-store/data-streams/logs-data-stream.md b/manage-data/data-store/data-streams/logs-data-stream.md index 2a08d26c12..7148309556 100644 --- a/manage-data/data-store/data-streams/logs-data-stream.md +++ b/manage-data/data-store/data-streams/logs-data-stream.md @@ -100,4 +100,4 @@ To enable logsdb mode for integration data streams, create or update a `@custom` - [Review mappings and sorting](/manage-data/data-store/data-streams/logs-data-stream-configure.md#logsdb-host-name) - [](/manage-data/data-store/data-streams/use-data-stream.md) - [](/manage-data/data-store/data-streams/logs-data-stream-integrations.md) -- [](/solutions/security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md) +- [](/solutions/security/detect-and-alert/advanced-data-source-configuration.md) diff --git a/redirects.yml b/redirects.yml index 5c4add9fc1..7a14dec600 100644 --- a/redirects.yml +++ b/redirects.yml @@ -701,3 +701,17 @@ redirects: # Related to https://github.com/elastic/docs-content/pull/5033 'solutions/observability/observability-ai-assistant.md': 'solutions/observability/ai/observability-ai-assistant.md' 'solutions/observability/llm-performance-matrix.md': 'solutions/observability/ai/llm-performance-matrix.md' + +# Related to Detect & Alert section restructure (issue #1210) + # Renamed files + 'solutions/security/detect-and-alert/detections-requirements.md': 'solutions/security/detect-and-alert/requirements-privileges.md' + 'solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md': 'solutions/security/detect-and-alert/install-manage-prebuilt-rules.md' + 'solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md': 'solutions/security/detect-and-alert/update-prebuilt-rules.md' + 'solutions/security/detect-and-alert/mitre-attandckr-coverage.md': 'solutions/security/detect-and-alert/mitre-attack-coverage.md' + 'solutions/security/detect-and-alert/about-detection-rules.md': 'solutions/security/detect-and-alert/choose-the-right-rule-type.md' + 'solutions/security/detect-and-alert/create-detection-rule.md': 'solutions/security/detect-and-alert/using-the-rule-builder.md' + 'solutions/security/detect-and-alert/launch-timeline-from-investigation-guides.md': 'solutions/security/detect-and-alert/write-investigation-guides.md' + 'solutions/security/detect-and-alert/exclude-cold-frozen-data-from-individual-rules.md': 'solutions/security/detect-and-alert/set-rule-data-sources.md' + 'solutions/security/detect-and-alert/suppress-detection-alerts.md': 'solutions/security/detect-and-alert/alert-suppression.md' + 'solutions/security/detect-and-alert/reduce-notifications-alerts.md': 'solutions/security/detect-and-alert/snooze-rule-actions.md' + 'solutions/security/detect-and-alert/reference.md': 'solutions/security/detect-and-alert/detections-reference.md' diff --git a/reference/glossary/index.md b/reference/glossary/index.md index 438e54ac3e..e9f2270501 100644 --- a/reference/glossary/index.md +++ b/reference/glossary/index.md @@ -259,7 +259,7 @@ $$$glossary-epr$$$ Elastic Package Registry (EPR) : A service hosted by Elastic that stores Elastic package definitions in a central location. See the [EPR GitHub repository](https://github.com/elastic/package-registry). $$$glossary-elastic-security-indices$$$ {{elastic-sec}} indices -: Indices containing host and network source events (such as `packetbeat-*`, `log-*`, and `winlogbeat-*`). When you [create a new rule in {{elastic-sec}}](/solutions/security/detect-and-alert/create-detection-rule.md), the default index pattern corresponds to the values defined in the `securitySolution:defaultIndex` advanced setting. +: Indices containing host and network source events (such as `packetbeat-*`, `log-*`, and `winlogbeat-*`). When you [create a new rule in {{elastic-sec}}](/solutions/security/detect-and-alert/using-the-rule-builder.md), the default index pattern corresponds to the values defined in the `securitySolution:defaultIndex` advanced setting. $$$glossary-elastic-stack$$$ {{stack}} : Also known as the *ELK Stack*, the {{stack}} is the combination of various Elastic products that integrate for a scalable and flexible way to manage your data. @@ -425,7 +425,7 @@ $$$glossary-indexer$$$ indexer : A {{ls}} instance that is tasked with interfacing with an {{es}} cluster in order to index [event](/reference/glossary/index.md#glossary-event) data. $$$glossary-indicator-index$$$ indicator index -: Indices containing suspect field values in {{elastic-sec}}. [Indicator match rules](/solutions/security/detect-and-alert/create-detection-rule.md#create-indicator-rule) use these indices to compare their field values with source event values contained in [{{elastic-sec}} indices](/reference/glossary/index.md#glossary-elastic-security-indices). +: Indices containing suspect field values in {{elastic-sec}}. [Indicator match rules](/solutions/security/detect-and-alert/indicator-match.md) use these indices to compare their field values with source event values contained in [{{elastic-sec}} indices](/reference/glossary/index.md#glossary-elastic-security-indices). $$$glossary-inference-aggregation$$$ inference aggregation : A pipeline aggregation that references a [trained model](/reference/glossary/index.md#glossary-trained-model) in an aggregation to infer on the results field of the parent bucket aggregation. It enables you to use supervised {{ml}} at search time. diff --git a/reference/security/fields-and-object-schemas/alert-schema.md b/reference/security/fields-and-object-schemas/alert-schema.md index 1b43ccebaf..dd84811722 100644 --- a/reference/security/fields-and-object-schemas/alert-schema.md +++ b/reference/security/fields-and-object-schemas/alert-schema.md @@ -80,8 +80,8 @@ The non-ECS fields listed below are beta and subject to change. | `kibana.alert.original_event.*` | Event information copied from the original source event.
Type: object | | `kibana.alert.original_time` | The value copied from the source event (`@timestamp`).
Type: date | | `kibana.alert.reason` | Type: keyword | -| `kibana.alert.rule.author` | The value of the `author` who created the rule. Refer to [configure advanced rule settings](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-advanced-params).
Type: keyword | -| `kibana.alert.building_block_type` | The value of `building_block_type` from the rule that generated this alert. Refer to [configure advanced rule settings](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-advanced-params).
Type: keyword | +| `kibana.alert.rule.author` | The value of the `author` who created the rule. Refer to [configure advanced rule settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params).
Type: keyword | +| `kibana.alert.building_block_type` | The value of `building_block_type` from the rule that generated this alert. Refer to [configure advanced rule settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params).
Type: keyword | | `kibana.alert.rule.created_at` | The value of `created.at` from the rule that generated this alert.
Type: date | | `kibana.alert.rule.created_by` | Type: keyword | | `kibana.alert.rule.description` | Type: keyword | diff --git a/solutions/security/advanced-entity-analytics/anomaly-detection.md b/solutions/security/advanced-entity-analytics/anomaly-detection.md index 924bde4806..f51604a40f 100644 --- a/solutions/security/advanced-entity-analytics/anomaly-detection.md +++ b/solutions/security/advanced-entity-analytics/anomaly-detection.md @@ -23,7 +23,7 @@ Anomaly detection jobs identify anomalous events or patterns in your data. In a ::::{tip} -Refer to [{{ml-cap}}: Anomaly detection](/explore-analyze/machine-learning/anomaly-detection.md) and [About detection rules](/solutions/security/detect-and-alert/about-detection-rules.md) for more background. +Refer to [{{ml-cap}}: Anomaly detection](/explore-analyze/machine-learning/anomaly-detection.md) and [About detection rules](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) for more background. :::: @@ -56,7 +56,7 @@ You can also check the status of {{ml}} detection rules, and start or stop their ::: ::::{tip} -* For instructions on creating {{ml}} rules, refer to [Create a machine learning rule](/solutions/security/detect-and-alert/create-detection-rule.md#create-ml-rule). +* For instructions on creating {{ml}} rules, refer to [Create a machine learning rule](/solutions/security/detect-and-alert/machine-learning.md). * Alerts generated by {{ml}} rules are displayed on the **Alerts** page. For more information, refer to [Manage detection alerts](/solutions/security/detect-and-alert/manage-detection-alerts.md). :::: diff --git a/solutions/security/advanced-entity-analytics/entity-risk-scoring.md b/solutions/security/advanced-entity-analytics/entity-risk-scoring.md index 26292ded5b..7e713b1572 100644 --- a/solutions/security/advanced-entity-analytics/entity-risk-scoring.md +++ b/solutions/security/advanced-entity-analytics/entity-risk-scoring.md @@ -41,7 +41,7 @@ Entities without any alerts, or with only `Closed` alerts, are not assigned a ri ## How are risk scores calculated? [how-is-risk-score-calculated] -1. The risk scoring engine runs hourly to aggregate `Open` and `Acknowledged` alerts from the last 30 days, including [building block alerts](/solutions/security/detect-and-alert/about-building-block-rules.md). For each entity, the engine processes up to 10,000 alerts. +1. The risk scoring engine runs hourly to aggregate `Open` and `Acknowledged` alerts from the last 30 days, including [building block alerts](/solutions/security/detect-and-alert/choose-the-right-rule-type.md). For each entity, the engine processes up to 10,000 alerts. ::::{note} When [turning on the risk engine](turn-on-risk-scoring-engine.md), you can choose to also include `Closed` alerts in risk scoring calculations. diff --git a/solutions/security/advanced-entity-analytics/machine-learning-job-rule-requirements.md b/solutions/security/advanced-entity-analytics/machine-learning-job-rule-requirements.md index bddd06f946..cf3821f76b 100644 --- a/solutions/security/advanced-entity-analytics/machine-learning-job-rule-requirements.md +++ b/solutions/security/advanced-entity-analytics/machine-learning-job-rule-requirements.md @@ -19,7 +19,7 @@ To run and create {{ml}} jobs and rules in serverless, you need the appropriate * There must be at least one {{ml}} node in your cluster * The `machine_learning_admin` user role -Additionally, to configure [alert suppression](/solutions/security/detect-and-alert/suppress-detection-alerts.md) for {{ml}} rules, your role needs the following [index privilege](/deploy-manage/users-roles/cluster-or-deployment-auth/kibana-role-management.md#adding_index_privileges): +Additionally, to configure [alert suppression](/solutions/security/detect-and-alert/alert-suppression.md) for {{ml}} rules, your role needs the following [index privilege](/deploy-manage/users-roles/cluster-or-deployment-auth/kibana-role-management.md#adding_index_privileges): * `read` permission for the `.ml-anomalies-*` index diff --git a/solutions/security/cloud/d4c/get-started-with-d4c.md b/solutions/security/cloud/d4c/get-started-with-d4c.md index 4e0920b9c2..e1a5c62854 100644 --- a/solutions/security/cloud/d4c/get-started-with-d4c.md +++ b/solutions/security/cloud/d4c/get-started-with-d4c.md @@ -52,7 +52,7 @@ First, you’ll need to deploy Elastic’s Defend for Containers integration to The [default D4C policy](d4c-policies.md#d4c-default-policies) provides threat detection capabilities. It is designed to send process telemetry events (`fork` and `exec`) to {{es}}. -To detect threats using this data, you’ll need active [detection rules](/solutions/security/detect-and-alert.md). You can use Elastic's prebuilt rules that are designed for this data or create [custom rules](/solutions/security/detect-and-alert/create-detection-rule.md). +To detect threats using this data, you’ll need active [detection rules](/solutions/security/detect-and-alert.md). You can use Elastic's prebuilt rules that are designed for this data or create [custom rules](/solutions/security/detect-and-alert/using-the-rule-builder.md). To set up threat detection, install and enable Elastic's prebuilt rules that use data ingested by D4C: diff --git a/solutions/security/detect-and-alert.md b/solutions/security/detect-and-alert.md index 45c3ef5363..20401e73f6 100644 --- a/solutions/security/detect-and-alert.md +++ b/solutions/security/detect-and-alert.md @@ -13,102 +13,27 @@ products: # Detections and alerts [security-detection-engine-overview] -Use the detection engine to create and manage rules and view the alerts these rules create. Rules periodically search indices (such as `logs-*` and `filebeat-*`) for suspicious source events and create alerts when a rule’s conditions are met. When an alert is created, its status is `Open`. To help track investigations, an alert’s [status](/solutions/security/detect-and-alert/manage-detection-alerts.md#detection-alert-status) can be set as `Open`, `Acknowledged`, or `Closed`. +{{elastic-sec}}'s detection engine evaluates your data against detection rules and generates alerts when rule criteria are met. Rules can correlate events across all connected data sources to surface threats that no single data stream would reveal on its own. {{elastic-sec}} provides several [rule types](/solutions/security/detect-and-alert/choose-the-right-rule-type.md), from field-value matches to event correlation, {{ml}} anomaly detection, and more. -:::{image} /solutions/images/security-alert-page.png -:alt: Alerts page -:screenshot: -::: +## Where to start -In addition to creating [your own rules](/solutions/security/detect-and-alert/create-detection-rule.md), enable [Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#load-prebuilt-rules) to immediately start detecting suspicious activity. For detailed information on all the prebuilt rules, see the [Prebuilt rule reference](detection-rules://index.md) section. Once the prebuilt rules are loaded and running, [Tune detection rules](/solutions/security/detect-and-alert/tune-detection-rules.md) and [Add and manage exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md) explain how to modify the rules to reduce false positives and get a better set of actionable alerts. You can also use exceptions and value lists when creating or modifying your own rules. +| Your goal | Start here | +|---|---| +| Set up detection for the first time | [Requirements and privileges](/solutions/security/detect-and-alert/requirements-privileges.md) → [Install prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md) | +| Take over an existing deployment | [MITRE ATT&CK coverage](/solutions/security/detect-and-alert/mitre-attack-coverage.md) → [Monitor rule executions](/solutions/security/detect-and-alert/monitor-rule-executions.md) | +| Build coverage for a specific threat | [Choose the right rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) → [Rule builder](/solutions/security/detect-and-alert/using-the-rule-builder.md) | +| Reduce noise from existing rules | [Tune detection rules](/solutions/security/detect-and-alert/tune-detection-rules.md) → [Exceptions](/solutions/security/detect-and-alert/rule-exceptions.md), [Suppression](/solutions/security/detect-and-alert/alert-suppression.md), or [Snooze](/solutions/security/detect-and-alert/snooze-rule-actions.md) | -There are several special prebuilt rules you need to know about: +## Detection program lifecycle -* [**Endpoint protection rules**](/solutions/security/manage-elastic-defend/endpoint-protection-rules.md): Automatically create alerts based on {{elastic-defend}}'s threat monitoring and prevention. -* [**External Alerts**](detection-rules://rules/promotions/external_alerts.md): Automatically creates an alert for all incoming third-party system alerts (for example, Suricata alerts). +The following stages represent the suggested path to a functioning detection program. Most deployments move through these stages roughly in order, though the boundaries are not strict: tuning and noise reduction are ongoing rather than a final stage. -If you want to receive notifications via external systems, such as Slack or email, when alerts are created, use the {{kib}} [Alerting and Actions](/explore-analyze/alerts-cases.md) framework. +1. **Confirm requirements.** Verify infrastructure, privileges, and data availability. +2. **Assess coverage gaps.** Use MITRE ATT&CK coverage to identify priority areas. +3. **Enable prebuilt rules.** Activate Elastic's maintained rule library for priority tactics. +4. **Build custom rules.** Fill remaining gaps with rules tailored to your environment. +5. **Validate before enabling.** Test rule logic against historical data before going live. +6. **Monitor rule health.** Confirm rules are executing correctly and generating alerts. +7. **Reduce noise.** Tune, add exceptions, suppress, or snooze as needed. -::::{note} -To use {{kib}} Alerting for detection alert notifications in the {{stack}}, you need the [appropriate license](https://www.elastic.co/subscriptions). -:::: - - -After rules have started running, you can monitor their executions to verify they are functioning correctly, as well as view, manage, and troubleshoot alerts (see [Manage detection alerts](/solutions/security/detect-and-alert/manage-detection-alerts.md) and [Monitor and troubleshoot rule executions](/troubleshoot/security/detection-rules.md)). - -You can create and manage rules and alerts via the UI or the [Detections API](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-security-detections-api). - -::::{important} -To make sure you can access Detections and manage rules, see [](/solutions/security/detect-and-alert/detections-privileges.md). - -:::: - - - -## Manage data in cold and frozen tiers [cold-tier-detections] - -```yaml {applies_to} -stack: -``` - -Cold [data tiers](/manage-data/lifecycle/data-tiers.md) store time series data that's accessed infrequently and rarely updated, while frozen data tiers hold time series data that's accessed even less frequently and never updated. If you're automating searches across different data tiers using rules, consider the following best practices and limitations. - -### Best practices [best-practices-data-tiers] - -* **Retention in hot tier**: We recommend keeping data in the hot tier ({{ilm-cap}} hot phase) for at least 24 hours. {{ilm-cap}} policies that move ingested data from the hot phase to another phase (for example, cold or frozen) in less than 24 hours may cause performance issues and/or rule execution errors. -* **Replicas for mission-critical data**: Your data should have replicas if it must be highly available. Since frozen tiers don't have replicas by default, shard unavailability can cause partial rule run failures. Shard unavailability may be also encountered during or after {{stack}} upgrades. If this happens, you can [manually rerun](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules) rules over the affected time period once the shards are available. - -### Limitations [limitations-data-tiers] - -Data tiers are a powerful and useful tool. When using them, keep the following in mind: - -* To avoid rule failures, do not modify {{ilm}} policies for {{elastic-sec}}-controlled indices, such as alert and list indices. -* Source data must have an {{ilm}} policy that keeps it in the hot or warm tiers for at least 24 hours before moving to cold or frozen tiers. - -## Limited support for indicator match rules [support-indicator-rules] - -Indicator match rules provide a powerful capability to search your security data; however, their queries can consume significant deployment resources. When creating an [indicator match rule](/solutions/security/detect-and-alert/create-detection-rule.md#create-indicator-rule), we recommend limiting the time range of the indicator index query to the minimum period necessary for the desired rule coverage. For example, the default indicator index query `@timestamp > "now-30d/d"` searches specified indicator indices for indicators ingested during the past 30 days and rounds the query start time down to the nearest day (resolves to UTC `00:00:00`). Without this limitation, the rule will include all of the indicators in your indicator indices, which may extend the time it takes for the indicator index query to complete. - -In addition, the following support restrictions are in place: - -* Indicator match rules don’t support cold or frozen data. Cold or frozen data in indices queried by indicator match rules must be older than the time range queried by the rule. If your data’s timestamps are unreliable, you can exclude cold and frozen tier data using a [Query DSL filter](/solutions/security/detect-and-alert/exclude-cold-frozen-data-from-individual-rules.md). -* Indicator match rules with an additional look-back time value greater than 24 hours are not supported. - - -## Detections configuration and index privilege prerequisites [detections-permissions] - -[](/solutions/security/detect-and-alert/detections-requirements.md) provides detailed information on all the permissions required to initiate and use the Detections feature. - -## Resolve UI error messages [_resolve_ui_error_messages] - -Depending on your privileges and whether detection system indices have already been created for the {{kib}} space, you might get one of these error messages when you open the **Alerts** or **Rules** page: - -* **`Let’s set up your detection engine`** - - If you get this message, a user with specific privileges must visit the **Alerts** or **Rules** page before you can view detection alerts and rules. Refer to [Enable and access detections](/solutions/security/detect-and-alert/detections-privileges.md) for a list of all the requirements. - - ::::{note} - For **self-managed** {{stack}} deployments only, this message may be displayed when the [`xpack.encryptedSavedObjects.encryptionKey`](/solutions/security/detect-and-alert.md#detections-permissions) setting has not been added to the [`kibana.yml`](/deploy-manage/stack-settings.md) file. For more information, refer to [](/solutions/security/detect-and-alert/detections-requirements.md) for self-managed {{stack}} deployments. - :::: - -* **`Detection engine permissions required`** - - If you get this message, you do not have the [required privileges](/solutions/security/detect-and-alert.md#detections-permissions) to view the **Detections** feature, and you should contact your {{kib}} administrator. - - ::::{note} - For **self-managed** {{stack}} deployments only, this message may be displayed when the [`xpack.security.enabled`](/solutions/security/detect-and-alert.md#detections-permissions) setting is not enabled in the `elasticsearch.yml` file. For more information, refer to [](/solutions/security/detect-and-alert/detections-requirements.md) for self-managed {{stack}} deployments. - :::: - - - -## Using logsdb index mode [detections-logsdb-index-mode] - -To learn how your rules and alerts are affected by using the [logsdb index mode](/manage-data/data-store/data-streams/logs-data-stream.md), refer to [Using logsdb index mode with {{elastic-sec}}](/solutions/security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md). - -## Manage rules as code [manage-rule-dac] - -Utilize the [Detection-as-Code](https://dac-reference.readthedocs.io/en/latest/dac_concept_and_workflows.html) (DaC) principles to externally manage your detection rules. - -The {{elastic-sec}} Labs team uses the [detection-rules](https://github.com/elastic/detection-rules) repo to develop, test, and release {{elastic-sec}}'s[ prebuilt rules](https://github.com/elastic/detection-rules/tree/main/rules). The repo provides DaC features and allows you to customize settings to simplify the setup for managing user rules with the DaC pipeline. - -To get started, refer to the [DaC documentation](https://github.com/elastic/detection-rules/blob/main/README.md#detections-as-code-dac). +A minimal viable detection program (prebuilt rules enabled for your highest-priority tactics, running correctly, with noise managed to an actionable level) is a meaningful outcome at any stage of this workflow. You do not need to complete every stage before your detection program delivers value. diff --git a/solutions/security/detect-and-alert/about-building-block-rules.md b/solutions/security/detect-and-alert/about-building-block-rules.md index f141d9f2ef..33e6111c58 100644 --- a/solutions/security/detect-and-alert/about-building-block-rules.md +++ b/solutions/security/detect-and-alert/about-building-block-rules.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Create rules that generate hidden alerts used as building blocks for downstream detection rules. --- # About building block rules [security-building-block-rules] @@ -19,7 +20,7 @@ Create building block rules when you do not want to see their generated alerts i * Rules that execute on the alert indices (`.alerts-security.alerts-`). You can then use building block rules to create hidden alerts that act as a basis for an *ordinary* rule to generate visible alerts. ::::{tip} -Add [rule notifications](/solutions/security/detect-and-alert/create-detection-rule.md#rule-notifications) to building block rules to notify you when building block alerts are generated. +Add [rule notifications](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) to building block rules to notify you when building block alerts are generated. :::: diff --git a/solutions/security/detect-and-alert/about-detection-rules.md b/solutions/security/detect-and-alert/about-detection-rules.md deleted file mode 100644 index 0af167345b..0000000000 --- a/solutions/security/detect-and-alert/about-detection-rules.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/security/current/about-rules.html - - https://www.elastic.co/guide/en/serverless/current/security-about-rules.html -applies_to: - stack: all - serverless: - security: all -products: - - id: security - - id: cloud-serverless ---- - -# About detection rules [security-about-rules] - -Rules run periodically and search for source events, matches, sequences, or {{ml}} job anomaly results that meet their criteria. When a rule’s criteria are met, a detection alert is created. - - -## Rule types [rule-types] - -You can create the following types of rules: - -* [**Custom query**](/solutions/security/detect-and-alert/create-detection-rule.md#create-custom-rule): Query-based rule, which searches the defined indices and creates an alert when one or more documents match the rule’s query. -* [**Machine learning**](/solutions/security/detect-and-alert/create-detection-rule.md#create-ml-rule): {{ml-cap}} rule, which creates an alert when a {{ml}} job discovers an anomaly above the defined threshold (see [Anomaly detection](/solutions/security/advanced-entity-analytics/anomaly-detection.md)). - - For {{ml}} rules, the associated {{ml}} job must be running. If the {{ml}} job isn’t running, the rule will: - - * Run and create alerts if existing anomaly results with scores above the defined threshold are discovered. - * Issue an error stating the {{ml}} job was not running when the rule executed. - -* [**Threshold**](/solutions/security/detect-and-alert/create-detection-rule.md#create-threshold-rule): Searches the defined indices and creates a detections alert when the number of times the specified field’s value is present and meets the threshold during a single execution. When multiple values meet the threshold, an alert is generated for each value. - - For example, if the threshold `field` is `source.ip` and its `value` is `10`, an alert is generated for every source IP address that appears in at least 10 of the rule’s search results. - -* [**Event correlation**](/solutions/security/detect-and-alert/create-detection-rule.md#create-eql-rule): Searches the defined indices and creates an alert when results match an [Event Query Language (EQL)](/explore-analyze/query-filter/languages/eql.md) query. -* [**Indicator match**](/solutions/security/detect-and-alert/create-detection-rule.md#create-indicator-rule): Creates an alert when {{elastic-sec}} index field values match field values defined in the specified indicator index patterns. For example, you can create an indicator index for IP addresses and use this index to create an alert whenever an event’s `destination.ip` equals a value in the index. Indicator index field mappings should be [ECS-compliant](ecs://reference/index.md). For information on creating {{es}} indices and field types, see [Index some documents](/manage-data/ingest.md), [Create index API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-create), and [Field data types](elasticsearch://reference/elasticsearch/mapping-reference/field-data-types.md). If you have indicators in a standard file format, such as CSV or JSON, you can also use the Machine Learning Data Visualizer to import your indicators into an indicator index. See [Explore the data in {{kib}}](/explore-analyze/machine-learning/anomaly-detection/ml-getting-started.md#sample-data-visualizer) and use the **Import Data** option to import your indicators. - - ::::{tip} - You can also use value lists as the indicator match index. See [Use value lists with indicator match rules](/solutions/security/detect-and-alert/create-detection-rule.md#indicator-value-lists) at the end of this topic for more information. - :::: - -* [**New terms**](/solutions/security/detect-and-alert/create-detection-rule.md#create-new-terms-rule): Generates an alert for each new term detected in source documents within a specified time range. You can also detect a combination of up to three new terms (for example, a `host.ip` and `host.id` that have never been observed together before). -* [**ES|QL**](/solutions/security/detect-and-alert/create-detection-rule.md#create-esql-rule): Searches the defined indices and creates an alert when results match an [Elasticsearch Query Language {{esql}}](elasticsearch://reference/query-languages/esql.md) query. - - ::::{note} - {{esql}} is enabled by default in {{kib}}. It can be disabled using the `enableESQL` setting from the [Advanced Settings](kibana://reference/advanced-settings.md). This will hide the {{esql}} user interface from various applications. However, users will be able to access existing {{esql}} artifacts like saved searches and visualizations. - :::: - - -:::{image} /solutions/images/security-all-rules.png -:alt: Shows the Rules page -:screenshot: -::: - - -## Data views and index patterns [views-index-patterns] - -When you create a rule, you must either specify the {{es}} index pattens for which you’d like the rule to run, or select a [data view field](/solutions/security/get-started/data-views-elastic-security.md) as the data source. If you select a data view, you can select [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md) associated with that data view to create a query for the rule (with the exception of {{ml}} rules, which do not use queries). - -::::{note} -To access data views in {{stack}}, you must have the [required permissions](/explore-analyze/find-and-organize/data-views.md#data-views-read-only-access). To access them in {{serverless-short}}, you must have the appropriate [predefined Security user role](/deploy-manage/users-roles/cloud-organization/user-roles.md#general-assign-user-roles) or a [custom role](../../../deploy-manage/users-roles/cloud-organization/user-roles.md) with the right privileges. -:::: - -::::{important} - -System indices, such as the alert indices, contain important configuration and internal data; do not change their mappings. Changes can lead to rule execution and alert indexing failures. Use [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md) instead, which allow you to add fields to existing alert and event documents. - -:::: - - -## Notifications [about-notifications] - -For both prebuilt and custom rules, you can send notifications when alerts are created. Notifications can be sent via {{jira}}, Microsoft Teams, PagerDuty, Slack, and others, and can be configured when you create or edit a rule. - - -## Authorization [alerting-authorization-model] - -Rules, including all background detection and the actions they generate, are authorized using an [API key](/deploy-manage/api-keys/elasticsearch-api-keys.md) associated with the last user to edit the rule. Upon creating or modifying a rule, an API key is generated for that user, capturing a snapshot of their privileges. The API key is then used to run all background tasks associated with the rule including detection checks and executing actions. - -::::{important} -If a rule requires certain privileges to run, such as index privileges, keep in mind that if a user without those privileges updates the rule, the rule will no longer function. - -:::: - - - -## Exceptions [about-exceptions] - -When modifying rules or managing detection alerts, you can [add exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md) that prevent a rule from generating alerts even when its criteria are met. This is useful for reducing noise, such as preventing alerts from trusted processes and internal IP addresses. - -::::{note} -You can add exceptions to custom query, machine learning, event correlation, and indicator match rule types. -:::: - - diff --git a/solutions/security/detect-and-alert/add-detection-alerts-to-cases.md b/solutions/security/detect-and-alert/add-detection-alerts-to-cases.md index 548196d18a..ba63edd182 100644 --- a/solutions/security/detect-and-alert/add-detection-alerts-to-cases.md +++ b/solutions/security/detect-and-alert/add-detection-alerts-to-cases.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Attach one or more detection alerts to new or existing cases from the Alerts table. --- # Add detection alerts to cases [security-signals-to-cases] diff --git a/solutions/security/detect-and-alert/add-manage-exceptions.md b/solutions/security/detect-and-alert/add-manage-exceptions.md index d14be5189d..1d4bae4ab4 100644 --- a/solutions/security/detect-and-alert/add-manage-exceptions.md +++ b/solutions/security/detect-and-alert/add-manage-exceptions.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Add and manage rule exceptions to prevent false positives and reduce alert noise. --- # Add and manage exceptions [add-exceptions] @@ -17,7 +18,7 @@ You can add exceptions to a rule from the rule details page, the Alerts table, t ::::{important} * To ensure an exception is successfully applied, ensure that the fields you’ve defined for its query are correctly and consistently mapped in their respective indices. Refer to [ECS](ecs://reference/index.md) to learn more about supported mappings. -* Be careful when adding exceptions to [event correlation](create-detection-rule.md#create-eql-rule) rules. Exceptions are evaluated against every event in the sequence, and if an exception matches any events that are necessary to complete the sequence, alerts are not created. +* Be careful when adding exceptions to [event correlation](eql.md) rules. Exceptions are evaluated against every event in the sequence, and if an exception matches any events that are necessary to complete the sequence, alerts are not created. To exclude values from a specific event in the sequence, update the rule’s EQL statement. For example: @@ -29,13 +30,13 @@ You can add exceptions to a rule from the rule details page, the Alerts table, t and process.name != "process-name.exe"]` ``` -* Be careful when adding exceptions to [indicator match](create-detection-rule.md#create-indicator-rule) rules. Exceptions are evaluated against source and indicator indices, so if the exception matches events in *either* index, alerts are not generated. +* Be careful when adding exceptions to [indicator match](indicator-match.md) rules. Exceptions are evaluated against source and indicator indices, so if the exception matches events in *either* index, alerts are not generated. :::: ## Requirements [exceptions-requirements] -To use exceptions ensure your role has the appropriate access. To learn how to access other detection features, refer to [](/solutions/security/detect-and-alert/detections-requirements.md). +To use exceptions ensure your role has the appropriate access. To learn how to access other detection features, refer to [](/solutions/security/detect-and-alert/requirements-privileges.md). ### Exceptions requirements @@ -190,7 +191,7 @@ For required privileges to view and manage {{elastic-endpoint}} exceptions, refe ## Add {{elastic-endpoint}} exceptions [endpoint-rule-exceptions] -You can add {{elastic-endpoint}} exceptions to [endpoint protection rules](../manage-elastic-defend/endpoint-protection-rules.md) or to rules that are associated with {{elastic-endpoint}} rule exceptions. To associate rules when creating or editing a rule, select the [**{{elastic-endpoint}} exceptions**](create-detection-rule.md#rule-ui-advanced-params) option. +You can add {{elastic-endpoint}} exceptions to [endpoint protection rules](../manage-elastic-defend/endpoint-protection-rules.md) or to rules that are associated with {{elastic-endpoint}} rule exceptions. To associate rules when creating or editing a rule, select the [**{{elastic-endpoint}} exceptions**](rule-settings-reference.md#rule-ui-advanced-params) option. Endpoint exceptions are added to the endpoint protection rules **and** the {{elastic-endpoint}} on your hosts. @@ -227,7 +228,7 @@ Additionally, to add an Endpoint exception to an endpoint protection rule, there 2. Expand the Endpoint Security Exception List or click the list name to open the list’s details page. Next, click **Add endpoint exception**. ::::{note} - The Endpoint Security Exception List is automatically created. By default, it’s associated with endpoint protection rules and any rules with the [**{{elastic-endpoint}} exceptions**](create-detection-rule.md#rule-ui-advanced-params) option selected. + The Endpoint Security Exception List is automatically created. By default, it’s associated with endpoint protection rules and any rules with the [**{{elastic-endpoint}} exceptions**](rule-settings-reference.md#rule-ui-advanced-params) option selected. :::: diff --git a/solutions/security/detect-and-alert/advanced-data-source-configuration.md b/solutions/security/detect-and-alert/advanced-data-source-configuration.md new file mode 100644 index 0000000000..8e046d4937 --- /dev/null +++ b/solutions/security/detect-and-alert/advanced-data-source-configuration.md @@ -0,0 +1,22 @@ +--- +applies_to: + stack: all + serverless: + security: all +products: + - id: security + - id: cloud-serverless +description: Deployment-level data source settings that affect detection rule behavior across your environment. +--- + +# Advanced data source configuration + +These pages cover deployment-level data settings that affect detection rule behavior. Unlike [per-rule data source settings](/solutions/security/detect-and-alert/set-rule-data-sources.md), which apply to individual rules, the configurations below affect how your entire environment interacts with the detection engine. + +Most users don't need these pages during initial setup. Review them if any of the following apply to your environment: + +**[{{ccs-cap}} and detection rules](/solutions/security/detect-and-alert/cross-cluster-search-detection-rules.md)** +: Relevant if your data is spread across multiple {{es}} clusters and you need detection rules on one cluster to query indices on another. Covers trust setup, remote cluster connections, and how to reference remote indices in rule index patterns. {{stack}} only. + +**[Using logsdb index mode with {{elastic-sec}}](/solutions/security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md)** +: Relevant if your indices use logsdb index mode (enabled by default in {{serverless-short}}). Explains how synthetic `_source` reconstruction can affect field formatting in alerts and rule queries, and what to watch for when writing rules against logsdb-backed indices. diff --git a/solutions/security/detect-and-alert/suppress-detection-alerts.md b/solutions/security/detect-and-alert/alert-suppression.md similarity index 90% rename from solutions/security/detect-and-alert/suppress-detection-alerts.md rename to solutions/security/detect-and-alert/alert-suppression.md index 36ce2cd20e..47175df3ac 100644 --- a/solutions/security/detect-and-alert/suppress-detection-alerts.md +++ b/solutions/security/detect-and-alert/alert-suppression.md @@ -14,7 +14,7 @@ description: Use alert suppression to reduce duplicate detection alerts by group # Suppress detection alerts [security-alert-suppression] -Alert suppression allows you to reduce the number of repeated or duplicate detection alerts created by [detection rules](/solutions/security/detect-and-alert/about-detection-rules.md). Normally, when a rule meets its criteria repeatedly, it creates multiple alerts, one for each time the rule’s criteria are met. When alert suppression is configured, alerts for duplicate events are not created. Instead, the qualifying events are grouped, and only one alert is created for each group. +Alert suppression allows you to reduce the number of repeated or duplicate detection alerts created by [detection rules](/solutions/security/detect-and-alert/choose-the-right-rule-type.md). Normally, when a rule meets its criteria repeatedly, it creates multiple alerts, one for each time the rule’s criteria are met. When alert suppression is configured, alerts for duplicate events are not created. Instead, the qualifying events are grouped, and only one alert is created for each group. Depending on the rule type, you can configure alert suppression to create alerts each time the rule runs, or once within a specified time window. You can also specify multiple fields to group events by unique combinations of values. @@ -28,7 +28,7 @@ The {{security-app}} displays several indicators in the Alerts table and the ale :::: -You can configure alert suppression when [creating](/solutions/security/detect-and-alert/create-detection-rule.md) or editing a rule. +You can configure alert suppression when [creating](/solutions/security/detect-and-alert/using-the-rule-builder.md) or editing a rule. 1. When configuring the rule (the **Define rule** step for a new rule, or the **Definition** tab for an existing rule), specify how you want to group alerts for alert suppression: @@ -44,7 +44,7 @@ You can configure alert suppression when [creating](/solutions/security/detect-a ::::{tip} - Refer to [Suppression for fields with an array of values](/solutions/security/detect-and-alert/suppress-detection-alerts.md#security-alert-suppression-fields-with-multiple-values) to learn how fields with multiple values are handled. + Refer to [Suppression for fields with an array of values](/solutions/security/detect-and-alert/alert-suppression.md#security-alert-suppression-fields-with-multiple-values) to learn how fields with multiple values are handled. :::: @@ -74,7 +74,7 @@ You can configure alert suppression when [creating](/solutions/security/detect-a ::::{tip} * Use the **Rule preview** before saving the rule to visualize how alert suppression will affect the alerts created, based on historical data. -* If a rule times out while suppression is turned on, try shortening the rule’s [look-back](/solutions/security/detect-and-alert/create-detection-rule.md#rule-schedule) time or turn off suppression to improve the rule’s performance. +* If a rule times out while suppression is turned on, try shortening the rule’s [look-back](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-schedule) time or turn off suppression to improve the rule’s performance. :::: @@ -94,7 +94,7 @@ When specifying fields to suppress alerts by, you can select fields that have mu The {{security-app}} displays several indicators of whether a detection alert was created with alert suppression enabled, and how many qualifying alerts were suppressed. ::::{important} -Changing an alert's status to `Closed` can affect suppression. Refer to [Impact of closing suppressed alerts](/solutions/security/detect-and-alert/suppress-detection-alerts.md#security-alert-suppression-impact-close-alerts) to learn more. +Changing an alert's status to `Closed` can affect suppression. Refer to [Impact of closing suppressed alerts](/solutions/security/detect-and-alert/alert-suppression.md#security-alert-suppression-impact-close-alerts) to learn more. :::: @@ -157,8 +157,8 @@ For example, say you set the suppression time period to 5 minutes and specify to Some rule types have a maximum number of alerts that can be suppressed (custom query rules don’t have a suppression limit): -* **Threshold, event correlation, {{esql}}, and {{ml}}:** The maximum number of alerts is the value you choose for the rule’s **Max alerts per run** [advanced setting](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-advanced-params), which is `100` by default. -* **Indicator match and new terms:** The maximum number is five times the value you choose for the rule’s **Max alerts per run** [advanced setting](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-advanced-params). The default value is `100`, which means the default maximum limit for indicator match rules and new terms rules is `500`. +* **Threshold, event correlation, {{esql}}, and {{ml}}:** The maximum number of alerts is the value you choose for the rule’s **Max alerts per run** [advanced setting](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params), which is `100` by default. +* **Indicator match and new terms:** The maximum number is five times the value you choose for the rule’s **Max alerts per run** [advanced setting](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params). The default value is `100`, which means the default maximum limit for indicator match rules and new terms rules is `500`. ## Bulk apply and remove alert suppression [security-alert-suppression-bulk-apply] diff --git a/solutions/security/detect-and-alert/before-you-begin.md b/solutions/security/detect-and-alert/before-you-begin.md new file mode 100644 index 0000000000..531bf3821a --- /dev/null +++ b/solutions/security/detect-and-alert/before-you-begin.md @@ -0,0 +1,28 @@ +--- +applies_to: + stack: all + serverless: + security: all +products: + - id: security + - id: cloud-serverless +description: Prerequisites and initial setup tasks before creating and running detection rules. +--- + +# Before you begin + +Before you can create and run detection rules, confirm that your environment meets the infrastructure requirements and that your users have the necessary privileges. Some tasks only need to be done once during initial setup, while others should be revisited as your environment evolves. + +## One-time setup + +These tasks are typically completed once when you first configure detection capabilities: + +- [**Turn on detections**](/solutions/security/detect-and-alert/requirements-privileges.md): Enable the Detections feature for your deployment type. On {{serverless-short}}, detections are on by default. +- [**Detections privileges**](/solutions/security/detect-and-alert/detections-privileges.md): Understand the cluster, index, and {{kib}} privileges required for detection features, and review predefined roles and the authorization model. + +## Revisit as your environment changes + +These tasks may need to be updated over time as you onboard new data sources, add users, or expand your detection coverage: + +- **User roles and privileges**: As your team grows or responsibilities shift, review and update role assignments to ensure analysts have the access they need. Refer to [Detections privileges](/solutions/security/detect-and-alert/detections-privileges.md). +- [**Advanced data source configuration**](/solutions/security/detect-and-alert/advanced-data-source-configuration.md): Revisit {{ccs}} setup, data tier exclusions, and index mode settings when you add new clusters, change data retention policies, or onboard data sources that use different index configurations. diff --git a/solutions/security/detect-and-alert/choose-the-right-rule-type.md b/solutions/security/detect-and-alert/choose-the-right-rule-type.md new file mode 100644 index 0000000000..0e0cc55626 --- /dev/null +++ b/solutions/security/detect-and-alert/choose-the-right-rule-type.md @@ -0,0 +1,123 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/security/current/about-rules.html + - https://www.elastic.co/guide/en/serverless/current/security-about-rules.html +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Compare detection rule types and select the best fit for your threat detection use case. +--- + +# Choose the right rule type [security-about-rules] + +{{elastic-sec}} offers several detection rule types, each designed for a different kind of threat signal. Selecting the right type is the single most important decision when creating a rule, because it determines what the rule can detect, how it performs, and how its alerts behave. + +## Rule type comparison [rule-types] + +Use the following table to narrow your selection based on what you want to detect: + +| If you want to detect... | Use this rule type | Learn more | +|---|---|---| +| A known field value, pattern, or boolean condition | [**Custom query**](/solutions/security/detect-and-alert/custom-query.md) | Matches events using KQL or Lucene. The most flexible and widely used type. | +| An ordered sequence of events or a missing event | [**Event correlation (EQL)**](/solutions/security/detect-and-alert/eql.md) | Uses EQL to correlate events by shared fields across time. Detects multi-step attack chains and gaps in expected activity. | +| A field value count exceeding a boundary | [**Threshold**](/solutions/security/detect-and-alert/threshold.md) | Fires when the number of matching events grouped by one or more fields meets or exceeds a threshold. Ideal for brute-force and volume-based patterns. | +| Events matching a known threat indicator | [**Indicator match**](/solutions/security/detect-and-alert/indicator-match.md) | Compares source event fields against threat intelligence indices. Alerts are enriched with indicator metadata. | +| A field value appearing for the first time | [**New terms**](/solutions/security/detect-and-alert/new-terms.md) | Fires when a value (or combination of up to three values) has never appeared in a configurable history window. Surfaces novel activity. | +| Aggregated, transformed, or computed conditions | [**{{esql}}**](/solutions/security/detect-and-alert/esql.md) | Uses pipe-based {{esql}} queries to aggregate, transform, and filter data before alerting. Each result row becomes an alert. | +| Behavioral anomalies without a fixed pattern | [**{{ml-cap}}**](/solutions/security/detect-and-alert/machine-learning.md) | Relies on {{ml}} anomaly detection jobs to model normal behavior and flag deviations. No query authoring required. | + +## Decision flowchart + +Ask these questions in order to identify the right rule type: + +1. **Can you describe the exact pattern?** If not, and the threat is a behavioral deviation from normal, use a **{{ml}}** rule. +2. **Does the detection require comparing events against an external threat feed?** If yes, use an **indicator match** rule. +3. **Is the signal the first-ever appearance of a value?** If yes, use a **new terms** rule. +4. **Does the detection require a sequence of events in a specific order, or detecting a missing event?** If yes, use an **EQL** rule. +5. **Does the detection require counting events and firing when a volume threshold is crossed?** If yes, use a **threshold** rule. +6. **Do you need aggregation, transformation, or computed fields within the query?** If yes, use an **{{esql}}** rule. +7. **None of the above?** Use a **custom query** rule. + +## Building block rules and detection chains [about-building-block-rules] + +Any rule type can be designated as a **building block** rule. Building block rules generate alerts that are hidden from the Alerts page by default. They serve as intermediate signals that feed into higher-level detection logic. + +### When to use building blocks + +Building block rules are useful when: + +* An individual event is too low-risk to warrant analyst attention, but a combination of such events is significant. +* You want to create a **detection chain**: a set of building block rules whose hidden alerts become the input for a downstream rule that produces a visible, high-confidence alert. +* You need a persistent record of low-severity signals for threat hunting or retrospective analysis without cluttering the Alerts page. + +### How detection chains work + +A detection chain typically has two layers: + +1. **Building block rules** query source event indices and produce hidden alerts. These alerts are written to the `.alerts-security.alerts-` index. +2. **A downstream rule** queries the alert index (`.alerts-security.alerts-*`) instead of source event indices. It correlates or aggregates the building block alerts and produces a visible alert when the combined pattern meets its criteria. + +For example, you might create three building block rules: + +* One that detects a suspicious registry modification. +* One that detects a new scheduled task creation. +* One that detects an outbound connection to a rare domain. + +Each of these individually produces low-confidence alerts. A downstream EQL sequence rule can then query the alert index to detect all three occurring on the same host within a short time window, producing a single high-confidence alert for a likely intrusion chain. + +::::{tip} +Add [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) to building block rules if you want notifications when building block alerts are generated, even though the alerts are hidden from the default Alerts view. +:::: + +### Viewing building block alerts + +Building block alerts are excluded from the Overview and Alerts pages by default. To include them: + +1. Navigate to the **Alerts** page. +2. Select **Additional filters** then **Include building block alerts**. + +On a building block rule's details page, the rule's alerts are always displayed. + +### Marking a rule as a building block + +Select the **Building block** option in the rule's [advanced settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) when creating or editing any rule type. + +## Shared concepts [shared-rule-concepts] + +The following concepts apply to all rule types. For the full settings reference, see [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). + +### Data sources [views-index-patterns] + +When you create a rule, you must specify either {{es}} index patterns or a {{data-source}} as the data source (except for {{ml}} rules, which do not use queries). If you select a {{data-source}}, you can use [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md) associated with that {{data-source}} in your rule query. + +::::{note} +To access {{data-source}}s in {{stack}}, you must have the [required permissions](/explore-analyze/find-and-organize/data-views.md#data-views-read-only-access). In {{serverless-short}}, you must have the appropriate [predefined Security user role](/deploy-manage/users-roles/cloud-organization/user-roles.md#general-assign-user-roles) or a [custom role](../../../deploy-manage/users-roles/cloud-organization/user-roles.md) with the right privileges. +:::: + +::::{important} +System indices, such as the alert indices, contain important configuration and internal data. Do not change their mappings. Changes can lead to rule execution and alert indexing failures. Use [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md) instead to add fields to existing alert and event documents. +:::: + +### Authorization [alerting-authorization-model] + +Rules are authorized using an [API key](/deploy-manage/api-keys/elasticsearch-api-keys.md) associated with the last user to edit the rule. When you create or modify a rule, an API key is generated that captures a snapshot of your privileges. This API key is used for all background tasks, including detection checks and action execution. + +::::{important} +If a user without the required privileges (such as index privileges) updates a rule, the rule stops functioning. Ensure that only users with appropriate access edit rules. +:::: + +### Exceptions [about-exceptions] + +You can [add exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md) to rules to prevent them from generating alerts even when their criteria are met. This is useful for reducing noise from trusted processes, internal IP addresses, or known-benign activity. + +::::{note} +Exceptions are supported for custom query, {{ml}}, event correlation, and indicator match rule types. +:::: + +### Notifications [about-notifications] + +For both prebuilt and custom rules, you can send notifications when alerts are created. Notifications can be sent through {{jira}}, Microsoft Teams, PagerDuty, Slack, and other connectors. Configure actions when you [create or edit a rule](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications). diff --git a/solutions/security/detect-and-alert/create-a-detection-rule.md b/solutions/security/detect-and-alert/create-a-detection-rule.md new file mode 100644 index 0000000000..b6258538c3 --- /dev/null +++ b/solutions/security/detect-and-alert/create-a-detection-rule.md @@ -0,0 +1,35 @@ +--- +applies_to: + stack: all + serverless: + security: all +products: + - id: security + - id: cloud-serverless +description: Overview and workflow for building custom detection rules tailored to your threat model. +--- + +# Create a detection rule + +Build custom detection rules tailored to your environment and threat model. The pages in this section walk you through selecting a rule type, writing rule logic, and configuring rule settings. + +**[Select the right rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md)** +: Start here if you're not sure which rule type fits your use case. Compares all rule types side by side and explains how building block rules fit into detection chains. + +**[Rule types](/solutions/security/detect-and-alert/rule-types.md)** +: Go here once you've selected a rule type. Each rule type page covers when to use it, how to write effective queries, real-world examples, and the field configuration specific to that type. + +**[Using the rule builder](/solutions/security/detect-and-alert/using-the-rule-builder.md)** +: The step-by-step workflow for creating rules in the {{kib}} UI. Covers the creation steps and links to rule settings and rule type guides. + +**[Using the API](/solutions/security/detect-and-alert/using-the-api.md)** +: Relevant if you need to create or manage rules programmatically, integrate rule management into CI/CD pipelines, or bulk-import rules. + +**[Set rule data sources](/solutions/security/detect-and-alert/set-rule-data-sources.md)** +: Relevant if you need to override the default index patterns for a specific rule, target a narrower set of indices, or exclude cold and frozen data tiers. + +**[Write investigation guides](/solutions/security/detect-and-alert/write-investigation-guides.md)** +: Use this when you want to add triage guidance to a rule. Covers Markdown syntax, Timeline query buttons, and Osquery integration for investigation guides. + +**[Validate and test rules](/solutions/security/detect-and-alert/validate-and-test-rules.md)** +: Relevant before enabling a new rule in production. Covers how to test rule logic against historical data and assess alert volume. diff --git a/solutions/security/detect-and-alert/create-detection-rule.md b/solutions/security/detect-and-alert/create-detection-rule.md deleted file mode 100644 index 58da53009b..0000000000 --- a/solutions/security/detect-and-alert/create-detection-rule.md +++ /dev/null @@ -1,880 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/security/current/rules-ui-create.html - - https://www.elastic.co/guide/en/serverless/current/security-rules-create.html -applies_to: - stack: all - serverless: - security: all -products: - - id: security - - id: cloud-serverless ---- - -# Create a detection rule [security-rules-create] - -Once the Detections feature is [turned on](/solutions/security/detect-and-alert/detections-requirements.md), follow these steps to create a detection rule: - -1. Define the [**rule type**](/solutions/security/detect-and-alert/about-detection-rules.md#rule-types). The configuration for this step varies depending on the rule type. -2. Configure basic rule settings. -3. Configure advanced rule settings (optional). -4. Set the rule’s schedule. -5. Set up rule actions (optional). -6. Set up response actions (optional). - -::::{tip} -* At any step, you can [preview the rule](/solutions/security/detect-and-alert/create-detection-rule.md#preview-rules) before saving it to see what kind of results you can expect. -* To ensure rules don’t search cold and frozen data when executing, either configure the `excludedDataTiersForRuleExecution` [advanced setting](/solutions/security/get-started/configure-advanced-settings.md#exclude-cold-frozen-data-rule-executions) (which applies to all rules in a space), or add a [Query DSL filter](/solutions/security/detect-and-alert/exclude-cold-frozen-data-from-individual-rules.md) to individual rules. These options are only available if you're on the {{stack}}. - -:::: - -## Detection rule requirements - -To create detection rules, you must have: - -* At least `Read` access to data views, which requires the `Data View Management` [{{kib}} privilege](/deploy-manage/users-roles/cluster-or-deployment-auth/defining-roles.md) in {{stack}} or the appropriate [user role](/deploy-manage/users-roles/cloud-organization/user-roles.md) in {{serverless-short}}. -* The required privileges to preview rules, manage rules, and manage alerts. Refer to [](/solutions/security/detect-and-alert/detections-privileges.md) for more details. - -::::{note} -Additional configuration is required for detection rules using cross-cluster search. Refer to [Cross-cluster search and detection rules](/solutions/security/detect-and-alert/cross-cluster-search-detection-rules.md). -:::: - -## Create a custom query rule [create-custom-rule] - -1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then click **Create new rule**. -2. To create a rule based on a KQL or Lucene query, select **Custom query** on the **Create new rule** page, then: - - 1. Define which {{es}} indices or data view the rule searches for alerts. - 2. Use the filter and query fields to create the criteria used for detecting alerts. - - The following example (based on the prebuilt rule [Volume Shadow Copy Deleted or Resized via VssAdmin](https://www.elastic.co/guide/en/security/8.17/prebuilt-rule-0-14-2-volume-shadow-copy-deleted-or-resized-via-vssadmin.html)) detects when the `vssadmin delete shadows` Windows command is executed: - - * **Index patterns**: `winlogbeat-*` - - Winlogbeat ships Windows event logs to {{elastic-sec}}. - - * **Custom query**: `event.action:"Process Create (rule: ProcessCreate)" and process.name:"vssadmin.exe" and process.args:("delete" and "shadows")` - - Searches the `winlogbeat-*` indices for `vssadmin.exe` executions with the `delete` and `shadow` arguments, which are used to delete a volume’s shadow copies. - - :::{image} /solutions/images/security-rule-query-example.png - :alt: Rule query example - :screenshot: - ::: - - 3. You can use {{kib}} saved queries (![Saved query menu](/solutions/images/security-saved-query-menu.png "title =20x20")) and queries from saved Timelines (**Import query from saved Timeline**) as rule conditions. - - When you use a saved query, the **Load saved query "*query name*" dynamically on each rule execution** check box appears: - - * Select this to use the saved query every time the rule runs. This links the rule to the saved query, and you won’t be able to modify the rule’s **Custom query** field or filters because the rule will only use settings from the saved query. To make changes, modify the saved query itself. - * Deselect this to load the saved query as a one-time way of populating the rule’s **Custom query** field and filters. This copies the settings from the saved query to the rule, so you can then further adjust the rule’s query and filters as needed. If the saved query is later changed, the rule will not inherit those changes. - -3. (Optional) Use **Suppress alerts by** to reduce the number of repeated or duplicate alerts created by the rule. Refer to [Suppress detection alerts](/solutions/security/detect-and-alert/suppress-detection-alerts.md) for more information. -4. (Optional) Create a list of **Required fields** that the rule needs to function. This list is informational only, to help users understand the rule; it doesn’t affect how the rule actually runs. - - 1. Click **Add required field**, then select a field from the index patterns or data view you specified for the rule. You can also start typing a field’s name to find it faster, or type in an entirely new custom field. - 2. Enter the field’s data type. - -5. (Optional) Add **Related integrations** to associate the rule with one or more [Elastic integrations](https://docs.elastic.co/en/integrations). This indicates the rule’s dependency on specific integrations and the data they generate, and allows users to confirm each integration’s [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites) when viewing the rule. - - 1. Click **Add integration**, then select an integration from the list. You can also start typing an integration’s name to find it faster. - 2. Enter the version of the integration you want to associate with the rule, using [semantic versioning](https://semver.org/). For version ranges, you must use tilde or caret syntax. For example, `~1.2.3` is from 1.2.3 to any patch version less than 1.3.0, and `^1.2.3` is from 1.2.3 to any minor and patch version less than 2.0.0. - -6. Click **Continue** to [configure basic rule settings](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-basic-params). - -## Create a machine learning rule [create-ml-rule] - -::::{admonition} Requirements -To create or edit {{ml}} rules, you need: -* The appropriate [{{stack}} subscription](https://www.elastic.co/pricing) or [{{serverless-short}} project feature tier](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). -* The [`machine_learning_admin`](elasticsearch://reference/elasticsearch/roles.md#built-in-roles-ml-admin) in {{stack}} or the appropriate [user role](/deploy-manage/users-roles/cloud-organization/user-roles.md) in {{serverless-short}}. -* The selected {{ml}} job to be running for the rule to function correctly. -:::: - -::::{tip} -For an overview of using {{ml}} with {{elastic-sec}}, refer to [Anomaly detection](/solutions/security/advanced-entity-analytics/anomaly-detection.md). -:::: - - -1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then click **Create new rule**. -2. To create a rule based on a {{ml}} anomaly threshold, select **Machine Learning** on the **Create new rule** page, then select: - - 1. The required {{ml}} jobs. - - ::::{note} - If a required job isn’t currently running, it will automatically start when you finish configuring and enable the rule. - :::: - - 2. The anomaly score threshold above which alerts are created. - -3. (Optional) Use **Suppress alerts by** to reduce the number of repeated or duplicate alerts created by the rule. Refer to [Suppress detection alerts](/solutions/security/detect-and-alert/suppress-detection-alerts.md) for more information. - - ::::{note} - Because {{ml}} rules generate alerts from anomalies, which don’t contain source event fields, you can only use anomaly fields when configuring alert suppression. - :::: - -4. (Optional) Add **Related integrations** to associate the rule with one or more [Elastic integrations](https://docs.elastic.co/en/integrations). This indicates the rule’s dependency on specific integrations and the data they generate, and allows users to confirm each integration’s [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites) when viewing the rule. - - 1. Click **Add integration**, then select an integration from the list. You can also start typing an integration’s name to find it faster. - 2. Enter the version of the integration you want to associate with the rule, using [semantic versioning](https://semver.org/). For version ranges, you must use tilde or caret syntax. For example, `~1.2.3` is from 1.2.3 to any patch version less than 1.3.0, and `^1.2.3` is from 1.2.3 to any minor and patch version less than 2.0.0. - -5. Click **Continue** to [configure basic rule settings](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-basic-params). - -::::{tip} -To filter noisy {{ml}} rules, use [rule exceptions](/solutions/security/detect-and-alert/rule-exceptions.md). -:::: - -## Create a threshold rule [create-threshold-rule] - -1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then click **Create new rule**. -2. To create a rule based on a source event field threshold, select **Threshold**, then: - - 1. Define which {{es}} indices the rule analyzes for alerts. - 2. Use the filter and query fields to create the criteria used for detecting alerts. - - ::::{note} - You can use {{kib}} saved queries (![Saved query menu](/solutions/images/security-saved-query-menu.png "title =20x20")) and queries from saved Timelines (**Import query from saved Timeline**) as rule conditions. - :::: - - 3. Use the **Group by** and **Threshold** fields to determine which source event field is used as a threshold and the threshold’s value. - - ::::{note} - Consider the following when using the **Group by** field: - - Nested fields are not supported. - - High cardinality in the fields or a high number of matching documents can result in a rule timeout or a circuit breaker error from {{es}}. - :::: - - 4. Use the **Count** field to limit alerts by cardinality of a certain field. - - For example, if **Group by** is `source.ip, destination.ip` and its **Threshold** is `10`, an alert is generated for every pair of source and destination IP addresses that appear in at least 10 of the rule’s search results. - - You can also leave the **Group by** field undefined. The rule then creates an alert when the number of search results is equal to or greater than the threshold value. If you set **Count** to limit the results by `process.name` >= 2, an alert will only be generated for source/destination IP pairs that appear with at least 2 unique process names across all events. - - ::::{important} - Alerts created by threshold rules are synthetic alerts that do not resemble the source documents: - - - The alert itself only contains data about the fields that were aggregated over (the **Group by** fields specified in the rule). - - All other fields are omitted and aren't available in the alert. This is because these fields can vary across all source documents that were counted toward the threshold. - - You can reference the actual count of documents that exceeded the threshold from the `kibana.alert.threshold_result.count` field. - - `context.alerts.kibana.alert.threshold_result.terms` contains fields and values from any **Group by** fields specified in the rule. For example: - ``` - {{#context.alerts}} - {{#kibana.alert.threshold_result.terms}} - {{field}}: {{value}} - {{/kibana.alert.threshold_result.terms}} - {{/context.alerts}} - ``` - :::: - -3. (Optional) Select **Suppress alerts** to reduce the number of repeated or duplicate alerts created by the rule. Refer to [Suppress detection alerts](/solutions/security/detect-and-alert/suppress-detection-alerts.md) for more information. -4. (Optional) Create a list of **Required fields** that the rule needs to function. This list is informational only, to help users understand the rule; it doesn’t affect how the rule actually runs. - - 1. Click **Add required field**, then select a field from the index patterns or data view you specified for the rule. You can also start typing a field’s name to find it faster, or type in an entirely new custom field. - 2. Enter the field’s data type. - -5. (Optional) Add **Related integrations** to associate the rule with one or more [Elastic integrations](https://docs.elastic.co/en/integrations). This indicates the rule’s dependency on specific integrations and the data they generate, and allows users to confirm each integration’s [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites) when viewing the rule. - - 1. Click **Add integration**, then select an integration from the list. You can also start typing an integration’s name to find it faster. - 2. Enter the version of the integration you want to associate with the rule, using [semantic versioning](https://semver.org). For version ranges, you must use tilde or caret syntax. For example, `~1.2.3` is from 1.2.3 to any patch version less than 1.3.0, and `^1.2.3` is from 1.2.3 to any minor and patch version less than 2.0.0. - -6. Click **Continue** to [configure basic rule settings](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-basic-params). - - -## Create an event correlation rule [create-eql-rule] - -1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then click **Create new rule**. -2. To create an event correlation rule using EQL, select **Event Correlation** on the **Create new rule** page, then: - - 1. Define which {{es}} indices or data view the rule searches when querying for events. - 2. Write an [EQL query](elasticsearch://reference/query-languages/eql/eql-syntax.md) that searches for matching events or a series of matching events. - - ::::{tip} - To find events that are missing in a sequence, use the [missing events](elasticsearch://reference/query-languages/eql/eql-syntax.md#eql-missing-events) syntax. - :::: - - - For example, the following rule detects when `msxsl.exe` makes an outbound network connection: - - * **Index patterns**: `winlogbeat-*` - - Winlogbeat ships Windows events to {{elastic-sec}}. - - * **EQL query**: - - ```eql - sequence by process.entity_id - [process - where event.type in ("start", "process_started") - and process.name == "msxsl.exe"] - [network - where event.type == "connection" - and process.name == "msxsl.exe" - and network.direction == "outgoing"] - ``` - - Searches the `winlogbeat-*` indices for sequences of a `msxsl.exe` process start event followed by an outbound network connection event that was started by the `msxsl.exe` process. - - :::{image} /solutions/images/security-eql-rule-query-example.png - :alt: eql rule query example - :screenshot: - ::: - - ::::{note} - For sequence events, the {{security-app}} generates a single alert when all events listed in the sequence are detected. To see the matched sequence events in more detail, you can view the alert in the Timeline, and, if all events came from the same process, open the alert in Analyze Event view. - :::: - -3. (Optional) Click the EQL settings icon (![EQL settings icon](/solutions/images/security-eql-settings-icon.png "title =20x20")) to configure additional fields used by [EQL search](/explore-analyze/query-filter/languages/eql.md#specify-a-timestamp-or-event-category-field): - - * **Event category field**: Contains the event classification, such as `process`, `file`, or `network`. This field is typically mapped as a field type in the [keyword family](elasticsearch://reference/elasticsearch/mapping-reference/keyword.md). Defaults to the `event.category` ECS field. - * **Tiebreaker field**: Sets a secondary field for sorting events (in ascending, lexicographic order) if they have the same timestamp. - * **Timestamp field**: Contains the event timestamp used for sorting a sequence of events. This is different from the **Timestamp override** advanced setting, which is used for querying events within a range. Defaults to the `@timestamp` ECS field. - -4. (Optional) Use **Suppress alerts by** to reduce the number of repeated or duplicate alerts created by the rule. Refer to [Suppress detection alerts](/solutions/security/detect-and-alert/suppress-detection-alerts.md) for more information. -5. (Optional) Create a list of **Required fields** that the rule needs to function. This list is informational only, to help users understand the rule; it doesn’t affect how the rule actually runs. - - 1. Click **Add required field**, then select a field from the index patterns or data view you specified for the rule. You can also start typing a field’s name to find it faster, or type in an entirely new custom field. - 2. Enter the field’s data type. - -6. (Optional) Add **Related integrations** to associate the rule with one or more [Elastic integrations](https://docs.elastic.co/en/integrations). This indicates the rule’s dependency on specific integrations and the data they generate, and allows users to confirm each integration’s [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites) when viewing the rule. - - 1. Click **Add integration**, then select an integration from the list. You can also start typing an integration’s name to find it faster. - 2. Enter the version of the integration you want to associate with the rule, using [semantic versioning](https://semver.org/). For version ranges, you must use tilde or caret syntax. For example, `~1.2.3` is from 1.2.3 to any patch version less than 1.3.0, and `^1.2.3` is from 1.2.3 to any minor and patch version less than 2.0.0. - -7. Click **Continue** to [configure basic rule settings](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-basic-params). - - -## Create an indicator match rule [create-indicator-rule] - -::::{note} -{{elastic-sec}} provides [limited support](/solutions/security/detect-and-alert.md#support-indicator-rules) for indicator match rules. -:::: - -1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then click **Create new rule**. - -2. To create a rule that continually compares your security source events with threat indicators and generates alerts when they meet the rule criteria that you specify, select **Indicator Match**, then configure the following: - - 1. **Source**: The index patterns or data view that store your source event documents. The **Index patterns** field is prepopulated with indices that are set in the [default {{elastic-sec}} indices](/solutions/security/get-started/configure-advanced-settings.md#update-sec-indices). If you choose to use a **Data View**, you must specify one from the drop-down. - - 2. **Custom query**: The query and filters used to retrieve documents from your source event indices. Field values in these documents are compared against indicator values, according to the threat mapping conditions that you set. - - The default KQL query `*:*` retrieves every document in the specified event indices. You can modify the query as needed. For example, if you only want to retrieve documents that contain a `destination.ip` address field, enter `destination.ip : *`. - - ::::{tip} - You can use saved queries and queries from saved Timelines (**Import query from saved Timeline**) as rule conditions. - :::: - - 3. **Indicator index patterns**: The index patterns that store your threat indicator documents. This field is prepopulated with indices specified in the [`securitySolution:defaultThreatIndex`](/solutions/security/get-started/configure-advanced-settings.md#update-threat-intel-indices) advanced setting. - - ::::{important} - Data in threat indicator indices must be [ECS compatible](/reference/security/fields-and-object-schemas/siem-field-reference.md), and must contain a `@timestamp` field. - :::: - - 4. **Indicator index query**: The query used to retrieve documents from your threat indicator indices. Field values in these documents are compared against source event values, according to the threat mapping conditions that you set. - - The default KQL query `@timestamp > "now-30d/d"` searches the threat indicator indices for threat intelligence indicators that were ingested during the past 30 days. The start time is rounded down to the nearest day (resolves to UTC `00:00:00`). - - 5. **Indicator mapping**: Set threat mapping conditions that compare values in source event fields with values in threat indicator fields. Alerts are generated if the conditions are met. - - ::::{note} - Only single-value fields are supported. - :::: - - To specify fields to compare from your specified source event and threat indicator indices, create a threat mapping entry and configure the following: - - * **Field**: Select a field from your source event indices for comparison. - * {applies_to}`stack: ga 9.2` **MATCHES/DOES NOT MATCH**: Choose whether the source event field value should match or not match the threat indicator field value that it's being compared to. - - ::::{note} - Define matching (`MATCHES`) conditions first, then narrow down your results even more by adding `DOES NOT MATCH` conditions to exclude field values that you want to ignore. Mapping entries that _only_ use the `DOES NOT MATCH` condition are not supported. When configuring your threat mappings, at least one entry must have a `MATCHES` condition. - :::: - - * **Indicator index field**: Select a field from your threat indicator index for comparison. - - 6. (Optional) Add more threat mapping entries and combine them with `AND` and `OR` clauses. - - For example, to create a rule that generates alerts when `host.name` **and** `destination.ip` field values in the `logs-*` or `packetbeat-*` {{elastic-sec}} indices are identical to the corresponding field values in the `logs-ti_*` indicator index, enter the rule parameters seen in the following image: - - :::{image} /solutions/images/security-indicator-rule-example.png - :alt: Indicator match rule settings - :screenshot: - ::: - - ::::{tip} - Before you create rules, create [Timeline templates](/solutions/security/investigate/timeline.md) so you can select them under **Timeline template** at the end of the **Define rule** section. When alerts generated by the rule are investigated in the Timeline, Timeline query values are replaced with their corresponding alert field values. - :::: - -3. (Optional) Select **Suppress alerts** to reduce the number of repeated or duplicate alerts created by the rule. Refer to [Suppress detection alerts](/solutions/security/detect-and-alert/suppress-detection-alerts.md) for more information. -4. (Optional) Create a list of **Required fields** that the rule needs to function. This list is informational only, to help users understand the rule; it doesn’t affect how the rule actually runs. - - 1. Click **Add required field**, then select a field from the index patterns or data view you specified for the rule. You can also start typing a field’s name to find it faster, or type in an entirely new custom field. - 2. Enter the field’s data type. - -5. (Optional) Add **Related integrations** to associate the rule with one or more [Elastic integrations](https://docs.elastic.co/en/integrations). This indicates the rule’s dependency on specific integrations and the data they generate, and allows users to confirm each integration’s [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites) when viewing the rule. - - 1. Click **Add integration**, then select an integration from the list. You can also start typing an integration’s name to find it faster. - 2. Enter the version of the integration you want to associate with the rule, using [semantic versioning](https://semver.org/). For version ranges, you must use tilde or caret syntax. For example, `~1.2.3` is from 1.2.3 to any patch version less than 1.3.0, and `^1.2.3` is from 1.2.3 to any minor and patch version less than 2.0.0. - -6. Click **Continue** to [configure basic rule settings](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-basic-params). - - -### Use value lists with indicator match rules [indicator-value-lists] - -While there are numerous ways you can add data into indicator indices, you can use value lists as the indicator match index in an indicator match rule. Take the following scenario, for example: - -You uploaded a value list of known ransomware domains, and you want to be notified if any of those domains matches a value contained in a domain field in your security event index pattern. - -1. Upload a value list of indicators. -2. Create an indicator match rule and fill in the following fields: - - 1. **Index patterns**: The Elastic Security event indices on which the rule runs. - 2. **Custom query**: The query and filters used to retrieve the required results from the Elastic Security event indices (e.g., `host.domain :*`). - 3. **Indicator index patterns**: Value lists are stored in a hidden index called `.items-`. Enter the name of the {{kib}} space in which this rule will run in this field. - 4. **Indicator index query**: Enter the value `list_id :`, followed by the name of the value list you want to use as your indicator index (uploaded in Step 1 above). - 5. **Indicator mapping** - - * **Field**: Enter the field from the Elastic Security event indices to be used for comparing values. - * **Indicator index field**: Enter the type of value list you created (i.e., `keyword`, `text`, or `IP`). - - ::::{tip} - If you don’t remember this information, refer to the appropriate [value list](/solutions/security/detect-and-alert/create-manage-value-lists.md) and find the list’s type in the **Type** column (for example, the type can be `Keywords`, `Text`, or `IP`). - :::: - - -:::{image} /solutions/images/security-indicator_value_list.png -:alt: indicator value list -:screenshot: -::: - - -## Create a new terms rule [create-new-terms-rule] - -1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then click **Create new rule**. -2. To create a rule that searches for each new term detected in source documents, select **New Terms** on the **Create new rule** page, then: - - 1. Specify what data to search by entering individual {{es}} index patterns or selecting an existing data view. - 2. Use the filter and query fields to create the criteria used for detecting alerts. - - ::::{note} - You can use saved queries and queries from saved Timelines (**Import query from saved Timeline**) as rule conditions. - :::: - - 3. Use the **Fields** menu to select a field to check for new terms. You can also select up to three fields to detect a combination of new terms (for example, a `host.ip` and `host.id` that have never been observed together before). - - ::::{important} - When checking multiple fields, each unique combination of values from those fields is evaluated separately. For example, a document with `host.name: ["host-1", "host-2", "host-3"]` and `user.name: ["user-1", "user-2", "user-3"]` has 9 (3x3) unique combinations of `host.name` and `user.name`. A document with 11 values in `host.name` and 10 values in `user.name` has 110 (11x10) unique combinations. The new terms rule only evaluates 100 unique combinations per document, so selecting fields with large arrays of values might cause incorrect results. - :::: - - 4. Use the **History Window Size** menu to specify the time range to search in minutes, hours, or days to determine if a term is new. The history window size must be larger than the rule interval plus additional look-back time, because the rule will look for terms where the only time(s) the term appears within the history window is *also* within the rule interval and additional look-back time. - - For example, if a rule has an interval of 5 minutes, no additional look-back time, and a history window size of 7 days, a term will be considered new only if the time it appears within the last 7 days is also within the last 5 minutes. Configure the rule interval and additional look-back time when you [set the rule’s schedule](/solutions/security/detect-and-alert/create-detection-rule.md#rule-schedule). - -3. (Optional) Use **Suppress alerts by** to reduce the number of repeated or duplicate alerts created by the rule. Refer to [Suppress detection alerts](/solutions/security/detect-and-alert/suppress-detection-alerts.md) for more information. -4. (Optional) Create a list of **Required fields** that the rule needs to function. This list is informational only, to help users understand the rule; it doesn’t affect how the rule actually runs. - - 1. Click **Add required field**, then select a field from the index patterns or data view you specified for the rule. You can also start typing a field’s name to find it faster, or type in an entirely new custom field. - 2. Enter the field’s data type. - -5. (Optional) Add **Related integrations** to associate the rule with one or more [Elastic integrations](https://docs.elastic.co/en/integrations). This indicates the rule’s dependency on specific integrations and the data they generate, and allows users to confirm each integration’s [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites) when viewing the rule. - - 1. Click **Add integration**, then select an integration from the list. You can also start typing an integration’s name to find it faster. - 2. Enter the version of the integration you want to associate with the rule, using [semantic versioning](https://semver.org). For version ranges, you must use tilde or caret syntax. For example, `~1.2.3` is from 1.2.3 to any patch version less than 1.3.0, and `^1.2.3` is from 1.2.3 to any minor and patch version less than 2.0.0. - -6. Click **Continue** to [configure basic rule settings](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-basic-params). - - -## Create an {{esql}} rule [create-esql-rule] - -Use [{{esql}}](elasticsearch://reference/query-languages/esql.md) to query your source events and aggregate event data. Query results are returned in a table with rows and columns. Each row becomes an alert. - -To create an {{esql}} rule: - -1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then click **Create new rule**. -2. Select **{{esql}}**, then write a query. - - ::::{note} - Refer to the sections below to learn more about [{{esql}} query types](/solutions/security/detect-and-alert/create-detection-rule.md#esql-rule-query-types), [query design considerations](/solutions/security/detect-and-alert/create-detection-rule.md#esql-query-design), and [rule limitations](/solutions/security/detect-and-alert/create-detection-rule.md#esql-rule-limitations). - :::: - - - ::::{tip} - Click the help icon (![Click the ES|QL help icon](/solutions/images/security-esql-help-ref-button.png "title =20x20")) to open the in-product reference documentation for all {{esql}} commands and functions. - :::: - -3. (Optional) Use **Suppress alerts by** to reduce the number of repeated or duplicate alerts created by the rule. Refer to [Suppress detection alerts](/solutions/security/detect-and-alert/suppress-detection-alerts.md) for more information. -4. (Optional) Create a list of **Required fields** that the rule needs to function. This list is informational only, to help users understand the rule; it doesn’t affect how the rule actually runs. - - 1. Click **Add required field**, then select a field from the index patterns or data view you specified for the rule. You can also start typing a field’s name to find it faster, or type in an entirely new custom field. - 2. Enter the field’s data type. - -5. (Optional) Add **Related integrations** to associate the rule with one or more [Elastic integrations](https://docs.elastic.co/en/integrations). This indicates the rule’s dependency on specific integrations and the data they generate, and allows users to confirm each integration’s [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites) when viewing the rule. - - 1. Click **Add integration**, then select an integration from the list. You can also start typing an integration’s name to find it faster. - 2. Enter the version of the integration you want to associate with the rule, using [semantic versioning](https://semver.org/). For version ranges, you must use tilde or caret syntax. For example, `~1.2.3` is from 1.2.3 to any patch version less than 1.3.0, and `^1.2.3` is from 1.2.3 to any minor and patch version less than 2.0.0. - -6. Click **Continue** to [configure basic rule settings](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-basic-params). - - -### {{esql}} query types [esql-rule-query-types] - -{{esql}} rule queries are loosely categorized into two types: aggregating and non-aggregating. - - -#### Aggregating query [esql-agg-query] - -Aggregating queries use [`STATS...BY`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md) functions to aggregate source event data. Alerts generated by a rule with an aggregating query only contain the fields that the {{esql}} query returns and any new fields that the query creates. - -::::{note} -A *new field* is a field that doesn’t exist in the query’s source index and is instead created when the rule runs. You can access new fields in the details of any alerts that are generated by the rule. For example, if you use the `STATS...BY` function to create a column with aggregated values, the column is created when the rule runs and is added as a new field to any alerts that are generated by the rule. -:::: - - -Here is an example aggregating query: - -```esql -FROM logs-* -| STATS host_count = COUNT(host.name) BY host.name -| SORT host_count DESC -| WHERE host_count > 20 -``` - -* This query starts by searching logs from indices that match the pattern `logs-*`. -* The query then aggregates the count of events by `host.name`. -* Next, it sorts the result by `host_count` in descending order. -* Then, it filters for events where the `host_count` field appears more than 20 times during the specified rule interval. - -::::{note} -Rules that use aggregating queries might create duplicate alerts. This can happen when events that occur in the additional look-back time are aggregated both in the current rule execution and in a previous rule execution. -:::: - - - -#### Non-aggregating query [esql-non-agg-query] - -Non-aggregating queries don’t use `STATS...BY` functions and don’t aggregate source event data. Alerts generated by a non-aggregating query contain source event fields that the query returns, new fields the query creates, and all other fields in the source event document. - -::::{note} -A *new field* is a field that doesn’t exist in the query’s source index and is instead created when the rule runs. You can access new fields in the details of any alerts that are generated by the rule. For example, if you use the [`EVAL`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-eval) command to append new columns with calculated values, the columns are created when the rule runs, and are added as new fields to any alerts generated by the rule. -:::: - - -Here is an example non-aggregating query: - -```esql -FROM logs-* METADATA _id, _index, _version -| WHERE event.category == "process" AND event.id == "8a4f500d" -| LIMIT 10 -``` - -* This query starts by querying logs from indices that match the pattern `logs-*`. The `METADATA _id, _index, _version` operator allows [alert deduplication](/solutions/security/detect-and-alert/create-detection-rule.md#esql-non-agg-query-dedupe). -* Next, the query filters events where the `event.category` is a process and the `event.id` is `8a4f500d`. -* Then, it limits the output to the top 10 results. - - -#### Turn on alert deduplication for rules using non-aggregating queries [esql-non-agg-query-dedupe] - -To deduplicate alerts, a query needs access to the `_id`, `_index`, and `_version` metadata fields of the queried source event documents. You can allow this by adding the `METADATA _id, _index, _version` operator after the `FROM` source command, for example: - -```esql -FROM logs-* METADATA _id, _index, _version -| WHERE event.category == "process" AND event.id == "8a4f500d" -| LIMIT 10 -``` - -When those metadata fields are provided, unique alert IDs are created for each alert generated by the query. - -When developing the query, make sure you don’t [`DROP`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-drop) or filter out the `_id`, `_index`, or `_version` metadata fields. - -Here is an example of a query that fails to deduplicate alerts. It uses the `DROP` command to omit the `_id` property from the results table: - -```esql -FROM logs-* METADATA _id, _index, _version -| WHERE event.category == "process" AND event.id == "8a4f500d" -| DROP _id -| LIMIT 10 -``` - -Here is another example of an invalid query that uses the `KEEP` command to only return `event.*` fields in the results table: - -```esql -FROM logs-* METADATA _id, _index, _version -| WHERE event.category == "process" AND event.id == "8a4f500d" -| KEEP event.* -| LIMIT 10 -``` - - -### Query design considerations [esql-query-design] - -When writing your query, consider the following: - -* The [`LIMIT`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-limit) command specifies the maximum number of rows an {{esql}} query returns and the maximum number of alerts created per rule execution. Similarly, a detection rule’s **Max alerts per run** setting specifies the maximum number of alerts it can create every time it runs. - - If the `LIMIT` value and **Max alerts per run** value are different, the rule uses the lower value to determine the maximum number of alerts the rule generates. - -* When writing an aggregating query, use the [`STATS...BY`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-stats-by) command with fields that you want to search and filter for after alerts are created. For example, using the `host.name`, `user.name`, `process.name` fields with the `BY` operator of the `STATS...BY` command returns these fields in alert documents, and allows you to search and filter for them from the Alerts table. -* When configuring alert suppression on a non-aggregating query, we recommend sorting results by ascending `@timestamp` order. Doing so ensures that alerts are properly suppressed, especially if the number of alerts generated is higher than the **Max alerts per run** value. - - -### {{esql}} rule limitations [esql-rule-limitations] - -If your {{esql}} query creates new fields that aren’t part of the ECS schema, they aren’t mapped to the alerts index, so you can’t search for or filter them in the Alerts table. As a workaround, create [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md). - - -### Highlight fields returned by the {{esql}} rule query [custom-highlighted-esql-fields] - -When configuring an {{esql}} rule’s **[Custom highlighted fields](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-advanced-params)**, you can specify any fields that the rule’s aggregating or non-aggregating query return. This can help ensure that returned fields are visible in the alert details flyout while you’re investigating alerts. - - -## Configure basic rule settings [rule-ui-basic-params] - -1. In the **About rule** pane, fill in the following fields: - - 1. **Name**: The rule’s name. - 2. **Description**: A description of what the rule does. - 3. **Default severity**: Select the severity level of alerts created by the rule: - - * **Low**: Alerts that are of interest but generally are not considered to be security incidents. Sometimes a combination of low severity alerts can indicate suspicious activity. - * **Medium**: Alerts that require investigation. - * **High**: Alerts that require an immediate investigation. - * **Critical**: Alerts that indicate it is highly likely a security incident has occurred. - - 4. **Severity override** (optional): Select to use source event values to override the **Default severity** in generated alerts. When selected, a UI component is displayed where you can map the source event field values to severity levels. The following example shows how to map severity levels to `host.name` values: - - :::{image} /solutions/images/security-severity-mapping-ui.png - :alt: severity mapping ui - :screenshot: - ::: - - ::::{note} - For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. Also note that overrides are not supported for event correlation rules. - :::: - - 5. **Default risk score**: A numerical value between 0 and 100 that indicates the risk of events detected by the rule. This setting changes to a default value when you change the **Severity** level, but you can adjust the risk score as needed. General guidelines are: - - * `0` - `21` represents low severity. - * `22` - `47` represents medium severity. - * `48` - `73` represents high severity. - * `74` - `100` represents critical severity. - - 6. **Risk score override** (optional): Select to use a source event value to override the **Default risk score** in generated alerts. When selected, a UI component is displayed to select the source field used for the risk score. For example, if you want to use the source event’s risk score in alerts: - - :::{image} /solutions/images/security-risk-source-field-ui.png - :alt: risk source field ui - :screenshot: - ::: - - ::::{note} - For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. - :::: - - 7. **Tags** (optional): Words and phrases used to categorize, filter, and search the rule. - -2. Continue with **one** of the following: - - * [Configure advanced rule settings (optional)](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-advanced-params) - * [Set the rule’s schedule](/solutions/security/detect-and-alert/create-detection-rule.md#rule-schedule) - - - -## Configure advanced rule settings (optional) [rule-ui-advanced-params] - -1. Click **Advanced settings** and fill in the following fields where applicable: - - 1. **Reference URLs** (optional): References to information that is relevant to the rule. For example, links to background information. - 2. **False positive examples** (optional): List of common scenarios that may produce false-positive alerts. - 3. **MITRE ATT&CKTM threats** (optional): Add relevant [MITRE](https://attack.mitre.org/) framework tactics, techniques, and subtechniques. - 4. **Custom highlighted fields** (optional): Specify highlighted fields for unique alert investigation flows. You can choose any fields that are available in the indices you selected for the rule’s data source. - - After you create the rule, you can find all custom highlighted fields in the About section of the rule details page. If the rule has alerts, you can find custom highlighted fields in the [Highlighted fields](/solutions/security/detect-and-alert/view-detection-alert-details.md#investigation-section) section of the alert details flyout. - - 5. **Setup guide** (optional): Instructions on rule prerequisites such as required integrations, configuration steps, and anything else needed for the rule to work correctly. - 6. **Investigation guide** (optional): Information for analysts investigating alerts created by the rule. You can also add action buttons to [run Osquery](/solutions/security/investigate/run-osquery-from-investigation-guides.md) or [launch Timeline investigations](/solutions/security/detect-and-alert/launch-timeline-from-investigation-guides.md) using alert data. - 7. **Author** (optional): The rule’s authors. - 8. **License** (optional): The rule’s license. - 9. **Elastic endpoint exceptions** (optional): Adds all [{{elastic-endpoint}} exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md#endpoint-rule-exceptions) to this rule. - - ::::{note} - If you select this option, you can add {{elastic-endpoint}} exceptions on the Rule details page. Additionally, all future exceptions added to [endpoint protection rules](/solutions/security/manage-elastic-defend/endpoint-protection-rules.md) will also affect this rule. - :::: - - 10. **Building block** (optional): Select to create a building-block rule. By default, alerts generated from a building-block rule are not displayed in the UI. See [About building block rules](/solutions/security/detect-and-alert/about-building-block-rules.md) for more information. - 11. **Max alerts per run** (optional): Specify the maximum number of alerts the rule can create each time it runs. Default is 100. - - ::::{note} - This setting can be superseded by the [{{kib}} configuration setting](kibana://reference/configuration-reference/alerting-settings.md#alert-settings) `xpack.alerting.rules.run.alerts.max`, which determines the maximum alerts generated by *any* rule in the {{kib}} alerting framework. For example, if `xpack.alerting.rules.run.alerts.max` is set to `1000`, the rule can generate no more than 1000 alerts even if **Max alerts per run** is set higher. - :::: - - 12. **Indicator prefix override**: Define the location of indicator data within the structure of indicator documents. When the indicator match rule executes, it queries specified indicator indices and references this setting to locate fields with indicator data. This data is used to enrich indicator match alerts with metadata about matched threat indicators. The default value for this setting is `threat.indicator`. - - ::::{important} - If your threat indicator data is at a different location, update this setting accordingly to ensure alert enrichment can still be performed. - :::: - - 13. **Rule name override** (optional): Select a source event field to use as the rule name in the UI (Alerts table). This is useful for exposing, at a glance, more information about an alert. For example, if the rule generates alerts from Suricata, selecting `event.action` lets you see what action (Suricata category) caused the event directly in the Alerts table. - - ::::{note} - For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. - :::: - - 14. **Timestamp override** (optional): Select a source event timestamp field. When selected, the rule’s query uses the selected field, instead of the default `@timestamp` field, to search for alerts. This can help reduce missing alerts due to network or server outages. Specifically, if your ingest pipeline adds a timestamp when events are sent to {{es}}, this can prevent missing alerts from ingestion delays. - - If the selected field is unavailable, the rule query will use the `@timestamp` field instead. In the case that you don’t want to use the `@timestamp` field because you know your data source has an inaccurate `@timestamp` value, we recommend selecting the **Do not use @timestamp as a fallback timestamp field** option instead. This will ensure that the rule query ignores the `@timestamp` field entirely. - - ::::{tip} - The [Microsoft](beats://reference/filebeat/filebeat-module-microsoft.md) and [Google Workspace](beats://reference/filebeat/filebeat-module-google_workspace.md) {{filebeat}} modules have an `event.ingested` timestamp field that can be used instead of the default `@timestamp` field. - :::: - -2. Click **Continue**. The **Schedule rule** pane is displayed. - - :::{image} /solutions/images/security-schedule-rule.png - :alt: schedule rule - :screenshot: - ::: - -3. Continue with [setting the rule’s schedule](/solutions/security/detect-and-alert/create-detection-rule.md#rule-schedule). - - -## Set the rule’s schedule [rule-schedule] - -1. Select how often the rule runs. -2. Optionally, add `Additional look-back time` to the rule. When defined, the rule searches indices with the additional time. - - For example, if you set a rule to run every 5 minutes with an additional look-back time of 1 minute, the rule runs every 5 minutes but analyzes the documents added to indices during the last 6 minutes. - - ::::{important} - It is recommended to set the `Additional look-back time` to at least 1 minute. This ensures there are no missing alerts when a rule does not run exactly at its scheduled time. - - {{elastic-sec}} prevents duplication. Any duplicate alerts that are discovered during the `Additional look-back time` are *not* created. - - :::: - -3. Click **Continue**. The **Rule actions** pane is displayed. -4. Do either of the following: - - * Continue onto [setting up alert notifications](/solutions/security/detect-and-alert/create-detection-rule.md#rule-notifications) and [Response Actions](/solutions/security/detect-and-alert/create-detection-rule.md#rule-response-action) (optional). - * Create the rule (with or without activation). - - - -## Set up rule actions (optional) [rule-notifications] - -Use actions to set up notifications sent via other systems when alerts are generated. - -::::{note} -To use actions for alert notifications, you need the [appropriate license](https://www.elastic.co/subscriptions). For more information, see [Cases requirements](/solutions/security/investigate/cases-requirements.md). -:::: - -::::{tip} -:applies_to: {stack: preview 9.3+, serverless: preview} -You can use [workflows](/explore-analyze/workflows.md) as a rule action to automate alert response processes. Workflows can create cases, route notifications, or perform other automated tasks when alerts are generated. To learn how to set up a workflow as a rule action, refer to [](/explore-analyze/workflows/triggers/alert-triggers.md). -:::: - -1. Select a connector type to determine how notifications are sent. For example, if you select the {{jira}} connector, notifications are sent to your {{jira}} system. - - ::::{note} - Each action type requires a connector. Connectors store the information required to send the notification from the external system. You can configure connectors while creating the rule or from the **{{connectors-ui}}** page. For more information, refer to [Action and connector types](/deploy-manage/manage-connectors.md). - - Some connectors that perform actions require less configuration. For example, you do not need to set the action frequency or variables for the [Cases connector](kibana://reference/connectors-kibana/cases-action-type.md) - - :::: - -2. After you select a connector, set its action frequency to define when notifications are sent: - - * **Summary of alerts**: Select this option to get a report that summarizes generated alerts, which you can review at your convenience. Alert summaries will be sent at the specified time intervals. - - ::::{note} - When setting a custom notification frequency, do not choose a time that is shorter than the rule’s execution schedule. - :::: - - * **For each alert**: Select this option to ensure notifications are sent every time new alerts are generated. - -3. (Optional) Specify additional conditions that need to be met for notifications to send. Click the toggle to enable a setting, then add the required details: - - * **If alert matches query**: Enter a KQL query that defines field-value pairs or query conditions that must be met for notifications to send. The query only searches alert documents in the indices specified for the rule. - * **If alert is generated during timeframe**: Set timeframe details. Notifications are only sent if alerts are generated within the timeframe you define. - -4. Complete the required connector type fields. Here is an example with {{jira}}: - - :::{image} /solutions/images/security-selected-action-type.png - :alt: selected action type - :screenshot: - ::: - -5. Use the default notification message or customize it. You can add more context to the message by clicking the icon above the message text box and selecting from a list of available [alert notification variables](/solutions/security/detect-and-alert/create-detection-rule.md#rule-action-variables). -6. Create the rule with or without activation. - - ::::{note} - When you activate a rule, it is queued, and its schedule is determined by its initial run time. For example, if you activate a rule that runs every 5 minutes at 14:03 but it does not run until 14:04, it will run again at 14:09. - :::: - - -::::{important} -After you activate a rule, you can check if it is running as expected using the [Monitoring tab](/troubleshoot/security/detection-rules.md) on the Rules page. If you see values in the `Gap` column, you can [Troubleshoot missing alerts](/troubleshoot/security/detection-rules.md#troubleshoot-signals). - -When a rule fails to run, the {{security-app}} tries to rerun it at its next scheduled run time. - -:::: - - - -### Alert notification placeholders [rule-action-variables] - -You can use [mustache syntax](http://mustache.github.io/) to add variables to notification messages. The action frequency you choose determines the variables you can select from. - -The following variables can be passed for all rules: - -::::{note} -Refer to [Action frequency: Summary of alerts](/explore-analyze/alerts-cases/alerts/rule-action-variables.md#alert-summary-action-variables) to learn about additional variables that can be passed if the rule’s action frequency is **Summary of alerts**. -:::: - - -* `{{context.alerts}}`: Array of detected alerts -* `{{{context.results_link}}}`: URL to the alerts in {{kib}} -* `{{context.rule.anomaly_threshold}}`: Anomaly threshold score above which alerts are generated ({{ml}} rules only) -* `{{context.rule.description}}`: Rule description -* `{{context.rule.false_positives}}`: Rule false positives -* `{{context.rule.filters}}`: Rule filters (query rules only) -* `{{context.rule.id}}`: Unique rule ID returned after creating the rule -* `{{context.rule.index}}`: Indices rule runs on (query rules only) -* `{{context.rule.language}}`: Rule query language (query rules only) -* `{{context.rule.machine_learning_job_id}}`: ID of associated {{ml}} job ({{ml}} rules only) -* `{{context.rule.max_signals}}`: Maximum allowed number of alerts per rule execution -* `{{context.rule.name}}`: Rule name -* `{{context.rule.query}}`: Rule query (query rules only) -* `{{context.rule.references}}`: Rule references -* `{{context.rule.risk_score}}`: Default rule risk score - - ::::{note} - This placeholder contains the rule’s default values even when the **Risk score override** option is used. - :::: - -* `{{context.rule.rule_id}}`: Generated or user-defined rule ID that can be used as an identifier across systems -* `{{context.rule.saved_id}}`: Saved search ID -* `{{context.rule.severity}}`: Default rule severity - - ::::{note} - This placeholder contains the rule’s default values even when the **Severity override** option is used. - :::: - -* `{{context.rule.threat}}`: Rule threat framework -* `{{context.rule.threshold}}`: Rule threshold values (threshold rules only) -* `{{context.rule.timeline_id}}`: Associated Timeline ID -* `{{context.rule.timeline_title}}`: Associated Timeline name -* `{{context.rule.type}}`: Rule type -* `{{context.rule.version}}`: Rule version -* `{{date}}`: Date the rule scheduled the action -* `{{kibanaBaseUrl}}`: Configured `server.publicBaseUrl` value, or empty string if not configured -* `{{rule.id}}`: ID of the rule -* `{{rule.name}}`: Name of the rule -* `{{rule.spaceId}}`: Space ID of the rule -* `{{rule.tags}}`: Tags of the rule -* `{{rule.type}}`: Type of rule -* `{{state.signals_count}}`: Number of alerts detected - -The following variables can only be passed if the rule’s action frequency is for each alert: - -* `{{alert.actionGroup}}`: Action group of the alert that scheduled actions for the rule -* `{{alert.actionGroupName}}`: Human-readable name of the action group of the alert that scheduled actions for the rule -* `{{alert.actionSubgroup}}`: Action subgroup of the alert that scheduled actions for the rule -* `{{alert.id}}`: ID of the alert that scheduled actions for the rule -* `{{alert.flapping}}`: A flag on the alert that indicates whether the alert status is changing repeatedly - - -#### Alert placeholder examples [placeholder-examples] - -To understand which fields to parse, see the [Detections API](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-security-detections-api) to view the JSON representation of rules. - -Example using `{{context.rule.filters}}` to output a list of filters: - -```json -{{#context.rule.filters}} -{{^meta.disabled}}{{meta.key}} {{#meta.negate}}NOT {{/meta.negate}}{{meta.type}} {{^exists}}{{meta.value}}{{meta.params.query}}{{/exists}}{{/meta.disabled}} -{{/context.rule.filters}} -``` - -Example using `{{context.alerts}}` as an array, which contains each alert generated since the last time the action was executed: - -```json -{{#context.alerts}} -Detection alert for user: {{user.name}} -{{/context.alerts}} -``` - -Example using the mustache "current element" notation `{{.}}` to output all the rule references in the `signal.rule.references` array: - -```json -{{#signal.rule.references}} {{.}} {{/signal.rule.references}} -``` - - -### Set up response actions (optional) [rule-response-action] - -Use response actions to set up additional functionality that will run whenever a rule executes: - -* **Osquery**: Include live Osquery queries with a custom query rule. When an alert is generated, Osquery automatically collects data on the system related to the alert. Refer to [Add Osquery Response Actions](/solutions/security/investigate/add-osquery-response-actions.md) to learn more. -* **{{elastic-defend}}**: Automatically run response actions on an endpoint when rule conditions are met. For example, you can automatically isolate a host or terminate a process when specific activities or events are detected on the host. Refer to [Automated response actions](/solutions/security/endpoint-response-actions/automated-response-actions.md) to learn more. - -::::{important} -Host isolation involves quarantining a host from the network to prevent further spread of threats and limit potential damage. Be aware that automatic host isolation can cause unintended consequences, such as disrupting legitimate user activities or blocking critical business processes. -:::: - - - -## Preview your rule (optional) [preview-rules] - -You can preview any custom or prebuilt rule to find out how noisy it will be. For a custom rule, you can then adjust the rule’s query or other settings. - -::::{note} -To preview rules, you must have the appropriate user role. Refer to [](/solutions/security/detect-and-alert/detections-privileges.md) for more information. -:::: - - -Click the **Rule preview** button while creating or editing a rule. The preview opens in a side panel, showing a histogram and table with the alerts you can expect, based on the defined rule settings and past events in your indices. - -:::{image} /solutions/images/security-preview-rule.png -:alt: Rule preview -:screenshot: -::: - -The preview also includes the effects of rule exceptions and override fields. In the histogram, alerts are stacked by `event.category` (or `host.name` for machine learning rules), and alerts with multiple values are counted more than once. - -To interact with the rule preview: - -* Use the date and time picker to define the preview’s time range. - - ::::{tip} - Avoid setting long time ranges with short rule intervals, or the rule preview might time out. - :::: - -* Click **Refresh** to update the preview. - - * When you edit the rule’s settings or the preview’s time range, the button changes from blue (![Blue circular refresh icon](/solutions/images/security-rule-preview-refresh-circle.png "title =20x20")) to green (![Green right-pointing arrow refresh icon](/solutions/images/security-rule-preview-refresh-arrow.png "title =20x20")) to indicate that the rule has been edited since the last preview. - * For a relative time range (such as `Last 1 hour`), refresh the preview to check for the latest results. (Previews don’t automatically refresh with new incoming data.) - -* Click the **View details** icon (![View details icon](/solutions/images/security-view-details-icon.png "title =20x20")) in the alerts table to view the details of a particular alert. -* To resize the preview, hover between the rule settings and preview, then click and drag the border. You can also click the border, then the collapse icon (![Collapse icon](/solutions/images/security-collapse-right-icon.png "title =20x20")) to collapse and expand the preview. -* To close the preview, click the **Rule preview** button again. - - -### View your rule’s {{es}} queries (optional) [view-rule-es-queries] - -::::{note} -This option is offered for all rule types except indicator match rules. -:::: - - -When previewing a rule, you can also examine the {{es}} queries that are submitted when the rule runs. Use this information to identify and troubleshoot potential rule issues and confirm that your rule is retrieving the expected data. - -To learn more about your rule’s {{es}} queries, preview its results and do the following: - -1. Select the **Show {{es}} requests, ran during rule executions** option below the preview’s date and time picker. The **Preview logged results** section displays under the histogram and alerts table. -2. Click the **Preview logged results** section to expand it. Within the section, each rule execution is shown on an individual row. -3. Expand each row to learn more about the {{es}} queries that the rule submits each time it executes. The following details are provided: - - * When the rule execution started, and how long it took to complete - * A brief explanation of what the {{es}} queries do - * The first two {{es}} queries that the rule submits to indices containing events that are used during the rule execution - - ::::{tip} - Run the queries in [Console](/explore-analyze/query-filter/tools/console.md) to determine if your rule is retrieving the expected data. For example, to test your rule’s exceptions, run the rule’s {{es}} queries, which will also contain exceptions added to the rule. If your rule’s exceptions are working as intended, the query will not return events that should be ignored. - :::: diff --git a/solutions/security/detect-and-alert/create-manage-shared-exception-lists.md b/solutions/security/detect-and-alert/create-manage-shared-exception-lists.md index 8ccf0e2461..b2a223cae0 100644 --- a/solutions/security/detect-and-alert/create-manage-shared-exception-lists.md +++ b/solutions/security/detect-and-alert/create-manage-shared-exception-lists.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Create and manage shared exception lists to apply the same exceptions to multiple rules. --- # Create and manage shared exception lists [shared-exception-lists] diff --git a/solutions/security/detect-and-alert/create-manage-value-lists.md b/solutions/security/detect-and-alert/create-manage-value-lists.md index 9327acc2eb..1fd3848d2f 100644 --- a/solutions/security/detect-and-alert/create-manage-value-lists.md +++ b/solutions/security/detect-and-alert/create-manage-value-lists.md @@ -26,12 +26,12 @@ Value lists are lists of items with the same {{es}} [data type](elasticsearch:// After creating value lists, you can use `is in list` and `is not in list` operators to [define exceptions](add-manage-exceptions.md). ::::{tip} -You can also use a value list as the [indicator match index](create-detection-rule.md#indicator-value-lists) when creating an indicator match rule. +You can also use a value list as the [indicator match index](indicator-match.md#using-value-lists) when creating an indicator match rule. :::: ## Value lists requirements -To manage value lists, your role must have the required privileges. Refer to [Manage exception value lists](/solutions/security/detect-and-alert/detections-privileges.md#detections-privileges-manage-value-lists) for details. +To manage value lists, your role must have the required privileges. Refer to [Manage exception value lists](/solutions/security/detect-and-alert/requirements-privileges.md#detections-privileges-manage-value-lists) for details. ## Create value lists [create-value-lists] diff --git a/solutions/security/detect-and-alert/cross-cluster-search-detection-rules.md b/solutions/security/detect-and-alert/cross-cluster-search-detection-rules.md index c2ad88396f..5d53c30d34 100644 --- a/solutions/security/detect-and-alert/cross-cluster-search-detection-rules.md +++ b/solutions/security/detect-and-alert/cross-cluster-search-detection-rules.md @@ -5,6 +5,7 @@ applies_to: stack: all products: - id: security +description: Configure detection rules to query data across remote clusters using cross-cluster search. --- # Cross-cluster search and detection rules [rules-cross-cluster-search] @@ -50,11 +51,11 @@ This section explains the general process for setting up cross-cluster search in ::::{important} * This step ensures that the privileges to read remote indices are applied from the user to the rule itself. When a user creates a new rule or saves edits to an existing rule, their current privileges are saved to the rule’s API key. If that user’s privileges change in the future, the rule’s API key will not update until you manually update it. Refer to [Update a rule’s API key](#update-api-key) for details. - * This user must also have the [appropriate privileges](detections-privileges.md) to manage and preview rules. + * This user must also have the [appropriate privileges](requirements-privileges.md) to manage and preview rules. :::: - 2. As this user, [configure a rule](create-detection-rule.md) that searches the remote indices: create or edit a rule, then enter the `:` pattern in the **Source** section. + 2. As this user, [configure a rule](using-the-rule-builder.md) that searches the remote indices: create or edit a rule, then enter the `:` pattern in the **Source** section. :::{image} /solutions/images/security-ccs-rule-source.png :alt: Rule source configuration @@ -65,7 +66,7 @@ This section explains the general process for setting up cross-cluster search in If the rule’s **Source** uses a data view instead of index patterns, you must define the data view for cross-cluster search separately, using the `:` pattern. Refer to [Use data views with cross-cluster search](../../../explore-analyze/find-and-organize/data-views.md#management-cross-cluster-search) for more on defining a data view. :::: - 3. (Optional) [Preview the rule](create-detection-rule.md#preview-rules) to test its expected results. + 3. (Optional) [Preview the rule](using-the-rule-builder.md) to test its expected results. ::::{important} The rule preview uses the current user’s cross-cluster search privileges, while the rule itself runs using the privileges snapshot saved in its API key the moment the key is created. The preview results could be different from the rule’s actual behavior if the user performing the preview has different privileges than what’s saved in the rule’s API key. diff --git a/solutions/security/detect-and-alert/custom-query.md b/solutions/security/detect-and-alert/custom-query.md new file mode 100644 index 0000000000..7d64acaf86 --- /dev/null +++ b/solutions/security/detect-and-alert/custom-query.md @@ -0,0 +1,107 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Create rules using KQL or Lucene queries to detect known field values and patterns. +--- + +# Custom query rules [custom-query-rule-type] + +## Overview + +Custom query rules search your {{es}} indices using a KQL or Lucene query and generate an alert whenever one or more documents match. They are the most flexible and widely used rule type in {{elastic-sec}}. + +### When to use a custom query rule + +Custom query rules are the right fit when: + +* You need to detect known indicators, field values, or simple boolean conditions, such as a specific process name, a registry path, or a combination of event fields. +* The detection logic can be expressed as a single query without requiring event ordering, aggregation, or comparison against external threat feeds. +* You want to reuse an existing {{kib}} saved query or Timeline query as the basis for a detection. + +Custom query rules are **not** the best fit when you need to: + +* Detect **sequences** of events in a specific order. Use an [event correlation (EQL) rule](/solutions/security/detect-and-alert/eql.md) instead. +* Fire only when a field value **exceeds a count**. Use a [threshold rule](/solutions/security/detect-and-alert/threshold.md) instead. +* Match events against an **external indicator feed**. Use an [indicator match rule](/solutions/security/detect-and-alert/indicator-match.md) instead. +* Detect a **field value that has never appeared before**. Use a [new terms rule](/solutions/security/detect-and-alert/new-terms.md) instead. + +### Data requirements + +Custom query rules require at least one {{es}} index pattern or [data view](/solutions/security/get-started/data-views-elastic-security.md) that contains the events you want to match. The indices must be accessible to the user who creates or last edits the rule, because the rule executes with that user's [API key privileges](/solutions/security/detect-and-alert/choose-the-right-rule-type.md#alerting-authorization-model). + +## Writing effective queries [craft-custom-query] + +### Query language + +Custom query rules accept either **KQL** (Kibana Query Language) or **Lucene** syntax. KQL is the default and is generally easier to read. Use Lucene when you need regular expressions, fuzzy matching, or other features KQL does not support. + +### Building the query + +A good custom query is precise enough to surface true positives without excessive noise. Follow these guidelines: + +* **Start narrow, then widen.** Begin with the most specific field-value pairs that identify the behavior, then relax constraints only if you miss true positives. +* **Anchor on stable fields.** Prefer fields that adversaries cannot easily change, such as `event.action`, `process.pe.original_file_name`, or `file.path`, over fields like `process.name` that can be trivially renamed. +* **Combine conditions with `and`.** Joining multiple conditions reduces false positives. For example, matching on both `process.name` and `process.args` is more precise than matching on either alone. +* **Use `or` for variant coverage.** If the same behavior can appear with different field values (for example, multiple process names), group them with `or` or use a wildcard. + +### Using saved queries and Timeline queries + +You can populate a custom query rule from a {{kib}} saved query or a saved Timeline: + +* **Saved query (dynamic):** Select **Load saved query dynamically on each rule execution** to link the rule to the saved query. The rule always uses the current version of the saved query. You cannot edit the rule's query directly while this option is active. +* **Saved query (one-time):** Deselect the dynamic option to copy the saved query into the rule. The rule's query becomes independent, and future changes to the saved query are not inherited. +* **Timeline query:** Click **Import query from saved Timeline** to copy a Timeline's query into the rule. + +### Annotated example + +The following query (adapted from the prebuilt rule *Volume Shadow Copy Deleted or Resized via VssAdmin*) detects when the `vssadmin delete shadows` Windows command is executed: + +``` +event.action:"Process Create (rule: ProcessCreate)" and process.name:"vssadmin.exe" and process.args:("delete" and "shadows") +``` + +| Clause | Purpose | +|---|---| +| `event.action:"Process Create (rule: ProcessCreate)"` | Anchors the query to process-creation events reported by Sysmon, filtering out unrelated event types. | +| `process.name:"vssadmin.exe"` | Narrows to the specific binary. | +| `process.args:("delete" and "shadows")` | Requires both arguments to be present, distinguishing destructive shadow-copy deletion from benign `vssadmin` usage such as `list shadows`. | + +**Index patterns:** `winlogbeat-*` (Winlogbeat ships Windows event logs to {{elastic-sec}}). + +::::{tip} +**See it in practice.** These prebuilt rules use custom queries and illustrate different detection patterns: + +* **Volume Shadow Copy Deleted or Resized via VssAdmin.** Matches a specific process with targeted arguments. A focused, low-noise pattern. +* **Clearing Windows Event Logs.** Uses `or` to cover multiple utilities (`wevtutil`, `powershell`) that can clear event logs. Demonstrates variant coverage. +* **Potential Process Injection via PowerShell.** Combines `process.name` with `powershell.file.script_block_text` field matching to detect in-memory injection techniques. An example of pairing process metadata with deeper content inspection. +:::: + +## Custom query field reference [custom-query-fields] + +The following settings are specific to custom query rules. For settings shared across all rule types (severity, risk score, schedule, actions, and so on), refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). + +**Index patterns or data view** +: The {{es}} indices or data view the rule queries when searching for events. Index patterns are prepopulated with the indices configured in the [default {{elastic-sec}} indices](/solutions/security/get-started/configure-advanced-settings.md#update-sec-indices) advanced setting. Alternatively, select a data view from the drop-down to use its associated index patterns and [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md). + +**Custom query** +: The KQL or Lucene query that defines the detection logic. Documents matching this query generate alerts. Use the filter bar to add structured filters alongside the query text. + +**Saved query** (optional) +: A {{kib}} saved query to use as the rule's detection logic. When loaded dynamically, the rule inherits changes to the saved query automatically. When loaded as a one-time copy, the query is embedded in the rule and can be edited independently. + +**Timeline query** (optional) +: Import a query from a saved Timeline to use as the rule's detection logic. The imported query populates the **Custom query** field. + +**Suppress alerts by** (optional) +: Reduce repeated or duplicate alerts by grouping them on one or more fields. For details, refer to [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md). + +**Required fields** (optional) +: An informational list of fields the rule needs to function. This does not affect rule execution. It helps other users understand the rule's data dependencies. + +**Related integrations** (optional) +: Associate the rule with one or more [Elastic integrations](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/detections-privileges.md b/solutions/security/detect-and-alert/detections-privileges.md index db8279cedc..6f8176f33d 100644 --- a/solutions/security/detect-and-alert/detections-privileges.md +++ b/solutions/security/detect-and-alert/detections-privileges.md @@ -19,7 +19,7 @@ Learn about the access requirements for detection features, including: - **Predefined roles**: {{serverless-full}} roles with detection privileges - **Authorization model**: How detection rules use API keys to run background tasks -For instructions on turning on the detections feature, refer to [Turn on detections](/solutions/security/detect-and-alert/detections-requirements.md). +For instructions on turning on the detections feature, refer to [Turn on detections](/solutions/security/detect-and-alert/requirements-privileges.md). ## About index privileges diff --git a/solutions/security/detect-and-alert/detections-reference.md b/solutions/security/detect-and-alert/detections-reference.md new file mode 100644 index 0000000000..4c960643a2 --- /dev/null +++ b/solutions/security/detect-and-alert/detections-reference.md @@ -0,0 +1,20 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Technical reference for detection rule settings, alert schema, and querying alert indices. +--- + +# Settings, fields, and indices + +Look up rule configuration settings, alert field definitions, and patterns for querying alert indices directly. These pages are designed for reference, not reading end to end. + +**[Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md)** +: Use this when you need to look up a specific rule setting, understand what a field does, or check valid values and defaults. Covers all shared settings (severity, risk score, schedule, actions, response actions, and notification variables) that apply across rule types. For rule-type-specific fields, refer to the individual [rule type](/solutions/security/detect-and-alert/rule-types.md) pages. + +**[Query alert indices](/solutions/security/detect-and-alert/query-alert-indices.md)** +: Relevant if you're building custom dashboards, visualizations, or SOAR integrations that query the `.alerts-security.alerts-*` index directly. Explains how to query alert indices safely, which fields are available, and links to the full [alert schema](/reference/security/fields-and-object-schemas/alert-schema.md). diff --git a/solutions/security/detect-and-alert/eql.md b/solutions/security/detect-and-alert/eql.md new file mode 100644 index 0000000000..b58a6e4da6 --- /dev/null +++ b/solutions/security/detect-and-alert/eql.md @@ -0,0 +1,120 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Create detection rules using Event Query Language (EQL) to detect event sequences and correlations. +--- + +# Event correlation (EQL) rules [eql-rule-type] + +## Overview + +Event correlation rules use [Event Query Language (EQL)](elasticsearch://reference/query-languages/eql/eql-syntax.md) to detect ordered sequences of events, single events with complex conditions, or the absence of expected events. EQL is purpose-built for event-based data and excels at expressing time-ordered relationships that other query languages cannot. + +### When to use an EQL rule + +EQL rules are the right fit when: + +* You need to detect a **sequence** of events that must occur in a specific order, such as a process creation followed by a network connection from the same process. +* You want to detect **missing events** in a sequence, for example, a login that is never followed by a logout within a time window. +* The detection logic involves correlating events by a **shared field** (such as `process.entity_id` or `host.id`) across time. +* You need richer event-level logic than KQL allows, such as filtering by event category within a sequence step. + +EQL rules are **not** the best fit when: + +* A single-event match is sufficient. Use a [custom query rule](/solutions/security/detect-and-alert/custom-query.md) instead. +* You need to count how often something happens. Use a [threshold rule](/solutions/security/detect-and-alert/threshold.md) instead. +* You need aggregation, transformations, or pipe-based processing. Use an [{{esql}} rule](/solutions/security/detect-and-alert/esql.md) instead. + +### Data requirements + +EQL rules require at least one {{es}} index pattern or [{{data-source}}](/solutions/security/get-started/data-views-elastic-security.md). The indexed data must include a timestamp field (defaults to `@timestamp`) and an event category field (defaults to `event.category`). Sequence queries also benefit from a tiebreaker field to resolve events that share the same timestamp. + +## Writing effective EQL queries [craft-eql] + +### Sequence queries + +Sequence queries are the signature capability of EQL. A sequence defines two or more event conditions that must occur in order, optionally joined by a shared field: + +```eql +sequence by process.entity_id + [process where event.type in ("start", "process_started") + and process.name == "msxsl.exe"] + [network where event.type == "connection" + and process.name == "msxsl.exe" + and network.direction == "outgoing"] +``` + +| Clause | Purpose | +|---|---| +| `sequence by process.entity_id` | Correlates events that share the same process entity, ensuring the network event came from the same process instance that started. | +| First `[ ]` bracket | Matches the process-start event for `msxsl.exe`. | +| Second `[ ]` bracket | Matches an outbound network connection from the same `msxsl.exe` process. | + +The rule generates a single alert when the full sequence is detected. + +### Single-event queries + +EQL also supports single-event queries when you need EQL-specific syntax features, such as function calls or the `wildcard` function: + +```eql +process where event.type == "start" + and process.name == "certutil.exe" + and process.args : "-urlcache" +``` + +### Missing event detection + +The `!` (missing events) syntax detects events that should have occurred but did not: + +```eql +sequence by user.name with maxspan=1h + [authentication where event.outcome == "success"] + ![authentication where event.action == "logout"] +``` + +This detects a successful login that is never followed by a logout within one hour. + +### Best practices + +* **Use `by` clauses for precision.** Joining on a shared field like `process.entity_id` or `host.id` prevents unrelated events on different hosts from matching. +* **Set `maxspan` to limit time windows.** Without a `maxspan`, a sequence can match events that are days apart, generating noisy alerts. +* **Order conditions from most specific to least.** Put the rarest event first to reduce the number of partial sequences the engine must track. + +::::{tip} +**See it in practice.** These prebuilt rules demonstrate different EQL patterns: + +* **Suspicious MSXSL Process**: A two-step sequence correlating process creation with outbound network activity. Demonstrates the core sequence-by-entity pattern. +* **Potential Credential Access through Renamed COM+ Services DLL**: A single-event EQL query using `process.pe.original_file_name` to catch renamed binaries. +* **Startup Persistence through DLL Search Order Hijacking**: A sequence that links file creation in a specific directory to a subsequent process load, illustrating file-to-process correlation. +:::: + +## EQL field reference [eql-fields] + +The following settings are specific to EQL rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). + +**Index patterns or {{data-source}}** +: The {{es}} indices or {{data-source}} the rule searches when querying for events. + +**EQL query** +: The [EQL query](elasticsearch://reference/query-languages/eql/eql-syntax.md) that defines the detection logic. Can be a single-event query, a sequence, or a sequence with missing events. Documents or sequences matching this query generate alerts. + +**EQL settings** (optional) +: Additional fields used by [EQL search](/explore-analyze/query-filter/languages/eql.md#specify-a-timestamp-or-event-category-field): + + * **Event category field**: The field containing event {{classification}} (such as `process`, `file`, or `network`). Typically a [keyword family](elasticsearch://reference/elasticsearch/mapping-reference/keyword.md) field. Defaults to `event.category`. + * **Tiebreaker field**: A secondary field for sorting events in ascending, lexicographic order when they share the same timestamp. + * **Timestamp field**: The field containing the event timestamp, used for ordering sequence events. Defaults to `@timestamp`. This is different from the **Timestamp override** advanced setting, which controls the query time range. + +**Suppress alerts by** (optional) +: Reduce repeated or duplicate alerts by grouping them on one or more fields. For details, refer to [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md). + +**Required fields** (optional) +: An informational list of fields the rule needs to function. This does not affect rule execution. + +**Related integrations** (optional) +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/esql.md b/solutions/security/detect-and-alert/esql.md new file mode 100644 index 0000000000..9dba4d68ad --- /dev/null +++ b/solutions/security/detect-and-alert/esql.md @@ -0,0 +1,118 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Create detection rules using Elasticsearch Query Language (ESQL) with aggregation and pipeline processing. +--- + +# {{esql}} rules [esql-rule-type] + +## Overview + +{{esql}} rules use [{{es}} Query Language ({{esql}})](elasticsearch://reference/query-languages/esql.md) to query source events and aggregate or transform data using a pipeline syntax. Query results are returned as a table where each row becomes an alert. {{esql}} rules combine the flexibility of a full query pipeline with the detection capabilities of {{elastic-sec}}. + +### When to use an {{esql}} rule + +{{esql}} rules are the right fit when: + +* You need **aggregation, transformation, or enrichment** within the query itself, such as computing statistics, renaming fields, or filtering on calculated values. +* The detection logic requires **pipe-based processing** that KQL and EQL cannot express, such as `STATS...BY` followed by `WHERE` to filter aggregated results. +* You want to create **new computed fields** (using `EVAL`) and alert on values derived from source data rather than raw field values. + +{{esql}} rules are **not** the best fit when: + +* A field-value match is sufficient. Use a [custom query rule](/solutions/security/detect-and-alert/custom-query.md) instead. +* You need to detect ordered event sequences. Use an [EQL rule](/solutions/security/detect-and-alert/eql.md) instead. +* You want {{anomaly-detect}} without explicit query logic. Use a [{{ml}} rule](/solutions/security/detect-and-alert/machine-learning.md) instead. + +### Data requirements + +{{esql}} rules query {{es}} indices directly using the `FROM` command. The indices must be accessible to the user who creates or last edits the rule. + +## Writing effective {{esql}} queries [craft-esql] + +### Query types + +{{esql}} rule queries fall into two categories that affect how alerts are generated: + +#### Aggregating queries + +Aggregating queries use [`STATS...BY`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md) to group and count events. Alerts contain **only** the fields returned by the query plus any new fields created during execution. + +```esql +FROM logs-* +| STATS host_count = COUNT(host.name) BY host.name +| SORT host_count DESC +| WHERE host_count > 20 +``` + +This query counts events per host and alerts on hosts with more than 20 events. Use the `BY` clause with fields you want available for searching and filtering in the Alerts table. + +::::{note} +Aggregating queries may create duplicate alerts when events in the additional look-back time are counted in both the current and previous rule execution. +:::: + +#### Non-aggregating queries + +Non-aggregating queries do not use `STATS...BY`. Alerts contain the returned source event fields, any new fields, and all other fields from the source document. + +```esql +FROM logs-* METADATA _id, _index, _version +| WHERE event.category == "process" AND event.id == "8a4f500d" +| LIMIT 10 +``` + +Adding `METADATA _id, _index, _version` enables [alert deduplication](#alert-deduplication). Without it, the same source event can generate duplicate alerts across executions. + +### Alert deduplication [alert-deduplication] + +For non-aggregating queries, add `METADATA _id, _index, _version` after the `FROM` command to enable deduplication: + +```esql +FROM logs-* METADATA _id, _index, _version +| WHERE event.category == "process" +| LIMIT 50 +``` + +Ensure you do not `DROP` or filter out `_id`, `_index`, or `_version` in subsequent pipeline steps, or deduplication fails silently. + +### Query design guidelines + +* **`LIMIT` and Max alerts per run interact.** The rule uses the lower of the two values. If `LIMIT` is 200 but **Max alerts per run** is 100, only 100 alerts are created. +* **Include fields you need in `BY` clauses.** For aggregating queries, only the `BY` fields appear in alerts. Fields not included are unavailable for filtering or investigation. +* **Sort non-aggregating results by `@timestamp` ascending** when using alert suppression. This ensures proper suppression when the alert count exceeds **Max alerts per run**. + +### Limitations + +If your {{esql}} query creates new fields that are not part of the ECS schema, they are not mapped to the alerts index. You cannot search for or filter them in the Alerts table. As a workaround, create [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md). + +### Custom highlighted fields + +When configuring an {{esql}} rule's **Custom highlighted fields** (in [advanced settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params)), you can specify any fields that the query returns. This ensures returned fields are visible in the alert details flyout during investigation. + +::::{tip} +**See it in practice.** These patterns demonstrate {{esql}} detection use cases: + +* **High event count per host.** An aggregating query that groups by `host.name` and fires when a host exceeds a threshold count. Useful for detecting noisy hosts or denial-of-service patterns. +* **Process execution with computed risk.** A non-aggregating query with `EVAL` that computes a custom risk score from multiple fields, alerting only when the computed score exceeds a threshold. +:::: + +## {{esql}} field reference [esql-fields] + +The following settings are specific to {{esql}} rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). + +**{{esql}} query** +: The [{{esql}} query](elasticsearch://reference/query-languages/esql.md) that defines the detection logic. Can be aggregating (with `STATS...BY`) or non-aggregating. Each row in the query result becomes an alert. + +**Suppress alerts by** (optional) +: Reduce repeated or duplicate alerts by grouping them on one or more fields. For details, refer to [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md). + +**Required fields** (optional) +: An informational list of fields the rule needs to function. This does not affect rule execution. + +**Related integrations** (optional) +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/exclude-cold-frozen-data-from-individual-rules.md b/solutions/security/detect-and-alert/exclude-cold-frozen-data-from-individual-rules.md deleted file mode 100644 index 2b7365f491..0000000000 --- a/solutions/security/detect-and-alert/exclude-cold-frozen-data-from-individual-rules.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/security/current/exclude-cold-frozen-data-individual-rules.html -applies_to: - stack: all -products: - - id: security ---- - -# Exclude cold and frozen data from individual rules [exclude-cold-frozen-data-individual-rules] - -Your rule might perform slower or fail if it queries data from cold or frozen [data tiers](../../../manage-data/lifecycle/data-tiers.md). To help Elasticsearch exclude cold and frozen data more efficiently, apply a Query DSL filter that ignores cold and frozen documents when your rule executes. You can add the filter when creating a new rule or updating an existing one. - -::::{tip} -To ensure that rules in your {{kib}} space exclude cold and frozen documents when executing, configure the `excludedDataTiersForRuleExecution` [advanced setting](../get-started/configure-advanced-settings.md#exclude-cold-frozen-data-rule-executions). This setting does not apply to {{ml}} rules. -:::: - - -::::{important} -* This method is not supported for {{esql}} and {{ml}} rules. -* Even when applying this filter, indicator match and event correlation rules may still fail if a frozen or cold shard that matches the rule’s specified index pattern is unavailable during rule executions. If failures occur, we recommend modifying the rule’s index patterns to only match indices containing hot tier data. - -:::: - - -Here is a sample Query DSL filter that excludes frozen tier documents during rule execution: - -```console -{ - "bool":{ - "must_not":{ - "terms":{ - "_tier":[ - "data_frozen" - ] - } - } - } -} -``` - -Here is another sample Query DSL filter that excludes cold and frozen tier documents during rule execution: - -```console -{ - "bool":{ - "must_not":{ - "terms":{ - "_tier":[ - "data_frozen", "data_cold" - ] - } - } - } -} -``` - diff --git a/solutions/security/detect-and-alert/indicator-match.md b/solutions/security/detect-and-alert/indicator-match.md new file mode 100644 index 0000000000..7e3c894e44 --- /dev/null +++ b/solutions/security/detect-and-alert/indicator-match.md @@ -0,0 +1,110 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Detect security events that match known threat indicators from threat intelligence feeds. +--- + +# Indicator match rules [indicator-match-rule-type] + +## Overview + +Indicator match rules continuously compare field values in your security event indices against field values in threat indicator indices. When a match is found, an alert is generated, enriched with metadata from the matched indicator. This makes indicator match rules the primary mechanism for operationalizing threat intelligence feeds within {{elastic-sec}}. + +### When to use an indicator match rule + +Indicator match rules are the right fit when: + +* You maintain an index of **known threat indicators** (IP addresses, domains, file hashes, URLs) and want to detect when any of them appear in your event data. +* You need to compare fields across **two separate indices** (source events and threat intelligence). +* You want alerts automatically **enriched** with indicator metadata, such as threat type, confidence, and source feed. + +Indicator match rules are **not** the best fit when: + +* You are matching against values within the same index. Use a [custom query rule](/solutions/security/detect-and-alert/custom-query.md) instead. +* You need to detect event sequences or ordering. Use an [EQL rule](/solutions/security/detect-and-alert/eql.md) instead. +* Your indicators are in a flat file rather than an {{es}} index. First [upload them as a value list](#using-value-lists) or import them through the {{ml-app}} {{data-viz}}. + +### Data requirements + +Indicator match rules require: + +* **Source event indices** containing the security events you want to scan. +* **Indicator indices** containing threat intelligence data. Data in these indices must be [ECS compatible](/reference/security/fields-and-object-schemas/siem-field-reference.md) and must contain a `@timestamp` field. + +## Writing effective indicator match rules [craft-indicator-match] + +### Designing threat mappings + +Threat mappings define which fields to compare between your source events and indicator indices. Good mappings are: + +* **Specific:** Use the most specific fields available when mapping. For example, map `destination.ip` to `threat.indicator.ip` rather than a generic text field. +* **Combined with `AND`:** Join multiple mapping entries to increase precision. Requiring both `source.ip` and `destination.port` to match narrows results to truly relevant hits. +* **Scoped with `DOES NOT MATCH`:** {applies_to}`stack: ga 9.2` After defining matching conditions, add `DOES NOT MATCH` entries to exclude known-safe values. At least one `MATCHES` entry is required. + +### Indicator index query + +The default indicator index query `@timestamp > "now-30d/d"` limits matches to indicators ingested in the past 30 days. Adjust this window based on your threat intelligence freshness requirements: + +* **Shorter window (7-14 days):** Reduces stale matches but may miss long-lived indicators. +* **Longer window (60-90 days):** Catches more indicators but increases the volume of matches and rule execution time. + +### Using value lists as indicator indices [using-value-lists] + +You can use [value lists](/solutions/security/detect-and-alert/create-manage-value-lists.md) as the indicator index. This is useful when you have a flat list of indicators (IPs, domains, hashes) that you want to match against without creating a full indicator index: + +1. Upload a value list of indicators. +2. In the **Indicator index patterns** field, enter `.items-<{{kib}} space>` (the hidden index where value lists are stored). +3. In the **Indicator index query** field, enter `list_id : `. +4. In **Indicator mapping**, set the **Indicator index field** to the list type (`keyword`, `text`, or `IP`). + +### Best practices + +* **Keep indicator indices current.** Stale indicators generate false positives and waste analyst time. +* **Use the Indicator prefix override** advanced setting if your indicator data uses a non-standard field structure (default is `threat.indicator`). +* **Create Timeline templates** before building indicator match rules. When investigating alerts in Timeline, query values are automatically replaced with corresponding alert field values. + +::::{tip} +**See it in practice.** These prebuilt rules use indicator matching: + +* **Threat Intel Indicator Match:** The foundational indicator match rule that compares source events against threat intelligence indices across multiple field types (IP, domain, hash). +* **Threat Intel Hash Indicator Match:** A focused variant matching file hashes (`file.hash.sha256`, `file.hash.md5`) against indicator indices for malware detection. +:::: + +## Indicator match field reference [indicator-match-fields] + +The following settings are specific to indicator match rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). + +**Source** +: The index patterns or {{data-source}} that store your source event documents. Prepopulated with indices from the [default {{elastic-sec}} indices](/solutions/security/get-started/configure-advanced-settings.md#update-sec-indices) advanced setting. + +**Custom query** +: The query and filters used to retrieve source event documents. Field values in matching documents are compared against indicator values according to the threat mapping. Defaults to `*:*` (all documents). + +**Indicator index patterns** +: The index patterns that store threat indicator documents. Prepopulated with indices from the [`securitySolution:defaultThreatIndex`](/solutions/security/get-started/configure-advanced-settings.md#update-threat-intel-indices) advanced setting. + +**Indicator index query** +: The query used to retrieve indicator documents. Defaults to `@timestamp > "now-30d/d"`, which searches for indicators ingested in the past 30 days. + +**Indicator mapping** +: Threat mapping conditions that compare values in source event fields with values in indicator fields. Configure: + + * **Field**: A field from your source event indices. + * **MATCHES / DOES NOT MATCH**: {applies_to}`stack: ga 9.2` Whether the values should match or not match. At least one `MATCHES` entry is required. + * **Indicator index field**: A field from your threat indicator index. + + Multiple mapping entries can be combined with `AND` and `OR` clauses. Only single-value fields are supported. + +**Suppress alerts** (optional) +: Reduce repeated or duplicate alerts. For details, refer to [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md). + +**Required fields** (optional) +: An informational list of fields the rule needs to function. This does not affect rule execution. + +**Related integrations** (optional) +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md b/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md similarity index 90% rename from solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md rename to solutions/security/detect-and-alert/install-manage-prebuilt-rules.md index 48191c981d..2e54c3f91e 100644 --- a/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md +++ b/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md @@ -10,22 +10,23 @@ applies_to: products: - id: security - id: cloud-serverless +description: Learn how to install, enable, update, and manage Elastic prebuilt detection rules. --- # Install and manage Elastic prebuilt rules [security-prebuilt-rules-management] Follow these guidelines to start using the {{security-app}}'s [prebuilt rules](detection-rules://index.md), keep them updated, and make sure they have the data needed to run successfully. -* [Install and enable Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#load-prebuilt-rules) -* [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#prebuilt-rule-tags) -* [Select and duplicate all prebuilt rules](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#select-all-prebuilt-rules) -* [Update Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#update-prebuilt-rules) +* [Install and enable Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#load-prebuilt-rules) +* [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#prebuilt-rule-tags) +* [Select and duplicate all prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#select-all-prebuilt-rules) +* [Update Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#update-prebuilt-rules) * [Confirm rule prerequisites](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites) ::::{note} * Most prebuilt rules don’t start running by default. You can use the **Install and enable** option to start running rules as you install them, or first install the rules, then enable them manually. After installation, only a few prebuilt rules will be enabled by default, such as the Endpoint Security rule. -* Without an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can't modify most settings on Elastic prebuilt rules. You must first duplicate them, then make your changes to the duplicated rules. Refer to [Select and duplicate all prebuilt rules](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#select-all-prebuilt-rules) to learn more. +* Without an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can't modify most settings on Elastic prebuilt rules. You must first duplicate them, then make your changes to the duplicated rules. Refer to [Select and duplicate all prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#select-all-prebuilt-rules) to learn more. * On {{stack}}, automatic updates of Elastic prebuilt rules are supported for the current {{elastic-sec}} version and the latest three previous minor releases. For example, if you’re on {{elastic-sec}} 9.0, you’ll be able to use the Rules UI to update your prebuilt rules until {{elastic-sec}} 9.4 is released. After that point, you can still manually download and install updated prebuilt rules, but you must upgrade to the latest {{elastic-sec}} version to receive automatic updates. :::: @@ -56,7 +57,7 @@ Follow these guidelines to start using the {{security-app}}'s [prebuilt rules](d * Install multiple rules: Select the rules, and then at the top of the page either click **Install *x* selected rule(s)** to install without enabling the rules, or click ![Vertical boxes button](/solutions/images/serverless-boxesVertical.svg "") → **Install and enable** to install and start running the rules. ::::{tip} - Use the search bar and **Tags** filter to find the rules you want to install. For example, filter by `OS: Windows` if your environment only includes Windows endpoints. For more on tag categories, refer to [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#prebuilt-rule-tags). + Use the search bar and **Tags** filter to find the rules you want to install. For example, filter by `OS: Windows` if your environment only includes Windows endpoints. For more on tag categories, refer to [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#prebuilt-rule-tags). :::: @@ -102,7 +103,7 @@ Each prebuilt rule includes several tags identifying the rule’s purpose, detec ## Select and duplicate prebuilt rules [select-all-prebuilt-rules] -Without an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can't modify most settings on Elastic prebuilt rules. You can only edit [rule actions](/solutions/security/detect-and-alert/create-detection-rule.md#rule-schedule) and [add exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md). If you want to modify other settings on a prebuilt rule, you must first duplicate it, then make your changes to the duplicated rule. Note that your customized rule is entirely separate from the original prebuilt rule, and will not get updates from Elastic if the prebuilt rule is updated. +Without an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can't modify most settings on Elastic prebuilt rules. You can only edit [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-schedule) and [add exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md). If you want to modify other settings on a prebuilt rule, you must first duplicate it, then make your changes to the duplicated rule. Note that your customized rule is entirely separate from the original prebuilt rule, and will not get updates from Elastic if the prebuilt rule is updated. 1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). 2. In the **Rules** table, select the **Elastic rules** filter. @@ -119,7 +120,7 @@ You can then modify the duplicated rules and, if required, delete the prebuilt o The following steps are only applicable if you have a [Platinum](https://www.elastic.co/pricing) subscription or lower on {{stack}} or a [Security Analytics Essentials project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}. -If you have an Enterprise subscription on {{stack}} or a Security Analytics Complete project on {{serverless-short}}, follow the guidelines in [Update modified and unmodified Elastic prebuilt rules](/solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md) instead. +If you have an Enterprise subscription on {{stack}} or a Security Analytics Complete project on {{serverless-short}}, follow the guidelines in [Update modified and unmodified Elastic prebuilt rules](/solutions/security/detect-and-alert/update-prebuilt-rules.md) instead. :::: Elastic regularly updates prebuilt rules to optimize their performance and ensure they detect the latest threats and techniques. When updated versions are available for your installed prebuilt rules, the **Rule Updates** tab appears on the **Rules** page, allowing you to update your installed rules with the latest versions. @@ -155,5 +156,5 @@ Elastic regularly updates prebuilt rules to optimize their performance and ensur * Update multiple rules: Select the rules and click **Update *x* selected rule(s)**. ::::{tip} - Use the search bar and **Tags** filter to find the rules you want to update. For example, filter by `OS: Windows` if your environment only includes Windows endpoints. For more on tag categories, refer to [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#prebuilt-rule-tags). + Use the search bar and **Tags** filter to find the rules you want to update. For example, filter by `OS: Windows` if your environment only includes Windows endpoints. For more on tag categories, refer to [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#prebuilt-rule-tags). :::: \ No newline at end of file diff --git a/solutions/security/detect-and-alert/launch-timeline-from-investigation-guides.md b/solutions/security/detect-and-alert/launch-timeline-from-investigation-guides.md deleted file mode 100644 index eedeef2dc1..0000000000 --- a/solutions/security/detect-and-alert/launch-timeline-from-investigation-guides.md +++ /dev/null @@ -1,140 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/security/current/interactive-investigation-guides.html - - https://www.elastic.co/guide/en/serverless/current/security-interactive-investigation-guides.html -applies_to: - stack: all - serverless: - security: all -products: - - id: security - - id: cloud-serverless ---- - -# Launch Timeline from investigation guides [security-interactive-investigation-guides] - -Detection rule investigation guides suggest steps for triaging, analyzing, and responding to potential security issues. For custom rules, you can create an interactive investigation guide that includes buttons for launching runtime queries in [Timeline](/solutions/security/investigate/timeline.md), using alert data and hard-coded literal values. This allows you to start detailed Timeline investigations directly from an alert using relevant data. - -:::{image} /solutions/images/security-ig-alert-flyout.png -:alt: Alert details flyout with interactive investigation guide -:screenshot: -::: - -Under the Investigation section, click **Show investigation guide** to open the **Investigation** tab in the left panel of the alert details flyout. - -:::{image} /solutions/images/security-ig-alert-flyout-invest-tab.png -:alt: Alert details flyout with interactive investigation guide -:screenshot: -::: - -The **Investigation** tab displays query buttons, and each query button displays the number of event documents found. Click the query button to automatically load the query in Timeline, based on configuration settings in the investigation guide. - -:::{image} /solutions/images/security-ig-timeline.png -:alt: Timeline with query pre-loaded from investigation guide action -:screenshot: -::: - - -## Add investigation guide actions to a rule [add-ig-actions-rule] - -::::{note} -You can only create interactive investigation guides with custom rules because Elastic prebuilt rules can’t be edited. However, you can duplicate a prebuilt rule, then configure the investigation guide for the duplicated rule. -:::: - - -You can configure an interactive investigation guide when you [create a new rule](/solutions/security/detect-and-alert/create-detection-rule.md) or [edit an existing rule](/solutions/security/detect-and-alert/manage-detection-rules.md#edit-rules-settings). - -1. When configuring the rule’s settings (the **About rule** step for a new rule, or the **About** tab for an existing rule), expand the **Advanced settings**, then scroll down to the **Investigation guide** Markdown editor. - - :::{image} /solutions/images/security-ig-investigation-guide-editor.png - :alt: Investigation guide editor field - :screenshot: - ::: - -2. Place the editor cursor where you want to add the query button in the investigation guide, then select the Investigate icon (![Investigate icon](/solutions/images/security-ig-investigate-icon.png "title =20x20")) in the toolbar. The **Add investigation query** builder form appears. - - :::{image} /solutions/images/security-ig-investigation-query-builder.png - :alt: Add investigation guide UI - :screenshot: - ::: - -3. Complete the query builder form to create an investigation query: - - 1. **Label**: Enter the text to appear on the query button. - 2. **Description**: (Optional) Enter additional text to include with the button. - 3. **Filters**: Select fields, operators, and values to build the query. Click **OR** or **AND** to create multiple filters and define their relationships. - - To use a field value from the alert as a query parameter, enter the field name surrounded by double curly brackets — such as `{{kibana.alert.example}}` — as a custom option for the filter value. - - :::{image} /solutions/images/security-ig-filters-field-custom-value.png - :alt: Add investigation guide UI - :screenshot: - ::: - - 4. **Relative time range**: (Optional) Select a time range to limit the query, relative to the alert’s creation time. - -4. Click **Save changes**. The syntax is added to the investigation guide editor. - - ::::{note} - If you need to change the query button’s configuration, you can either edit the syntax directly in the editor (refer to the [syntax reference](/solutions/security/detect-and-alert/launch-timeline-from-investigation-guides.md#query-button-syntax) below), or delete the syntax and use the query builder form to recreate the query. - :::: - -5. Save and enable the rule. - - -### Query button syntax [query-button-syntax] - -The following syntax defines a query button in an interactive investigation guide. - -| Field | Description | -| --- | --- | -| `!{investigate{ }}` | The container object holding all the query button’s configuration attributes. | -| `label` | Identifying text on the button. | -| `description` | Additional text included with the button. | -| `providers` | A two-level nested array that defines the query to run in Timeline. Similar to the structure of queries in Timeline, items in the outer level are joined by an `OR` relationship, and items in the inner level are joined by an `AND` relationship.

Each item in `providers` corresponds to a filter created in the query builder UI and is defined by these attributes:

- `field`: The name of the field to query.
- `excluded`: Whether the query result is excluded (such as **is not one of**) or included (**is one of**).
- `queryType`: The query type used to filter events, based on the filter’s operator. For example, `phrase` or `range`.
- `value`: The value to search for. Either a hard-coded literal value, or the name of an alert field (in double curly brackets) whose value you want to use as a query parameter.
- `valueType`: The data type of `value`, such as `string` or `boolean`.
| -| `relativeFrom`, `relativeTo` | (Optional) The start and end, respectively, of the relative time range for the query. Times are relative to the alert’s creation time, represented as `now` in [date math](elasticsearch://reference/elasticsearch/rest-apis/common-options.md#date-math) format. For example, selecting **Last 15 minutes** in the query builder form creates the syntax `"relativeFrom": "now-15m", "relativeTo": "now"`. | - -::::{note} -Some characters must be escaped with a backslash, such as `\"` for a quotation mark and `\\` for a literal backslash. Divide Windows paths with double backslashes (for example, `C:\\Windows\\explorer.exe`), and paths that already include double backslashes might require four backslashes for each divider. A clickable error icon (![Error icon](/solutions/images/security-ig-error-icon.png "title =20x20")) displays below the Markdown editor if there are any syntax errors. -:::: - - - -### Example syntax [_example_syntax] - -```json -!{investigate{ - "label": "Test action", - "description": "Click to investigate.", - "providers": [ - [ - {"field": "event.id", "excluded": false, "queryType": "phrase", "value": "{{event.id}}", "valueType": "string"} - ], - [ - {"field": "event.action", "excluded": false, "queryType": "phrase", "value": "rename", "valueType": "string"}, - {"field": "process.pid", "excluded": false, "queryType": "phrase", "value": "{{process.pid}}", "valueType": "string"} - ] - ], - "relativeFrom": "now-15m", - "relativeTo": "now" -}} -``` - -This example creates the following Timeline query, as illustrated below: - -`(event.id : )`
`OR (event.action : "rename" AND process.pid : )` - -:::{image} /solutions/images/security-ig-timeline-query.png -:alt: Timeline query -:screenshot: -::: - - -### Timeline template fields [_timeline_template_fields] - -When viewing an interactive investigation guide in contexts unconnected to a specific alert (such a rule’s details page), queries open as [Timeline templates](/solutions/security/investigate/timeline-templates.md), and `parameter` fields are treated as Timeline template fields. - -:::{image} /solutions/images/security-ig-timeline-template-fields.png -:alt: Timeline template -:screenshot: -::: diff --git a/solutions/security/detect-and-alert/machine-learning.md b/solutions/security/detect-and-alert/machine-learning.md new file mode 100644 index 0000000000..23133e0f31 --- /dev/null +++ b/solutions/security/detect-and-alert/machine-learning.md @@ -0,0 +1,103 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Create detection rules that trigger on machine learning anomaly detection job results. +--- + +# {{ml-cap}} rules [ml-rule-type] + +## Overview + +{{ml-cap}} rules generate alerts when a {{ml}} {{anomaly-job}} discovers an anomaly that exceeds a defined score threshold. Unlike other rule types, {{ml}} rules do not require you to write a query. Instead, they rely on {{ml}} jobs that continuously model normal behavior and flag deviations. + +### When to use a {{ml}} rule + +{{ml}} rules are the right fit when: + +* You want to detect **behavioral anomalies** that are difficult to express as static queries, such as unusual login times, atypical data transfer volumes, or rare process executions for a given user or host. +* A {{ml}} job is already active (or you plan to enable one) that models the relevant behavior. +* You want **adaptive detection** that automatically adjusts to changing baselines without manual threshold tuning. + +{{ml}} rules are **not** the best fit when: + +* You can define the exact pattern you're looking for. Use a [custom query rule](/solutions/security/detect-and-alert/custom-query.md) or [EQL rule](/solutions/security/detect-and-alert/eql.md) instead. +* You need a count-based threshold. Use a [threshold rule](/solutions/security/detect-and-alert/threshold.md) instead. +* You need to compare events against threat intelligence. Use an [indicator match rule](/solutions/security/detect-and-alert/indicator-match.md) instead. + +### Requirements + +::::{admonition} Prerequisites +To create or edit {{ml}} rules, you need: +* The appropriate [{{stack}} subscription](https://www.elastic.co/pricing) or [{{serverless-short}} project feature tier](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). +* The [`machine_learning_admin`](elasticsearch://reference/elasticsearch/roles.md#built-in-roles-ml-admin) role in {{stack}} or the appropriate [user role](/deploy-manage/users-roles/cloud-organization/user-roles.md) in {{serverless-short}}. +* At least one {{ml}} {{anomaly-job}} active or configured to start. +:::: + +For an overview of using {{ml}} with {{elastic-sec}}, refer to [{{anomaly-detect-cap}}](/solutions/security/advanced-entity-analytics/anomaly-detection.md). + +## Configuring effective {{ml}} rules [craft-ml] + +### Selecting {{ml}} jobs + +Select one or more {{ml}} jobs that model the behavior you want to detect. {{elastic-sec}} ships with prebuilt jobs covering common use cases: + +* **Unusual login activity.** Detects logins at unusual times or from unusual locations. +* **Rare process execution.** Surfaces processes that rarely execute on a given host. +* **DNS tunneling.** Identifies unusually high DNS query volumes or rare query patterns. + +If a selected job is not currently active, it starts automatically when you enable the rule. + +### Setting the anomaly score threshold + +The anomaly score ranges from 0 to 100, where higher scores indicate stronger deviations from normal behavior. Guidelines for setting the threshold: + +| Threshold range | Effect | +|---|---| +| 25-50 | Casts a wide net. Generates more alerts, including moderate anomalies. Suitable for initial exploration of a new job. | +| 50-75 | Balanced. Surfaces significant anomalies while filtering out low-confidence results. A good starting point for most rules. | +| 75-100 | High confidence only. Generates fewer alerts but each one represents a strong deviation. Best for mature jobs with well-understood baselines. | + +### Tuning and noise reduction + +{{ml}} rules can be noisy when a job is newly deployed or the underlying data shifts. Use these techniques to reduce false positives: + +* **Raise the anomaly score threshold** if the job is generating too many low-confidence alerts. +* **Add [rule exceptions](/solutions/security/detect-and-alert/rule-exceptions.md)** to suppress alerts from known-benign anomalies, such as scheduled maintenance windows or expected batch processes. +* **Allow the job time to learn.** Most jobs need at least two weeks of data before their baselines stabilize. Anomalies flagged during the initial learning period are less reliable. + +### Understanding {{ml}} alerts + +{{ml}} alerts differ from other alert types: + +* Alerts are generated from **anomaly results**, not raw source events. The alert contains anomaly metadata (score, job ID, influencers, bucket time) rather than the original event fields. +* When configuring **alert suppression**, only anomaly fields are available because source event fields are not present in the anomaly results. +* **Severity and risk score overrides** based on source event fields are not applicable. Consider mapping the anomaly score to severity using severity override on anomaly-specific fields. + +::::{tip} +**See it in practice.** These prebuilt rules use {{ml}} detection: + +* **Anomalous Process For a Linux Population.** Uses a rare-process {{ml}} job to surface processes that are unusual across a fleet of Linux hosts. +* **Unusual Login Activity.** Triggers when a user logs in at unusual times or from unusual source IPs, based on a login-activity {{ml}} job. +* **Spike in Network Traffic To a Country.** Detects sudden increases in outbound network traffic to a specific country, useful for identifying data exfiltration. +:::: + +## {{ml}} field reference [ml-fields] + +The following settings are specific to {{ml}} rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). + +**{{ml-cap}} jobs** +: The {{anomaly-jobs}} whose results the rule evaluates. Select one or more jobs. If a selected job is not active, it starts automatically when the rule is enabled. + +**Anomaly score threshold** +: The minimum anomaly score (0-100) above which the rule generates alerts. Only anomalies that meet or exceed this score trigger alerts. + +**Suppress alerts by** (optional) +: Reduce repeated or duplicate alerts by grouping them on one or more fields. Only anomaly fields are available for suppression because {{ml}} alerts do not contain source event fields. For details, refer to [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md). + +**Related integrations** (optional) +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/manage-detection-alerts.md b/solutions/security/detect-and-alert/manage-detection-alerts.md index 379009cc0c..3f2553de14 100644 --- a/solutions/security/detect-and-alert/manage-detection-alerts.md +++ b/solutions/security/detect-and-alert/manage-detection-alerts.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Filter, triage, and take actions on detection alerts from the Alerts page. --- # Manage detection alerts [security-alerts-manage] @@ -45,7 +46,7 @@ The Alerts page offers various ways for you to organize and triage detection ale :screenshot: ::: -* Filter alert results to include building block alerts or to only show alerts from indicator match rules by selecting the **Additional filters** drop-down. By default, [building block alerts](/solutions/security/detect-and-alert/about-building-block-rules.md) are excluded from the Overview and Alerts pages. You can choose to include building block alerts on the Alerts page, which expands the number of alerts. +* Filter alert results to include building block alerts or to only show alerts from indicator match rules by selecting the **Additional filters** drop-down. By default, [building block alerts](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) are excluded from the Overview and Alerts pages. You can choose to include building block alerts on the Alerts page, which expands the number of alerts. ::::{note} When updating alert results to include building block alerts, the Security app searches the `.alerts-security.alerts-` index for the `kibana.alert.building_block_type` field. When looking for alerts created from indicator match rules, the app searches the same index for `kibana.alert.rule.type:'threat_match'`. @@ -302,7 +303,7 @@ For information about exceptions and how to use them, refer to [Add and manage e ::::{tip} -When you send an alert generated by a [threshold rule](/solutions/security/detect-and-alert/create-detection-rule.md) to Timeline, all matching events are listed in the Timeline, even ones that did not reach the threshold value. For example, if you have an alert generated by a threshold rule that detects 10 failed login attempts, when you send that alert to Timeline, all failed login attempts detected by the rule are listed. +When you send an alert generated by a [threshold rule](/solutions/security/detect-and-alert/using-the-rule-builder.md) to Timeline, all matching events are listed in the Timeline, even ones that did not reach the threshold value. For example, if you have an alert generated by a threshold rule that detects 10 failed login attempts, when you send that alert to Timeline, all failed login attempts detected by the rule are listed. :::: @@ -313,7 +314,7 @@ Suppose the rule that generated the alert uses a Timeline template. In this case This Timeline template uses the `host.name: "{host.name}"` dropzone filter in the rule. When alerts generated by the rule are investigated in Timeline, the `{host.name}` value is replaced with the alert’s `host.name` value. If the alerts’s `host.name` value is `Windows-ArsenalFC`, the Timeline dropzone query is `host.name: "Windows-ArsenalFC"`. ::::{note} -Refer to [Timeline](/solutions/security/investigate/timeline.md) for information on creating Timelines and Timeline templates. For information on how to add Timeline templates to rules, refer to [Create a detection rule](/solutions/security/detect-and-alert/create-detection-rule.md). +Refer to [Timeline](/solutions/security/investigate/timeline.md) for information on creating Timelines and Timeline templates. For information on how to add Timeline templates to rules, refer to [Create a detection rule](/solutions/security/detect-and-alert/using-the-rule-builder.md). :::: ## Clean up alerts [clean-up-alerts-sec] diff --git a/solutions/security/detect-and-alert/manage-detection-rules.md b/solutions/security/detect-and-alert/manage-detection-rules.md index 46c6d04fc1..a88f20e08a 100644 --- a/solutions/security/detect-and-alert/manage-detection-rules.md +++ b/solutions/security/detect-and-alert/manage-detection-rules.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: View, edit, enable, duplicate, and manage detection rules from the Rules page. --- # Manage detection rules [security-rules-ui-management] @@ -31,7 +32,7 @@ On the Rules page, you can: * Perform actions on multiple rules with [bulk actions](/solutions/security/detect-and-alert/manage-detection-rules.md#bulk-actions-reference) :::{note} -For information about the role privileges required to manage rules, refer to [Detections prerequisites and requirements](/solutions/security/detect-and-alert/detections-privileges.md). +For information about the role privileges required to manage rules, refer to [Detections prerequisites and requirements](/solutions/security/detect-and-alert/requirements-privileges.md). ::: @@ -72,7 +73,7 @@ Edit rule settings to modify detection logic, notifications, schedules, and othe ### Requirements * **Custom rules**: You can edit and bulk-modify custom rules with any [{{stack}} subscription](https://www.elastic.co/pricing) or [{{serverless-short}} project tier](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). -* **Prebuilt rules**: You can edit [rule notifications](/solutions/security/detect-and-alert/create-detection-rule.md#rule-notifications) with any subscription or project tier. Editing all other prebuilt rule settings (except **Author** and **License**) or bulk-modifying prebuilt rules requires an [Enterprise subscription](https://www.elastic.co/pricing) or [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). +* **Prebuilt rules**: You can edit [rule notifications](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) with any subscription or project tier. Editing all other prebuilt rule settings (except **Author** and **License**) or bulk-modifying prebuilt rules requires an [Enterprise subscription](https://www.elastic.co/pricing) or [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). ### Edit a single rule [edit-single-rule] @@ -80,7 +81,7 @@ Edit rule settings to modify detection logic, notifications, schedules, and othe 2. Do one of the following: * In the Rules table, select the **All actions** menu {icon}`boxes_horizontal` on a rule, then select **Edit rule settings**. * Click on a rule's name to open its details page, then click **Edit rule settings**. -3. The **Edit rule settings** view opens, where you can modify the [rule's settings](/solutions/security/detect-and-alert/create-detection-rule.md). To [snooze](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) rule actions, go to the **Actions** tab and click the bell icon {icon}`bell`. +3. The **Edit rule settings** view opens, where you can modify the [rule's settings](/solutions/security/detect-and-alert/using-the-rule-builder.md). To [snooze](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) rule actions, go to the **Actions** tab and click the bell icon {icon}`bell`. 4. Click **Save changes**. ::::{admonition} Prebuilt rules @@ -100,8 +101,8 @@ Use bulk editing to update settings on multiple rules simultaneously. Rules that * **Index patterns**: Add or delete the index patterns used by all selected rules. * **Tags**: Add or delete tags on all selected rules. * **Custom highlighted fields**: Add custom highlighted fields on all selected rules. You can choose any fields that are available in the [default {{elastic-sec}} indices](/solutions/security/get-started/configure-advanced-settings.md#update-sec-indices), or enter field names from other indices. To overwrite a rule's current set of custom highlighted fields, select the **Overwrite all selected rules' custom highlighted fields** option, then click **Save**. - * **Add rule actions**: Add [rule actions](/solutions/security/detect-and-alert/create-detection-rule.md#rule-notifications) on all selected rules. If you add multiple actions, you can specify an action frequency for each of them. To overwrite the frequency of existing actions, select the option to **Overwrite all selected rules actions**. Keep in mind that rule actions won't run during a [maintenance window](/explore-analyze/alerts-cases/alerts/maintenance-windows.md); they'll resume after the maintenance window ends. - * **Update rule schedules**: Update the [schedules](/solutions/security/detect-and-alert/create-detection-rule.md#rule-schedule) and look-back times on all selected rules. + * **Add rule actions**: Add [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) on all selected rules. If you add multiple actions, you can specify an action frequency for each of them. To overwrite the frequency of existing actions, select the option to **Overwrite all selected rules actions**. Keep in mind that rule actions won't run during a [maintenance window](/explore-analyze/alerts-cases/alerts/maintenance-windows.md); they'll resume after the maintenance window ends. + * **Update rule schedules**: Update the [schedules](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-schedule) and look-back times on all selected rules. * **Apply Timeline template**: Apply a specified [Timeline template](/solutions/security/investigate/timeline-templates.md) to the selected rules. You can also choose **None** to remove Timeline templates from the selected rules. 4. On the page or flyout that opens, update the rule settings. @@ -210,7 +211,7 @@ Before manually running rules, make sure you properly understand and plan for ru 3. Specify when the manual run starts and ends. The default selection is the current day starting three hours in the past. The rule will search for events during the selected time range. 4. Click **Run** to manually run the rule. -The rule will run over the time range that you selected. Note that all [rule actions](/solutions/security/detect-and-alert/create-detection-rule.md#rule-notifications) will also be activated, except for **Summary of alerts** actions that run at a custom frequency. +The rule will run over the time range that you selected. Note that all [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) will also be activated, except for **Summary of alerts** actions that run at a custom frequency. Go to the [Manual runs table](/solutions/security/detect-and-alert/monitor-rule-executions.md#manual-runs-table) on the **Execution results** tab to track the manual rule executions. If you manually ran the rule over a gap, you can also monitor the gap fill's progress from the [Gaps table](/solutions/security/detect-and-alert/monitor-rule-executions.md#gaps-table). @@ -265,7 +266,7 @@ You can snooze rule notifications from the **Installed Rules** tab, the rule det ::::{admonition} Requirements * You can export and import custom rules and prebuilt rules (modified and unmodified) with any [{{stack}} subscription](https://www.elastic.co/pricing) or [{{serverless-short}} project feature tier](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). -* At minimum, your role needs `Read` privileges for the **Action and Connectors** feature to import rules with actions. To overwrite or add new connectors, you need `All` privileges. Refer to [Enable and access detections](/solutions/security/detect-and-alert/detections-privileges.md) to learn more about the required privileges for managing rules. +* At minimum, your role needs `Read` privileges for the **Action and Connectors** feature to import rules with actions. To overwrite or add new connectors, you need `All` privileges. Refer to [Enable and access detections](/solutions/security/detect-and-alert/requirements-privileges.md) to learn more about the required privileges for managing rules. :::: You can export custom detection rules to an `.ndjson` file, which you can then import into another {{elastic-sec}} environment. diff --git a/solutions/security/detect-and-alert/mitre-attack-coverage.md b/solutions/security/detect-and-alert/mitre-attack-coverage.md new file mode 100644 index 0000000000..fb6761e8ec --- /dev/null +++ b/solutions/security/detect-and-alert/mitre-attack-coverage.md @@ -0,0 +1,84 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/security/current/rules-coverage.html + - https://www.elastic.co/guide/en/serverless/current/security-rules-coverage.html +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: View which MITRE ATT&CK tactics and techniques your detection rules cover. +--- + +# MITRE ATT&CK coverage [security-rules-coverage] + +The **MITRE ATT&CK coverage** page shows which [MITRE ATT&CK](https://attack.mitre.org) adversary tactics and techniques are covered by your installed and enabled detection rules, including both Elastic prebuilt rules and custom rules. + +How you use this page depends on your goal: + +- **Identifying coverage gaps**: Filter to view only enabled rules, then look for light or empty cells to find tactics and techniques where your detection coverage is thin. Use this to prioritize which [prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md) to install or which [custom rules](/solutions/security/detect-and-alert/using-the-rule-builder.md) to build next. +- **Activating coverage quickly**: Find techniques you care about and [enable installed rules](#security-rules-coverage-enable-rules) directly from the coverage grid, without leaving the page. + +## Access the page [access-mitre-page] + +Find **Detection rules (SIEM)** in the navigation menu or look for "Detection rules (SIEM)" using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then go to **MITRE ATT&CK coverage**. + +:::{image} /solutions/images/security-rules-coverage.png +:alt: MITRE ATT&CK coverage page +:screenshot: +::: + +## Read the coverage grid [read-coverage-grid] + +Mirroring the MITRE ATT&CK framework, columns represent major tactics, and cells within each column represent a tactic's related techniques. Cells are darker when a technique has more rules matching the current filters, as indicated in the **Legend** at the top. + +::::{note} +The coverage grid only includes detection rules you currently have installed, and only rules that are mapped to MITRE ATT&CK. Prebuilt rules that are not installed, and custom rules that are unmapped or mapped to a deprecated tactic or technique, do not appear. You can map custom rules to tactics in **Advanced settings** when creating or editing a rule. +:::: + +## Supported MITRE ATT&CK versions [supported-versions] + +The coverage page maps detections to [MITRE ATT&CK versions](https://attack.mitre.org/resources/updates/) used by {{elastic-sec}}. + +| MITRE ATT\&CK version | {{elastic-sec}} version | +| :---- | :---- | +| [v16.1](https://attack.mitre.org/resources/updates/updates-october-2024/) | 9.0.0-9.0.6, 9.1.0-9.1.3| +| [v17.1](https://attack.mitre.org/resources/updates/updates-april-2025/) | 9.0.7, 9.1.4, 9.2.0| +| [v18.1](https://attack.mitre.org/resources/updates/updates-october-2025/) | 9.1.10, 9.2.4, {applies_to}`stack: ga 9.3.0`, {{serverless-short}}| + + +## Filter rules [security-rules-coverage-filter-rules] + +Use the drop-down filters at the top of the page to control which of your installed detection rules are included in calculating coverage. + +* **Installed rule status**: Select to include **Enabled rules**, **Not enabled rules**, or both. +* **Installed rule type**: Select to include **Elastic rules** (prebuilt rules), **Custom rules** (user-created rules), or both. + +You can also search for a tactic or technique name, technique number, or rule name in the search bar. The search bar acts as a filter for the coverage grid: only rules matching the search term are included. + +::::{note} +Searches for tactics and techniques must match exactly, are case sensitive, and do *not* support wildcards. +:::: + + + +## Expand and collapse cells [security-rules-coverage-expand-and-collapse-cells] + +Select **Collapse cells** or **Expand cells** to change how much information the cells display. Cells always include the technique's name and the number of sub-techniques covered by enabled rules. Expand the cells to also display counts of not-enabled and enabled rules for each technique. + +::::{note} +The counts inside cells are affected by how you filter the page. For example, if you filter the **Installed rule status** to only include **Enabled rules**, then all not-enabled rule counts are 0 because those rules are filtered out. +:::: + + + +## Enable rules [security-rules-coverage-enable-rules] + +You can quickly enable all the rules for a specific technique that you've installed but not yet enabled. Select the technique's cell, then select **Enable all disabled** in the popup that appears. + + +## Learn more about techniques and sub-techniques [security-rules-coverage-learn-more-about-techniques-and-sub-techniques] + +For more information on a specific technique and its sub-techniques, select the technique's cell, then select the title in the popup that appears. This opens a new browser tab with the technique's MITRE ATT&CK documentation. diff --git a/solutions/security/detect-and-alert/mitre-attandckr-coverage.md b/solutions/security/detect-and-alert/mitre-attandckr-coverage.md deleted file mode 100644 index bb585920a9..0000000000 --- a/solutions/security/detect-and-alert/mitre-attandckr-coverage.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/security/current/rules-coverage.html - - https://www.elastic.co/guide/en/serverless/current/security-rules-coverage.html -applies_to: - stack: all - serverless: - security: all -products: - - id: security - - id: cloud-serverless ---- - -# MITRE ATT&CK® coverage [security-rules-coverage] - -The **MITRE ATT&CK® coverage** page shows which [MITRE ATT&CK®](https://attack.mitre.org) adversary tactics and techniques are covered by your installed and enabled detection rules. This includes both Elastic prebuilt rules and custom rules. - -Mirroring the MITRE ATT&CK® framework, columns represent major tactics, and cells within each column represent a tactic’s related techniques. Cells are darker when a technique has more rules matching the current filters, as indicated in the **Legend** at the top. - -To access the **MITRE ATT&CK® coverage** page, find **Detection rules (SIEM)** in the navigation menu or look for “Detection rules (SIEM)” using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then go to **MITRE ATT&CK® coverage**. - -::::{note} -This page only includes the detection rules you currently have installed, and only rules that are mapped to MITRE ATT&CK®. The coverage page maps detections to [MITRE ATT&CK® versions](https://attack.mitre.org/resources/updates/) used by {{elastic-sec}}. - - -Elastic prebuilt rules that aren’t installed and custom rules that are either unmapped or mapped to a deprecated tactic or technique will not appear on the coverage map. You can map custom rules to tactics in **Advanced settings** when creating or editing a rule. -:::: - - -:::{image} /solutions/images/security-rules-coverage.png -:alt: MITRE ATT&CK® coverage page -:screenshot: -::: - -Refer to the following table to find the MITRE ATT&CK® version that's mapped to your version of {{elastic-sec}}. - -| MITRE ATT\&CK® version | {{elastic-sec}} version | -| :---- | :---- | -| [v16.1](https://attack.mitre.org/resources/updates/updates-october-2024/) | • 9.0.0-9.0.6
• 9.1.0-9.1.3| -| [v17.1](https://attack.mitre.org/resources/updates/updates-april-2025/) | • 9.0.7
• 9.1.4
• 9.2.0| -| [v18.1](https://attack.mitre.org/resources/updates/updates-october-2025/) | • 9.1.10
• 9.2.4
• {applies_to}`stack: ga 9.3.0`
• {{serverless-short}}| - - -## Filter rules [security-rules-coverage-filter-rules] - -Use the drop-down filters at the top of the page to control which of your installed detection rules are included in calculating coverage. - -* **Installed rule status**: Select to include **Enabled rules**, **Disabled rules**, or both. -* **Installed rule type**: Select to include **Elastic rules** (prebuilt rules), **Custom rules** (user-created rules), or both. - -You can also search for a tactic or technique name, technique number, or rule name in the search bar. The search bar acts as a filter for the coverage grid: only rules matching the search term will be included. - -::::{note} -Searches for tactics and techniques must match exactly, are case sensitive, and do *not* support wildcards. -:::: - - - -## Expand and collapse cells [security-rules-coverage-expand-and-collapse-cells] - -Click **Collapse cells** or **Expand cells** to change how much information the cells display. Cells always include the technique’s name and the number of sub-techniques covered by enabled rules. Expand the cells to also display counts of disabled and enabled rules for each technique. - -::::{note} -The counts inside cells are affected by how you filter the page. For example, if you filter the **Installed rule status** to only include **Enabled rules**, then all disabled rule counts will be 0 because disabled rules are filtered out. -:::: - - - -## Enable rules [security-rules-coverage-enable-rules] - -You can quickly enable all the rules for a specific technique that you’ve installed, but not enabled. Click the technique’s cell, then click **Enable all disabled** in the popup that appears. - - -## Learn more about techniques and sub-techniques [security-rules-coverage-learn-more-about-techniques-and-sub-techniques] - -For more information on a specific technique and its sub-techniques, click the technique’s cell, then click the title in the popup that appears. This opens a new browser tab with the technique’s MITRE ATT&CK® documentation. diff --git a/solutions/security/detect-and-alert/monitor-rule-executions.md b/solutions/security/detect-and-alert/monitor-rule-executions.md index a09c01242b..18d3db496b 100644 --- a/solutions/security/detect-and-alert/monitor-rule-executions.md +++ b/solutions/security/detect-and-alert/monitor-rule-executions.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Monitor rule executions, execution logs, gaps, and manual runs for detection rules. --- # Monitor rule executions [alerts-ui-monitor] diff --git a/solutions/security/detect-and-alert/new-terms.md b/solutions/security/detect-and-alert/new-terms.md new file mode 100644 index 0000000000..ae6ae5047d --- /dev/null +++ b/solutions/security/detect-and-alert/new-terms.md @@ -0,0 +1,100 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Create detection rules that alert when a field value appears for the first time in a history window. +--- + +# New terms rules [new-terms-rule-type] + +## Overview + +New terms rules detect the first appearance of a field value (or combination of field values) within a configurable history window. The rule compares each value it finds during its current execution against a historical baseline, and generates an alert only when the value has never appeared in that baseline period. + +### When to use a new terms rule + +New terms rules are the right fit when: + +* You want to detect **first-seen activity**, such as a user logging in from a new country, a process executing for the first time on a host, or a new domain appearing in DNS logs. +* The threat signal is **novelty**, not a specific pattern or count. +* You want to combine up to three fields to detect **novel combinations**, such as a `host.ip` and `user.name` pair that has never been observed together before. + +New terms rules are **not** the best fit when: + +* You are looking for a known bad value. Use a [custom query rule](/solutions/security/detect-and-alert/custom-query.md) instead. +* You need to detect spikes in volume. Use a [threshold rule](/solutions/security/detect-and-alert/threshold.md) instead. +* You want statistical {{anomaly-detect}} without defining explicit fields. Use a [{{ml}} rule](/solutions/security/detect-and-alert/machine-learning.md) instead. + +### Data requirements + +New terms rules require at least one {{es}} index pattern or [{{data-source}}](/solutions/security/get-started/data-views-elastic-security.md) with a sufficient history of events. The history window must contain enough data to establish a reliable baseline. A window that is too short leads to excessive false positives as many values appear "new". + +## Writing effective new terms rules [craft-new-terms] + +### Selecting fields + +Select the field whose novelty you want to detect. Fields with moderate cardinality work best: + +* **Good candidates:** `source.geo.country_name`, `process.executable`, `dns.question.name`, `user.name` +* **Poor candidates:** High-cardinality fields like `event.id` or `_id` (nearly every value is unique, so nearly every event triggers an alert). + +### Multi-field combinations + +You can select up to three fields to detect novel combinations. For example, selecting `host.name` and `user.name` generates an alert when a user logs in to a host they have never used before, even if both the user and host are individually well-known. + +::::{important} +Each unique combination of values is evaluated separately. A document with `host.name: ["host-1", "host-2"]` and `user.name: ["user-1", "user-2", "user-3"]` produces 6 (2x3) combinations. The rule evaluates a maximum of 100 unique combinations per document. Fields with large arrays of values may produce incorrect results. +:::: + +### Sizing the history window + +The **History Window Size** determines how far back the rule looks to decide whether a term is new. The window must be larger than the rule interval plus any additional look-back time. + +| Window size | Best for | Trade-off | +|---|---|---| +| 1-7 days | Fast-moving environments, high-volume data | More false positives from values that are infrequent | +| 7-30 days | General-purpose detection | Balanced coverage | +| 30-90 days | Detecting truly rare first appearances | Higher resource usage, slower execution | + +### Best practices + +* **Start with a longer history window and shorten it** as you understand your data's baseline patterns. +* **Pair with exceptions** to suppress known-benign first appearances, such as expected new employees or scheduled system changes. +* **Monitor rule execution time.** Large history windows on high-volume indices can increase execution time significantly. + +::::{tip} +**See it in practice.** These prebuilt rules use new terms detection: + +* **First Time Seen Commonly Exploited Remote Access Tool Execution.** Detects the first execution of a known remote access tool on any host, surfacing tools that may have been recently deployed by an adversary. +* **First Occurrence of User Agent For a GitHub Personal Access Token (PAT).** Detects a new user agent string associated with a GitHub PAT, which may indicate token compromise. +* **First Time Seen Google Workspace OAuth Login from Third-Party Application.** Detects novel third-party OAuth application logins, surfacing potential unauthorized integrations. +:::: + +## New terms field reference [new-terms-fields] + +The following settings are specific to new terms rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). + +**Index patterns or {{data-source}}** +: The {{es}} indices or {{data-source}} the rule searches. + +**Custom query** +: The KQL or Lucene query used to filter events before evaluating new terms. Only matching documents are checked for new field values. + +**Fields** +: The field (or up to three fields) to check for new terms. When multiple fields are selected, the rule detects novel combinations of their values. + +**History Window Size** +: The time range (in minutes, hours, or days) to search for historical occurrences of a term. A term is considered new only if it does not appear in the history window outside of the current rule interval and additional look-back time. + +**Suppress alerts by** (optional) +: Reduce repeated or duplicate alerts by grouping them on one or more fields. For details, refer to [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md). + +**Required fields** (optional) +: An informational list of fields the rule needs to function. This does not affect rule execution. + +**Related integrations** (optional) +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/prebuilt-rules.md b/solutions/security/detect-and-alert/prebuilt-rules.md new file mode 100644 index 0000000000..25d40da887 --- /dev/null +++ b/solutions/security/detect-and-alert/prebuilt-rules.md @@ -0,0 +1,23 @@ +--- +applies_to: + stack: all + serverless: + security: all +products: + - id: security + - id: cloud-serverless +description: Overview of Elastic's prebuilt detection rules library mapped to MITRE ATT&CK. +--- + +# Prebuilt rules + +Elastic maintains a library of prebuilt detection rules mapped to the MITRE ATT&CK framework. Enabling prebuilt rules is the fastest path to detection coverage and the recommended starting point before building custom rules. You can browse the full [prebuilt rule catalog](detection-rules://index.md) to see what's available. + +**[Install and manage prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md)** +: Start here if you're setting up prebuilt rules for the first time, or if you need to install new rules, enable or turn off existing ones, or review what data sources each rule requires. Also covers rule tags, customization options, and how to export or duplicate prebuilt rules. + +**[Update prebuilt rules](/solutions/security/detect-and-alert/update-prebuilt-rules.md)** +: Relevant when Elastic releases rule updates and you need to decide how to apply them. Explains how the update process works for rules you haven't modified versus rules you've customized, and how to resolve conflicts between your changes and Elastic's updates. Requires an Enterprise subscription on {{stack}} or a Security Analytics Complete project on {{serverless-short}}. + +**[MITRE ATT&CK coverage](/solutions/security/detect-and-alert/mitre-attack-coverage.md)** +: Use this page to visualize which MITRE ATT&CK tactics and techniques your installed rules cover. Helpful for identifying gaps in your detection posture and deciding which additional prebuilt rules to enable or where to build custom rules. diff --git a/solutions/security/detect-and-alert/query-alert-indices.md b/solutions/security/detect-and-alert/query-alert-indices.md index 8f9bd45ceb..29cadc3eda 100644 --- a/solutions/security/detect-and-alert/query-alert-indices.md +++ b/solutions/security/detect-and-alert/query-alert-indices.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Query alert indices when building rule queries, dashboards, or visualizations in Elastic Security. --- # Query alert indices [security-query-alert-indices] diff --git a/solutions/security/detect-and-alert/reduce-noise-and-false-positives.md b/solutions/security/detect-and-alert/reduce-noise-and-false-positives.md new file mode 100644 index 0000000000..0cd19eeeb9 --- /dev/null +++ b/solutions/security/detect-and-alert/reduce-noise-and-false-positives.md @@ -0,0 +1,103 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: "Compare and choose the right mechanism to reduce alert noise: tuning, exceptions, suppression, or snoozing." +--- + +# Reduce noise and false positives + +{{elastic-sec}} provides four distinct mechanisms for managing alert noise, each operating at a different point in the pipeline from raw events to analyst notification. Selecting the wrong tool for your situation reduces effectiveness and can obscure real threats. This page explains when to use each mechanism and how they work together. + +## Select the right approach + +Use the following table to identify which mechanism matches your situation. If multiple rows apply, refer to [Using them together](#using-them-together). + +| Your situation | Use this | What it does | +|---|---|---| +| The rule fires on activity that isn't suspicious in any environment (for example, the query is too broad and catches normal admin behavior) | [Tune rule logic](/solutions/security/detect-and-alert/tune-detection-rules.md) | Refine the query, threshold, or schedule so the rule only matches what it should | +| The rule correctly identifies a pattern, but a specific known-safe case keeps firing in your environment (for example, an SCCM server triggers your remote service creation rule nightly) | [Exception](/solutions/security/detect-and-alert/rule-exceptions.md) | Permanently suppress alerts matching specific field values. The rule logic stays unchanged. | +| The rule generates many alerts for the same entity in a short window (for example, a compromised host triggers the same rule every 5 minutes for an hour) | [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md) | After the first alert for a given field value, suppress further alerts for a defined time window | +| You need to temporarily stop receiving notifications during scheduled maintenance or a known event | [Snooze actions](/solutions/security/detect-and-alert/snooze-rule-actions.md) | The rule runs and alerts are still created and stored. Only notifications (email, Slack, webhooks) are paused. | +| A specific known-safe process or user keeps appearing across many different rules (for example, your vulnerability scanner triggers dozens of rules) | [Shared exception list](/solutions/security/detect-and-alert/create-manage-shared-exception-lists.md) | Apply one exception list across multiple rules simultaneously, avoiding duplicate exceptions rule by rule | +| You're deploying a new rule and want to observe alert volume before enabling notifications | [Snooze actions](/solutions/security/detect-and-alert/snooze-rule-actions.md) | Enable the rule fully, review alerts in the UI, and only enable notifications once you're confident in the signal | + +## How each mechanism works + +Each mechanism intervenes at a different stage of the detection pipeline. They are listed here in pipeline order, from earliest to latest. + +### Tune rule logic + +Acts on: **the rule query, before events are evaluated** + +*"The rule is catching the wrong things."* + +Modify the rule query, threshold count, look-back window, or schedule to improve detection precision. Tuning is the only mechanism that improves the underlying signal rather than filtering output. Alerts are still created for events matching the revised logic, and change history requires version tracking. Tuning affects all environments using the rule. + +Refer to [Tune detection rules](/solutions/security/detect-and-alert/tune-detection-rules.md) for detailed guidance. + +### Rule exceptions + +Acts on: **alert creation, after the rule evaluates but before the alert is written** + +*"This specific case is acceptable in my environment."* + +Permanently prevent alert creation for events matching defined field conditions. The rule logic stays unchanged. Exceptions are auditable and removable at any time. They can apply to a single rule or be shared across multiple rules through [shared exception lists](/solutions/security/detect-and-alert/create-manage-shared-exception-lists.md). + +Refer to [Rule exceptions](/solutions/security/detect-and-alert/rule-exceptions.md) for detailed guidance. + +### Alert suppression + +Acts on: **alert deduplication, after the first alert is created** + +*"I already know about this. Don't repeat it."* + +After the first alert fires for a given field value (for example, a specific host or user), suppress subsequent duplicate alerts for a defined time window. Unlike exceptions, suppression is temporary and time-bounded. It acknowledges the signal is real but reduces repetitive noise. Configured per rule, grouped by field value. + +Refer to [Suppress detection alerts](/solutions/security/detect-and-alert/alert-suppression.md) for detailed guidance. + +### Snooze rule actions + +Acts on: **notifications, after alerts are created and stored** + +*"Don't page anyone right now. I'll review later."* + +Temporarily pause the rule's notification actions (emails, Slack messages, webhooks) without affecting rule execution or alert creation. Alerts generated during a snooze period are stored normally and visible in the Alerts UI. Snoozing expires automatically or can be canceled manually. For space-wide pausing, use a [maintenance window](/explore-analyze/alerts-cases/alerts/maintenance-windows.md). + +Refer to [Snooze rule actions](/solutions/security/detect-and-alert/snooze-rule-actions.md) for detailed guidance. + +## Key distinctions + +| | Tuning logic | Rule exceptions | Alert suppression | Snooze rule actions | +|---|---|---|---|---| +| Rule still runs | Yes | Yes | Yes | Yes | +| Alert written to index | Modified | No | First only | Yes | +| Analyst notified | For matches | No | First only | No | +| Alert visible in UI | For matches | No | First only | Yes, all alerts | +| Time-bounded | No | No, permanent | Yes, configurable window | Yes, configurable duration | +| Can span multiple rules | No | Yes, shared lists | No | No | + +::::{important} +Exceptions and suppression have different forensic implications. Exceptions permanently prevent alert records from being created. If you later need to investigate whether a specific event occurred, the alert data won't be there. Suppression writes the first alert but drops subsequent ones. For high-value assets or regulated environments, prefer suppression over exceptions so there's always at least one alert record for any real detection event. Reserve exceptions for activity that is definitively never relevant. +:::: + +## Using them together [using-them-together] + +These four mechanisms are not mutually exclusive. A well-tuned ruleset typically uses all of them simultaneously for different purposes. The following scenario illustrates how they layer on top of each other in practice. + +**Scenario: A noisy lateral movement rule in a real environment** + +| Situation | Action | Mechanism | +|---|---|---| +| The base query matches too broadly, catching any remote service creation including from `C:\Windows\` | Narrow the query to exclude standard install paths. The rule now only fires on services installed to non-standard locations. | Tune rule logic | +| The SCCM deployment server still triggers it every deployment cycle from a known management IP | Add an exception: `source.ip: "10.1.5.20" AND user.name: "svc-sccm"`. Those specific alerts are never created. | Exception | +| A compromised host generates 30 alerts in an hour. Analysts only need to know about the first one. | Enable alert suppression grouped by `host.id` with a 1-hour window. One alert per host per hour. | Suppression | +| Planned maintenance window tonight. Legitimate admin activity will trigger alerts, but no one should be paged. | Snooze rule actions for 4 hours. Alerts are created and stored. Notifications are paused. Review them in the morning. | Snooze actions | + +::::{note} +Order of application matters. Tuning and exceptions are evaluated before an alert is created. Suppression deduplicates after creation. Snoozing acts after suppression, only on notifications. Applying them in the wrong conceptual order leads to gaps. For example, using suppression to handle what should be an exception means the first alert is still stored and may page someone, and the suppression window may expire before the condition is resolved. +:::: diff --git a/solutions/security/detect-and-alert/detections-requirements.md b/solutions/security/detect-and-alert/requirements-privileges.md similarity index 85% rename from solutions/security/detect-and-alert/detections-requirements.md rename to solutions/security/detect-and-alert/requirements-privileges.md index 2f54904d29..f0f02d8bd4 100644 --- a/solutions/security/detect-and-alert/detections-requirements.md +++ b/solutions/security/detect-and-alert/requirements-privileges.md @@ -33,7 +33,7 @@ Refer to [Predefined roles](/solutions/security/detect-and-alert/detections-priv ::::::{tab-item} {{ecloud}} -The detection engine initializes automatically when a user with [sufficient privileges](/solutions/security/detect-and-alert/detections-privileges.md) visits the **Rules** page. To open the page, find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). +The detection engine initializes automatically when a user with [sufficient privileges](/solutions/security/detect-and-alert/requirements-privileges.md) visits the **Rules** page. To open the page, find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). No additional configuration is required. @@ -71,7 +71,7 @@ In your [`elasticsearch.yml`](/deploy-manage/deploy/self-managed/configure-elast ::::{step} Enable detections 1. Go to the **Rules** page. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). -2. The detection engine initializes when a user with [sufficient privileges](/solutions/security/detect-and-alert/detections-privileges.md) visits the page. +2. The detection engine initializes when a user with [sufficient privileges](/solutions/security/detect-and-alert/requirements-privileges.md) visits the page. ::::{note} To enable detections in multiple spaces, visit the **Rules** page in each space. @@ -90,13 +90,13 @@ To enable detections in multiple spaces, visit the **Rules** page in each space. With detections enabled, you're ready to create rules and start responding to threats. Do the following: - **Add detection rules** - - [Install Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md): Get started quickly with hundreds of rules that detect common threats - - [Create custom rules](/solutions/security/detect-and-alert/create-detection-rule.md): Write rules tailored to your environment + - [Install Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md): Get started quickly with hundreds of rules that detect common threats + - [Create custom rules](/solutions/security/detect-and-alert/using-the-rule-builder.md): Write rules tailored to your environment - [Configure {{anomaly-detect}} rules](/solutions/security/advanced-entity-analytics/machine-learning-job-rule-requirements.md): Use {{ml}} to detect unusual behavior - **Respond to and manage alerts** - [Manage detection alerts](/solutions/security/detect-and-alert/manage-detection-alerts.md): Triage, investigate, and resolve alerts - - [Set up alert notifications](/solutions/security/detect-and-alert/create-detection-rule.md#rule-notifications): Send alerts to external systems like Slack, email, or ticketing tools + - [Set up alert notifications](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications): Send alerts to external systems like Slack, email, or ticketing tools - [Tune rules to reduce noise](/solutions/security/detect-and-alert/tune-detection-rules.md): Add exceptions and adjust rules to minimize false positives ::::{tip} diff --git a/solutions/security/detect-and-alert/rule-exceptions.md b/solutions/security/detect-and-alert/rule-exceptions.md index 87b70f204e..06d5d1dc1e 100644 --- a/solutions/security/detect-and-alert/rule-exceptions.md +++ b/solutions/security/detect-and-alert/rule-exceptions.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Associate rule exceptions with detection rules to prevent trusted activity from generating unnecessary alerts. --- # Rule exceptions [detections-ui-exceptions] diff --git a/solutions/security/detect-and-alert/rule-settings-reference.md b/solutions/security/detect-and-alert/rule-settings-reference.md new file mode 100644 index 0000000000..1684f43501 --- /dev/null +++ b/solutions/security/detect-and-alert/rule-settings-reference.md @@ -0,0 +1,303 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Reference for all detection rule settings including basic, advanced, schedule, actions, and notification variables. +--- + +# Rule settings reference [detection-rule-settings-reference] + +All detection rules share a common set of settings for describing the rule, controlling its schedule, configuring actions, and setting up response actions. These settings apply regardless of the [rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) you select. + +For rule-type-specific settings (query definitions, index patterns, {{ml}} jobs, and so on), refer to [Using the rule builder](/solutions/security/detect-and-alert/using-the-rule-builder.md). + +## Basic settings [rule-ui-basic-params] + +Configure these settings in the **About rule** pane. + +**Name** +: The rule's name. + +**Description** +: A description of what the rule does. + +**Default severity** +: The severity level of alerts created by the rule: + + * **Low**: Alerts that are of interest but generally are not considered to be security incidents. Sometimes a combination of low severity alerts can indicate suspicious activity. + * **Medium**: Alerts that require investigation. + * **High**: Alerts that require an immediate investigation. + * **Critical**: Alerts that indicate it is highly likely a security incident has occurred. + +**Severity override** (optional) +: Select to use source event values to override the **Default severity** in generated alerts. When selected, a UI component is displayed where you can map the source event field values to severity levels. + + :::{image} /solutions/images/security-severity-mapping-ui.png + :alt: severity mapping ui + :screenshot: + ::: + + ::::{note} + For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. Overrides are not supported for event correlation rules. + :::: + +**Default risk score** +: A numerical value between 0 and 100 that indicates the risk of events detected by the rule. This setting changes to a default value when you change the **Severity** level, but you can adjust the risk score as needed. General guidelines are: + + * `0` - `21` represents low severity. + * `22` - `47` represents medium severity. + * `48` - `73` represents high severity. + * `74` - `100` represents critical severity. + +**Risk score override** (optional) +: Select to use a source event value to override the **Default risk score** in generated alerts. When selected, a UI component is displayed to select the source field used for the risk score. + + :::{image} /solutions/images/security-risk-source-field-ui.png + :alt: risk source field ui + :screenshot: + ::: + + ::::{note} + For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. + :::: + +**Tags** (optional) +: Words and phrases used to categorize, filter, and search the rule. + +## Advanced settings [rule-ui-advanced-params] + +Configure these settings by clicking **Advanced settings** in the **About rule** pane. + +**Reference URLs** (optional) +: References to information that is relevant to the rule. For example, links to background information. + +**False positive examples** (optional) +: List of common scenarios that may produce false-positive alerts. + +**MITRE ATT&CKTM threats** (optional) +: Add relevant [MITRE](https://attack.mitre.org/) framework tactics, techniques, and subtechniques. + +**Custom highlighted fields** (optional) +: Specify highlighted fields for unique alert investigation flows. You can select any fields that are available in the indices you selected for the rule's data source. + + After you create the rule, you can find all custom highlighted fields in the About section of the rule details page. If the rule has alerts, you can find custom highlighted fields in the [Highlighted fields](/solutions/security/detect-and-alert/view-detection-alert-details.md#investigation-section) section of the alert details flyout. + +**Setup guide** (optional) +: Instructions on rule prerequisites such as required integrations, configuration steps, and anything else needed for the rule to work correctly. + +**Investigation guide** (optional) +: Information for analysts investigating alerts created by the rule. You can also add action buttons to [run Osquery](/solutions/security/investigate/run-osquery-from-investigation-guides.md) or [start Timeline investigations](/solutions/security/detect-and-alert/write-investigation-guides.md) using alert data. + +**Author** (optional) +: The rule's authors. + +**License** (optional) +: The rule's license. + +**Elastic endpoint exceptions** (optional) +: Adds all [{{elastic-endpoint}} exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md#endpoint-rule-exceptions) to this rule. + + ::::{note} + If you select this option, you can add {{elastic-endpoint}} exceptions on the Rule details page. Additionally, all future exceptions added to [endpoint protection rules](/solutions/security/manage-elastic-defend/endpoint-protection-rules.md) will also affect this rule. + :::: + +**Building block** (optional) +: Select to create a building-block rule. By default, alerts generated from a building-block rule are not displayed in the UI. See [About building block rules](/solutions/security/detect-and-alert/about-building-block-rules.md) for more information. + +**Max alerts per run** (optional) +: Specify the maximum number of alerts the rule can create each time it executes. Default is 100. + + ::::{note} + This setting can be superseded by the [{{kib}} configuration setting](kibana://reference/configuration-reference/alerting-settings.md#alert-settings) `xpack.alerting.rules.run.alerts.max`, which determines the maximum alerts generated by *any* rule in the {{kib}} alerting framework. For example, if `xpack.alerting.rules.run.alerts.max` is set to `1000`, the rule can generate no more than 1000 alerts even if **Max alerts per run** is set higher. + :::: + +**Indicator prefix override** (indicator match rules only) +: Define the location of indicator data within the structure of indicator documents. When the indicator match rule executes, it queries specified indicator indices and references this setting to locate fields with indicator data. This data is used to enrich indicator match alerts with metadata about matched threat indicators. The default value for this setting is `threat.indicator`. + + ::::{important} + If your threat indicator data is at a different location, update this setting accordingly to ensure alert enrichment can still be performed. + :::: + +**Rule name override** (optional) +: Select a source event field to use as the rule name in the UI (Alerts table). This is useful for exposing, at a glance, more information about an alert. For example, if the rule generates alerts from Suricata, selecting `event.action` lets you see what action (Suricata category) caused the event directly in the Alerts table. + + ::::{note} + For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. + :::: + +**Timestamp override** (optional) +: Select a source event timestamp field. When selected, the rule's query uses the selected field, instead of the default `@timestamp` field, to search for alerts. This can help reduce missing alerts due to network or server outages. Specifically, if your ingest pipeline adds a timestamp when events are sent to {{es}}, this can prevent missing alerts from ingestion delays. + + If the selected field is unavailable, the rule query will use the `@timestamp` field instead. If you don't want to use the `@timestamp` field because your data source has an inaccurate `@timestamp` value, select the **Do not use @timestamp as a fallback timestamp field** option instead. This ensures the rule query ignores the `@timestamp` field entirely. + + ::::{tip} + The [Microsoft](beats://reference/filebeat/filebeat-module-microsoft.md) and [Google Workspace](beats://reference/filebeat/filebeat-module-google_workspace.md) {{filebeat}} modules have an `event.ingested` timestamp field that can be used instead of the default `@timestamp` field. + :::: + +## Schedule settings [rule-schedule] + +**Runs every** +: How often the rule runs. + +**Additional look-back time** (optional) +: When defined, the rule searches indices with the additional time. + + For example, if you set a rule to run every 5 minutes with an additional look-back time of 1 minute, the rule runs every 5 minutes but analyzes the documents added to indices during the last 6 minutes. + + ::::{important} + It is recommended to set the `Additional look-back time` to at least 1 minute. This ensures there are no missing alerts when a rule does not run exactly at its scheduled time. + + {{elastic-sec}} prevents duplication. Any duplicate alerts that are discovered during the `Additional look-back time` are *not* created. + :::: + +## Rule actions [rule-notifications] + +Use actions to set up notifications sent through other systems when alerts are generated. + +::::{note} +To use actions for alert notifications, you need the [appropriate license]({{subscriptions}}). For more information, see [Cases requirements](/solutions/security/investigate/cases-requirements.md). +:::: + +::::{tip} +:applies_to: {stack: preview 9.3+, serverless: preview} +You can use [workflows](/explore-analyze/workflows.md) as a rule action to automate alert response processes. Workflows can create cases, route notifications, or perform other automated tasks when alerts are generated. To learn how to set up a workflow as a rule action, refer to [](/explore-analyze/workflows/triggers/alert-triggers.md). +:::: + +**Connector type** +: Determines how notifications are sent. For example, if you select the {{jira}} connector, notifications are sent to your {{jira}} system. + + ::::{note} + Each action type requires a connector. Connectors store the information required to send the notification from the external system. You can configure connectors while creating the rule or from the **{{connectors-ui}}** page. For more information, refer to [Action and connector types](/deploy-manage/manage-connectors.md). + + Some connectors that perform actions require less configuration. For example, you do not need to set the action frequency or variables for the [Cases connector](kibana://reference/connectors-kibana/cases-action-type.md). + :::: + +**Action frequency** +: Defines when notifications are sent: + + * **Summary of alerts**: Sends a report that summarizes generated alerts at the specified time intervals. + + ::::{note} + When setting a custom notification frequency, do not select a time that is shorter than the rule's execution schedule. + :::: + + * **For each alert**: Sends notifications every time new alerts are generated. + +**Conditional actions** (optional) +: Specify additional conditions that need to be met for notifications to send: + + * **If alert matches query**: Enter a KQL query that defines field-value pairs or query conditions that must be met for notifications to send. The query only searches alert documents in the indices specified for the rule. + * **If alert is generated during timeframe**: Set timeframe details. Notifications are only sent if alerts are generated within the timeframe you define. + +**Notification message** +: Use the default notification message or customize it. You can add more context to the message by clicking the icon above the message text box and selecting from a list of available [alert notification variables](#rule-action-variables). + +::::{important} +After you activate a rule, you can check if it is executing as expected using the [Monitoring tab](/troubleshoot/security/detection-rules.md) on the {{rules-ui}} page. If you see values in the `Gap` column, you can [Troubleshoot missing alerts](/troubleshoot/security/detection-rules.md#troubleshoot-signals). + +When a rule fails to execute, the {{security-app}} tries to rerun it at its next scheduled time. +:::: + +## Response actions [rule-response-action] + +Use response actions to set up additional functionality that executes whenever a rule triggers: + +* **Osquery**: Include live Osquery queries with a custom query rule. When an alert is generated, Osquery automatically collects data on the system related to the alert. Refer to [Add Osquery Response Actions](/solutions/security/investigate/add-osquery-response-actions.md) to learn more. +* **{{elastic-defend}}**: Automatically execute response actions on an endpoint when rule conditions are met. For example, you can automatically isolate a host or end a process when specific activities or events are detected on the host. Refer to [Automated response actions](/solutions/security/endpoint-response-actions/automated-response-actions.md) to learn more. + +::::{important} +Host isolation involves quarantining a host from the network to prevent further proliferation of threats and limit potential damage. Be aware that automatic host isolation can cause unintended consequences, such as disrupting legitimate user activities or blocking critical business processes. +:::: + +## Alert notification variables [rule-action-variables] + +You can use [mustache syntax](http://mustache.github.io/) to add variables to notification messages. The action frequency you select determines the available variables. + +::::{note} +Refer to [Action frequency: Summary of alerts](/explore-analyze/alerts-cases/alerts/rule-action-variables.md#alert-summary-action-variables) to learn about additional variables that can be passed if the rule's action frequency is **Summary of alerts**. +:::: + +### Variables for all rules [all-rule-variables] + +* `{{context.alerts}}`: Array of detected alerts +* `{{{context.results_link}}}`: URL to the alerts in {{kib}} +* `{{context.rule.anomaly_threshold}}`: Anomaly threshold score above which alerts are generated ({{ml}} rules only) +* `{{context.rule.description}}`: Rule description +* `{{context.rule.false_positives}}`: Rule false positives +* `{{context.rule.filters}}`: Rule filters (query rules only) +* `{{context.rule.id}}`: Unique rule ID returned after creating the rule +* `{{context.rule.index}}`: Indices rule runs on (query rules only) +* `{{context.rule.language}}`: Rule query language (query rules only) +* `{{context.rule.machine_learning_job_id}}`: ID of associated {{ml}} job ({{ml}} rules only) +* `{{context.rule.max_signals}}`: Maximum allowed number of alerts per rule execution +* `{{context.rule.name}}`: Rule name +* `{{context.rule.query}}`: Rule query (query rules only) +* `{{context.rule.references}}`: Rule references +* `{{context.rule.risk_score}}`: Default rule risk score + + ::::{note} + This placeholder contains the rule's default values even when the **Risk score override** option is used. + :::: + +* `{{context.rule.rule_id}}`: Generated or user-defined rule ID that can be used as an identifier across systems +* `{{context.rule.saved_id}}`: Saved search ID +* `{{context.rule.severity}}`: Default rule severity + + ::::{note} + This placeholder contains the rule's default values even when the **Severity override** option is used. + :::: + +* `{{context.rule.threat}}`: Rule threat framework +* `{{context.rule.threshold}}`: Rule threshold values (threshold rules only) +* `{{context.rule.timeline_id}}`: Associated Timeline ID +* `{{context.rule.timeline_title}}`: Associated Timeline name +* `{{context.rule.type}}`: Rule type +* `{{context.rule.version}}`: Rule version +* `{{date}}`: Date the rule scheduled the action +* `{{kibanaBaseUrl}}`: Configured `server.publicBaseUrl` value, or empty string if not configured +* `{{rule.id}}`: ID of the rule +* `{{rule.name}}`: Name of the rule +* `{{rule.spaceId}}`: Space ID of the rule +* `{{rule.tags}}`: Tags of the rule +* `{{rule.type}}`: Type of rule +* `{{state.signals_count}}`: Number of alerts detected + +### Per-alert variables [per-alert-variables] + +The following variables can only be passed if the rule's action frequency is **For each alert**: + +* `{{alert.actionGroup}}`: Action group of the alert that scheduled actions for the rule +* `{{alert.actionGroupName}}`: Human-readable name of the action group of the alert that scheduled actions for the rule +* `{{alert.actionSubgroup}}`: Action subgroup of the alert that scheduled actions for the rule +* `{{alert.id}}`: ID of the alert that scheduled actions for the rule +* `{{alert.flapping}}`: A flag on the alert that indicates whether the alert status is changing repeatedly + +### Placeholder examples [placeholder-examples] + +To understand which fields to parse, see the [Detections API](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-security-detections-api) to view the JSON representation of rules. + +Example using `{{context.rule.filters}}` to output a list of filters: + +```json +{{#context.rule.filters}} +{{^meta.disabled}}{{meta.key}} {{#meta.negate}}NOT {{/meta.negate}}{{meta.type}} {{^exists}}{{meta.value}}{{meta.params.query}}{{/exists}}{{/meta.disabled}} +{{/context.rule.filters}} +``` + +Example using `{{context.alerts}}` as an array, which contains each alert generated since the last time the action was executed: + +```json +{{#context.alerts}} +Detection alert for user: {{user.name}} +{{/context.alerts}} +``` + +Example using the mustache "current element" notation `{{.}}` to output all the rule references in the `signal.rule.references` array: + +```json +{{#signal.rule.references}} {{.}} {{/signal.rule.references}} +``` diff --git a/solutions/security/detect-and-alert/rule-types.md b/solutions/security/detect-and-alert/rule-types.md new file mode 100644 index 0000000000..a905074bf1 --- /dev/null +++ b/solutions/security/detect-and-alert/rule-types.md @@ -0,0 +1,26 @@ +--- +applies_to: + stack: all + serverless: + security: all +products: + - id: security + - id: cloud-serverless +description: Learn when to use each detection rule type and access detailed guides for custom query, EQL, threshold, and more. +--- + +# Rule type guides + +{{elastic-sec}} provides several rule types for building detections. Each rule type page covers when to use it, how to write effective queries, real-world examples, and field configuration specific to that type. + +Not sure which rule type fits your use case? Refer to [Select the right rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) for a decision guide comparing all rule types. + +| If you want to detect... | Rule type | +|---|---| +| A known field value, pattern, or boolean condition | [Custom query](/solutions/security/detect-and-alert/custom-query.md) | +| An ordered sequence of events or a missing event | [Event correlation (EQL)](/solutions/security/detect-and-alert/eql.md) | +| A field value count exceeding a boundary | [Threshold](/solutions/security/detect-and-alert/threshold.md) | +| Events matching a known threat indicator | [Indicator match](/solutions/security/detect-and-alert/indicator-match.md) | +| A field value appearing for the first time | [New terms](/solutions/security/detect-and-alert/new-terms.md) | +| Aggregated, transformed, or computed conditions | [{{esql}}](/solutions/security/detect-and-alert/esql.md) | +| Behavioral anomalies without a fixed pattern | [{{ml-cap}}](/solutions/security/detect-and-alert/machine-learning.md) | diff --git a/solutions/security/detect-and-alert/set-rule-data-sources.md b/solutions/security/detect-and-alert/set-rule-data-sources.md new file mode 100644 index 0000000000..c19722ffbf --- /dev/null +++ b/solutions/security/detect-and-alert/set-rule-data-sources.md @@ -0,0 +1,91 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/security/current/exclude-cold-frozen-data-individual-rules.html +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Configure which Elasticsearch indices rules query and exclude cold or frozen data from rule execution. +--- + +# Set rule data sources [exclude-cold-frozen-data-individual-rules] + +Every detection rule needs a data source that tells it which {{es}} indices to query. By default, rules inherit the index patterns defined in the [`securitySolution:defaultIndex`](/solutions/security/get-started/configure-advanced-settings.md#update-sec-indices) advanced setting. You can override this default on a per-rule basis to target specific indices, exclude data tiers, or use a {{data-source}} with runtime fields. + +## Per-rule index patterns [per-rule-index-patterns] + +When you create or edit a rule, the **Index patterns** field (or **Data view** selector) controls which {{es}} indices the rule queries. This field is prepopulated with the space-level defaults, but you can change it for any individual rule. + +Common reasons to override the defaults: + +**Target a narrower set of indices.** If a rule only applies to Windows endpoint data, restricting its index patterns to `winlogbeat-*` or `logs-endpoint.events.process-*` reduces the volume of data the rule scans and improves performance. + +**Broaden to additional indices.** If a rule needs data from a source that isn't in the space-level defaults (for example, a custom integration or a third-party feed), add the relevant index pattern. + +**Use a {{data-source}}.** Instead of specifying index patterns directly, you can select a {{data-source}} from the drop-down. The rule then uses the {{data-source}}'s index patterns and any [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md) defined on it, which can be useful for enrichment or field normalization. + +::::{tip} +For indicator match rules, the **Indicator index patterns** field controls which threat intelligence indices the rule queries separately from the main source index patterns. By default, this uses the [`securitySolution:defaultThreatIndex`](/solutions/security/get-started/configure-advanced-settings.md) setting (`logs-ti_*`). +:::: + +::::{note} +{{esql}} and {{ml}} rules do not use the index patterns field. {{esql}} rules define their data source within the query itself (using the `FROM` command). {{ml}} rules rely on the {{ml}} job's datafeed configuration. +:::: + +## Exclude cold and frozen data [exclude-cold-frozen-tier] + +Rules may perform slower or time out if they query data stored in cold or frozen [data tiers](../../../manage-data/lifecycle/data-tiers.md). You have two options for excluding this data: + +**Space-level setting (all rules).** Configure the `excludedDataTiersForRuleExecution` [advanced setting](../get-started/configure-advanced-settings.md#exclude-cold-frozen-data-rule-executions) to exclude cold or frozen data from all rules in a {{kib}} space. This does not apply to {{ml}} rules. Only available on {{stack}}. + +**Per-rule Query DSL filter (individual rules).** Add a Query DSL filter to the rule that ignores cold or frozen documents at query time. This gives you per-rule control and is described below. + +::::{important} +* Per-rule Query DSL filters are not supported for {{esql}} and {{ml}} rules. +* Even with this filter applied, indicator match and event correlation rules may still fail if a frozen or cold shard that matches the rule's index pattern is unavailable during rule execution. If failures occur, modify the rule's index patterns to only match indices containing hot-tier data. +:::: + +### Sample Query DSL filters [query-dsl-filter-examples] + +Exclude frozen-tier documents: + +```console +{ + "bool":{ + "must_not":{ + "terms":{ + "_tier":[ + "data_frozen" + ] + } + } + } +} +``` + +Exclude cold and frozen-tier documents: + +```console +{ + "bool":{ + "must_not":{ + "terms":{ + "_tier":[ + "data_frozen", "data_cold" + ] + } + } + } +} +``` + +To apply a filter, paste the Query DSL into the **Custom query** filter bar when creating or editing a rule. + +## Related pages + +* [Advanced data source configuration](/solutions/security/detect-and-alert/advanced-data-source-configuration.md): Deployment-level settings that affect data sources, including {{ccs}} and logsdb index mode. +* [Update default {{elastic-sec}} indices](/solutions/security/get-started/configure-advanced-settings.md#update-sec-indices): Change the space-level default index patterns inherited by all rules. +* [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md): Shared rule settings including timestamp override, which controls which timestamp field the rule uses when querying data. diff --git a/solutions/security/detect-and-alert/reduce-notifications-alerts.md b/solutions/security/detect-and-alert/snooze-rule-actions.md similarity index 70% rename from solutions/security/detect-and-alert/reduce-notifications-alerts.md rename to solutions/security/detect-and-alert/snooze-rule-actions.md index bd1adcdb24..bc110effa8 100644 --- a/solutions/security/detect-and-alert/reduce-notifications-alerts.md +++ b/solutions/security/detect-and-alert/snooze-rule-actions.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Compare rule snoozing, maintenance windows, alert suppression, and exceptions for reducing notifications. --- # Reduce notifications and alerts [security-reduce-notifications-alerts] @@ -17,7 +18,7 @@ products: | | | | --- | --- | -| [Rule action snoozing](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) | **Stops a specific rule’s notification actions from running**.

Use to avoid unnecessary notifications from a specific rule. The rule continues to run and generate alerts during the snooze period, but its [notification actions](/solutions/security/detect-and-alert/create-detection-rule.md#rule-response-action) don’t run.
| -| [Maintenance window](/explore-analyze/alerts-cases/alerts/maintenance-windows.md) | **Prevents all rules' notification actions from running**.

Use to avoid false alarms and unnecessary notifications during planned outages. All rules continue to run and generate alerts during the maintenance window, but their [notification actions](/solutions/security/detect-and-alert/create-detection-rule.md#rule-notifications) don’t run.

**Note**: Maintenance windows are a {{kib}} feature, configured outside of the {{security-app}} in **Stack Management**.

| -| [Alert suppression](/solutions/security/detect-and-alert/suppress-detection-alerts.md) | **Reduces repeated or duplicate alerts**.

Use to reduce the number of alerts created when a rule meets its criteria repeatedly. Duplicate qualifying events are grouped, and only one alert is created for each group.
| +| [Rule action snoozing](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) | **Stops a specific rule’s notification actions from running**.

Use to avoid unnecessary notifications from a specific rule. The rule continues to run and generate alerts during the snooze period, but its [notification actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-response-action) don’t run.
| +| [Maintenance window](/explore-analyze/alerts-cases/alerts/maintenance-windows.md) | **Prevents all rules' notification actions from running**.

Use to avoid false alarms and unnecessary notifications during planned outages. All rules continue to run and generate alerts during the maintenance window, but their [notification actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) don’t run.

**Note**: Maintenance windows are a {{kib}} feature, configured outside of the {{security-app}} in **Stack Management**.

| +| [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md) | **Reduces repeated or duplicate alerts**.

Use to reduce the number of alerts created when a rule meets its criteria repeatedly. Duplicate qualifying events are grouped, and only one alert is created for each group.
| | [Rule exception](/solutions/security/detect-and-alert/rule-exceptions.md) | **Prevents a rule from creating alerts under specific conditions**.

Use to reduce false positive alerts by preventing trusted processes and network activity from generating unnecessary alerts. You can configure an exception to be used by a single rule or shared among multiple rules, but they typically don’t affect *all* rules.
| diff --git a/solutions/security/detect-and-alert/threshold.md b/solutions/security/detect-and-alert/threshold.md new file mode 100644 index 0000000000..bd6f30a54a --- /dev/null +++ b/solutions/security/detect-and-alert/threshold.md @@ -0,0 +1,103 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Create threshold rules to alert when the number of matching events exceeds a specified count within a rule run. +--- + +# Threshold rules [threshold-rule-type] + +## Overview + +Threshold rules search your {{es}} indices and generate an alert when the number of events matching a query meets or exceeds a specified threshold within a single rule execution. Optionally, events can be grouped by one or more fields so that each unique combination is evaluated independently. + +### When to use a threshold rule + +Threshold rules are the right fit when: + +* You want to detect **volume-based anomalies**, such as a brute-force attack (many failed logins from a single source IP) or a data exfiltration attempt (an unusually high number of outbound connections). +* The signal is not a single event but a **count crossing a boundary** within a time window. +* You need to group counts by fields like `source.ip`, `user.name`, or `destination.ip` and alert on each group independently. + +Threshold rules are **not** the best fit when: + +* A single matching event is sufficient. Use a [custom query rule](/solutions/security/detect-and-alert/custom-query.md) instead. +* You need to detect a specific **order** of events. Use an [EQL rule](/solutions/security/detect-and-alert/eql.md) instead. +* You need full aggregation pipelines with transformations. Use an [{{esql}} rule](/solutions/security/detect-and-alert/esql.md) instead. + +### Data requirements + +Threshold rules require at least one {{es}} index pattern or {{data-source}}. The data should contain the fields you plan to group by and enough event volume for meaningful threshold evaluation. + +## Writing effective threshold rules [craft-threshold] + +### Choosing Group by fields + +The **Group by** field determines how events are bucketed before counting. Select fields that represent the entity you want to monitor: + +* **Single field:** `source.ip` with a threshold of `100` alerts on any source IP that generates 100 or more matching events. +* **Multiple fields:** `source.ip, destination.ip` with a threshold of `10` alerts on every unique source-destination pair that appears at least 10 times. +* **No field:** Omit **Group by** to count all matching events together. The rule fires when the total count meets the threshold. + +### Adding cardinality constraints + +Use the **Count** field to add a cardinality requirement. This is useful when volume alone is not enough and you need to ensure diversity across a field. + +For example, with **Group by** set to `source.ip`, **Threshold** set to `5`, and **Count** limiting by `destination.port` >= `3`, an alert fires only for source IPs that connect to at least three unique destination ports across five or more events. This pattern surfaces port-scanning behavior while filtering out noisy but benign repeated connections. + +### Understanding threshold alerts + +Threshold alerts are **synthetic**. They do not contain the original source document fields: + +* Only the **Group by** fields and the count appear in the alert. +* All other source fields are omitted because they can vary across the counted documents. +* The actual count is available in `kibana.alert.threshold_result.count`. +* The grouped field values are in `kibana.alert.threshold_result.terms`. + +Keep this in mind when configuring severity overrides, risk score overrides, or rule name overrides, as only the aggregated fields contain usable data. + +### Best practices + +* **Set thresholds conservatively at first.** Start with a higher threshold to understand baseline volumes, then lower it as you tune out false positives. +* **Combine with additional look-back time.** A look-back buffer of at least 1 minute helps avoid gaps between executions. +* **Be cautious with high-cardinality Group by fields.** Fields with many unique values can cause rule timeouts or circuit-breaker errors. + +::::{tip} +**See it in practice.** These prebuilt rules use thresholds effectively: + +* **Potential Brute Force Attack** groups failed authentication attempts by `source.ip` and fires when the count crosses a threshold, detecting credential-stuffing patterns. +* **High Number of Process Terminations** counts process termination events per host, surfacing hosts where mass process termination may indicate ransomware activity. +* **Multiple Alerts Involving a Single User** groups existing alerts by `user.name` to detect users generating an unusually high volume of alerts, a useful meta-detection pattern. +:::: + +## Threshold field reference [threshold-fields] + +The following settings are specific to threshold rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). + +**Index patterns or {{data-source}}** +: The {{es}} indices or {{data-source}} the rule searches. + +**Custom query** +: The KQL or Lucene query used to filter events before counting. Only matching documents are evaluated against the threshold. + +**Group by** (optional) +: One or more fields to group events by. Each unique combination of field values is evaluated independently against the threshold. Nested fields are not supported. + +**Threshold** +: The minimum number of matching events required to generate an alert. If **Group by** is defined, each group must independently meet this count. + +**Count** (optional) +: A cardinality constraint on an additional field. Limits alerts to groups where the specified field has at least the given number of unique values. + +**Suppress alerts** (optional) +: Reduce repeated or duplicate alerts. For details, refer to [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md). + +**Required fields** (optional) +: An informational list of fields the rule needs to function. This does not affect rule execution. + +**Related integrations** (optional) +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/tune-detection-rules.md b/solutions/security/detect-and-alert/tune-detection-rules.md index b21ac3762d..e1fc203145 100644 --- a/solutions/security/detect-and-alert/tune-detection-rules.md +++ b/solutions/security/detect-and-alert/tune-detection-rules.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Optimize detection rules to reduce noise by adding exceptions, disabling rules, cloning rules, or enabling suppression. --- # Tune detection rules [security-tune-detection-signals] @@ -24,7 +25,7 @@ Using the {{security-app}}, you can tune prebuilt and custom detection rules to * Disable detection rules that rarely produce actionable alerts because they match expected local behavior, workflows, or policy exceptions. * [Clone and modify](/solutions/security/detect-and-alert/manage-detection-rules.md#duplicate-rules) detection rule queries so they are aligned with local policy exceptions. This reduces noise while retaining actionable alerts. * Clone and modify detection rule risk scores, and use branching logic to map higher risk scores to higher priority workflows. -* Enable [alert suppression](/solutions/security/detect-and-alert/suppress-detection-alerts.md) for custom query rules to reduce the number of repeated or duplicate alerts. +* Enable [alert suppression](/solutions/security/detect-and-alert/alert-suppression.md) for custom query rules to reduce the number of repeated or duplicate alerts. For details about tuning rules for specific categories: @@ -137,7 +138,7 @@ Take the following steps to tune indicator match rules: * Avoid cluster performance issues by scheduling your rule to run in one-hour intervals or longer. For example, avoid scheduling an indicator match rule to check for indicators every five minutes. ::::{note} -{{elastic-sec}} provides limited support for indicator match rules. Visit [support limitations](/solutions/security/detect-and-alert.md#support-indicator-rules) for more information. +{{elastic-sec}} provides limited support for indicator match rules. Visit [Indicator match rules](/solutions/security/detect-and-alert/indicator-match.md) for more information. :::: diff --git a/solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md b/solutions/security/detect-and-alert/update-prebuilt-rules.md similarity index 85% rename from solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md rename to solutions/security/detect-and-alert/update-prebuilt-rules.md index 244ea61597..106cc05da5 100644 --- a/solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md +++ b/solutions/security/detect-and-alert/update-prebuilt-rules.md @@ -10,6 +10,7 @@ products: - id: cloud-enterprise - id: cloud-kubernetes - id: elastic-stack +description: Update modified and unmodified Elastic prebuilt rules, resolve conflicts, and understand rule field update statuses. --- # Update modified and unmodified Elastic prebuilt rules [prebuilt-rules-update-modified-unmodified] @@ -18,11 +19,11 @@ products: You must have an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}} to access this feature. -If you have a Platinum subscription or lower on {{stack}} or a Security Analytics Essentials project on {{serverless-short}}, follow the guidelines in [Update Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#update-prebuilt-rules) instead. +If you have a Platinum subscription or lower on {{stack}} or a Security Analytics Essentials project on {{serverless-short}}, follow the guidelines in [Update Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#update-prebuilt-rules) instead. :::: -This page provides instructions for updating modified and unmodified prebuilt rules. You can also find information about [statuses](/solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md#rule-field-update-statuses) or [conflicts](/solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md#resolve-reduce-rule-conflicts) that you might encounter when updating rules. +This page provides instructions for updating modified and unmodified prebuilt rules. You can also find information about [statuses](/solutions/security/detect-and-alert/update-prebuilt-rules.md#rule-field-update-statuses) or [conflicts](/solutions/security/detect-and-alert/update-prebuilt-rules.md#resolve-reduce-rule-conflicts) that you might encounter when updating rules. To update rules: @@ -48,9 +49,9 @@ To update rules: If you haven't updated the rule in a while, its original version might be unavailable for comparison. Instead, you will only have access to the rule's current version and the incoming Elastic update. You can avoid this by updating prebuilt rules more often. :::: - * Check the update status: View the status of the entire rule update and for [each field that's being changed](/solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md#rule-field-update-statuses). + * Check the update status: View the status of the entire rule update and for [each field that's being changed](/solutions/security/detect-and-alert/update-prebuilt-rules.md#rule-field-update-statuses). - * Address update conflicts: Find and address conflicts that [need additional attention](/solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md#resolve-reduce-rule-conflicts). + * Address update conflicts: Find and address conflicts that [need additional attention](/solutions/security/detect-and-alert/update-prebuilt-rules.md#resolve-reduce-rule-conflicts). * Edit the final update: Change the update that will be applied to the field when you update the rule. To change the update, go to the **Final update** section, make your changes, and then save them. @@ -66,7 +67,7 @@ To update rules: 4. From the **Rule Updates** tab, do one of the following to update prebuilt rules: - * Update all available rules: Click **Update all**. If any rules have conflicts, you will be prompted to take [additional action](/solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md#resolve-reduce-rule-conflicts). + * Update all available rules: Click **Update all**. If any rules have conflicts, you will be prompted to take [additional action](/solutions/security/detect-and-alert/update-prebuilt-rules.md#resolve-reduce-rule-conflicts). * Update a single rule without conflicts: Click **Update rule** for that rule. * Update multiple rules: Select the rules and click **Update _x_ selected rule(s)**. If any rules have conflicts, you will be prompted to take additional action. @@ -76,7 +77,7 @@ To update rules: To find specific rules to update: * Use the **Modified/Unmodified** drop-down menu to only display modified or unmodified prebuilt rules. - * Use the search bar and **Tags** filter to find the rules you want to update. For example, filter by `OS: Windows` if your environment only includes Windows endpoints. For more on tag categories, refer to [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#prebuilt-rule-tags). + * Use the search bar and **Tags** filter to find the rules you want to update. For example, filter by `OS: Windows` if your environment only includes Windows endpoints. For more on tag categories, refer to [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#prebuilt-rule-tags). :::: @@ -88,8 +89,8 @@ This table describes statuses that might appear for rule fields being updated. | --- | --- | | **Ready for update** | Displays when there are no conflicts to resolve.

Further action is not required for the field. It is ready to be updated.
| | **No update** | Displays when the field is not being updated by Elastic, but the current field value differs from the original one. This typically happens when the field's value was changed after the prebuilt rule was initially installed.

Further action is not required for the field. It is ready to be updated.

**Tip**: You can still change the final field update, if needed. To do so, make your changes in the **Final update** section and save them.

| -| **Review required** | Displays when Elastic auto-resolves a conflict between the current field value and the value from the incoming Elastic update.

You must accept or edit the field's final update and save the changes. Refer to [Resolve and reduce update conflicts](/solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md#resolve-reduce-rule-conflicts) to learn more about auto-resolved conflicts and how to reduce future conflicts.
| -| **Action required** | Displays when Elastic could not auto-resolve the conflict between the current field value and the value from the incoming Elastic update.

You must manually set and save the field's final update. Refer to [Resolve and reduce update conflicts](/solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md#resolve-reduce-rule-conflicts) to learn more about conflicts that need manual fixes and how to reduce future conflicts.
| +| **Review required** | Displays when Elastic auto-resolves a conflict between the current field value and the value from the incoming Elastic update.

You must accept or edit the field's final update and save the changes. Refer to [Resolve and reduce update conflicts](/solutions/security/detect-and-alert/update-prebuilt-rules.md#resolve-reduce-rule-conflicts) to learn more about auto-resolved conflicts and how to reduce future conflicts.
| +| **Action required** | Displays when Elastic could not auto-resolve the conflict between the current field value and the value from the incoming Elastic update.

You must manually set and save the field's final update. Refer to [Resolve and reduce update conflicts](/solutions/security/detect-and-alert/update-prebuilt-rules.md#resolve-reduce-rule-conflicts) to learn more about conflicts that need manual fixes and how to reduce future conflicts.
| ## Resolve and reduce update conflicts [resolve-reduce-rule-conflicts] diff --git a/solutions/security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md b/solutions/security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md index c12b62f6ed..9ecb38ff52 100644 --- a/solutions/security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md +++ b/solutions/security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Understand how logsdb index mode affects alerts, rule actions, and runtime fields in Elastic Security. --- # Using logsdb index mode with Elastic Security [detections-logsdb-index-mode-impact] diff --git a/solutions/security/detect-and-alert/using-the-api.md b/solutions/security/detect-and-alert/using-the-api.md new file mode 100644 index 0000000000..4dfe17da23 --- /dev/null +++ b/solutions/security/detect-and-alert/using-the-api.md @@ -0,0 +1,45 @@ +--- +applies_to: + stack: all + serverless: + security: all +products: + - id: security + - id: cloud-serverless +description: Create and manage detection rules programmatically using the Security detections API for CI/CD and bulk operations. +--- + +# Using the API + +You can create and manage detection rules programmatically instead of using the {{kib}} UI. This is useful for CI/CD pipelines, bulk rule management, rule-as-code workflows, and integrating detection management with external tooling. + +## API reference + +The detection APIs are part of the {{kib}} API. Use the appropriate reference for your deployment type: + +**{{stack}}** +: [Security detections API](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-security-detections-api): Create, read, update, delete, and bulk-manage detection rules. Also covers alert management (status, tags, assignees) and prebuilt rule installation. + +**{{serverless-full}}** +: [Security detections API (Serverless)](https://www.elastic.co/docs/api/doc/kibana-serverless/group/endpoint-security-detections-api): The same detection operations, scoped to {{serverless-short}} projects. + +For rule exceptions and value lists, use these additional APIs: + +- [Exceptions API](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-security-exceptions-api): Create and manage rule exceptions and shared exception lists. +- [Endpoint exceptions API](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-security-endpoint-exceptions-api): Manage endpoint-specific exceptions. +- [Lists API](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-security-lists-api): Create source event value lists for use with rule exceptions. + +For a complete list of {{elastic-sec}} APIs, refer to [{{elastic-sec}} APIs](/solutions/security/apis.md). + +## Common operations + +| Task | Endpoint | +|---|---| +| Create a rule | [`POST /api/detection_engine/rules`](https://www.elastic.co/docs/api/doc/kibana/operation/operation-createrule) | +| List all rules | [`GET /api/detection_engine/rules/_find`](https://www.elastic.co/docs/api/doc/kibana/operation/operation-findrules) | +| Update a rule | [`PUT /api/detection_engine/rules`](https://www.elastic.co/docs/api/doc/kibana/operation/operation-updaterule) | +| Bulk actions (enable, export, duplicate, delete) | [`POST /api/detection_engine/rules/_bulk_action`](https://www.elastic.co/docs/api/doc/kibana/operation/operation-performrulesbulkaction) | +| Import rules | [`POST /api/detection_engine/rules/_import`](https://www.elastic.co/docs/api/doc/kibana/operation/operation-importrules) | +| Export rules | [`POST /api/detection_engine/rules/_export`](https://www.elastic.co/docs/api/doc/kibana/operation/operation-exportrules) | +| Install prebuilt rules | [`PUT /api/detection_engine/rules/prepackaged`](https://www.elastic.co/docs/api/doc/kibana/operation/operation-installprebuiltrulesandtimelines) | +| Set alert status | [`POST /api/detection_engine/signals/status`](https://www.elastic.co/docs/api/doc/kibana/operation/operation-setalertsstatus) | diff --git a/solutions/security/detect-and-alert/using-the-rule-builder.md b/solutions/security/detect-and-alert/using-the-rule-builder.md new file mode 100644 index 0000000000..225c6cfe0b --- /dev/null +++ b/solutions/security/detect-and-alert/using-the-rule-builder.md @@ -0,0 +1,61 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/security/current/rules-ui-create.html + - https://www.elastic.co/guide/en/serverless/current/security-rules-create.html +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +navigation_title: Using the UI +description: Step-by-step guide to create detection rules using the Kibana rule builder UI. +--- + +# Create a detection rule using the UI [security-rules-create] + +Once the Detections feature is [turned on](/solutions/security/detect-and-alert/requirements-privileges.md), follow these steps to create a detection rule: + +1. Define the [rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md#rule-types). The configuration for this step varies depending on the rule type. +2. Configure [basic rule settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-basic-params). +3. Configure [advanced rule settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) (optional). +4. Set the [rule's schedule](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-schedule). +5. Set up [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) (optional). +6. Set up [response actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-response-action) (optional). + +::::{tip} +* At any step, you can preview the rule before saving it to see what kind of results you can expect. +* To ensure rules don't search cold and frozen data when executing, either configure the `excludedDataTiersForRuleExecution` [advanced setting](/solutions/security/get-started/configure-advanced-settings.md#exclude-cold-frozen-data-rule-executions) (which applies to all rules in a space), or add a [Query DSL filter](/solutions/security/detect-and-alert/set-rule-data-sources.md) to individual rules. These options are only available if you're on the {{stack}}. +:::: + +## Detection rule requirements + +To create detection rules, you must have: + +* At least `Read` access to {{data-source}}s, which requires the `Data View {{manage-app}}` [{{kib}} privilege](/deploy-manage/users-roles/cluster-or-deployment-auth/defining-roles.md) in {{stack}} or the appropriate [user role](/deploy-manage/users-roles/cloud-organization/user-roles.md) in {{serverless-short}}. +* The required privileges to preview rules, manage rules, and manage alerts. Refer to [](/solutions/security/detect-and-alert/requirements-privileges.md) for more details. + +::::{note} +Additional configuration is required for detection rules using {{ccs}}. Refer to [{{ccs-cap}} and detection rules](/solutions/security/detect-and-alert/advanced-data-source-configuration.md). +:::: + +## Rule type guides + +Each rule type has its own configuration and query requirements. Refer to the appropriate guide for type-specific instructions: + +* [Custom query](/solutions/security/detect-and-alert/custom-query.md) +* [Event correlation (EQL)](/solutions/security/detect-and-alert/eql.md) +* [Threshold](/solutions/security/detect-and-alert/threshold.md) +* [Indicator match](/solutions/security/detect-and-alert/indicator-match.md) +* [New terms](/solutions/security/detect-and-alert/new-terms.md) +* [{{esql}}](/solutions/security/detect-and-alert/esql.md) +* [{{ml-cap}}](/solutions/security/detect-and-alert/machine-learning.md) + +To understand which type to use, refer to [Select the right rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md). + +## Related pages + +* [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md): All shared rule settings, including severity, risk score, schedule, actions, and notification variables. +* [Using the API](/solutions/security/detect-and-alert/using-the-api.md): Create and manage rules programmatically. +* [Manage detection rules](/solutions/security/detect-and-alert/manage-detection-rules.md): Enable, export, duplicate, and bulk-edit rules. diff --git a/solutions/security/detect-and-alert/validate-and-test-rules.md b/solutions/security/detect-and-alert/validate-and-test-rules.md new file mode 100644 index 0000000000..9d9a9ff518 --- /dev/null +++ b/solutions/security/detect-and-alert/validate-and-test-rules.md @@ -0,0 +1,16 @@ +--- +applies_to: + stack: all + serverless: + security: all +products: + - id: security + - id: cloud-serverless +description: Verify rule logic against historical data, assess alert volume, and use shadow deployment before enabling notifications. +--- + +# Validate and test rules + +% This is a stub. Content is coming soon. + +Verify rule logic against historical data, assess alert volume, and use shadow deployment before enabling notifications. diff --git a/solutions/security/detect-and-alert/view-detection-alert-details.md b/solutions/security/detect-and-alert/view-detection-alert-details.md index 87edb49962..d514dde96a 100644 --- a/solutions/security/detect-and-alert/view-detection-alert-details.md +++ b/solutions/security/detect-and-alert/view-detection-alert-details.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Use the alert details flyout to investigate, manage, and respond to detection alerts in Elastic Security. --- # View detection alert details [security-view-alert-details] @@ -103,10 +104,10 @@ The Investigation section provides the following information: * **Investigation guide**: The **Show investigation guide** button displays if the rule associated with the alert has an investigation guide. Click the button to open the expanded Investigation view in the left panel. ::::{tip} - Add an [investigation guide](/solutions/security/detect-and-alert/launch-timeline-from-investigation-guides.md#add-ig-actions-rule) to a rule when creating a new custom rule or modifying an existing custom rule’s settings. + Add an [investigation guide](/solutions/security/detect-and-alert/write-investigation-guides.md#add-ig-actions-rule) to a rule when creating a new custom rule or modifying an existing custom rule’s settings. :::: -* **Highlighted fields**: Shows relevant fields for the alert and any [custom highlighted fields](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-advanced-params) you added to the rule. Custom highlighted fields with values are added to this section. Those without values aren’t added. +* **Highlighted fields**: Shows relevant fields for the alert and any [custom highlighted fields](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) you added to the rule. Custom highlighted fields with values are added to this section. Those without values aren’t added. ::::{tip} {applies_to}`stack: ga 9.1` You can quickly add and remove custom highlighted fields from the rule by clicking **Add field** in the Highlighted fields table. @@ -147,7 +148,7 @@ The Threat intelligence overview shows matched indicators, which provide threat The Threat intelligence overview provides the following information: -* **Threat match detected**: Only available when examining an alert generated from an [indicator match](/solutions/security/detect-and-alert/create-detection-rule.md#create-indicator-rule) rule. Shows the number of matched indicators that are present in the alert document. Shows zero if there are no matched indicators or you’re examining an alert generated by another type of rule. +* **Threat match detected**: Only available when examining an alert generated from an [indicator match](/solutions/security/detect-and-alert/indicator-match.md) rule. Shows the number of matched indicators that are present in the alert document. Shows zero if there are no matched indicators or you’re examining an alert generated by another type of rule. * **Fields enriched with threat intelligence**: Shows the number of matched indicators that are present on an alert that *wasn’t* generated from an indicator match rule. If none exist, the total number of matched indicators is zero. @@ -214,7 +215,7 @@ From the right panel, click **Correlations** to open the expanded Correlations v In the expanded view, corelation data is organized into several tables: -* **Suppressed alerts**: Shows how many duplicate alerts were suppressed. This information only appears if [alert suppression](/solutions/security/detect-and-alert/suppress-detection-alerts.md) is enabled for the rule. +* **Suppressed alerts**: Shows how many duplicate alerts were suppressed. This information only appears if [alert suppression](/solutions/security/detect-and-alert/alert-suppression.md) is enabled for the rule. * **Related cases**: Shows cases to which the alert has been added. Click a case’s name to open its details. * **Alerts related by source event**: Shows alerts created by the same source event. This can help you find alerts with a shared origin and provide more context about the source event. Click the **Investigate in timeline** button to examine related alerts in Timeline. * **Alerts related by session**: Shows alerts generated during the same [session](/solutions/security/investigate/session-view.md). These alerts share the same session ID, which is a unique ID for tracking a given Linux session. To use this feature, you must enable the **Collect session data** setting in your {{elastic-defend}} integration policy. Refer to [Enable Session View data](/solutions/security/investigate/session-view.md#enable-session-view) for more information. @@ -249,7 +250,7 @@ The following features require a [Platinum subscription](https://www.elastic.co/ ## Response [response-overview] -The **Response** section is located on the **Overview** tab in the right panel. It shows [response actions](/solutions/security/detect-and-alert/create-detection-rule.md) that were added to the rule associated with the alert. Click **Response** to display the response action’s results in the left panel. +The **Response** section is located on the **Overview** tab in the right panel. It shows [response actions](/solutions/security/detect-and-alert/using-the-rule-builder.md) that were added to the rule associated with the alert. Click **Response** to display the response action’s results in the left panel. ## Notes [expanded-notes-view] diff --git a/solutions/security/detect-and-alert/visualize-detection-alerts.md b/solutions/security/detect-and-alert/visualize-detection-alerts.md index 524d8de67b..1cfca32128 100644 --- a/solutions/security/detect-and-alert/visualize-detection-alerts.md +++ b/solutions/security/detect-and-alert/visualize-detection-alerts.md @@ -9,6 +9,7 @@ applies_to: products: - id: security - id: cloud-serverless +description: Visualize and group detection alerts using Summary, Trend, Counts, and Treemap views on the Alerts page. --- # Visualize detection alerts [security-visualize-alerts] diff --git a/solutions/security/detect-and-alert/write-investigation-guides.md b/solutions/security/detect-and-alert/write-investigation-guides.md new file mode 100644 index 0000000000..c7040f46c2 --- /dev/null +++ b/solutions/security/detect-and-alert/write-investigation-guides.md @@ -0,0 +1,242 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/security/current/interactive-investigation-guides.html + - https://www.elastic.co/guide/en/serverless/current/security-interactive-investigation-guides.html +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Create Markdown investigation guides with Timeline and Osquery buttons to help analysts triage and respond to alerts. +--- + +# Write investigation guides [security-interactive-investigation-guides] + +Investigation guides are Markdown documents attached to detection rules that help analysts triage, analyze, and respond to alerts. A well-written guide reduces mean time to respond by giving analysts the context and queries they need without leaving the alert details flyout. + +You author an investigation guide in the **Investigation guide** Markdown editor, which is available in the rule's [advanced settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) (under the **About** step when creating a rule, or the **About** tab when editing one). The guide renders in the **Investigation** tab of the alert details flyout whenever an analyst opens an alert produced by that rule. + +:::{image} /solutions/images/security-ig-alert-flyout.png +:alt: Alert details flyout with investigation guide +:screenshot: +::: + +::::{note} +You can only author investigation guides for custom rules. Elastic prebuilt rules include their own investigation guides that cannot be edited. To customize a prebuilt rule's guide, duplicate the rule first, then edit the duplicate. +:::: + +## Supported Markdown syntax [ig-markdown-syntax] + +The investigation guide editor supports standard Markdown with several extensions. Use the toolbar above the editor for quick formatting, or write the syntax directly. + +| Syntax | Result | +|---|---| +| `# Heading` | Section heading (levels 1–6) | +| `**bold**` | **Bold text** | +| `*italic*` | *Italic text* | +| `` `code` `` | Inline code | +| `[text](url)` | Hyperlink | +| `- item` | Unordered list | +| `1. item` | Ordered list | +| `> quote` | Block quote | +| `---` | Horizontal rule | +| `![alt](url)` | Image | +| ` ``` ` | Fenced code block | +| `\| col \| col \|` | Table | + +::::{tip} +Use Markdown headings to break the guide into clear sections (for example, **Triage**, **Analysis**, **Response**) so analysts can scan quickly. +:::: + +## Best practices for alert triage [ig-best-practices] + +Effective investigation guides share several characteristics: + +**Start with context, not commands.** Open with a one- or two-sentence summary of what the rule detects and why it matters. Analysts who understand the threat make better decisions. + +**Structure the guide around a workflow.** Organize content into sequential sections: + +1. **Triage.** Quick checks to confirm whether the alert is a true positive. Include field values to inspect and known false-positive conditions. +2. **Analysis.** Deeper investigation steps such as Timeline queries, Osquery lookups, and correlated data sources. +3. **Response.** Recommended actions if the alert is confirmed, including escalation paths and containment steps. + +**Reference alert fields directly.** Use double curly brackets (for example, `{{host.name}}`, `{{user.name}}`) to surface dynamic alert data in Timeline query buttons, making the guide immediately actionable. + +**Keep it scannable.** Use bullet lists, bold key terms, and short paragraphs. Analysts read investigation guides under time pressure. + +**Link to related resources.** Include links to relevant dashboards, runbooks, or external threat intelligence references so analysts don't have to search for them. + +## Timeline query buttons [add-ig-actions-rule] + +You can embed interactive query buttons that open pre-populated [Timeline](/solutions/security/investigate/timeline.md) investigations directly from an alert. Each button runs a query using alert field values and hard-coded literals, and displays the number of matching event documents. + +:::{image} /solutions/images/security-ig-alert-flyout-invest-tab.png +:alt: Alert details flyout with interactive investigation guide +:screenshot: +::: + +Click a query button to load the query in Timeline automatically. + +:::{image} /solutions/images/security-ig-timeline.png +:alt: Timeline with query pre-loaded from investigation guide action +:screenshot: +::: + +### Add a Timeline query button [add-timeline-query-button] + +1. Open the **Investigation guide** Markdown editor (in the rule's **Advanced settings**). + + :::{image} /solutions/images/security-ig-investigation-guide-editor.png + :alt: Investigation guide editor field + :screenshot: + ::: + +2. Place the cursor where you want the button to appear, then select the Investigate icon (![Investigate icon](/solutions/images/security-ig-investigate-icon.png "title =20x20")) in the toolbar. The **Add investigation query** builder form appears. + + :::{image} /solutions/images/security-ig-investigation-query-builder.png + :alt: Add investigation guide UI + :screenshot: + ::: + +3. Complete the form: + + 1. **Label**: Text displayed on the button. + 2. **Description**: (Optional) Additional text shown with the button. + 3. **Filters**: Select fields, operators, and values to build the query. Click **OR** or **AND** to create multiple filters and define their relationships. + + To use a field value from the alert as a query parameter, enter the field name surrounded by double curly brackets (for example, `{{kibana.alert.example}}`) as a custom option for the filter value. + + :::{image} /solutions/images/security-ig-filters-field-custom-value.png + :alt: Custom filter value using alert field + :screenshot: + ::: + + 4. **Relative time range**: (Optional) A time range that limits the query relative to the alert's creation time. + +4. Click **Save changes**. The syntax is added to the editor. + + ::::{note} + To modify a query button, either edit the syntax directly in the editor (refer to the [syntax reference](/solutions/security/detect-and-alert/write-investigation-guides.md#query-button-syntax) below) or delete the syntax and recreate it using the builder form. + :::: + +5. Save and enable the rule. + +### Query button syntax [query-button-syntax] + +The following syntax defines a query button in an investigation guide. + +| Field | Description | +|---|---| +| `!{investigate{ }}` | The container object holding all the query button's configuration attributes. | +| `label` | Identifying text on the button. | +| `description` | Additional text included with the button. | +| `providers` | A two-level nested array that defines the query to run in Timeline. Items in the outer level are joined by an `OR` relationship, and items in the inner level are joined by an `AND` relationship. Each item is defined by these attributes: `field` (field name), `excluded` (whether the result is excluded), `queryType` (filter operator, such as `phrase` or `range`), `value` (a literal value or an alert field name in double curly brackets), and `valueType` (data type such as `string` or `boolean`). | +| `relativeFrom`, `relativeTo` | (Optional) The start and end of a relative time range for the query. Times are relative to the alert's creation time, represented as `now` in [date math](elasticsearch://reference/elasticsearch/rest-apis/common-options.md#date-math) format. For example, `"relativeFrom": "now-15m", "relativeTo": "now"`. | + +::::{note} +Some characters must be escaped with a backslash, such as `\"` for a quotation mark and `\\` for a literal backslash. Divide Windows paths with double backslashes (for example, `C:\\Windows\\explorer.exe`), and paths that already include double backslashes might require four backslashes for each divider. A clickable error icon (![Error icon](/solutions/images/security-ig-error-icon.png "title =20x20")) displays below the Markdown editor if there are any syntax errors. +:::: + +### Example [ig-timeline-example] + +```json +!{investigate{ + "label": "Test action", + "description": "Click to investigate.", + "providers": [ + [ + {"field": "event.id", "excluded": false, "queryType": "phrase", "value": "{{event.id}}", "valueType": "string"} + ], + [ + {"field": "event.action", "excluded": false, "queryType": "phrase", "value": "rename", "valueType": "string"}, + {"field": "process.pid", "excluded": false, "queryType": "phrase", "value": "{{process.pid}}", "valueType": "string"} + ] + ], + "relativeFrom": "now-15m", + "relativeTo": "now" +}} +``` + +This creates the following Timeline query: + +`(event.id : )`
`OR (event.action : "rename" AND process.pid : )` + +:::{image} /solutions/images/security-ig-timeline-query.png +:alt: Timeline query +:screenshot: +::: + +### Timeline template fields [ig-timeline-template-fields] + +When viewing an investigation guide outside the context of a specific alert (such as on a rule's details page), queries open as [Timeline templates](/solutions/security/investigate/timeline-templates.md), and parameter fields are treated as Timeline template fields. + +:::{image} /solutions/images/security-ig-timeline-template-fields.png +:alt: Timeline template +:screenshot: +::: + +## Osquery buttons [ig-osquery-syntax] + +You can embed Osquery buttons that let analysts run live queries against {{agent}}s directly from the investigation guide. This is useful for gathering host-level context, such as running processes, installed software, or open network connections, while triaging an alert. + +::::{admonition} Requirements +* The [Osquery manager integration](/solutions/security/investigate/manage-integration.md) must be installed. +* {{agent}}'s [status](/reference/fleet/monitor-elastic-agent.md) must be `Healthy`. Refer to [](/troubleshoot/ingest/fleet/common-problems.md) if it isn't. +* In {{stack}}, your role must have [Osquery feature privileges](/solutions/security/investigate/osquery.md). +* In {{serverless-short}}, you must have the appropriate user role. +:::: + +:::{image} /solutions/images/security-osquery-investigation-guide.png +:alt: A live query in an investigation guide +:screenshot: +::: + +### Add an Osquery button [add-osquery-button] + +1. Open the **Investigation guide** Markdown editor (in the rule's **Advanced settings**). +2. In the toolbar, click the **Osquery** button (![Osquery button](/solutions/images/security-osquery-button.png "title =20x20")). + + 1. Add a descriptive label (for example, `Search for executables`). + 2. Select a saved query or enter a new one. + + ::::{tip} + Use [placeholder fields](/solutions/security/investigate/use-placeholder-fields-in-osquery-queries.md) to dynamically add existing alert data to your query. + :::: + + 3. Expand the **Advanced** section to set a timeout period for the query and view or set [mapped ECS fields](/solutions/security/investigate/osquery.md#osquery-map-fields) included in the results (optional). + + ::::{note} + Overwriting the query's default timeout period allows you to support queries that take longer to run. The default and minimum supported value for the **Timeout** field is `60`. The maximum supported value is `86400` (24 hours). + :::: + + :::{image} /solutions/images/security-setup-osquery-investigation-guide.png + :alt: Setting up an Osquery investigation guide query + :screenshot: + ::: + +3. Click **Save changes** to add the query to the rule's investigation guide. + +### Run an Osquery from an investigation guide [run-osquery-from-ig] + +1. Open an alert's details flyout and go to the **Investigation** tab. +2. Click the Osquery button. The **Run Osquery** pane opens with the **Query** field autofilled. + + 1. Select one or more {{agent}}s or groups to query. Start typing in the search field to get suggestions for {{agent}}s by name, ID, platform, and policy. + 2. (Optional) Expand the **Advanced** section to set a timeout period and view or set [mapped ECS fields](/solutions/security/investigate/osquery.md#osquery-map-fields). + +3. Click **Submit** to run the query. Results display in the flyout. + + ::::{note} + Refer to [Examine Osquery results](/solutions/security/investigate/examine-osquery-results.md) for more about query results. + :::: + +4. (Optional) Click **Save for later** to save the query for future use. + + :::{image} /solutions/images/security-run-query-investigation-guide.png + :alt: Results from running an Osquery query from an investigation guide + :screenshot: + ::: + +For the complete Osquery reference, refer to [Run Osquery from investigation guides](/solutions/security/investigate/run-osquery-from-investigation-guides.md). diff --git a/solutions/security/endpoint-response-actions/configure-third-party-response-actions.md b/solutions/security/endpoint-response-actions/configure-third-party-response-actions.md index a62652e12e..441eae070e 100644 --- a/solutions/security/endpoint-response-actions/configure-third-party-response-actions.md +++ b/solutions/security/endpoint-response-actions/configure-third-party-response-actions.md @@ -83,7 +83,7 @@ Expand a section below for your endpoint security system: 4. Click **Save**. -4. **Create and enable detection rules to generate {{elastic-sec}} alerts.** (Optional) Create [detection rules](/solutions/security/detect-and-alert/create-detection-rule.md) to generate {{elastic-sec}} alerts based on CrowdStrike events and data. The [CrowdStrike integration docs](https://docs.elastic.co/en/integrations/crowdstrike) list the available ingested logs and fields you can use to build a rule query. +4. **Create and enable detection rules to generate {{elastic-sec}} alerts.** (Optional) Create [detection rules](/solutions/security/detect-and-alert/using-the-rule-builder.md) to generate {{elastic-sec}} alerts based on CrowdStrike events and data. The [CrowdStrike integration docs](https://docs.elastic.co/en/integrations/crowdstrike) list the available ingested logs and fields you can use to build a rule query. This gives you visibility into CrowdStrike without needing to leave {{elastic-sec}}. You can perform supported endpoint response actions directly from alerts that a rule creates, by using the **Take action** menu in the alert details flyout. :::: @@ -140,7 +140,7 @@ Expand a section below for your endpoint security system: 4. (Optional) If necessary, adjust the default values populated for the other configuration parameters. 5. Click **Save**. -4. **Create and enable detection rules to generate {{elastic-sec}} alerts.** Create [detection rules](/solutions/security/detect-and-alert/create-detection-rule.md) to generate {{elastic-sec}} alerts based on Microsoft Defender for Endpoint events and data. +4. **Create and enable detection rules to generate {{elastic-sec}} alerts.** Create [detection rules](/solutions/security/detect-and-alert/using-the-rule-builder.md) to generate {{elastic-sec}} alerts based on Microsoft Defender for Endpoint events and data. This gives you visibility into Microsoft Defender hosts without needing to leave {{elastic-sec}}. You can perform supported endpoint response actions directly from alerts that a rule creates, by using the **Take action** menu in the alert details flyout. @@ -194,7 +194,7 @@ Expand a section below for your endpoint security system: 4. Click **Save**. -4. **Create and enable detection rules to generate {{elastic-sec}} alerts.** Create [detection rules](/solutions/security/detect-and-alert/create-detection-rule.md#create-custom-rule) to generate {{elastic-sec}} alerts based on SentinelOne events and data. +4. **Create and enable detection rules to generate {{elastic-sec}} alerts.** Create [detection rules](/solutions/security/detect-and-alert/custom-query.md) to generate {{elastic-sec}} alerts based on SentinelOne events and data. This gives you visibility into SentinelOne without needing to leave {{elastic-sec}}. You can perform supported endpoint response actions directly from alerts that a rule creates, by using the **Take action** menu in the alert details flyout. diff --git a/solutions/security/endpoint-response-actions/isolate-host.md b/solutions/security/endpoint-response-actions/isolate-host.md index bcc4f8a409..14b2a411a9 100644 --- a/solutions/security/endpoint-response-actions/isolate-host.md +++ b/solutions/security/endpoint-response-actions/isolate-host.md @@ -95,7 +95,7 @@ Be aware that automatic host isolation can result in unintended consequences, su 1. Add an endpoint response action to a new or existing custom query rule. The endpoint response action will run whenever rule conditions are met: - * **New rule**: On the last step of [custom query rule](/solutions/security/detect-and-alert/create-detection-rule.md#create-custom-rule) creation, go to the **Response Actions** section and select **{{elastic-defend}}**. + * **New rule**: On the last step of [custom query rule](/solutions/security/detect-and-alert/custom-query.md) creation, go to the **Response Actions** section and select **{{elastic-defend}}**. * **Existing rule**: Edit the rule’s settings, then go to the **Actions** tab. In the tab, select **{{elastic-defend}}** under the **Response Actions** section. 2. In the **Response action** field, select **Isolate**. diff --git a/solutions/security/esql-for-security.md b/solutions/security/esql-for-security.md index 3e2d0900df..7286270e9b 100644 --- a/solutions/security/esql-for-security.md +++ b/solutions/security/esql-for-security.md @@ -21,5 +21,5 @@ Learn how to: - [Generate and understand {{esql}} queries](/solutions/security/ai/generate-customize-learn-about-esorql-queries.md) using the AI Assistant - [Investigate events in Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline) using {{esql}} -- [Create detection rules](/solutions/security/detect-and-alert/create-detection-rule.md#create-esql-rule) using {{esql}} +- [Create detection rules](/solutions/security/detect-and-alert/esql.md) using {{esql}} - [Convert Splunk SPL rules to {{esql}}](/solutions/security/get-started/automatic-migration.md) with Automatic Migration \ No newline at end of file diff --git a/solutions/security/get-started/automatic-migration.md b/solutions/security/get-started/automatic-migration.md index f10e2dfda5..6c1e581e90 100644 --- a/solutions/security/get-started/automatic-migration.md +++ b/solutions/security/get-started/automatic-migration.md @@ -138,7 +138,7 @@ The table's fields are as follows: * `Failed`: Translation failed. Refer to the the error for details. * **Risk Score:** For Elastic-authored rules, risk scores are predefined. For custom translated rules, risk scores are defined as follows: * If the source rule has a field comparable to Elastic's `risk score`, we use that value. - * Otherwise, if the source rule has a field comparable to Elastic's `rule severity` field, we base the risk score on that value according to [these guidelines](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-basic-params). + * Otherwise, if the source rule has a field comparable to Elastic's `rule severity` field, we base the risk score on that value according to [these guidelines](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-basic-params). * If neither of the above apply, we assign a default value. * **Rule severity:** For Elastic-authored rules, severity scores are predefined. For custom translated rules, risk scores are based on the source rule's severity field. Splunk severity scores are translated to Elastic rule severity scores as follows: diff --git a/solutions/security/get-started/configure-advanced-settings.md b/solutions/security/get-started/configure-advanced-settings.md index 77b693ceeb..6ae77914bd 100644 --- a/solutions/security/get-started/configure-advanced-settings.md +++ b/solutions/security/get-started/configure-advanced-settings.md @@ -93,7 +93,7 @@ The `securitySolution:defaultThreatIndex` advanced setting specifies threat inte You can specify one or more threat intelligence indices; multiple indices must be separated by commas. By default, only the `logs-ti_*` index pattern is specified. Do not remove or overwrite this index pattern, as it is used by {{agent}} integrations. ::::{important} -Threat intelligence indices aren’t required to be ECS-compatible for use in indicator match rules. However, we strongly recommend compatibility if you want your alerts to be enriched with relevant threat indicator information. When searching for threat indicator data, indicator match rules use the threat indicator path specified in the **Indicator prefix override** advanced setting. Visit [Configure advanced rule settings](/solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-advanced-params) for more information. +Threat intelligence indices aren’t required to be ECS-compatible for use in indicator match rules. However, we strongly recommend compatibility if you want your alerts to be enriched with relevant threat indicator information. When searching for threat indicator data, indicator match rules use the threat indicator path specified in the **Indicator prefix override** advanced setting. Visit [Configure advanced rule settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) for more information. :::: @@ -210,7 +210,7 @@ If you’ve ensured that your detection rules have the required privileges acros stack: ga 9.2 ``` -To control whether alert suppression continues after you close a supressed alert during an [active suppression window](/solutions/security/detect-and-alert/suppress-detection-alerts.md#security-alert-suppression-impact-close-alerts), configure the `securitySolution:suppressionBehaviorOnAlertClosure` advanced setting. This setting lets you choose whether suppression continues or restarts when the next qualifying alert meets the suppression criteria. The default selection is **Restart suppression**. +To control whether alert suppression continues after you close a supressed alert during an [active suppression window](/solutions/security/detect-and-alert/alert-suppression.md#security-alert-suppression-impact-close-alerts), configure the `securitySolution:suppressionBehaviorOnAlertClosure` advanced setting. This setting lets you choose whether suppression continues or restarts when the next qualifying alert meets the suppression criteria. The default selection is **Restart suppression**. ## Show/hide related integrations in Rules page tables [show-related-integrations] @@ -238,7 +238,7 @@ To ensure the rules in your {{kib}} space exclude query results from cold and fr This setting does not apply to {{ml}} rules because {{ml}} anomalies are not stored in cold or frozen data tiers. ::::{tip} -To only exclude cold and frozen data from specific rules, add a [Query DSL filter](/solutions/security/detect-and-alert/exclude-cold-frozen-data-from-individual-rules.md) to the rules you want affected. +To only exclude cold and frozen data from specific rules, add a [Query DSL filter](/solutions/security/detect-and-alert/set-rule-data-sources.md) to the rules you want affected. :::: diff --git a/solutions/security/get-started/elastic-security-requirements.md b/solutions/security/get-started/elastic-security-requirements.md index 009908658a..962fc58bcc 100644 --- a/solutions/security/get-started/elastic-security-requirements.md +++ b/solutions/security/get-started/elastic-security-requirements.md @@ -58,7 +58,7 @@ For more information about index privileges, refer to [{{es}} security privilege There are some additional requirements for specific features: -* [Detections requirements](/solutions/security/detect-and-alert/detections-requirements.md) +* [Detections requirements](/solutions/security/detect-and-alert/requirements-privileges.md) * [Cases requirements](/solutions/security/investigate/cases-requirements.md) * [Entity risk scoring requirements](/solutions/security/advanced-entity-analytics/entity-risk-scoring-requirements.md) * [Machine learning job and rule requirements](/solutions/security/advanced-entity-analytics/machine-learning-job-rule-requirements.md) diff --git a/solutions/security/get-started/elastic-security-ui.md b/solutions/security/get-started/elastic-security-ui.md index 6218164e88..5458e820be 100644 --- a/solutions/security/get-started/elastic-security-ui.md +++ b/solutions/security/get-started/elastic-security-ui.md @@ -111,7 +111,7 @@ Expand this section to access the following pages: * [Shared Exception Lists](/solutions/security/detect-and-alert/rule-exceptions.md#shared-exception-list-intro): View and manage rule exceptions and shared exception lists. -* [MITRE ATT&CK® coverage](/solutions/security/detect-and-alert/mitre-attandckr-coverage.md): Review your coverage of MITRE ATT&CK® tactics and techniques, based on installed rules. +* [MITRE ATT&CK® coverage](/solutions/security/detect-and-alert/mitre-attack-coverage.md): Review your coverage of MITRE ATT&CK® tactics and techniques, based on installed rules. ### Alerts [_alerts] diff --git a/solutions/security/get-started/get-started-cloud-security.md b/solutions/security/get-started/get-started-cloud-security.md index 818f9d60ff..9896fb657f 100644 --- a/solutions/security/get-started/get-started-cloud-security.md +++ b/solutions/security/get-started/get-started-cloud-security.md @@ -80,7 +80,7 @@ You can create a detection rule directly from the **Findings** page: 1. Click the arrow to the left of a finding to open the findings flyout. 2. Click **Take action**, then **Create a detection rule**. -3. To review or customize the new rule, click **View rule**. For example, you might want to set up a rule action—like an email or Slack notification—when alerts are generated. To learn more about rule actions, refer to [](/solutions/security/detect-and-alert/create-detection-rule.md#rule-notifications). +3. To review or customize the new rule, click **View rule**. For example, you might want to set up a rule action—like an email or Slack notification—when alerts are generated. To learn more about rule actions, refer to [](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications). ## More resources diff --git a/solutions/security/get-started/get-started-detect-with-siem.md b/solutions/security/get-started/get-started-detect-with-siem.md index b9722ee086..bb333315f1 100644 --- a/solutions/security/get-started/get-started-detect-with-siem.md +++ b/solutions/security/get-started/get-started-detect-with-siem.md @@ -19,7 +19,7 @@ In this quickstart guide, we'll learn how to use some of {{elastic-sec}}'s SIEM * If you're using the recommended integration in this guide, {{elastic-defend}}, then: * Ensure you have the minimum [system requirements](/solutions/security/configure-elastic-defend/elastic-defend-requirements.md) to install {{elastic-defend}}. * Ensure you grant the appropriate [{{elastic-defend}} sub-feature privileges](/solutions/security/configure-elastic-defend/elastic-defend-feature-privileges.md). At the least, you need `All` access for the **Endpoint List** and **Elastic Defend Policy Management** sub-features. -* We recommend `manage` and `write` access to manage rules and alerts. Refer to [](/solutions/security/detect-and-alert/detections-privileges.md) for the required cluster, index, and space privileges. +* We recommend `manage` and `write` access to manage rules and alerts. Refer to [](/solutions/security/detect-and-alert/requirements-privileges.md) for the required cluster, index, and space privileges. ## Add data using {{elastic-defend}} @@ -110,7 +110,7 @@ Detection rules allow you to monitor your environment by searching for source ev ::: ::::{tip} -{{elastic-sec}} regularly updates prebuilt rules to ensure they detect the latest threats. However, you must manually update these rules to the latest version. To learn how to do this, refer to [Update prebuilt rules](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#update-prebuilt-rules). To learn how to view and manage all detection rules, refer to [Manage detection rules](/solutions/security/detect-and-alert/manage-detection-rules.md). +{{elastic-sec}} regularly updates prebuilt rules to ensure they detect the latest threats. However, you must manually update these rules to the latest version. To learn how to do this, refer to [Update prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#update-prebuilt-rules). To learn how to view and manage all detection rules, refer to [Manage detection rules](/solutions/security/detect-and-alert/manage-detection-rules.md). :::: ::: diff --git a/solutions/security/get-started/get-started-endpoint-security.md b/solutions/security/get-started/get-started-endpoint-security.md index 71c1efcdb2..064d7f1ca7 100644 --- a/solutions/security/get-started/get-started-endpoint-security.md +++ b/solutions/security/get-started/get-started-endpoint-security.md @@ -120,5 +120,5 @@ You can apply trusted applications, blocklist entries, and host isolation except After your hosts are secure and your environment has all the appropriate security settings configured, we recommend taking these next steps: * Check out the [Hosts page](/solutions/security/explore/hosts-page.md) for a comprehensive overview of all hosts and host-related security events. This page is also useful to identify uncommon processes and anomalies discovered by {{ml}} jobs. -* Install and turn on prebuilt detection rules. You're already set to receive endpoint threat alerts from {{elastic-defend}}, but did you know {{elastic-sec}} ships with several out-of-the-box rules you can turn on? Check out our [SIEM quick start guide](/solutions/security/get-started/get-started-detect-with-siem.md#add-elastic-prebuilt-detection-rules) or our [documentation](/solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md#load-prebuilt-rules). +* Install and turn on prebuilt detection rules. You're already set to receive endpoint threat alerts from {{elastic-defend}}, but did you know {{elastic-sec}} ships with several out-of-the-box rules you can turn on? Check out our [SIEM quick start guide](/solutions/security/get-started/get-started-detect-with-siem.md#add-elastic-prebuilt-detection-rules) or our [documentation](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#load-prebuilt-rules). * Discover all the other tools available to [manage {{elastic-defend}}](/solutions/security/manage-elastic-defend.md). diff --git a/solutions/security/investigate/add-osquery-response-actions.md b/solutions/security/investigate/add-osquery-response-actions.md index 0d877fdda8..43fa335388 100644 --- a/solutions/security/investigate/add-osquery-response-actions.md +++ b/solutions/security/investigate/add-osquery-response-actions.md @@ -42,7 +42,7 @@ You can add Osquery Response Actions to new or existing custom query rules. Quer 1. Choose one of the following: - * **New rule**: When you are on the last step of [custom query rule](/solutions/security/detect-and-alert/create-detection-rule.md#create-custom-rule) creation, go to the Response Actions section and click the **Osquery** icon. + * **New rule**: When you are on the last step of [custom query rule](/solutions/security/detect-and-alert/custom-query.md) creation, go to the Response Actions section and click the **Osquery** icon. * **Existing rule**: Edit the rule’s settings, then go to the **Actions** tab. In the tab, click the **Osquery** icon under the Response Actions section. ::::{note} diff --git a/solutions/security/investigate/cases-requirements.md b/solutions/security/investigate/cases-requirements.md index 5986cffb3c..5c8557e679 100644 --- a/solutions/security/investigate/cases-requirements.md +++ b/solutions/security/investigate/cases-requirements.md @@ -16,7 +16,7 @@ products: ::::{note} - To send cases to external systems, ensure you have the appropriate [{{stack}} subscription](https://www.elastic.co/pricing) or [{{serverless-short}} project feature tier](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). -- You need particular subscriptions and privileges to manage case attachments. For example in {{stack}}, to add alerts to cases, you must have privileges for [managing alerts](/solutions/security/detect-and-alert/detections-privileges.md). In {{serverless-short}}, you need the Security Analytics Complete [project feature tier](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). +- You need particular subscriptions and privileges to manage case attachments. For example in {{stack}}, to add alerts to cases, you must have privileges for [managing alerts](/solutions/security/detect-and-alert/requirements-privileges.md). In {{serverless-short}}, you need the Security Analytics Complete [project feature tier](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). - If you have an on-premises deployment and want email notifications and external incident management systems to contain links back to {{kib}}, you must configure the [server.publicBaseUrl](kibana://reference/configuration-reference/general-settings.md#server-publicbaseurl) setting. :::: diff --git a/solutions/security/investigate/timeline-templates.md b/solutions/security/investigate/timeline-templates.md index 1cf9b03ba0..52bd62d484 100644 --- a/solutions/security/investigate/timeline-templates.md +++ b/solutions/security/investigate/timeline-templates.md @@ -23,7 +23,7 @@ Templates can include two types of filters: For example, if you define the `host.name: "{host.name}"` template filter, when alerts generated by the rule are investigated in Timeline, the alert’s `host.name` value is used in the filter. If the alert’s `host.name` value is `Linux_stafordshire-061`, the Timeline filter is: `host.name: "Linux_stafordshire-061"`. ::::{note} -For information on how to add Timeline templates to rules, refer to [Create a detection rule](/solutions/security/detect-and-alert/create-detection-rule.md). +For information on how to add Timeline templates to rules, refer to [Create a detection rule](/solutions/security/detect-and-alert/using-the-rule-builder.md). :::: diff --git a/solutions/security/manage-elastic-defend/endpoint-protection-rules.md b/solutions/security/manage-elastic-defend/endpoint-protection-rules.md index b571a4c621..6e48f530dc 100644 --- a/solutions/security/manage-elastic-defend/endpoint-protection-rules.md +++ b/solutions/security/manage-elastic-defend/endpoint-protection-rules.md @@ -13,7 +13,7 @@ products: # Endpoint protection rules [endpoint-protection-rules] -Endpoint protection rules are [prebuilt rules](../detect-and-alert/install-manage-elastic-prebuilt-rules.md) designed to help you manage and respond to alerts generated by {{elastic-endpoint}}, the installed component that performs {{elastic-defend}}'s threat monitoring and prevention. These rules include the Endpoint Security ({{elastic-defend}}) rule as well as additional detection and prevention rules for different {{elastic-defend}} protection features. +Endpoint protection rules are [prebuilt rules](../detect-and-alert/install-manage-prebuilt-rules.md) designed to help you manage and respond to alerts generated by {{elastic-endpoint}}, the installed component that performs {{elastic-defend}}'s threat monitoring and prevention. These rules include the Endpoint Security ({{elastic-defend}}) rule as well as additional detection and prevention rules for different {{elastic-defend}} protection features. ::::{important} To receive {{elastic-endpoint}} alerts, you must install {{agent}} and the {{elastic-defend}} integration on your hosts (refer to [Install {{elastic-defend}}](../configure-elastic-defend/install-elastic-defend.md)). @@ -54,7 +54,7 @@ If you choose to use the feature-specific protection rules, we recommend that yo :::: -To use these rules, you need to manually enable them from the **Rules** page in the {{security-app}}. Follow the instructions for [installing and enabling Elastic prebuilt rules](../detect-and-alert/install-manage-elastic-prebuilt-rules.md#load-prebuilt-rules). +To use these rules, you need to manually enable them from the **Rules** page in the {{security-app}}. Follow the instructions for [installing and enabling Elastic prebuilt rules](../detect-and-alert/install-manage-prebuilt-rules.md#load-prebuilt-rules). ## Endpoint security exception handling [_endpoint_security_exception_handling] diff --git a/solutions/toc.yml b/solutions/toc.yml index 26f2e93f4a..7b7cc85eca 100644 --- a/solutions/toc.yml +++ b/solutions/toc.yml @@ -571,36 +571,60 @@ toc: - file: security/ai/ease/ease-value-report.md - file: security/detect-and-alert.md children: - - file: security/detect-and-alert/detections-requirements.md - - file: security/detect-and-alert/detections-privileges.md - - file: security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md - - file: security/detect-and-alert/about-detection-rules.md - - file: security/detect-and-alert/create-detection-rule.md - children: - - file: security/detect-and-alert/cross-cluster-search-detection-rules.md - - file: security/detect-and-alert/launch-timeline-from-investigation-guides.md - - file: security/detect-and-alert/exclude-cold-frozen-data-from-individual-rules.md - - file: security/detect-and-alert/install-manage-elastic-prebuilt-rules.md - children: - - file: security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md + - file: security/detect-and-alert/before-you-begin.md + children: + - file: security/detect-and-alert/requirements-privileges.md + - file: security/detect-and-alert/detections-privileges.md + - file: security/detect-and-alert/advanced-data-source-configuration.md + children: + - file: security/detect-and-alert/cross-cluster-search-detection-rules.md + - file: security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md + - file: security/detect-and-alert/prebuilt-rules.md + children: + - file: security/detect-and-alert/install-manage-prebuilt-rules.md + - file: security/detect-and-alert/update-prebuilt-rules.md + - file: security/detect-and-alert/mitre-attack-coverage.md + - file: security/detect-and-alert/create-a-detection-rule.md + children: + - file: security/detect-and-alert/choose-the-right-rule-type.md + children: + - file: security/detect-and-alert/about-building-block-rules.md + - file: security/detect-and-alert/rule-types.md + children: + - file: security/detect-and-alert/eql.md + - file: security/detect-and-alert/custom-query.md + - file: security/detect-and-alert/threshold.md + - file: security/detect-and-alert/indicator-match.md + - file: security/detect-and-alert/new-terms.md + - file: security/detect-and-alert/esql.md + - file: security/detect-and-alert/machine-learning.md + - file: security/detect-and-alert/using-the-rule-builder.md + - file: security/detect-and-alert/using-the-api.md + + - file: security/detect-and-alert/set-rule-data-sources.md + - file: security/detect-and-alert/write-investigation-guides.md + - file: security/detect-and-alert/validate-and-test-rules.md - file: security/detect-and-alert/manage-detection-rules.md - file: security/detect-and-alert/monitor-rule-executions.md - - file: security/detect-and-alert/rule-exceptions.md + - file: security/detect-and-alert/reduce-noise-and-false-positives.md children: - - file: security/detect-and-alert/create-manage-value-lists.md - - file: security/detect-and-alert/add-manage-exceptions.md - - file: security/detect-and-alert/create-manage-shared-exception-lists.md - - file: security/detect-and-alert/about-building-block-rules.md - - file: security/detect-and-alert/mitre-attandckr-coverage.md + - file: security/detect-and-alert/tune-detection-rules.md + - file: security/detect-and-alert/rule-exceptions.md + children: + - file: security/detect-and-alert/create-manage-value-lists.md + - file: security/detect-and-alert/add-manage-exceptions.md + - file: security/detect-and-alert/create-manage-shared-exception-lists.md + - file: security/detect-and-alert/alert-suppression.md + - file: security/detect-and-alert/snooze-rule-actions.md - file: security/detect-and-alert/manage-detection-alerts.md children: - file: security/detect-and-alert/visualize-detection-alerts.md - file: security/detect-and-alert/view-detection-alert-details.md - file: security/detect-and-alert/add-detection-alerts-to-cases.md - - file: security/detect-and-alert/suppress-detection-alerts.md - - file: security/detect-and-alert/reduce-notifications-alerts.md - - file: security/detect-and-alert/query-alert-indices.md - - file: security/detect-and-alert/tune-detection-rules.md + - file: security/detect-and-alert/detections-reference.md + children: + - file: security/detect-and-alert/rule-settings-reference.md + - file: security/detect-and-alert/query-alert-indices.md - file: security/configure-elastic-defend.md children: - file: security/configure-elastic-defend/elastic-defend-requirements.md diff --git a/troubleshoot/security/detection-rules.md b/troubleshoot/security/detection-rules.md index a931bab567..62486cde21 100644 --- a/troubleshoot/security/detection-rules.md +++ b/troubleshoot/security/detection-rules.md @@ -17,7 +17,7 @@ products: # Troubleshoot detection rules [ts-detection-rules] -This topic covers common troubleshooting issues when creating or managing [detection rules](../../solutions/security/detect-and-alert/create-detection-rule.md). +This topic covers common troubleshooting issues when creating or managing [detection rules](../../solutions/security/detect-and-alert/using-the-rule-builder.md). ## {{ml-cap}} rules [ML-rules-ts] @@ -150,7 +150,7 @@ You can also use Task Manager in {{kib}} to troubleshoot background tasks and pr When a rule reaches the maximum number of alerts it can generate during a single rule execution, the following warning appears on the rule’s details page and in the rule execution log: `This rule reached the maximum alert limit for the rule execution. Some alerts were not created.` -If you receive this warning, go to the rule’s **Alerts** tab and check for anything unexpected. Unexpected alerts might be created from data source issues or queries that are too broadly scoped. To further reduce alert volume, you can also add [rule exceptions](../../solutions/security/detect-and-alert/add-manage-exceptions.md) or [suppress alerts](../../solutions/security/detect-and-alert/suppress-detection-alerts.md). +If you receive this warning, go to the rule’s **Alerts** tab and check for anything unexpected. Unexpected alerts might be created from data source issues or queries that are too broadly scoped. To further reduce alert volume, you can also add [rule exceptions](../../solutions/security/detect-and-alert/add-manage-exceptions.md) or [suppress alerts](../../solutions/security/detect-and-alert/alert-suppression.md). ### Troubleshoot gaps [troubleshoot-gaps] @@ -162,7 +162,7 @@ It’s recommended to set the `Additional look-back time` to at least 1 minute. {{elastic-sec}} prevents duplication. Any duplicate alerts that are discovered during the `Additional look-back time` are *not* created. ::::{note} -If the rule that experiences gaps is an indicator match rule, see [how to tune indicator match rules](../../solutions/security/detect-and-alert/tune-detection-rules.md#tune-indicator-rules). Also note that {{elastic-sec}} provides [limited support for indicator match rules](../../solutions/security/detect-and-alert.md#support-indicator-rules). +If the rule that experiences gaps is an indicator match rule, see [how to tune indicator match rules](../../solutions/security/detect-and-alert/tune-detection-rules.md#tune-indicator-rules). Also note that {{elastic-sec}} provides [limited support for indicator match rules](../../solutions/security/detect-and-alert/indicator-match.md). :::: @@ -178,7 +178,7 @@ Even if your rule runs at its scheduled time, there might still be missing alert In addition, use caution when creating custom rule schedules to ensure that the specified interval + additional look-back time is greater than your deployment’s ingestion pipeline delay. -You can reduce the number of missed alerts due to ingestion pipeline delay by specifying the `Timestamp override` field value to `event.ingested` in [advanced settings](../../solutions/security/detect-and-alert/create-detection-rule.md#rule-ui-advanced-params) during rule creation or editing. The detection engine uses the value from the `event.ingested` field as the timestamp when executing the rule. +You can reduce the number of missed alerts due to ingestion pipeline delay by specifying the `Timestamp override` field value to `event.ingested` in [advanced settings](../../solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) during rule creation or editing. The detection engine uses the value from the `event.ingested` field as the timestamp when executing the rule. For example, say an event occurred at 10:00 but wasn’t ingested into {{es}} until 10:10 due to an ingestion pipeline delay. If you created a rule to detect that event with an interval + additional look-back time of 6 minutes, and the rule executes at 10:12, it would still detect the event because the `event.ingested` timestamp was from 10:10, only 2 minutes before the rule executed and well within the rule’s 6-minute interval + additional look-back time. From 778ef005765f6a29812873e0c7a7d215748215b8 Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Sat, 21 Feb 2026 02:08:30 -0500 Subject: [PATCH 02/24] Fix ref --- .../create-manage-value-lists.md | 2 +- .../validate-and-test-rules.md | 91 ++++++++++++++++++- 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/solutions/security/detect-and-alert/create-manage-value-lists.md b/solutions/security/detect-and-alert/create-manage-value-lists.md index 1fd3848d2f..867745e4f2 100644 --- a/solutions/security/detect-and-alert/create-manage-value-lists.md +++ b/solutions/security/detect-and-alert/create-manage-value-lists.md @@ -31,7 +31,7 @@ You can also use a value list as the [indicator match index](indicator-match.md# ## Value lists requirements -To manage value lists, your role must have the required privileges. Refer to [Manage exception value lists](/solutions/security/detect-and-alert/requirements-privileges.md#detections-privileges-manage-value-lists) for details. +To manage value lists, your role must have the required privileges. Refer to [Manage exception value lists](/solutions/security/detect-and-alert/detections-privileges.md#detections-privileges-manage-value-lists) for details. ## Create value lists [create-value-lists] diff --git a/solutions/security/detect-and-alert/validate-and-test-rules.md b/solutions/security/detect-and-alert/validate-and-test-rules.md index 9d9a9ff518..2be47c9855 100644 --- a/solutions/security/detect-and-alert/validate-and-test-rules.md +++ b/solutions/security/detect-and-alert/validate-and-test-rules.md @@ -11,6 +11,93 @@ description: Verify rule logic against historical data, assess alert volume, and # Validate and test rules -% This is a stub. Content is coming soon. +Before enabling a new detection rule in production, validate that it detects what you intend, at a volume your team can handle, without generating excessive false positives. The steps below apply to any [rule type](/solutions/security/detect-and-alert/rule-types.md). -Verify rule logic against historical data, assess alert volume, and use shadow deployment before enabling notifications. +## Validate against historical data [validate-historical-data] + +Use the rule preview feature to test your rule's query against a historical time range before enabling it. This shows you what the rule would have detected without creating actual alerts. + +1. While creating or editing a rule, select **Preview** in the rule builder. +2. Select a time range that represents normal activity in your environment. A range of 7 to 14 days captures both weekday and weekend patterns. +3. Review the preview results. Look for: + - **Expected true positives.** Does the rule detect the activity it's designed to catch? If you have known-good test data (for example, from red team exercises), confirm those events appear in the results. + - **Obvious false positives.** Do any results represent legitimate activity? If so, refine the query or plan to add [exceptions](/solutions/security/detect-and-alert/rule-exceptions.md) before enabling the rule. + - **Missing detections.** If the rule produces no results and you expected it to, check that the required data sources are being ingested and that your index patterns are correct. + +::::{tip} +If the rule uses [alert suppression](/solutions/security/detect-and-alert/alert-suppression.md), use the rule preview to visualize how suppression affects the alert output. This helps you confirm that suppression is grouping events as expected before the rule goes live. +:::: + +## Run a manual test [manual-test-run] + +For rules that are already enabled, you can [manually run](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules) them over a specific time range to test behavior against real data. Unlike preview, manual runs create actual alerts and trigger rule actions. + +Manual runs are useful when: + +- You want to test a rule against a specific incident window where you know what happened. +- You need to fill a gap in rule coverage after a rule was temporarily not running. +- You want to verify that a rule change produces the expected results in production. + +::::{important} +Manual runs activate all configured [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) except **Summary of alerts** actions that run at a custom frequency. If you want to test without sending notifications, [snooze the rule's actions](/solutions/security/detect-and-alert/snooze-rule-actions.md) first. +:::: + +## Estimate alert volume [estimate-alert-volume] + +High-volume rules can overwhelm analysts and degrade rule execution performance. Before enabling a rule, estimate its expected alert rate. + +1. Review the preview result count from the historical validation step. Divide by the number of days in your preview range to get a daily estimate. +2. Compare this estimate against your team's capacity. A rule generating hundreds of alerts per day may need to be narrowed before it delivers value. +3. If volume is too high, consider: + - Narrowing the query to be more specific. + - Adding [alert suppression](/solutions/security/detect-and-alert/alert-suppression.md) to deduplicate repeated alerts for the same entity. + - Adjusting the rule's schedule interval if a longer interval is acceptable for the threat you're detecting. + +## Assess false positive rate [assess-false-positives] + +A rule that produces mostly false positives trains analysts to ignore it. Evaluate the signal-to-noise ratio before going live. + +1. Sample 20 to 50 results from your preview and classify each as a true positive, false positive, or uncertain. +2. If more than half are false positives, refine the rule before enabling it. Common adjustments include: + - Adding field constraints to the query (for example, excluding known service accounts or internal IP ranges). + - Preparing [exceptions](/solutions/security/detect-and-alert/rule-exceptions.md) for specific known-safe cases. + - Switching to a more targeted [rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) if the current type is too broad for the detection goal. +3. Document the false positive patterns you find. This helps other analysts understand the rule's limitations and speeds up triage. + +## Shadow deployment [shadow-deployment] + +Shadow deployment means enabling a rule in production but suppressing its notifications so you can observe real alert output before analysts are paged. + +1. Create and enable the rule normally. +2. [Snooze the rule's actions](/solutions/security/detect-and-alert/snooze-rule-actions.md) to suppress notifications. The rule runs on its schedule and writes alerts to the index, but no emails, Slack messages, or webhook calls are sent. +3. Let the rule run for a representative period (typically 3 to 7 days). +4. Review the alerts it generates in the **Alerts** table. Evaluate: + - Is the volume manageable? + - Are the alerts actionable? + - Do any patterns suggest the query needs further tuning or exceptions? +5. Once you're confident in the rule's output, unsnooze the actions to begin sending notifications. + +::::{tip} +Shadow deployment is especially useful for rules that are difficult to validate with historical data alone, such as [threshold rules](/solutions/security/detect-and-alert/threshold.md) where volume depends on live traffic patterns, or [{{ml}} rules](/solutions/security/detect-and-alert/machine-learning.md) where anomaly baselines evolve over time. +:::: + +## Post-enablement monitoring [post-enablement-monitoring] + +Validation doesn't end when a rule goes live. Monitor newly enabled rules closely during their first weeks in production. + +- **Check execution health.** Use the [Rule Monitoring tab](/solutions/security/detect-and-alert/monitor-rule-executions.md) to confirm the rule is executing successfully and not timing out or failing. +- **Track alert volume trends.** A sudden spike or drop in alert volume can indicate a change in the data source, a rule misconfiguration, or an emerging incident. +- **Collect analyst feedback.** Ask the analysts triaging the rule's alerts whether they find them actionable. If a rule consistently produces alerts that are closed without investigation, it needs further tuning. +- **Review after one to two weeks.** Revisit the rule's configuration after it has run through a full operational cycle. Adjust the query, exceptions, suppression, or schedule based on what you've learned. + +## Test rules externally with Detection-as-Code [test-rules-dac] + +If you manage rules outside of the {{kib}} UI, you can use [Detection-as-Code](https://dac-reference.readthedocs.io/en/latest/dac_concept_and_workflows.html) (DaC) workflows to test rules before deploying them. The Elastic Security Labs team maintains the [detection-rules](https://github.com/elastic/detection-rules) repo, which provides tooling for developing, testing, and releasing detection rules programmatically. + +DaC workflows let you: + +- Validate rule syntax and schema before deployment. +- Run unit tests against rule logic in a CI/CD pipeline. +- Track rule changes in version control for auditability. + +To get started, refer to the [DaC documentation](https://github.com/elastic/detection-rules/blob/main/README.md#detections-as-code-dac). For managing rules through the API, refer to [Using the API](/solutions/security/detect-and-alert/using-the-api.md). From 65909720d9de3aa6ff233e4ec91372bbfa6bb7c4 Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Mon, 23 Feb 2026 15:46:01 -0500 Subject: [PATCH 03/24] Continued moving content around --- redirects.yml | 8 +- .../fields-and-object-schemas/alert-schema.md | 4 +- solutions/security/detect-and-alert.md | 2 +- .../about-building-block-rules.md | 2 +- .../detect-and-alert/add-manage-exceptions.md | 4 +- .../detect-and-alert/alert-suppression.md | 6 +- .../choose-the-right-rule-type.md | 8 +- .../detect-and-alert/common-rule-settings.md | 303 ++++++++++++++++++ .../security/detect-and-alert/custom-query.md | 2 +- ...te-a-detection-rule.md => custom-rules.md} | 2 +- .../detect-and-alert/detections-reference.md | 2 +- solutions/security/detect-and-alert/eql.md | 2 +- solutions/security/detect-and-alert/esql.md | 4 +- .../detect-and-alert/indicator-match.md | 11 +- .../install-manage-prebuilt-rules.md | 7 +- .../detect-and-alert/machine-learning.md | 2 +- .../manage-detection-rules.md | 18 +- .../security/detect-and-alert/new-terms.md | 2 +- .../reduce-noise-and-false-positives.md | 6 +- .../requirements-privileges.md | 2 +- .../rule-settings-reference.md | 2 +- .../detect-and-alert/set-rule-data-sources.md | 18 +- .../detect-and-alert/snooze-rule-actions.md | 24 -- .../security/detect-and-alert/threshold.md | 2 +- .../using-the-rule-builder.md | 12 +- .../validate-and-test-rules.md | 154 ++++----- .../view-detection-alert-details.md | 2 +- .../write-investigation-guides.md | 2 +- .../get-started/automatic-migration.md | 2 +- .../configure-advanced-settings.md | 2 +- .../get-started/get-started-cloud-security.md | 2 +- solutions/toc.yml | 9 +- troubleshoot/security/detection-rules.md | 29 +- 33 files changed, 493 insertions(+), 164 deletions(-) create mode 100644 solutions/security/detect-and-alert/common-rule-settings.md rename solutions/security/detect-and-alert/{create-a-detection-rule.md => custom-rules.md} (98%) delete mode 100644 solutions/security/detect-and-alert/snooze-rule-actions.md diff --git a/redirects.yml b/redirects.yml index 7a14dec600..3b94db6b66 100644 --- a/redirects.yml +++ b/redirects.yml @@ -713,5 +713,11 @@ redirects: 'solutions/security/detect-and-alert/launch-timeline-from-investigation-guides.md': 'solutions/security/detect-and-alert/write-investigation-guides.md' 'solutions/security/detect-and-alert/exclude-cold-frozen-data-from-individual-rules.md': 'solutions/security/detect-and-alert/set-rule-data-sources.md' 'solutions/security/detect-and-alert/suppress-detection-alerts.md': 'solutions/security/detect-and-alert/alert-suppression.md' - 'solutions/security/detect-and-alert/reduce-notifications-alerts.md': 'solutions/security/detect-and-alert/snooze-rule-actions.md' + 'solutions/security/detect-and-alert/reduce-notifications-alerts.md': 'solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions' + 'solutions/security/detect-and-alert/snooze-rule-actions.md': 'solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions' 'solutions/security/detect-and-alert/reference.md': 'solutions/security/detect-and-alert/detections-reference.md' + # Anchor redirect for cold/frozen data content moved to set-rule-data-sources.md + 'solutions/security/detect-and-alert/requirements-privileges.md': + to: 'solutions/security/detect-and-alert/requirements-privileges.md' + anchors: + 'cold-frozen-data': 'solutions/security/detect-and-alert/set-rule-data-sources.md#exclude-cold-frozen-tier' diff --git a/reference/security/fields-and-object-schemas/alert-schema.md b/reference/security/fields-and-object-schemas/alert-schema.md index dd84811722..d6aadf6e49 100644 --- a/reference/security/fields-and-object-schemas/alert-schema.md +++ b/reference/security/fields-and-object-schemas/alert-schema.md @@ -80,8 +80,8 @@ The non-ECS fields listed below are beta and subject to change. | `kibana.alert.original_event.*` | Event information copied from the original source event.
Type: object | | `kibana.alert.original_time` | The value copied from the source event (`@timestamp`).
Type: date | | `kibana.alert.reason` | Type: keyword | -| `kibana.alert.rule.author` | The value of the `author` who created the rule. Refer to [configure advanced rule settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params).
Type: keyword | -| `kibana.alert.building_block_type` | The value of `building_block_type` from the rule that generated this alert. Refer to [configure advanced rule settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params).
Type: keyword | +| `kibana.alert.rule.author` | The value of the `author` who created the rule. Refer to [configure advanced rule settings](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params).
Type: keyword | +| `kibana.alert.building_block_type` | The value of `building_block_type` from the rule that generated this alert. Refer to [configure advanced rule settings](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params).
Type: keyword | | `kibana.alert.rule.created_at` | The value of `created.at` from the rule that generated this alert.
Type: date | | `kibana.alert.rule.created_by` | Type: keyword | | `kibana.alert.rule.description` | Type: keyword | diff --git a/solutions/security/detect-and-alert.md b/solutions/security/detect-and-alert.md index 20401e73f6..f62e626392 100644 --- a/solutions/security/detect-and-alert.md +++ b/solutions/security/detect-and-alert.md @@ -22,7 +22,7 @@ products: | Set up detection for the first time | [Requirements and privileges](/solutions/security/detect-and-alert/requirements-privileges.md) → [Install prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md) | | Take over an existing deployment | [MITRE ATT&CK coverage](/solutions/security/detect-and-alert/mitre-attack-coverage.md) → [Monitor rule executions](/solutions/security/detect-and-alert/monitor-rule-executions.md) | | Build coverage for a specific threat | [Choose the right rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) → [Rule builder](/solutions/security/detect-and-alert/using-the-rule-builder.md) | -| Reduce noise from existing rules | [Tune detection rules](/solutions/security/detect-and-alert/tune-detection-rules.md) → [Exceptions](/solutions/security/detect-and-alert/rule-exceptions.md), [Suppression](/solutions/security/detect-and-alert/alert-suppression.md), or [Snooze](/solutions/security/detect-and-alert/snooze-rule-actions.md) | +| Reduce noise from existing rules | [Tune detection rules](/solutions/security/detect-and-alert/tune-detection-rules.md) → [Exceptions](/solutions/security/detect-and-alert/rule-exceptions.md), [Suppression](/solutions/security/detect-and-alert/alert-suppression.md), or [Snooze](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) | ## Detection program lifecycle diff --git a/solutions/security/detect-and-alert/about-building-block-rules.md b/solutions/security/detect-and-alert/about-building-block-rules.md index 33e6111c58..9c03005af4 100644 --- a/solutions/security/detect-and-alert/about-building-block-rules.md +++ b/solutions/security/detect-and-alert/about-building-block-rules.md @@ -20,7 +20,7 @@ Create building block rules when you do not want to see their generated alerts i * Rules that execute on the alert indices (`.alerts-security.alerts-`). You can then use building block rules to create hidden alerts that act as a basis for an *ordinary* rule to generate visible alerts. ::::{tip} -Add [rule notifications](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) to building block rules to notify you when building block alerts are generated. +Add [rule notifications](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications) to building block rules to notify you when building block alerts are generated. :::: diff --git a/solutions/security/detect-and-alert/add-manage-exceptions.md b/solutions/security/detect-and-alert/add-manage-exceptions.md index 1d4bae4ab4..86ca1d4cca 100644 --- a/solutions/security/detect-and-alert/add-manage-exceptions.md +++ b/solutions/security/detect-and-alert/add-manage-exceptions.md @@ -191,7 +191,7 @@ For required privileges to view and manage {{elastic-endpoint}} exceptions, refe ## Add {{elastic-endpoint}} exceptions [endpoint-rule-exceptions] -You can add {{elastic-endpoint}} exceptions to [endpoint protection rules](../manage-elastic-defend/endpoint-protection-rules.md) or to rules that are associated with {{elastic-endpoint}} rule exceptions. To associate rules when creating or editing a rule, select the [**{{elastic-endpoint}} exceptions**](rule-settings-reference.md#rule-ui-advanced-params) option. +You can add {{elastic-endpoint}} exceptions to [endpoint protection rules](../manage-elastic-defend/endpoint-protection-rules.md) or to rules that are associated with {{elastic-endpoint}} rule exceptions. To associate rules when creating or editing a rule, select the [**{{elastic-endpoint}} exceptions**](common-rule-settings.md#rule-ui-advanced-params) option. Endpoint exceptions are added to the endpoint protection rules **and** the {{elastic-endpoint}} on your hosts. @@ -228,7 +228,7 @@ Additionally, to add an Endpoint exception to an endpoint protection rule, there 2. Expand the Endpoint Security Exception List or click the list name to open the list’s details page. Next, click **Add endpoint exception**. ::::{note} - The Endpoint Security Exception List is automatically created. By default, it’s associated with endpoint protection rules and any rules with the [**{{elastic-endpoint}} exceptions**](rule-settings-reference.md#rule-ui-advanced-params) option selected. + The Endpoint Security Exception List is automatically created. By default, it’s associated with endpoint protection rules and any rules with the [**{{elastic-endpoint}} exceptions**](common-rule-settings.md#rule-ui-advanced-params) option selected. :::: diff --git a/solutions/security/detect-and-alert/alert-suppression.md b/solutions/security/detect-and-alert/alert-suppression.md index 47175df3ac..3fc9ecf42b 100644 --- a/solutions/security/detect-and-alert/alert-suppression.md +++ b/solutions/security/detect-and-alert/alert-suppression.md @@ -74,7 +74,7 @@ You can configure alert suppression when [creating](/solutions/security/detect-a ::::{tip} * Use the **Rule preview** before saving the rule to visualize how alert suppression will affect the alerts created, based on historical data. -* If a rule times out while suppression is turned on, try shortening the rule’s [look-back](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-schedule) time or turn off suppression to improve the rule’s performance. +* If a rule times out while suppression is turned on, try shortening the rule’s [look-back](/solutions/security/detect-and-alert/common-rule-settings.md#rule-schedule) time or turn off suppression to improve the rule’s performance. :::: @@ -157,8 +157,8 @@ For example, say you set the suppression time period to 5 minutes and specify to Some rule types have a maximum number of alerts that can be suppressed (custom query rules don’t have a suppression limit): -* **Threshold, event correlation, {{esql}}, and {{ml}}:** The maximum number of alerts is the value you choose for the rule’s **Max alerts per run** [advanced setting](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params), which is `100` by default. -* **Indicator match and new terms:** The maximum number is five times the value you choose for the rule’s **Max alerts per run** [advanced setting](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params). The default value is `100`, which means the default maximum limit for indicator match rules and new terms rules is `500`. +* **Threshold, event correlation, {{esql}}, and {{ml}}:** The maximum number of alerts is the value you choose for the rule’s **Max alerts per run** [advanced setting](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params), which is `100` by default. +* **Indicator match and new terms:** The maximum number is five times the value you choose for the rule’s **Max alerts per run** [advanced setting](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params). The default value is `100`, which means the default maximum limit for indicator match rules and new terms rules is `500`. ## Bulk apply and remove alert suppression [security-alert-suppression-bulk-apply] diff --git a/solutions/security/detect-and-alert/choose-the-right-rule-type.md b/solutions/security/detect-and-alert/choose-the-right-rule-type.md index 0e0cc55626..dd26490993 100644 --- a/solutions/security/detect-and-alert/choose-the-right-rule-type.md +++ b/solutions/security/detect-and-alert/choose-the-right-rule-type.md @@ -70,7 +70,7 @@ For example, you might create three building block rules: Each of these individually produces low-confidence alerts. A downstream EQL sequence rule can then query the alert index to detect all three occurring on the same host within a short time window, producing a single high-confidence alert for a likely intrusion chain. ::::{tip} -Add [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) to building block rules if you want notifications when building block alerts are generated, even though the alerts are hidden from the default Alerts view. +Add [rule actions](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications) to building block rules if you want notifications when building block alerts are generated, even though the alerts are hidden from the default Alerts view. :::: ### Viewing building block alerts @@ -84,11 +84,11 @@ On a building block rule's details page, the rule's alerts are always displayed. ### Marking a rule as a building block -Select the **Building block** option in the rule's [advanced settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) when creating or editing any rule type. +Select the **Building block** option in the rule's [advanced settings](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params) when creating or editing any rule type. ## Shared concepts [shared-rule-concepts] -The following concepts apply to all rule types. For the full settings reference, see [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). +The following concepts apply to all rule types. For the full settings reference, see [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md). ### Data sources [views-index-patterns] @@ -120,4 +120,4 @@ Exceptions are supported for custom query, {{ml}}, event correlation, and indica ### Notifications [about-notifications] -For both prebuilt and custom rules, you can send notifications when alerts are created. Notifications can be sent through {{jira}}, Microsoft Teams, PagerDuty, Slack, and other connectors. Configure actions when you [create or edit a rule](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications). +For both prebuilt and custom rules, you can send notifications when alerts are created. Notifications can be sent through {{jira}}, Microsoft Teams, PagerDuty, Slack, and other connectors. Configure actions when you [create or edit a rule](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications). diff --git a/solutions/security/detect-and-alert/common-rule-settings.md b/solutions/security/detect-and-alert/common-rule-settings.md new file mode 100644 index 0000000000..8321dd0f74 --- /dev/null +++ b/solutions/security/detect-and-alert/common-rule-settings.md @@ -0,0 +1,303 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Reference for all detection rule settings including basic, advanced, schedule, actions, and notification variables. +--- + +# Common rule settings [common-detection-rule-settings] + +All detection rules share a common set of settings for describing the rule, controlling its schedule, configuring actions, and setting up response actions. These settings apply regardless of the [rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) you select. + +For rule-type-specific settings (query definitions, index patterns, {{ml}} jobs, and so on), refer to [Using the rule builder](/solutions/security/detect-and-alert/using-the-rule-builder.md). + +## Basic settings [rule-ui-basic-params] + +Configure these settings in the **About rule** pane. + +**Name** +: The rule's name. + +**Description** +: A description of what the rule does. + +**Default severity** +: The severity level of alerts created by the rule: + + * **Low**: Alerts that are of interest but generally are not considered to be security incidents. Sometimes a combination of low severity alerts can indicate suspicious activity. + * **Medium**: Alerts that require investigation. + * **High**: Alerts that require an immediate investigation. + * **Critical**: Alerts that indicate it is highly likely a security incident has occurred. + +**Severity override** (optional) +: Select to use source event values to override the **Default severity** in generated alerts. When selected, a UI component is displayed where you can map the source event field values to severity levels. + + :::{image} /solutions/images/security-severity-mapping-ui.png + :alt: severity mapping ui + :screenshot: + ::: + + ::::{note} + For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. Overrides are not supported for event correlation rules. + :::: + +**Default risk score** +: A numerical value between 0 and 100 that indicates the risk of events detected by the rule. This setting changes to a default value when you change the **Severity** level, but you can adjust the risk score as needed. General guidelines are: + + * `0` - `21` represents low severity. + * `22` - `47` represents medium severity. + * `48` - `73` represents high severity. + * `74` - `100` represents critical severity. + +**Risk score override** (optional) +: Select to use a source event value to override the **Default risk score** in generated alerts. When selected, a UI component is displayed to select the source field used for the risk score. + + :::{image} /solutions/images/security-risk-source-field-ui.png + :alt: risk source field ui + :screenshot: + ::: + + ::::{note} + For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. + :::: + +**Tags** (optional) +: Words and phrases used to categorize, filter, and search the rule. + +## Advanced settings [rule-ui-advanced-params] + +Configure these settings by clicking **Advanced settings** in the **About rule** pane. + +**Reference URLs** (optional) +: References to information that is relevant to the rule. For example, links to background information. + +**False positive examples** (optional) +: List of common scenarios that may produce false-positive alerts. + +**MITRE ATT&CKTM threats** (optional) +: Add relevant [MITRE](https://attack.mitre.org/) framework tactics, techniques, and subtechniques. + +**Custom highlighted fields** (optional) +: Specify highlighted fields for unique alert investigation flows. You can select any fields that are available in the indices you selected for the rule's data source. + + After you create the rule, you can find all custom highlighted fields in the About section of the rule details page. If the rule has alerts, you can find custom highlighted fields in the [Highlighted fields](/solutions/security/detect-and-alert/view-detection-alert-details.md#investigation-section) section of the alert details flyout. + +**Setup guide** (optional) +: Instructions on rule prerequisites such as required integrations, configuration steps, and anything else needed for the rule to work correctly. + +**Investigation guide** (optional) +: Information for analysts investigating alerts created by the rule. You can also add action buttons to [run Osquery](/solutions/security/investigate/run-osquery-from-investigation-guides.md) or [start Timeline investigations](/solutions/security/detect-and-alert/write-investigation-guides.md) using alert data. + +**Author** (optional) +: The rule's authors. + +**License** (optional) +: The rule's license. + +**Elastic endpoint exceptions** (optional) +: Adds all [{{elastic-endpoint}} exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md#endpoint-rule-exceptions) to this rule. + + ::::{note} + If you select this option, you can add {{elastic-endpoint}} exceptions on the Rule details page. Additionally, all future exceptions added to [endpoint protection rules](/solutions/security/manage-elastic-defend/endpoint-protection-rules.md) will also affect this rule. + :::: + +**Building block** (optional) +: Select to create a building-block rule. By default, alerts generated from a building-block rule are not displayed in the UI. See [About building block rules](/solutions/security/detect-and-alert/about-building-block-rules.md) for more information. + +**Max alerts per run** (optional) +: Specify the maximum number of alerts the rule can create each time it executes. Default is 100. + + ::::{note} + This setting can be superseded by the [{{kib}} configuration setting](kibana://reference/configuration-reference/alerting-settings.md#alert-settings) `xpack.alerting.rules.run.alerts.max`, which determines the maximum alerts generated by *any* rule in the {{kib}} alerting framework. For example, if `xpack.alerting.rules.run.alerts.max` is set to `1000`, the rule can generate no more than 1000 alerts even if **Max alerts per run** is set higher. + :::: + +**Indicator prefix override** (indicator match rules only) +: Define the location of indicator data within the structure of indicator documents. When the indicator match rule executes, it queries specified indicator indices and references this setting to locate fields with indicator data. This data is used to enrich indicator match alerts with metadata about matched threat indicators. The default value for this setting is `threat.indicator`. + + ::::{important} + If your threat indicator data is at a different location, update this setting accordingly to ensure alert enrichment can still be performed. + :::: + +**Rule name override** (optional) +: Select a source event field to use as the rule name in the UI (Alerts table). This is useful for exposing, at a glance, more information about an alert. For example, if the rule generates alerts from Suricata, selecting `event.action` lets you see what action (Suricata category) caused the event directly in the Alerts table. + + ::::{note} + For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. + :::: + +**Timestamp override** (optional) +: Select a source event timestamp field. When selected, the rule's query uses the selected field, instead of the default `@timestamp` field, to search for alerts. This can help reduce missing alerts due to network or server outages. Specifically, if your ingest pipeline adds a timestamp when events are sent to {{es}}, this can prevent missing alerts from ingestion delays. + + If the selected field is unavailable, the rule query will use the `@timestamp` field instead. If you don't want to use the `@timestamp` field because your data source has an inaccurate `@timestamp` value, select the **Do not use @timestamp as a fallback timestamp field** option instead. This ensures the rule query ignores the `@timestamp` field entirely. + + ::::{tip} + The [Microsoft](beats://reference/filebeat/filebeat-module-microsoft.md) and [Google Workspace](beats://reference/filebeat/filebeat-module-google_workspace.md) {{filebeat}} modules have an `event.ingested` timestamp field that can be used instead of the default `@timestamp` field. + :::: + +## Schedule settings [rule-schedule] + +**Runs every** +: How often the rule runs. + +**Additional look-back time** (optional) +: When defined, the rule searches indices with the additional time. + + For example, if you set a rule to run every 5 minutes with an additional look-back time of 1 minute, the rule runs every 5 minutes but analyzes the documents added to indices during the last 6 minutes. + + ::::{important} + It is recommended to set the `Additional look-back time` to at least 1 minute. This ensures there are no missing alerts when a rule does not run exactly at its scheduled time. + + {{elastic-sec}} prevents duplication. Any duplicate alerts that are discovered during the `Additional look-back time` are *not* created. + :::: + +## Rule actions [rule-notifications] + +Use actions to set up notifications sent through other systems when alerts are generated. + +::::{note} +To use actions for alert notifications, you need the [appropriate license]({{subscriptions}}). For more information, see [Cases requirements](/solutions/security/investigate/cases-requirements.md). +:::: + +::::{tip} +:applies_to: {stack: preview 9.3+, serverless: preview} +You can use [workflows](/explore-analyze/workflows.md) as a rule action to automate alert response processes. Workflows can create cases, route notifications, or perform other automated tasks when alerts are generated. To learn how to set up a workflow as a rule action, refer to [](/explore-analyze/workflows/triggers/alert-triggers.md). +:::: + +**Connector type** +: Determines how notifications are sent. For example, if you select the {{jira}} connector, notifications are sent to your {{jira}} system. + + ::::{note} + Each action type requires a connector. Connectors store the information required to send the notification from the external system. You can configure connectors while creating the rule or from the **{{connectors-ui}}** page. For more information, refer to [Action and connector types](/deploy-manage/manage-connectors.md). + + Some connectors that perform actions require less configuration. For example, you do not need to set the action frequency or variables for the [Cases connector](kibana://reference/connectors-kibana/cases-action-type.md). + :::: + +**Action frequency** +: Defines when notifications are sent: + + * **Summary of alerts**: Sends a report that summarizes generated alerts at the specified time intervals. + + ::::{note} + When setting a custom notification frequency, do not select a time that is shorter than the rule's execution schedule. + :::: + + * **For each alert**: Sends notifications every time new alerts are generated. + +**Conditional actions** (optional) +: Specify additional conditions that need to be met for notifications to send: + + * **If alert matches query**: Enter a KQL query that defines field-value pairs or query conditions that must be met for notifications to send. The query only searches alert documents in the indices specified for the rule. + * **If alert is generated during timeframe**: Set timeframe details. Notifications are only sent if alerts are generated within the timeframe you define. + +**Notification message** +: Use the default notification message or customize it. You can add more context to the message by clicking the icon above the message text box and selecting from a list of available [alert notification variables](#rule-action-variables). + +::::{important} +After you activate a rule, you can check if it is executing as expected using the [Monitoring tab](/troubleshoot/security/detection-rules.md) on the {{rules-ui}} page. If you see values in the `Gap` column, you can [Troubleshoot missing alerts](/troubleshoot/security/detection-rules.md#troubleshoot-signals). + +When a rule fails to execute, the {{security-app}} tries to rerun it at its next scheduled time. +:::: + +## Response actions [rule-response-action] + +Use response actions to set up additional functionality that executes whenever a rule triggers: + +* **Osquery**: Include live Osquery queries with a custom query rule. When an alert is generated, Osquery automatically collects data on the system related to the alert. Refer to [Add Osquery Response Actions](/solutions/security/investigate/add-osquery-response-actions.md) to learn more. +* **{{elastic-defend}}**: Automatically execute response actions on an endpoint when rule conditions are met. For example, you can automatically isolate a host or end a process when specific activities or events are detected on the host. Refer to [Automated response actions](/solutions/security/endpoint-response-actions/automated-response-actions.md) to learn more. + +::::{important} +Host isolation involves quarantining a host from the network to prevent further proliferation of threats and limit potential damage. Be aware that automatic host isolation can cause unintended consequences, such as disrupting legitimate user activities or blocking critical business processes. +:::: + +## Alert notification variables [rule-action-variables] + +You can use [mustache syntax](http://mustache.github.io/) to add variables to notification messages. The action frequency you select determines the available variables. + +::::{note} +Refer to [Action frequency: Summary of alerts](/explore-analyze/alerts-cases/alerts/rule-action-variables.md#alert-summary-action-variables) to learn about additional variables that can be passed if the rule's action frequency is **Summary of alerts**. +:::: + +### Variables for all rules [all-rule-variables] + +* `{{context.alerts}}`: Array of detected alerts +* `{{{context.results_link}}}`: URL to the alerts in {{kib}} +* `{{context.rule.anomaly_threshold}}`: Anomaly threshold score above which alerts are generated ({{ml}} rules only) +* `{{context.rule.description}}`: Rule description +* `{{context.rule.false_positives}}`: Rule false positives +* `{{context.rule.filters}}`: Rule filters (query rules only) +* `{{context.rule.id}}`: Unique rule ID returned after creating the rule +* `{{context.rule.index}}`: Indices rule runs on (query rules only) +* `{{context.rule.language}}`: Rule query language (query rules only) +* `{{context.rule.machine_learning_job_id}}`: ID of associated {{ml}} job ({{ml}} rules only) +* `{{context.rule.max_signals}}`: Maximum allowed number of alerts per rule execution +* `{{context.rule.name}}`: Rule name +* `{{context.rule.query}}`: Rule query (query rules only) +* `{{context.rule.references}}`: Rule references +* `{{context.rule.risk_score}}`: Default rule risk score + + ::::{note} + This placeholder contains the rule's default values even when the **Risk score override** option is used. + :::: + +* `{{context.rule.rule_id}}`: Generated or user-defined rule ID that can be used as an identifier across systems +* `{{context.rule.saved_id}}`: Saved search ID +* `{{context.rule.severity}}`: Default rule severity + + ::::{note} + This placeholder contains the rule's default values even when the **Severity override** option is used. + :::: + +* `{{context.rule.threat}}`: Rule threat framework +* `{{context.rule.threshold}}`: Rule threshold values (threshold rules only) +* `{{context.rule.timeline_id}}`: Associated Timeline ID +* `{{context.rule.timeline_title}}`: Associated Timeline name +* `{{context.rule.type}}`: Rule type +* `{{context.rule.version}}`: Rule version +* `{{date}}`: Date the rule scheduled the action +* `{{kibanaBaseUrl}}`: Configured `server.publicBaseUrl` value, or empty string if not configured +* `{{rule.id}}`: ID of the rule +* `{{rule.name}}`: Name of the rule +* `{{rule.spaceId}}`: Space ID of the rule +* `{{rule.tags}}`: Tags of the rule +* `{{rule.type}}`: Type of rule +* `{{state.signals_count}}`: Number of alerts detected + +### Per-alert variables [per-alert-variables] + +The following variables can only be passed if the rule's action frequency is **For each alert**: + +* `{{alert.actionGroup}}`: Action group of the alert that scheduled actions for the rule +* `{{alert.actionGroupName}}`: Human-readable name of the action group of the alert that scheduled actions for the rule +* `{{alert.actionSubgroup}}`: Action subgroup of the alert that scheduled actions for the rule +* `{{alert.id}}`: ID of the alert that scheduled actions for the rule +* `{{alert.flapping}}`: A flag on the alert that indicates whether the alert status is changing repeatedly + +### Placeholder examples [placeholder-examples] + +To understand which fields to parse, see the [Detections API](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-security-detections-api) to view the JSON representation of rules. + +Example using `{{context.rule.filters}}` to output a list of filters: + +```json +{{#context.rule.filters}} +{{^meta.disabled}}{{meta.key}} {{#meta.negate}}NOT {{/meta.negate}}{{meta.type}} {{^exists}}{{meta.value}}{{meta.params.query}}{{/exists}}{{/meta.disabled}} +{{/context.rule.filters}} +``` + +Example using `{{context.alerts}}` as an array, which contains each alert generated since the last time the action was executed: + +```json +{{#context.alerts}} +Detection alert for user: {{user.name}} +{{/context.alerts}} +``` + +Example using the mustache "current element" notation `{{.}}` to output all the rule references in the `signal.rule.references` array: + +```json +{{#signal.rule.references}} {{.}} {{/signal.rule.references}} +``` diff --git a/solutions/security/detect-and-alert/custom-query.md b/solutions/security/detect-and-alert/custom-query.md index 7d64acaf86..0b752cd577 100644 --- a/solutions/security/detect-and-alert/custom-query.md +++ b/solutions/security/detect-and-alert/custom-query.md @@ -83,7 +83,7 @@ event.action:"Process Create (rule: ProcessCreate)" and process.name:"vssadmin.e ## Custom query field reference [custom-query-fields] -The following settings are specific to custom query rules. For settings shared across all rule types (severity, risk score, schedule, actions, and so on), refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). +The following settings are specific to custom query rules. For settings shared across all rule types (severity, risk score, schedule, actions, and so on), refer to [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md). **Index patterns or data view** : The {{es}} indices or data view the rule queries when searching for events. Index patterns are prepopulated with the indices configured in the [default {{elastic-sec}} indices](/solutions/security/get-started/configure-advanced-settings.md#update-sec-indices) advanced setting. Alternatively, select a data view from the drop-down to use its associated index patterns and [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md). diff --git a/solutions/security/detect-and-alert/create-a-detection-rule.md b/solutions/security/detect-and-alert/custom-rules.md similarity index 98% rename from solutions/security/detect-and-alert/create-a-detection-rule.md rename to solutions/security/detect-and-alert/custom-rules.md index b6258538c3..d25ff662d8 100644 --- a/solutions/security/detect-and-alert/create-a-detection-rule.md +++ b/solutions/security/detect-and-alert/custom-rules.md @@ -9,7 +9,7 @@ products: description: Overview and workflow for building custom detection rules tailored to your threat model. --- -# Create a detection rule +# Custom rules Build custom detection rules tailored to your environment and threat model. The pages in this section walk you through selecting a rule type, writing rule logic, and configuring rule settings. diff --git a/solutions/security/detect-and-alert/detections-reference.md b/solutions/security/detect-and-alert/detections-reference.md index 4c960643a2..fd6df8e765 100644 --- a/solutions/security/detect-and-alert/detections-reference.md +++ b/solutions/security/detect-and-alert/detections-reference.md @@ -13,7 +13,7 @@ description: Technical reference for detection rule settings, alert schema, and Look up rule configuration settings, alert field definitions, and patterns for querying alert indices directly. These pages are designed for reference, not reading end to end. -**[Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md)** +**[Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md)** : Use this when you need to look up a specific rule setting, understand what a field does, or check valid values and defaults. Covers all shared settings (severity, risk score, schedule, actions, response actions, and notification variables) that apply across rule types. For rule-type-specific fields, refer to the individual [rule type](/solutions/security/detect-and-alert/rule-types.md) pages. **[Query alert indices](/solutions/security/detect-and-alert/query-alert-indices.md)** diff --git a/solutions/security/detect-and-alert/eql.md b/solutions/security/detect-and-alert/eql.md index b58a6e4da6..129ab63a7b 100644 --- a/solutions/security/detect-and-alert/eql.md +++ b/solutions/security/detect-and-alert/eql.md @@ -95,7 +95,7 @@ This detects a successful login that is never followed by a logout within one ho ## EQL field reference [eql-fields] -The following settings are specific to EQL rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). +The following settings are specific to EQL rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md). **Index patterns or {{data-source}}** : The {{es}} indices or {{data-source}} the rule searches when querying for events. diff --git a/solutions/security/detect-and-alert/esql.md b/solutions/security/detect-and-alert/esql.md index 9dba4d68ad..12277c1ded 100644 --- a/solutions/security/detect-and-alert/esql.md +++ b/solutions/security/detect-and-alert/esql.md @@ -92,7 +92,7 @@ If your {{esql}} query creates new fields that are not part of the ECS schema, t ### Custom highlighted fields -When configuring an {{esql}} rule's **Custom highlighted fields** (in [advanced settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params)), you can specify any fields that the query returns. This ensures returned fields are visible in the alert details flyout during investigation. +When configuring an {{esql}} rule's **Custom highlighted fields** (in [advanced settings](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params)), you can specify any fields that the query returns. This ensures returned fields are visible in the alert details flyout during investigation. ::::{tip} **See it in practice.** These patterns demonstrate {{esql}} detection use cases: @@ -103,7 +103,7 @@ When configuring an {{esql}} rule's **Custom highlighted fields** (in [advanced ## {{esql}} field reference [esql-fields] -The following settings are specific to {{esql}} rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). +The following settings are specific to {{esql}} rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md). **{{esql}} query** : The [{{esql}} query](elasticsearch://reference/query-languages/esql.md) that defines the detection logic. Can be aggregating (with `STATS...BY`) or non-aggregating. Each row in the query result becomes an alert. diff --git a/solutions/security/detect-and-alert/indicator-match.md b/solutions/security/detect-and-alert/indicator-match.md index 7e3c894e44..15d06a4a7f 100644 --- a/solutions/security/detect-and-alert/indicator-match.md +++ b/solutions/security/detect-and-alert/indicator-match.md @@ -15,6 +15,15 @@ description: Detect security events that match known threat indicators from thre Indicator match rules continuously compare field values in your security event indices against field values in threat indicator indices. When a match is found, an alert is generated, enriched with metadata from the matched indicator. This makes indicator match rules the primary mechanism for operationalizing threat intelligence feeds within {{elastic-sec}}. +### Limitations + +Indicator match rules provide a powerful capability to search your security data; however, their queries can consume significant deployment resources. When creating an indicator match rule, limit the time range of the indicator index query to the minimum period necessary for the desired rule coverage. For example, the default indicator index query `@timestamp > "now-30d/d"` searches specified indicator indices for indicators ingested during the past 30 days and rounds the query start time down to the nearest day (resolves to UTC `00:00:00`). Without this limitation, the rule includes all of the indicators in your indicator indices, which may extend the time it takes for the indicator index query to complete. + +In addition, the following support restrictions are in place: + +* Indicator match rules don't support cold or frozen data. Cold or frozen data in indices queried by indicator match rules must be older than the time range queried by the rule. If your data's timestamps are unreliable, you can exclude cold and frozen tier data using a Query DSL filter. +* Indicator match rules with an additional look-back time value greater than 24 hours are not supported. + ### When to use an indicator match rule Indicator match rules are the right fit when: @@ -77,7 +86,7 @@ You can use [value lists](/solutions/security/detect-and-alert/create-manage-val ## Indicator match field reference [indicator-match-fields] -The following settings are specific to indicator match rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). +The following settings are specific to indicator match rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md). **Source** : The index patterns or {{data-source}} that store your source event documents. Prepopulated with indices from the [default {{elastic-sec}} indices](/solutions/security/get-started/configure-advanced-settings.md#update-sec-indices) advanced setting. diff --git a/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md b/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md index 2e54c3f91e..1b336965b6 100644 --- a/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md +++ b/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md @@ -17,6 +17,11 @@ description: Learn how to install, enable, update, and manage Elastic prebuilt d Follow these guidelines to start using the {{security-app}}'s [prebuilt rules](detection-rules://index.md), keep them updated, and make sure they have the data needed to run successfully. +There are several special prebuilt rules you should know about: + +* **Endpoint protection rules**: Create alerts based on {{elastic-defend}}'s threat monitoring and prevention. +* **External Alerts**: Create an alert for all incoming third-party system alerts (for example, Suricata alerts). + * [Install and enable Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#load-prebuilt-rules) * [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#prebuilt-rule-tags) * [Select and duplicate all prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#select-all-prebuilt-rules) @@ -103,7 +108,7 @@ Each prebuilt rule includes several tags identifying the rule’s purpose, detec ## Select and duplicate prebuilt rules [select-all-prebuilt-rules] -Without an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can't modify most settings on Elastic prebuilt rules. You can only edit [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-schedule) and [add exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md). If you want to modify other settings on a prebuilt rule, you must first duplicate it, then make your changes to the duplicated rule. Note that your customized rule is entirely separate from the original prebuilt rule, and will not get updates from Elastic if the prebuilt rule is updated. +Without an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can't modify most settings on Elastic prebuilt rules. You can only edit [rule actions](/solutions/security/detect-and-alert/common-rule-settings.md#rule-schedule) and [add exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md). If you want to modify other settings on a prebuilt rule, you must first duplicate it, then make your changes to the duplicated rule. Note that your customized rule is entirely separate from the original prebuilt rule, and will not get updates from Elastic if the prebuilt rule is updated. 1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). 2. In the **Rules** table, select the **Elastic rules** filter. diff --git a/solutions/security/detect-and-alert/machine-learning.md b/solutions/security/detect-and-alert/machine-learning.md index 23133e0f31..09202317d2 100644 --- a/solutions/security/detect-and-alert/machine-learning.md +++ b/solutions/security/detect-and-alert/machine-learning.md @@ -88,7 +88,7 @@ The anomaly score ranges from 0 to 100, where higher scores indicate stronger de ## {{ml}} field reference [ml-fields] -The following settings are specific to {{ml}} rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). +The following settings are specific to {{ml}} rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md). **{{ml-cap}} jobs** : The {{anomaly-jobs}} whose results the rule evaluates. Select one or more jobs. If a selected job is not active, it starts automatically when the rule is enabled. diff --git a/solutions/security/detect-and-alert/manage-detection-rules.md b/solutions/security/detect-and-alert/manage-detection-rules.md index a88f20e08a..5f1b8b432b 100644 --- a/solutions/security/detect-and-alert/manage-detection-rules.md +++ b/solutions/security/detect-and-alert/manage-detection-rules.md @@ -73,7 +73,7 @@ Edit rule settings to modify detection logic, notifications, schedules, and othe ### Requirements * **Custom rules**: You can edit and bulk-modify custom rules with any [{{stack}} subscription](https://www.elastic.co/pricing) or [{{serverless-short}} project tier](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). -* **Prebuilt rules**: You can edit [rule notifications](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) with any subscription or project tier. Editing all other prebuilt rule settings (except **Author** and **License**) or bulk-modifying prebuilt rules requires an [Enterprise subscription](https://www.elastic.co/pricing) or [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). +* **Prebuilt rules**: You can edit [rule notifications](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications) with any subscription or project tier. Editing all other prebuilt rule settings (except **Author** and **License**) or bulk-modifying prebuilt rules requires an [Enterprise subscription](https://www.elastic.co/pricing) or [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md). ### Edit a single rule [edit-single-rule] @@ -101,8 +101,8 @@ Use bulk editing to update settings on multiple rules simultaneously. Rules that * **Index patterns**: Add or delete the index patterns used by all selected rules. * **Tags**: Add or delete tags on all selected rules. * **Custom highlighted fields**: Add custom highlighted fields on all selected rules. You can choose any fields that are available in the [default {{elastic-sec}} indices](/solutions/security/get-started/configure-advanced-settings.md#update-sec-indices), or enter field names from other indices. To overwrite a rule's current set of custom highlighted fields, select the **Overwrite all selected rules' custom highlighted fields** option, then click **Save**. - * **Add rule actions**: Add [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) on all selected rules. If you add multiple actions, you can specify an action frequency for each of them. To overwrite the frequency of existing actions, select the option to **Overwrite all selected rules actions**. Keep in mind that rule actions won't run during a [maintenance window](/explore-analyze/alerts-cases/alerts/maintenance-windows.md); they'll resume after the maintenance window ends. - * **Update rule schedules**: Update the [schedules](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-schedule) and look-back times on all selected rules. + * **Add rule actions**: Add [rule actions](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications) on all selected rules. If you add multiple actions, you can specify an action frequency for each of them. To overwrite the frequency of existing actions, select the option to **Overwrite all selected rules actions**. Keep in mind that rule actions won't run during a [maintenance window](/explore-analyze/alerts-cases/alerts/maintenance-windows.md); they'll resume after the maintenance window ends. + * **Update rule schedules**: Update the [schedules](/solutions/security/detect-and-alert/common-rule-settings.md#rule-schedule) and look-back times on all selected rules. * **Apply Timeline template**: Apply a specified [Timeline template](/solutions/security/investigate/timeline-templates.md) to the selected rules. You can also choose **None** to remove Timeline templates from the selected rules. 4. On the page or flyout that opens, update the rule settings. @@ -211,7 +211,7 @@ Before manually running rules, make sure you properly understand and plan for ru 3. Specify when the manual run starts and ends. The default selection is the current day starting three hours in the past. The rule will search for events during the selected time range. 4. Click **Run** to manually run the rule. -The rule will run over the time range that you selected. Note that all [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) will also be activated, except for **Summary of alerts** actions that run at a custom frequency. +The rule will run over the time range that you selected. Note that all [rule actions](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications) will also be activated, except for **Summary of alerts** actions that run at a custom frequency. Go to the [Manual runs table](/solutions/security/detect-and-alert/monitor-rule-executions.md#manual-runs-table) on the **Execution results** tab to track the manual rule executions. If you manually ran the rule over a gap, you can also monitor the gap fill's progress from the [Gaps table](/solutions/security/detect-and-alert/monitor-rule-executions.md#gaps-table). @@ -351,12 +351,4 @@ You can also check rules' related integrations in the **Installed Rules** and ** ::::{tip} You can hide the **integrations** badge in the Rules tables by turning off the `securitySolution:showRelatedIntegrations` [advanced setting](/solutions/security/get-started/configure-advanced-settings.md#show-related-integrations). -:::: - -## Manage rules as code [manage-rule-dac] - -Utilize the [Detection-as-Code](https://dac-reference.readthedocs.io/en/latest/dac_concept_and_workflows.html) (DaC) principles to externally manage your detection rules. - -The {{elastic-sec}} Labs team uses the [detection-rules](https://github.com/elastic/detection-rules) repo to develop, test, and release {{elastic-sec}}'s[ prebuilt rules](https://github.com/elastic/detection-rules/tree/main/rules). The repo provides DaC features and allows you to customize settings to simplify the setup for managing user rules with the DaCe pipeline. - -To get started, refer to the [DaC documentation](https://github.com/elastic/detection-rules/blob/main/README.md#detections-as-code-dac). \ No newline at end of file +:::: \ No newline at end of file diff --git a/solutions/security/detect-and-alert/new-terms.md b/solutions/security/detect-and-alert/new-terms.md index ae6ae5047d..33c91adbdc 100644 --- a/solutions/security/detect-and-alert/new-terms.md +++ b/solutions/security/detect-and-alert/new-terms.md @@ -76,7 +76,7 @@ The **History Window Size** determines how far back the rule looks to decide whe ## New terms field reference [new-terms-fields] -The following settings are specific to new terms rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). +The following settings are specific to new terms rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md). **Index patterns or {{data-source}}** : The {{es}} indices or {{data-source}} the rule searches. diff --git a/solutions/security/detect-and-alert/reduce-noise-and-false-positives.md b/solutions/security/detect-and-alert/reduce-noise-and-false-positives.md index 0cd19eeeb9..0ea07e300c 100644 --- a/solutions/security/detect-and-alert/reduce-noise-and-false-positives.md +++ b/solutions/security/detect-and-alert/reduce-noise-and-false-positives.md @@ -22,9 +22,9 @@ Use the following table to identify which mechanism matches your situation. If m | The rule fires on activity that isn't suspicious in any environment (for example, the query is too broad and catches normal admin behavior) | [Tune rule logic](/solutions/security/detect-and-alert/tune-detection-rules.md) | Refine the query, threshold, or schedule so the rule only matches what it should | | The rule correctly identifies a pattern, but a specific known-safe case keeps firing in your environment (for example, an SCCM server triggers your remote service creation rule nightly) | [Exception](/solutions/security/detect-and-alert/rule-exceptions.md) | Permanently suppress alerts matching specific field values. The rule logic stays unchanged. | | The rule generates many alerts for the same entity in a short window (for example, a compromised host triggers the same rule every 5 minutes for an hour) | [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md) | After the first alert for a given field value, suppress further alerts for a defined time window | -| You need to temporarily stop receiving notifications during scheduled maintenance or a known event | [Snooze actions](/solutions/security/detect-and-alert/snooze-rule-actions.md) | The rule runs and alerts are still created and stored. Only notifications (email, Slack, webhooks) are paused. | +| You need to temporarily stop receiving notifications during scheduled maintenance or a known event | [Snooze actions](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) | The rule runs and alerts are still created and stored. Only notifications (email, Slack, webhooks) are paused. | | A specific known-safe process or user keeps appearing across many different rules (for example, your vulnerability scanner triggers dozens of rules) | [Shared exception list](/solutions/security/detect-and-alert/create-manage-shared-exception-lists.md) | Apply one exception list across multiple rules simultaneously, avoiding duplicate exceptions rule by rule | -| You're deploying a new rule and want to observe alert volume before enabling notifications | [Snooze actions](/solutions/security/detect-and-alert/snooze-rule-actions.md) | Enable the rule fully, review alerts in the UI, and only enable notifications once you're confident in the signal | +| You're deploying a new rule and want to observe alert volume before enabling notifications | [Snooze actions](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) | Enable the rule fully, review alerts in the UI, and only enable notifications once you're confident in the signal | ## How each mechanism works @@ -68,7 +68,7 @@ Acts on: **notifications, after alerts are created and stored** Temporarily pause the rule's notification actions (emails, Slack messages, webhooks) without affecting rule execution or alert creation. Alerts generated during a snooze period are stored normally and visible in the Alerts UI. Snoozing expires automatically or can be canceled manually. For space-wide pausing, use a [maintenance window](/explore-analyze/alerts-cases/alerts/maintenance-windows.md). -Refer to [Snooze rule actions](/solutions/security/detect-and-alert/snooze-rule-actions.md) for detailed guidance. +Refer to [Snooze rule actions](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) for detailed guidance. ## Key distinctions diff --git a/solutions/security/detect-and-alert/requirements-privileges.md b/solutions/security/detect-and-alert/requirements-privileges.md index f0f02d8bd4..f0a306affa 100644 --- a/solutions/security/detect-and-alert/requirements-privileges.md +++ b/solutions/security/detect-and-alert/requirements-privileges.md @@ -96,7 +96,7 @@ With detections enabled, you're ready to create rules and start responding to th - **Respond to and manage alerts** - [Manage detection alerts](/solutions/security/detect-and-alert/manage-detection-alerts.md): Triage, investigate, and resolve alerts - - [Set up alert notifications](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications): Send alerts to external systems like Slack, email, or ticketing tools + - [Set up alert notifications](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications): Send alerts to external systems like Slack, email, or ticketing tools - [Tune rules to reduce noise](/solutions/security/detect-and-alert/tune-detection-rules.md): Add exceptions and adjust rules to minimize false positives ::::{tip} diff --git a/solutions/security/detect-and-alert/rule-settings-reference.md b/solutions/security/detect-and-alert/rule-settings-reference.md index 1684f43501..8321dd0f74 100644 --- a/solutions/security/detect-and-alert/rule-settings-reference.md +++ b/solutions/security/detect-and-alert/rule-settings-reference.md @@ -9,7 +9,7 @@ products: description: Reference for all detection rule settings including basic, advanced, schedule, actions, and notification variables. --- -# Rule settings reference [detection-rule-settings-reference] +# Common rule settings [common-detection-rule-settings] All detection rules share a common set of settings for describing the rule, controlling its schedule, configuring actions, and setting up response actions. These settings apply regardless of the [rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) you select. diff --git a/solutions/security/detect-and-alert/set-rule-data-sources.md b/solutions/security/detect-and-alert/set-rule-data-sources.md index c19722ffbf..480d347c73 100644 --- a/solutions/security/detect-and-alert/set-rule-data-sources.md +++ b/solutions/security/detect-and-alert/set-rule-data-sources.md @@ -37,7 +37,21 @@ For indicator match rules, the **Indicator index patterns** field controls which ## Exclude cold and frozen data [exclude-cold-frozen-tier] -Rules may perform slower or time out if they query data stored in cold or frozen [data tiers](../../../manage-data/lifecycle/data-tiers.md). You have two options for excluding this data: +Cold data tiers store time series data that's accessed infrequently and rarely updated, while frozen data tiers hold time series data that's accessed even less frequently and never updated. Rules may perform slower or time out if they query data stored in these [data tiers](../../../manage-data/lifecycle/data-tiers.md). + +### Best practices + +* **Retention in hot tier**: Keep data in the hot tier ({{ilm}} hot phase) for at least 24 hours. {{ilm-cap}} policies that move ingested data from the hot phase to another phase (for example, cold or frozen) in less than 24 hours may cause performance issues or rule execution errors. +* **Replicas for mission-critical data**: Your data should have replicas if it must be highly available. Since frozen tiers don't have replicas by default, shard unavailability can cause partial rule run failures. Shard unavailability may also be encountered during or after {{stack}} upgrades. If this happens, you can manually rerun rules over the affected time period once the shards are available. + +### Limitations + +* To avoid rule failures, do not modify {{ilm}} policies for {{elastic-sec}}-controlled indices, such as alert and list indices. +* Source data must have an {{ilm}} policy that keeps it in the hot or warm tiers for at least 24 hours before moving to cold or frozen tiers. + +### Exclusion options + +You have two options for excluding cold and frozen data from rules: **Space-level setting (all rules).** Configure the `excludedDataTiersForRuleExecution` [advanced setting](../get-started/configure-advanced-settings.md#exclude-cold-frozen-data-rule-executions) to exclude cold or frozen data from all rules in a {{kib}} space. This does not apply to {{ml}} rules. Only available on {{stack}}. @@ -88,4 +102,4 @@ To apply a filter, paste the Query DSL into the **Custom query** filter bar when * [Advanced data source configuration](/solutions/security/detect-and-alert/advanced-data-source-configuration.md): Deployment-level settings that affect data sources, including {{ccs}} and logsdb index mode. * [Update default {{elastic-sec}} indices](/solutions/security/get-started/configure-advanced-settings.md#update-sec-indices): Change the space-level default index patterns inherited by all rules. -* [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md): Shared rule settings including timestamp override, which controls which timestamp field the rule uses when querying data. +* [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md): Shared rule settings including timestamp override, which controls which timestamp field the rule uses when querying data. diff --git a/solutions/security/detect-and-alert/snooze-rule-actions.md b/solutions/security/detect-and-alert/snooze-rule-actions.md deleted file mode 100644 index bc110effa8..0000000000 --- a/solutions/security/detect-and-alert/snooze-rule-actions.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/security/current/reduce-notifications-alerts.html - - https://www.elastic.co/guide/en/serverless/current/security-reduce-notifications-alerts.html -applies_to: - stack: all - serverless: - security: all -products: - - id: security - - id: cloud-serverless -description: Compare rule snoozing, maintenance windows, alert suppression, and exceptions for reducing notifications. ---- - -# Reduce notifications and alerts [security-reduce-notifications-alerts] - -{{elastic-sec}} offers several features to help reduce the number of notifications and alerts generated by your detection rules. This table provides a general comparison of these features, with links for more details: - -| | | -| --- | --- | -| [Rule action snoozing](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) | **Stops a specific rule’s notification actions from running**.

Use to avoid unnecessary notifications from a specific rule. The rule continues to run and generate alerts during the snooze period, but its [notification actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-response-action) don’t run.
| -| [Maintenance window](/explore-analyze/alerts-cases/alerts/maintenance-windows.md) | **Prevents all rules' notification actions from running**.

Use to avoid false alarms and unnecessary notifications during planned outages. All rules continue to run and generate alerts during the maintenance window, but their [notification actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) don’t run.

**Note**: Maintenance windows are a {{kib}} feature, configured outside of the {{security-app}} in **Stack Management**.

| -| [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md) | **Reduces repeated or duplicate alerts**.

Use to reduce the number of alerts created when a rule meets its criteria repeatedly. Duplicate qualifying events are grouped, and only one alert is created for each group.
| -| [Rule exception](/solutions/security/detect-and-alert/rule-exceptions.md) | **Prevents a rule from creating alerts under specific conditions**.

Use to reduce false positive alerts by preventing trusted processes and network activity from generating unnecessary alerts. You can configure an exception to be used by a single rule or shared among multiple rules, but they typically don’t affect *all* rules.
| diff --git a/solutions/security/detect-and-alert/threshold.md b/solutions/security/detect-and-alert/threshold.md index bd6f30a54a..c3ea352d96 100644 --- a/solutions/security/detect-and-alert/threshold.md +++ b/solutions/security/detect-and-alert/threshold.md @@ -76,7 +76,7 @@ Keep this in mind when configuring severity overrides, risk score overrides, or ## Threshold field reference [threshold-fields] -The following settings are specific to threshold rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md). +The following settings are specific to threshold rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md). **Index patterns or {{data-source}}** : The {{es}} indices or {{data-source}} the rule searches. diff --git a/solutions/security/detect-and-alert/using-the-rule-builder.md b/solutions/security/detect-and-alert/using-the-rule-builder.md index 225c6cfe0b..08357c53f4 100644 --- a/solutions/security/detect-and-alert/using-the-rule-builder.md +++ b/solutions/security/detect-and-alert/using-the-rule-builder.md @@ -18,11 +18,11 @@ description: Step-by-step guide to create detection rules using the Kibana rule Once the Detections feature is [turned on](/solutions/security/detect-and-alert/requirements-privileges.md), follow these steps to create a detection rule: 1. Define the [rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md#rule-types). The configuration for this step varies depending on the rule type. -2. Configure [basic rule settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-basic-params). -3. Configure [advanced rule settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) (optional). -4. Set the [rule's schedule](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-schedule). -5. Set up [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) (optional). -6. Set up [response actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-response-action) (optional). +2. Configure [basic rule settings](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-basic-params). +3. Configure [advanced rule settings](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params) (optional). +4. Set the [rule's schedule](/solutions/security/detect-and-alert/common-rule-settings.md#rule-schedule). +5. Set up [rule actions](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications) (optional). +6. Set up [response actions](/solutions/security/detect-and-alert/common-rule-settings.md#rule-response-action) (optional). ::::{tip} * At any step, you can preview the rule before saving it to see what kind of results you can expect. @@ -56,6 +56,6 @@ To understand which type to use, refer to [Select the right rule type](/solution ## Related pages -* [Rule settings reference](/solutions/security/detect-and-alert/rule-settings-reference.md): All shared rule settings, including severity, risk score, schedule, actions, and notification variables. +* [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md): All shared rule settings, including severity, risk score, schedule, actions, and notification variables. * [Using the API](/solutions/security/detect-and-alert/using-the-api.md): Create and manage rules programmatically. * [Manage detection rules](/solutions/security/detect-and-alert/manage-detection-rules.md): Enable, export, duplicate, and bulk-edit rules. diff --git a/solutions/security/detect-and-alert/validate-and-test-rules.md b/solutions/security/detect-and-alert/validate-and-test-rules.md index 2be47c9855..7a7e02b6de 100644 --- a/solutions/security/detect-and-alert/validate-and-test-rules.md +++ b/solutions/security/detect-and-alert/validate-and-test-rules.md @@ -13,83 +13,6 @@ description: Verify rule logic against historical data, assess alert volume, and Before enabling a new detection rule in production, validate that it detects what you intend, at a volume your team can handle, without generating excessive false positives. The steps below apply to any [rule type](/solutions/security/detect-and-alert/rule-types.md). -## Validate against historical data [validate-historical-data] - -Use the rule preview feature to test your rule's query against a historical time range before enabling it. This shows you what the rule would have detected without creating actual alerts. - -1. While creating or editing a rule, select **Preview** in the rule builder. -2. Select a time range that represents normal activity in your environment. A range of 7 to 14 days captures both weekday and weekend patterns. -3. Review the preview results. Look for: - - **Expected true positives.** Does the rule detect the activity it's designed to catch? If you have known-good test data (for example, from red team exercises), confirm those events appear in the results. - - **Obvious false positives.** Do any results represent legitimate activity? If so, refine the query or plan to add [exceptions](/solutions/security/detect-and-alert/rule-exceptions.md) before enabling the rule. - - **Missing detections.** If the rule produces no results and you expected it to, check that the required data sources are being ingested and that your index patterns are correct. - -::::{tip} -If the rule uses [alert suppression](/solutions/security/detect-and-alert/alert-suppression.md), use the rule preview to visualize how suppression affects the alert output. This helps you confirm that suppression is grouping events as expected before the rule goes live. -:::: - -## Run a manual test [manual-test-run] - -For rules that are already enabled, you can [manually run](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules) them over a specific time range to test behavior against real data. Unlike preview, manual runs create actual alerts and trigger rule actions. - -Manual runs are useful when: - -- You want to test a rule against a specific incident window where you know what happened. -- You need to fill a gap in rule coverage after a rule was temporarily not running. -- You want to verify that a rule change produces the expected results in production. - -::::{important} -Manual runs activate all configured [rule actions](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications) except **Summary of alerts** actions that run at a custom frequency. If you want to test without sending notifications, [snooze the rule's actions](/solutions/security/detect-and-alert/snooze-rule-actions.md) first. -:::: - -## Estimate alert volume [estimate-alert-volume] - -High-volume rules can overwhelm analysts and degrade rule execution performance. Before enabling a rule, estimate its expected alert rate. - -1. Review the preview result count from the historical validation step. Divide by the number of days in your preview range to get a daily estimate. -2. Compare this estimate against your team's capacity. A rule generating hundreds of alerts per day may need to be narrowed before it delivers value. -3. If volume is too high, consider: - - Narrowing the query to be more specific. - - Adding [alert suppression](/solutions/security/detect-and-alert/alert-suppression.md) to deduplicate repeated alerts for the same entity. - - Adjusting the rule's schedule interval if a longer interval is acceptable for the threat you're detecting. - -## Assess false positive rate [assess-false-positives] - -A rule that produces mostly false positives trains analysts to ignore it. Evaluate the signal-to-noise ratio before going live. - -1. Sample 20 to 50 results from your preview and classify each as a true positive, false positive, or uncertain. -2. If more than half are false positives, refine the rule before enabling it. Common adjustments include: - - Adding field constraints to the query (for example, excluding known service accounts or internal IP ranges). - - Preparing [exceptions](/solutions/security/detect-and-alert/rule-exceptions.md) for specific known-safe cases. - - Switching to a more targeted [rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) if the current type is too broad for the detection goal. -3. Document the false positive patterns you find. This helps other analysts understand the rule's limitations and speeds up triage. - -## Shadow deployment [shadow-deployment] - -Shadow deployment means enabling a rule in production but suppressing its notifications so you can observe real alert output before analysts are paged. - -1. Create and enable the rule normally. -2. [Snooze the rule's actions](/solutions/security/detect-and-alert/snooze-rule-actions.md) to suppress notifications. The rule runs on its schedule and writes alerts to the index, but no emails, Slack messages, or webhook calls are sent. -3. Let the rule run for a representative period (typically 3 to 7 days). -4. Review the alerts it generates in the **Alerts** table. Evaluate: - - Is the volume manageable? - - Are the alerts actionable? - - Do any patterns suggest the query needs further tuning or exceptions? -5. Once you're confident in the rule's output, unsnooze the actions to begin sending notifications. - -::::{tip} -Shadow deployment is especially useful for rules that are difficult to validate with historical data alone, such as [threshold rules](/solutions/security/detect-and-alert/threshold.md) where volume depends on live traffic patterns, or [{{ml}} rules](/solutions/security/detect-and-alert/machine-learning.md) where anomaly baselines evolve over time. -:::: - -## Post-enablement monitoring [post-enablement-monitoring] - -Validation doesn't end when a rule goes live. Monitor newly enabled rules closely during their first weeks in production. - -- **Check execution health.** Use the [Rule Monitoring tab](/solutions/security/detect-and-alert/monitor-rule-executions.md) to confirm the rule is executing successfully and not timing out or failing. -- **Track alert volume trends.** A sudden spike or drop in alert volume can indicate a change in the data source, a rule misconfiguration, or an emerging incident. -- **Collect analyst feedback.** Ask the analysts triaging the rule's alerts whether they find them actionable. If a rule consistently produces alerts that are closed without investigation, it needs further tuning. -- **Review after one to two weeks.** Revisit the rule's configuration after it has run through a full operational cycle. Adjust the query, exceptions, suppression, or schedule based on what you've learned. - ## Test rules externally with Detection-as-Code [test-rules-dac] If you manage rules outside of the {{kib}} UI, you can use [Detection-as-Code](https://dac-reference.readthedocs.io/en/latest/dac_concept_and_workflows.html) (DaC) workflows to test rules before deploying them. The Elastic Security Labs team maintains the [detection-rules](https://github.com/elastic/detection-rules) repo, which provides tooling for developing, testing, and releasing detection rules programmatically. @@ -101,3 +24,80 @@ DaC workflows let you: - Track rule changes in version control for auditability. To get started, refer to the [DaC documentation](https://github.com/elastic/detection-rules/blob/main/README.md#detections-as-code-dac). For managing rules through the API, refer to [Using the API](/solutions/security/detect-and-alert/using-the-api.md). + +% ## Validate against historical data [validate-historical-data] +% +% Use the rule preview feature to test your rule's query against a historical time range before enabling it. This shows you what the rule would have detected without creating actual alerts. +% +% 1. While creating or editing a rule, select **Preview** in the rule builder. +% 2. Select a time range that represents normal activity in your environment. A range of 7 to 14 days captures both weekday and weekend patterns. +% 3. Review the preview results. Look for: +% - **Expected true positives.** Does the rule detect the activity it's designed to catch? If you have known-good test data (for example, from red team exercises), confirm those events appear in the results. +% - **Obvious false positives.** Do any results represent legitimate activity? If so, refine the query or plan to add [exceptions](/solutions/security/detect-and-alert/rule-exceptions.md) before enabling the rule. +% - **Missing detections.** If the rule produces no results and you expected it to, check that the required data sources are being ingested and that your index patterns are correct. +% +% ::::{tip} +% If the rule uses [alert suppression](/solutions/security/detect-and-alert/alert-suppression.md), use the rule preview to visualize how suppression affects the alert output. This helps you confirm that suppression is grouping events as expected before the rule goes live. +% :::: +% +% ## Run a manual test [manual-test-run] +% +% For rules that are already enabled, you can [manually run](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules) them over a specific time range to test behavior against real data. Unlike preview, manual runs create actual alerts and trigger rule actions. +% +% Manual runs are useful when: +% +% - You want to test a rule against a specific incident window where you know what happened. +% - You need to fill a gap in rule coverage after a rule was temporarily not running. +% - You want to verify that a rule change produces the expected results in production. +% +% ::::{important} +% Manual runs activate all configured [rule actions](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications) except **Summary of alerts** actions that run at a custom frequency. If you want to test without sending notifications, [snooze the rule's actions](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) first. +% :::: +% +% ## Estimate alert volume [estimate-alert-volume] +% +% High-volume rules can overwhelm analysts and degrade rule execution performance. Before enabling a rule, estimate its expected alert rate. +% +% 1. Review the preview result count from the historical validation step. Divide by the number of days in your preview range to get a daily estimate. +% 2. Compare this estimate against your team's capacity. A rule generating hundreds of alerts per day may need to be narrowed before it delivers value. +% 3. If volume is too high, consider: +% - Narrowing the query to be more specific. +% - Adding [alert suppression](/solutions/security/detect-and-alert/alert-suppression.md) to deduplicate repeated alerts for the same entity. +% - Adjusting the rule's schedule interval if a longer interval is acceptable for the threat you're detecting. +% +% ## Assess false positive rate [assess-false-positives] +% +% A rule that produces mostly false positives trains analysts to ignore it. Evaluate the signal-to-noise ratio before going live. +% +% 1. Sample 20 to 50 results from your preview and classify each as a true positive, false positive, or uncertain. +% 2. If more than half are false positives, refine the rule before enabling it. Common adjustments include: +% - Adding field constraints to the query (for example, excluding known service accounts or internal IP ranges). +% - Preparing [exceptions](/solutions/security/detect-and-alert/rule-exceptions.md) for specific known-safe cases. +% - Switching to a more targeted [rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) if the current type is too broad for the detection goal. +% 3. Document the false positive patterns you find. This helps other analysts understand the rule's limitations and speeds up triage. +% +% ## Shadow deployment [shadow-deployment] +% +% Shadow deployment means enabling a rule in production but suppressing its notifications so you can observe real alert output before analysts are paged. +% +% 1. Create and enable the rule normally. +% 2. [Snooze the rule's actions](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) to suppress notifications. The rule runs on its schedule and writes alerts to the index, but no emails, Slack messages, or webhook calls are sent. +% 3. Let the rule run for a representative period (typically 3 to 7 days). +% 4. Review the alerts it generates in the **Alerts** table. Evaluate: +% - Is the volume manageable? +% - Are the alerts actionable? +% - Do any patterns suggest the query needs further tuning or exceptions? +% 5. Once you're confident in the rule's output, unsnooze the actions to begin sending notifications. +% +% ::::{tip} +% Shadow deployment is especially useful for rules that are difficult to validate with historical data alone, such as [threshold rules](/solutions/security/detect-and-alert/threshold.md) where volume depends on live traffic patterns, or [{{ml}} rules](/solutions/security/detect-and-alert/machine-learning.md) where anomaly baselines evolve over time. +% :::: +% +% ## Post-enablement monitoring [post-enablement-monitoring] +% +% Validation doesn't end when a rule goes live. Monitor newly enabled rules closely during their first weeks in production. +% +% - **Check execution health.** Use the [Rule Monitoring tab](/solutions/security/detect-and-alert/monitor-rule-executions.md) to confirm the rule is executing successfully and not timing out or failing. +% - **Track alert volume trends.** A sudden spike or drop in alert volume can indicate a change in the data source, a rule misconfiguration, or an emerging incident. +% - **Collect analyst feedback.** Ask the analysts triaging the rule's alerts whether they find them actionable. If a rule consistently produces alerts that are closed without investigation, it needs further tuning. +% - **Review after one to two weeks.** Revisit the rule's configuration after it has run through a full operational cycle. Adjust the query, exceptions, suppression, or schedule based on what you've learned. diff --git a/solutions/security/detect-and-alert/view-detection-alert-details.md b/solutions/security/detect-and-alert/view-detection-alert-details.md index d514dde96a..73f8780da9 100644 --- a/solutions/security/detect-and-alert/view-detection-alert-details.md +++ b/solutions/security/detect-and-alert/view-detection-alert-details.md @@ -107,7 +107,7 @@ The Investigation section provides the following information: Add an [investigation guide](/solutions/security/detect-and-alert/write-investigation-guides.md#add-ig-actions-rule) to a rule when creating a new custom rule or modifying an existing custom rule’s settings. :::: -* **Highlighted fields**: Shows relevant fields for the alert and any [custom highlighted fields](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) you added to the rule. Custom highlighted fields with values are added to this section. Those without values aren’t added. +* **Highlighted fields**: Shows relevant fields for the alert and any [custom highlighted fields](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params) you added to the rule. Custom highlighted fields with values are added to this section. Those without values aren’t added. ::::{tip} {applies_to}`stack: ga 9.1` You can quickly add and remove custom highlighted fields from the rule by clicking **Add field** in the Highlighted fields table. diff --git a/solutions/security/detect-and-alert/write-investigation-guides.md b/solutions/security/detect-and-alert/write-investigation-guides.md index c7040f46c2..f0e7b55f7d 100644 --- a/solutions/security/detect-and-alert/write-investigation-guides.md +++ b/solutions/security/detect-and-alert/write-investigation-guides.md @@ -16,7 +16,7 @@ description: Create Markdown investigation guides with Timeline and Osquery butt Investigation guides are Markdown documents attached to detection rules that help analysts triage, analyze, and respond to alerts. A well-written guide reduces mean time to respond by giving analysts the context and queries they need without leaving the alert details flyout. -You author an investigation guide in the **Investigation guide** Markdown editor, which is available in the rule's [advanced settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) (under the **About** step when creating a rule, or the **About** tab when editing one). The guide renders in the **Investigation** tab of the alert details flyout whenever an analyst opens an alert produced by that rule. +You author an investigation guide in the **Investigation guide** Markdown editor, which is available in the rule's [advanced settings](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params) (under the **About** step when creating a rule, or the **About** tab when editing one). The guide renders in the **Investigation** tab of the alert details flyout whenever an analyst opens an alert produced by that rule. :::{image} /solutions/images/security-ig-alert-flyout.png :alt: Alert details flyout with investigation guide diff --git a/solutions/security/get-started/automatic-migration.md b/solutions/security/get-started/automatic-migration.md index 6c1e581e90..2698327900 100644 --- a/solutions/security/get-started/automatic-migration.md +++ b/solutions/security/get-started/automatic-migration.md @@ -138,7 +138,7 @@ The table's fields are as follows: * `Failed`: Translation failed. Refer to the the error for details. * **Risk Score:** For Elastic-authored rules, risk scores are predefined. For custom translated rules, risk scores are defined as follows: * If the source rule has a field comparable to Elastic's `risk score`, we use that value. - * Otherwise, if the source rule has a field comparable to Elastic's `rule severity` field, we base the risk score on that value according to [these guidelines](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-basic-params). + * Otherwise, if the source rule has a field comparable to Elastic's `rule severity` field, we base the risk score on that value according to [these guidelines](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-basic-params). * If neither of the above apply, we assign a default value. * **Rule severity:** For Elastic-authored rules, severity scores are predefined. For custom translated rules, risk scores are based on the source rule's severity field. Splunk severity scores are translated to Elastic rule severity scores as follows: diff --git a/solutions/security/get-started/configure-advanced-settings.md b/solutions/security/get-started/configure-advanced-settings.md index 6ae77914bd..8e236a0983 100644 --- a/solutions/security/get-started/configure-advanced-settings.md +++ b/solutions/security/get-started/configure-advanced-settings.md @@ -93,7 +93,7 @@ The `securitySolution:defaultThreatIndex` advanced setting specifies threat inte You can specify one or more threat intelligence indices; multiple indices must be separated by commas. By default, only the `logs-ti_*` index pattern is specified. Do not remove or overwrite this index pattern, as it is used by {{agent}} integrations. ::::{important} -Threat intelligence indices aren’t required to be ECS-compatible for use in indicator match rules. However, we strongly recommend compatibility if you want your alerts to be enriched with relevant threat indicator information. When searching for threat indicator data, indicator match rules use the threat indicator path specified in the **Indicator prefix override** advanced setting. Visit [Configure advanced rule settings](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) for more information. +Threat intelligence indices aren’t required to be ECS-compatible for use in indicator match rules. However, we strongly recommend compatibility if you want your alerts to be enriched with relevant threat indicator information. When searching for threat indicator data, indicator match rules use the threat indicator path specified in the **Indicator prefix override** advanced setting. Visit [Configure advanced rule settings](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params) for more information. :::: diff --git a/solutions/security/get-started/get-started-cloud-security.md b/solutions/security/get-started/get-started-cloud-security.md index 9896fb657f..54a1b61d2c 100644 --- a/solutions/security/get-started/get-started-cloud-security.md +++ b/solutions/security/get-started/get-started-cloud-security.md @@ -80,7 +80,7 @@ You can create a detection rule directly from the **Findings** page: 1. Click the arrow to the left of a finding to open the findings flyout. 2. Click **Take action**, then **Create a detection rule**. -3. To review or customize the new rule, click **View rule**. For example, you might want to set up a rule action—like an email or Slack notification—when alerts are generated. To learn more about rule actions, refer to [](/solutions/security/detect-and-alert/rule-settings-reference.md#rule-notifications). +3. To review or customize the new rule, click **View rule**. For example, you might want to set up a rule action—like an email or Slack notification—when alerts are generated. To learn more about rule actions, refer to [](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications). ## More resources diff --git a/solutions/toc.yml b/solutions/toc.yml index 7b7cc85eca..0d0b2b8816 100644 --- a/solutions/toc.yml +++ b/solutions/toc.yml @@ -584,7 +584,7 @@ toc: - file: security/detect-and-alert/install-manage-prebuilt-rules.md - file: security/detect-and-alert/update-prebuilt-rules.md - file: security/detect-and-alert/mitre-attack-coverage.md - - file: security/detect-and-alert/create-a-detection-rule.md + - file: security/detect-and-alert/custom-rules.md children: - file: security/detect-and-alert/choose-the-right-rule-type.md children: @@ -604,6 +604,8 @@ toc: - file: security/detect-and-alert/set-rule-data-sources.md - file: security/detect-and-alert/write-investigation-guides.md - file: security/detect-and-alert/validate-and-test-rules.md + - file: security/detect-and-alert/common-rule-settings.md + - file: security/detect-and-alert/query-alert-indices.md - file: security/detect-and-alert/manage-detection-rules.md - file: security/detect-and-alert/monitor-rule-executions.md - file: security/detect-and-alert/reduce-noise-and-false-positives.md @@ -615,16 +617,11 @@ toc: - file: security/detect-and-alert/add-manage-exceptions.md - file: security/detect-and-alert/create-manage-shared-exception-lists.md - file: security/detect-and-alert/alert-suppression.md - - file: security/detect-and-alert/snooze-rule-actions.md - file: security/detect-and-alert/manage-detection-alerts.md children: - file: security/detect-and-alert/visualize-detection-alerts.md - file: security/detect-and-alert/view-detection-alert-details.md - file: security/detect-and-alert/add-detection-alerts-to-cases.md - - file: security/detect-and-alert/detections-reference.md - children: - - file: security/detect-and-alert/rule-settings-reference.md - - file: security/detect-and-alert/query-alert-indices.md - file: security/configure-elastic-defend.md children: - file: security/configure-elastic-defend/elastic-defend-requirements.md diff --git a/troubleshoot/security/detection-rules.md b/troubleshoot/security/detection-rules.md index 62486cde21..db2b82fff4 100644 --- a/troubleshoot/security/detection-rules.md +++ b/troubleshoot/security/detection-rules.md @@ -20,6 +20,33 @@ products: This topic covers common troubleshooting issues when creating or managing [detection rules](../../solutions/security/detect-and-alert/using-the-rule-builder.md). +## Setup errors [setup-errors-ts] + +Depending on your privileges and whether detection system indices have already been created for the {{kib}} space, you might get one of these error messages when you open the **Alerts** or **Rules** page: + +::::{dropdown} Let's set up your detection engine +:name: setup-detection-engine-ts + +If you get this message, a user with specific privileges must visit the **Alerts** or **Rules** page before you can view detection alerts and rules. Refer to [Detections prerequisites and requirements](../../solutions/security/detect-and-alert/detections-privileges.md) for a list of all the requirements. + +:::{note} +For **self-managed** {{stack}} deployments only, this message may be displayed when the `xpack.encryptedSavedObjects.encryptionKey` setting has not been added to the `kibana.yml` file. For more information, refer to [Turn on detections](../../solutions/security/detect-and-alert/requirements-privileges.md). +::: + +:::: + +::::{dropdown} Detection engine permissions required +:name: detection-permissions-ts + +If you get this message, you do not have the required privileges to view the **Detections** feature, and you should contact your {{kib}} administrator. + +:::{note} +For **self-managed** {{stack}} deployments only, this message may be displayed when the `xpack.security.enabled` setting is not enabled in the `elasticsearch.yml` file. For more information, refer to [Turn on detections](../../solutions/security/detect-and-alert/requirements-privileges.md). +::: + +:::: + + ## {{ml-cap}} rules [ML-rules-ts] ::::{dropdown} {{ml-cap}} rule is failing and a required {{ml}} job is stopped @@ -178,7 +205,7 @@ Even if your rule runs at its scheduled time, there might still be missing alert In addition, use caution when creating custom rule schedules to ensure that the specified interval + additional look-back time is greater than your deployment’s ingestion pipeline delay. -You can reduce the number of missed alerts due to ingestion pipeline delay by specifying the `Timestamp override` field value to `event.ingested` in [advanced settings](../../solutions/security/detect-and-alert/rule-settings-reference.md#rule-ui-advanced-params) during rule creation or editing. The detection engine uses the value from the `event.ingested` field as the timestamp when executing the rule. +You can reduce the number of missed alerts due to ingestion pipeline delay by specifying the `Timestamp override` field value to `event.ingested` in [advanced settings](../../solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params) during rule creation or editing. The detection engine uses the value from the `event.ingested` field as the timestamp when executing the rule. For example, say an event occurred at 10:00 but wasn’t ingested into {{es}} until 10:10 due to an ingestion pipeline delay. If you created a rule to detect that event with an interval + additional look-back time of 6 minutes, and the rule executes at 10:12, it would still detect the event because the `event.ingested` timestamp was from 10:10, only 2 minutes before the rule executed and well within the rule’s 6-minute interval + additional look-back time. From 0a854a00dd19f1c302544927ad41709bfe1824ff Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Mon, 23 Feb 2026 15:51:25 -0500 Subject: [PATCH 04/24] Fix error --- redirects.yml | 1 + .../rule-settings-reference.md | 303 ------------------ 2 files changed, 1 insertion(+), 303 deletions(-) delete mode 100644 solutions/security/detect-and-alert/rule-settings-reference.md diff --git a/redirects.yml b/redirects.yml index 3b94db6b66..927e188069 100644 --- a/redirects.yml +++ b/redirects.yml @@ -716,6 +716,7 @@ redirects: 'solutions/security/detect-and-alert/reduce-notifications-alerts.md': 'solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions' 'solutions/security/detect-and-alert/snooze-rule-actions.md': 'solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions' 'solutions/security/detect-and-alert/reference.md': 'solutions/security/detect-and-alert/detections-reference.md' + 'solutions/security/detect-and-alert/rule-settings-reference.md': 'solutions/security/detect-and-alert/common-rule-settings.md' # Anchor redirect for cold/frozen data content moved to set-rule-data-sources.md 'solutions/security/detect-and-alert/requirements-privileges.md': to: 'solutions/security/detect-and-alert/requirements-privileges.md' diff --git a/solutions/security/detect-and-alert/rule-settings-reference.md b/solutions/security/detect-and-alert/rule-settings-reference.md deleted file mode 100644 index 8321dd0f74..0000000000 --- a/solutions/security/detect-and-alert/rule-settings-reference.md +++ /dev/null @@ -1,303 +0,0 @@ ---- -applies_to: - stack: ga all - serverless: - security: ga all -products: - - id: security - - id: cloud-serverless -description: Reference for all detection rule settings including basic, advanced, schedule, actions, and notification variables. ---- - -# Common rule settings [common-detection-rule-settings] - -All detection rules share a common set of settings for describing the rule, controlling its schedule, configuring actions, and setting up response actions. These settings apply regardless of the [rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) you select. - -For rule-type-specific settings (query definitions, index patterns, {{ml}} jobs, and so on), refer to [Using the rule builder](/solutions/security/detect-and-alert/using-the-rule-builder.md). - -## Basic settings [rule-ui-basic-params] - -Configure these settings in the **About rule** pane. - -**Name** -: The rule's name. - -**Description** -: A description of what the rule does. - -**Default severity** -: The severity level of alerts created by the rule: - - * **Low**: Alerts that are of interest but generally are not considered to be security incidents. Sometimes a combination of low severity alerts can indicate suspicious activity. - * **Medium**: Alerts that require investigation. - * **High**: Alerts that require an immediate investigation. - * **Critical**: Alerts that indicate it is highly likely a security incident has occurred. - -**Severity override** (optional) -: Select to use source event values to override the **Default severity** in generated alerts. When selected, a UI component is displayed where you can map the source event field values to severity levels. - - :::{image} /solutions/images/security-severity-mapping-ui.png - :alt: severity mapping ui - :screenshot: - ::: - - ::::{note} - For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. Overrides are not supported for event correlation rules. - :::: - -**Default risk score** -: A numerical value between 0 and 100 that indicates the risk of events detected by the rule. This setting changes to a default value when you change the **Severity** level, but you can adjust the risk score as needed. General guidelines are: - - * `0` - `21` represents low severity. - * `22` - `47` represents medium severity. - * `48` - `73` represents high severity. - * `74` - `100` represents critical severity. - -**Risk score override** (optional) -: Select to use a source event value to override the **Default risk score** in generated alerts. When selected, a UI component is displayed to select the source field used for the risk score. - - :::{image} /solutions/images/security-risk-source-field-ui.png - :alt: risk source field ui - :screenshot: - ::: - - ::::{note} - For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. - :::: - -**Tags** (optional) -: Words and phrases used to categorize, filter, and search the rule. - -## Advanced settings [rule-ui-advanced-params] - -Configure these settings by clicking **Advanced settings** in the **About rule** pane. - -**Reference URLs** (optional) -: References to information that is relevant to the rule. For example, links to background information. - -**False positive examples** (optional) -: List of common scenarios that may produce false-positive alerts. - -**MITRE ATT&CKTM threats** (optional) -: Add relevant [MITRE](https://attack.mitre.org/) framework tactics, techniques, and subtechniques. - -**Custom highlighted fields** (optional) -: Specify highlighted fields for unique alert investigation flows. You can select any fields that are available in the indices you selected for the rule's data source. - - After you create the rule, you can find all custom highlighted fields in the About section of the rule details page. If the rule has alerts, you can find custom highlighted fields in the [Highlighted fields](/solutions/security/detect-and-alert/view-detection-alert-details.md#investigation-section) section of the alert details flyout. - -**Setup guide** (optional) -: Instructions on rule prerequisites such as required integrations, configuration steps, and anything else needed for the rule to work correctly. - -**Investigation guide** (optional) -: Information for analysts investigating alerts created by the rule. You can also add action buttons to [run Osquery](/solutions/security/investigate/run-osquery-from-investigation-guides.md) or [start Timeline investigations](/solutions/security/detect-and-alert/write-investigation-guides.md) using alert data. - -**Author** (optional) -: The rule's authors. - -**License** (optional) -: The rule's license. - -**Elastic endpoint exceptions** (optional) -: Adds all [{{elastic-endpoint}} exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md#endpoint-rule-exceptions) to this rule. - - ::::{note} - If you select this option, you can add {{elastic-endpoint}} exceptions on the Rule details page. Additionally, all future exceptions added to [endpoint protection rules](/solutions/security/manage-elastic-defend/endpoint-protection-rules.md) will also affect this rule. - :::: - -**Building block** (optional) -: Select to create a building-block rule. By default, alerts generated from a building-block rule are not displayed in the UI. See [About building block rules](/solutions/security/detect-and-alert/about-building-block-rules.md) for more information. - -**Max alerts per run** (optional) -: Specify the maximum number of alerts the rule can create each time it executes. Default is 100. - - ::::{note} - This setting can be superseded by the [{{kib}} configuration setting](kibana://reference/configuration-reference/alerting-settings.md#alert-settings) `xpack.alerting.rules.run.alerts.max`, which determines the maximum alerts generated by *any* rule in the {{kib}} alerting framework. For example, if `xpack.alerting.rules.run.alerts.max` is set to `1000`, the rule can generate no more than 1000 alerts even if **Max alerts per run** is set higher. - :::: - -**Indicator prefix override** (indicator match rules only) -: Define the location of indicator data within the structure of indicator documents. When the indicator match rule executes, it queries specified indicator indices and references this setting to locate fields with indicator data. This data is used to enrich indicator match alerts with metadata about matched threat indicators. The default value for this setting is `threat.indicator`. - - ::::{important} - If your threat indicator data is at a different location, update this setting accordingly to ensure alert enrichment can still be performed. - :::: - -**Rule name override** (optional) -: Select a source event field to use as the rule name in the UI (Alerts table). This is useful for exposing, at a glance, more information about an alert. For example, if the rule generates alerts from Suricata, selecting `event.action` lets you see what action (Suricata category) caused the event directly in the Alerts table. - - ::::{note} - For threshold rules, not all source event values can be used for overrides; only the fields that were aggregated over (the `Group by` fields) will contain data. - :::: - -**Timestamp override** (optional) -: Select a source event timestamp field. When selected, the rule's query uses the selected field, instead of the default `@timestamp` field, to search for alerts. This can help reduce missing alerts due to network or server outages. Specifically, if your ingest pipeline adds a timestamp when events are sent to {{es}}, this can prevent missing alerts from ingestion delays. - - If the selected field is unavailable, the rule query will use the `@timestamp` field instead. If you don't want to use the `@timestamp` field because your data source has an inaccurate `@timestamp` value, select the **Do not use @timestamp as a fallback timestamp field** option instead. This ensures the rule query ignores the `@timestamp` field entirely. - - ::::{tip} - The [Microsoft](beats://reference/filebeat/filebeat-module-microsoft.md) and [Google Workspace](beats://reference/filebeat/filebeat-module-google_workspace.md) {{filebeat}} modules have an `event.ingested` timestamp field that can be used instead of the default `@timestamp` field. - :::: - -## Schedule settings [rule-schedule] - -**Runs every** -: How often the rule runs. - -**Additional look-back time** (optional) -: When defined, the rule searches indices with the additional time. - - For example, if you set a rule to run every 5 minutes with an additional look-back time of 1 minute, the rule runs every 5 minutes but analyzes the documents added to indices during the last 6 minutes. - - ::::{important} - It is recommended to set the `Additional look-back time` to at least 1 minute. This ensures there are no missing alerts when a rule does not run exactly at its scheduled time. - - {{elastic-sec}} prevents duplication. Any duplicate alerts that are discovered during the `Additional look-back time` are *not* created. - :::: - -## Rule actions [rule-notifications] - -Use actions to set up notifications sent through other systems when alerts are generated. - -::::{note} -To use actions for alert notifications, you need the [appropriate license]({{subscriptions}}). For more information, see [Cases requirements](/solutions/security/investigate/cases-requirements.md). -:::: - -::::{tip} -:applies_to: {stack: preview 9.3+, serverless: preview} -You can use [workflows](/explore-analyze/workflows.md) as a rule action to automate alert response processes. Workflows can create cases, route notifications, or perform other automated tasks when alerts are generated. To learn how to set up a workflow as a rule action, refer to [](/explore-analyze/workflows/triggers/alert-triggers.md). -:::: - -**Connector type** -: Determines how notifications are sent. For example, if you select the {{jira}} connector, notifications are sent to your {{jira}} system. - - ::::{note} - Each action type requires a connector. Connectors store the information required to send the notification from the external system. You can configure connectors while creating the rule or from the **{{connectors-ui}}** page. For more information, refer to [Action and connector types](/deploy-manage/manage-connectors.md). - - Some connectors that perform actions require less configuration. For example, you do not need to set the action frequency or variables for the [Cases connector](kibana://reference/connectors-kibana/cases-action-type.md). - :::: - -**Action frequency** -: Defines when notifications are sent: - - * **Summary of alerts**: Sends a report that summarizes generated alerts at the specified time intervals. - - ::::{note} - When setting a custom notification frequency, do not select a time that is shorter than the rule's execution schedule. - :::: - - * **For each alert**: Sends notifications every time new alerts are generated. - -**Conditional actions** (optional) -: Specify additional conditions that need to be met for notifications to send: - - * **If alert matches query**: Enter a KQL query that defines field-value pairs or query conditions that must be met for notifications to send. The query only searches alert documents in the indices specified for the rule. - * **If alert is generated during timeframe**: Set timeframe details. Notifications are only sent if alerts are generated within the timeframe you define. - -**Notification message** -: Use the default notification message or customize it. You can add more context to the message by clicking the icon above the message text box and selecting from a list of available [alert notification variables](#rule-action-variables). - -::::{important} -After you activate a rule, you can check if it is executing as expected using the [Monitoring tab](/troubleshoot/security/detection-rules.md) on the {{rules-ui}} page. If you see values in the `Gap` column, you can [Troubleshoot missing alerts](/troubleshoot/security/detection-rules.md#troubleshoot-signals). - -When a rule fails to execute, the {{security-app}} tries to rerun it at its next scheduled time. -:::: - -## Response actions [rule-response-action] - -Use response actions to set up additional functionality that executes whenever a rule triggers: - -* **Osquery**: Include live Osquery queries with a custom query rule. When an alert is generated, Osquery automatically collects data on the system related to the alert. Refer to [Add Osquery Response Actions](/solutions/security/investigate/add-osquery-response-actions.md) to learn more. -* **{{elastic-defend}}**: Automatically execute response actions on an endpoint when rule conditions are met. For example, you can automatically isolate a host or end a process when specific activities or events are detected on the host. Refer to [Automated response actions](/solutions/security/endpoint-response-actions/automated-response-actions.md) to learn more. - -::::{important} -Host isolation involves quarantining a host from the network to prevent further proliferation of threats and limit potential damage. Be aware that automatic host isolation can cause unintended consequences, such as disrupting legitimate user activities or blocking critical business processes. -:::: - -## Alert notification variables [rule-action-variables] - -You can use [mustache syntax](http://mustache.github.io/) to add variables to notification messages. The action frequency you select determines the available variables. - -::::{note} -Refer to [Action frequency: Summary of alerts](/explore-analyze/alerts-cases/alerts/rule-action-variables.md#alert-summary-action-variables) to learn about additional variables that can be passed if the rule's action frequency is **Summary of alerts**. -:::: - -### Variables for all rules [all-rule-variables] - -* `{{context.alerts}}`: Array of detected alerts -* `{{{context.results_link}}}`: URL to the alerts in {{kib}} -* `{{context.rule.anomaly_threshold}}`: Anomaly threshold score above which alerts are generated ({{ml}} rules only) -* `{{context.rule.description}}`: Rule description -* `{{context.rule.false_positives}}`: Rule false positives -* `{{context.rule.filters}}`: Rule filters (query rules only) -* `{{context.rule.id}}`: Unique rule ID returned after creating the rule -* `{{context.rule.index}}`: Indices rule runs on (query rules only) -* `{{context.rule.language}}`: Rule query language (query rules only) -* `{{context.rule.machine_learning_job_id}}`: ID of associated {{ml}} job ({{ml}} rules only) -* `{{context.rule.max_signals}}`: Maximum allowed number of alerts per rule execution -* `{{context.rule.name}}`: Rule name -* `{{context.rule.query}}`: Rule query (query rules only) -* `{{context.rule.references}}`: Rule references -* `{{context.rule.risk_score}}`: Default rule risk score - - ::::{note} - This placeholder contains the rule's default values even when the **Risk score override** option is used. - :::: - -* `{{context.rule.rule_id}}`: Generated or user-defined rule ID that can be used as an identifier across systems -* `{{context.rule.saved_id}}`: Saved search ID -* `{{context.rule.severity}}`: Default rule severity - - ::::{note} - This placeholder contains the rule's default values even when the **Severity override** option is used. - :::: - -* `{{context.rule.threat}}`: Rule threat framework -* `{{context.rule.threshold}}`: Rule threshold values (threshold rules only) -* `{{context.rule.timeline_id}}`: Associated Timeline ID -* `{{context.rule.timeline_title}}`: Associated Timeline name -* `{{context.rule.type}}`: Rule type -* `{{context.rule.version}}`: Rule version -* `{{date}}`: Date the rule scheduled the action -* `{{kibanaBaseUrl}}`: Configured `server.publicBaseUrl` value, or empty string if not configured -* `{{rule.id}}`: ID of the rule -* `{{rule.name}}`: Name of the rule -* `{{rule.spaceId}}`: Space ID of the rule -* `{{rule.tags}}`: Tags of the rule -* `{{rule.type}}`: Type of rule -* `{{state.signals_count}}`: Number of alerts detected - -### Per-alert variables [per-alert-variables] - -The following variables can only be passed if the rule's action frequency is **For each alert**: - -* `{{alert.actionGroup}}`: Action group of the alert that scheduled actions for the rule -* `{{alert.actionGroupName}}`: Human-readable name of the action group of the alert that scheduled actions for the rule -* `{{alert.actionSubgroup}}`: Action subgroup of the alert that scheduled actions for the rule -* `{{alert.id}}`: ID of the alert that scheduled actions for the rule -* `{{alert.flapping}}`: A flag on the alert that indicates whether the alert status is changing repeatedly - -### Placeholder examples [placeholder-examples] - -To understand which fields to parse, see the [Detections API](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-security-detections-api) to view the JSON representation of rules. - -Example using `{{context.rule.filters}}` to output a list of filters: - -```json -{{#context.rule.filters}} -{{^meta.disabled}}{{meta.key}} {{#meta.negate}}NOT {{/meta.negate}}{{meta.type}} {{^exists}}{{meta.value}}{{meta.params.query}}{{/exists}}{{/meta.disabled}} -{{/context.rule.filters}} -``` - -Example using `{{context.alerts}}` as an array, which contains each alert generated since the last time the action was executed: - -```json -{{#context.alerts}} -Detection alert for user: {{user.name}} -{{/context.alerts}} -``` - -Example using the mustache "current element" notation `{{.}}` to output all the rule references in the `signal.rule.references` array: - -```json -{{#signal.rule.references}} {{.}} {{/signal.rule.references}} -``` From 5b0ff43b4da4f30c2294f7c725491f0a4afcf67d Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Mon, 23 Feb 2026 16:00:47 -0500 Subject: [PATCH 05/24] Comment out craft sections --- redirects.yml | 15 ++++++++------- .../security/detect-and-alert/custom-query.md | 2 ++ solutions/security/detect-and-alert/eql.md | 2 ++ solutions/security/detect-and-alert/esql.md | 2 ++ .../security/detect-and-alert/indicator-match.md | 2 ++ .../security/detect-and-alert/machine-learning.md | 6 ++++-- solutions/security/detect-and-alert/new-terms.md | 2 ++ solutions/security/detect-and-alert/threshold.md | 2 ++ solutions/toc.yml | 6 ++++-- 9 files changed, 28 insertions(+), 11 deletions(-) diff --git a/redirects.yml b/redirects.yml index 927e188069..76fc70ed69 100644 --- a/redirects.yml +++ b/redirects.yml @@ -713,12 +713,13 @@ redirects: 'solutions/security/detect-and-alert/launch-timeline-from-investigation-guides.md': 'solutions/security/detect-and-alert/write-investigation-guides.md' 'solutions/security/detect-and-alert/exclude-cold-frozen-data-from-individual-rules.md': 'solutions/security/detect-and-alert/set-rule-data-sources.md' 'solutions/security/detect-and-alert/suppress-detection-alerts.md': 'solutions/security/detect-and-alert/alert-suppression.md' - 'solutions/security/detect-and-alert/reduce-notifications-alerts.md': 'solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions' - 'solutions/security/detect-and-alert/snooze-rule-actions.md': 'solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions' + 'solutions/security/detect-and-alert/reduce-notifications-alerts.md': + to: 'solutions/security/detect-and-alert/manage-detection-rules.md' + anchors: + '': 'snooze-rule-actions' + 'solutions/security/detect-and-alert/snooze-rule-actions.md': + to: 'solutions/security/detect-and-alert/manage-detection-rules.md' + anchors: + '': 'snooze-rule-actions' 'solutions/security/detect-and-alert/reference.md': 'solutions/security/detect-and-alert/detections-reference.md' 'solutions/security/detect-and-alert/rule-settings-reference.md': 'solutions/security/detect-and-alert/common-rule-settings.md' - # Anchor redirect for cold/frozen data content moved to set-rule-data-sources.md - 'solutions/security/detect-and-alert/requirements-privileges.md': - to: 'solutions/security/detect-and-alert/requirements-privileges.md' - anchors: - 'cold-frozen-data': 'solutions/security/detect-and-alert/set-rule-data-sources.md#exclude-cold-frozen-tier' diff --git a/solutions/security/detect-and-alert/custom-query.md b/solutions/security/detect-and-alert/custom-query.md index 0b752cd577..6c479b5ead 100644 --- a/solutions/security/detect-and-alert/custom-query.md +++ b/solutions/security/detect-and-alert/custom-query.md @@ -34,6 +34,7 @@ Custom query rules are **not** the best fit when you need to: Custom query rules require at least one {{es}} index pattern or [data view](/solutions/security/get-started/data-views-elastic-security.md) that contains the events you want to match. The indices must be accessible to the user who creates or last edits the rule, because the rule executes with that user's [API key privileges](/solutions/security/detect-and-alert/choose-the-right-rule-type.md#alerting-authorization-model). + ## Custom query field reference [custom-query-fields] diff --git a/solutions/security/detect-and-alert/eql.md b/solutions/security/detect-and-alert/eql.md index 129ab63a7b..19384e9973 100644 --- a/solutions/security/detect-and-alert/eql.md +++ b/solutions/security/detect-and-alert/eql.md @@ -34,6 +34,7 @@ EQL rules are **not** the best fit when: EQL rules require at least one {{es}} index pattern or [{{data-source}}](/solutions/security/get-started/data-views-elastic-security.md). The indexed data must include a timestamp field (defaults to `@timestamp`) and an event category field (defaults to `event.category`). Sequence queries also benefit from a tiebreaker field to resolve events that share the same timestamp. + ## EQL field reference [eql-fields] diff --git a/solutions/security/detect-and-alert/esql.md b/solutions/security/detect-and-alert/esql.md index 12277c1ded..aa65a2fe31 100644 --- a/solutions/security/detect-and-alert/esql.md +++ b/solutions/security/detect-and-alert/esql.md @@ -33,6 +33,7 @@ description: Create detection rules using Elasticsearch Query Language (ESQL) wi {{esql}} rules query {{es}} indices directly using the `FROM` command. The indices must be accessible to the user who creates or last edits the rule. + ## {{esql}} field reference [esql-fields] diff --git a/solutions/security/detect-and-alert/indicator-match.md b/solutions/security/detect-and-alert/indicator-match.md index 15d06a4a7f..937a17c85a 100644 --- a/solutions/security/detect-and-alert/indicator-match.md +++ b/solutions/security/detect-and-alert/indicator-match.md @@ -45,6 +45,7 @@ Indicator match rules require: * **Source event indices** containing the security events you want to scan. * **Indicator indices** containing threat intelligence data. Data in these indices must be [ECS compatible](/reference/security/fields-and-object-schemas/siem-field-reference.md) and must contain a `@timestamp` field. + ## Indicator match field reference [indicator-match-fields] diff --git a/solutions/security/detect-and-alert/machine-learning.md b/solutions/security/detect-and-alert/machine-learning.md index 09202317d2..a05c2e8c0d 100644 --- a/solutions/security/detect-and-alert/machine-learning.md +++ b/solutions/security/detect-and-alert/machine-learning.md @@ -17,7 +17,7 @@ description: Create detection rules that trigger on machine learning anomaly det ### When to use a {{ml}} rule -{{ml}} rules are the right fit when: +{{ml-cap}} rules are the right fit when: * You want to detect **behavioral anomalies** that are difficult to express as static queries, such as unusual login times, atypical data transfer volumes, or rare process executions for a given user or host. * A {{ml}} job is already active (or you plan to enable one) that models the relevant behavior. @@ -40,6 +40,7 @@ To create or edit {{ml}} rules, you need: For an overview of using {{ml}} with {{elastic-sec}}, refer to [{{anomaly-detect-cap}}](/solutions/security/advanced-entity-analytics/anomaly-detection.md). + -## {{ml}} field reference [ml-fields] +## {{ml-cap}} field reference [ml-fields] The following settings are specific to {{ml}} rules. For settings shared across all rule types, refer to [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md). diff --git a/solutions/security/detect-and-alert/new-terms.md b/solutions/security/detect-and-alert/new-terms.md index 33c91adbdc..574ab16fa7 100644 --- a/solutions/security/detect-and-alert/new-terms.md +++ b/solutions/security/detect-and-alert/new-terms.md @@ -33,6 +33,7 @@ New terms rules are **not** the best fit when: New terms rules require at least one {{es}} index pattern or [{{data-source}}](/solutions/security/get-started/data-views-elastic-security.md) with a sufficient history of events. The history window must contain enough data to establish a reliable baseline. A window that is too short leads to excessive false positives as many values appear "new". + ## New terms field reference [new-terms-fields] diff --git a/solutions/security/detect-and-alert/threshold.md b/solutions/security/detect-and-alert/threshold.md index c3ea352d96..188b33aa95 100644 --- a/solutions/security/detect-and-alert/threshold.md +++ b/solutions/security/detect-and-alert/threshold.md @@ -33,6 +33,7 @@ Threshold rules are **not** the best fit when: Threshold rules require at least one {{es}} index pattern or {{data-source}}. The data should contain the fields you plan to group by and enough event volume for meaningful threshold evaluation. + ## Threshold field reference [threshold-fields] diff --git a/solutions/toc.yml b/solutions/toc.yml index 0d0b2b8816..230feb6f44 100644 --- a/solutions/toc.yml +++ b/solutions/toc.yml @@ -604,8 +604,10 @@ toc: - file: security/detect-and-alert/set-rule-data-sources.md - file: security/detect-and-alert/write-investigation-guides.md - file: security/detect-and-alert/validate-and-test-rules.md - - file: security/detect-and-alert/common-rule-settings.md - - file: security/detect-and-alert/query-alert-indices.md + - file: security/detect-and-alert/detections-reference.md + children: + - file: security/detect-and-alert/common-rule-settings.md + - file: security/detect-and-alert/query-alert-indices.md - file: security/detect-and-alert/manage-detection-rules.md - file: security/detect-and-alert/monitor-rule-executions.md - file: security/detect-and-alert/reduce-noise-and-false-positives.md From b55d975c442c4fb541efd5cfecf18233280fcdbd Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Mon, 23 Feb 2026 16:05:48 -0500 Subject: [PATCH 06/24] req rewrite --- .../detect-and-alert/indicator-match.md | 18 +++++++++--------- .../detect-and-alert/machine-learning.md | 7 +------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/solutions/security/detect-and-alert/indicator-match.md b/solutions/security/detect-and-alert/indicator-match.md index 937a17c85a..182cfe3a96 100644 --- a/solutions/security/detect-and-alert/indicator-match.md +++ b/solutions/security/detect-and-alert/indicator-match.md @@ -45,6 +45,15 @@ Indicator match rules require: * **Source event indices** containing the security events you want to scan. * **Indicator indices** containing threat intelligence data. Data in these indices must be [ECS compatible](/reference/security/fields-and-object-schemas/siem-field-reference.md) and must contain a `@timestamp` field. +## Using value lists as indicator indices [using-value-lists] + +You can use [value lists](/solutions/security/detect-and-alert/create-manage-value-lists.md) as the indicator index. This is useful when you have a flat list of indicators (IPs, domains, hashes) that you want to match against without creating a full indicator index: + +1. Upload a value list of indicators. +2. In the **Indicator index patterns** field, enter `.items-<{{kib}} space>` (the hidden index where value lists are stored). +3. In the **Indicator index query** field, enter `list_id : `. +4. In **Indicator mapping**, set the **Indicator index field** to the list type (`keyword`, `text`, or `IP`). + ## Shared concepts [shared-rule-concepts] diff --git a/solutions/security/detect-and-alert/detections-reference.md b/solutions/security/detect-and-alert/detections-reference.md deleted file mode 100644 index fd6df8e765..0000000000 --- a/solutions/security/detect-and-alert/detections-reference.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -applies_to: - stack: ga all - serverless: - security: ga all -products: - - id: security - - id: cloud-serverless -description: Technical reference for detection rule settings, alert schema, and querying alert indices. ---- - -# Settings, fields, and indices - -Look up rule configuration settings, alert field definitions, and patterns for querying alert indices directly. These pages are designed for reference, not reading end to end. - -**[Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md)** -: Use this when you need to look up a specific rule setting, understand what a field does, or check valid values and defaults. Covers all shared settings (severity, risk score, schedule, actions, response actions, and notification variables) that apply across rule types. For rule-type-specific fields, refer to the individual [rule type](/solutions/security/detect-and-alert/rule-types.md) pages. - -**[Query alert indices](/solutions/security/detect-and-alert/query-alert-indices.md)** -: Relevant if you're building custom dashboards, visualizations, or SOAR integrations that query the `.alerts-security.alerts-*` index directly. Explains how to query alert indices safely, which fields are available, and links to the full [alert schema](/reference/security/fields-and-object-schemas/alert-schema.md). diff --git a/solutions/toc.yml b/solutions/toc.yml index 230feb6f44..5004014e7f 100644 --- a/solutions/toc.yml +++ b/solutions/toc.yml @@ -600,14 +600,11 @@ toc: - file: security/detect-and-alert/machine-learning.md - file: security/detect-and-alert/using-the-rule-builder.md - file: security/detect-and-alert/using-the-api.md - - file: security/detect-and-alert/set-rule-data-sources.md - file: security/detect-and-alert/write-investigation-guides.md - file: security/detect-and-alert/validate-and-test-rules.md - - file: security/detect-and-alert/detections-reference.md - children: - - file: security/detect-and-alert/common-rule-settings.md - - file: security/detect-and-alert/query-alert-indices.md + - file: security/detect-and-alert/common-rule-settings.md + - file: security/detect-and-alert/query-alert-indices.md - file: security/detect-and-alert/manage-detection-rules.md - file: security/detect-and-alert/monitor-rule-executions.md - file: security/detect-and-alert/reduce-noise-and-false-positives.md From e3ff6734de9ddb5df25ad5d8f8eb8eafa1bdcbcd Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Mon, 23 Feb 2026 16:46:19 -0500 Subject: [PATCH 08/24] restructure and rename author rules --- redirects.yml | 1 + .../security/detect-and-alert/author-rules.md | 40 +++++++++++++++++++ .../security/detect-and-alert/custom-rules.md | 27 ++++++++----- solutions/toc.yml | 6 +-- 4 files changed, 60 insertions(+), 14 deletions(-) create mode 100644 solutions/security/detect-and-alert/author-rules.md diff --git a/redirects.yml b/redirects.yml index 38f346b921..a40da4daa9 100644 --- a/redirects.yml +++ b/redirects.yml @@ -724,3 +724,4 @@ redirects: 'solutions/security/detect-and-alert/reference.md': 'solutions/security/detect-and-alert/common-rule-settings.md' 'solutions/security/detect-and-alert/detections-reference.md': 'solutions/security/detect-and-alert/common-rule-settings.md' 'solutions/security/detect-and-alert/rule-settings-reference.md': 'solutions/security/detect-and-alert/common-rule-settings.md' + 'solutions/security/detect-and-alert/custom-rules.md': 'solutions/security/detect-and-alert/author-rules.md' diff --git a/solutions/security/detect-and-alert/author-rules.md b/solutions/security/detect-and-alert/author-rules.md new file mode 100644 index 0000000000..0d4793a36d --- /dev/null +++ b/solutions/security/detect-and-alert/author-rules.md @@ -0,0 +1,40 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/security/current/rules-ui-create.html +applies_to: + stack: all + serverless: + security: all +products: + - id: security + - id: cloud-serverless +description: Create and configure detection rules tailored to your environment and threat model. +--- + +# Author rules [rules-ui-create] + +Create detection rules tailored to your environment and threat model. Whether you're writing rules from scratch or customizing prebuilt rules, the pages in this section guide you through selecting a rule type, writing rule logic, and configuring settings. + +**[Choose the right rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md)** +: Start here if you're not sure which rule type fits your use case. Compares all rule types side by side. + +**[Rule types](/solutions/security/detect-and-alert/rule-types.md)** +: Detailed guidance for each rule type, including when to use it and field configuration specific to that type. + +**[Using the rule builder](/solutions/security/detect-and-alert/using-the-rule-builder.md)** +: Step-by-step workflow for creating rules in the {{kib}} UI. + +**[Using the API](/solutions/security/detect-and-alert/using-the-api.md)** +: Create or manage rules programmatically, integrate with CI/CD pipelines, or bulk-import rules. + +**[Common rule settings](/solutions/security/detect-and-alert/common-rule-settings.md)** +: Reference for all shared rule settings: severity, risk score, schedule, actions, and notification variables. + +**[Set rule data sources](/solutions/security/detect-and-alert/set-rule-data-sources.md)** +: Override default index patterns, target specific indices, or exclude cold and frozen data tiers. + +**[Write investigation guides](/solutions/security/detect-and-alert/write-investigation-guides.md)** +: Add triage guidance to rules using Markdown, Timeline query buttons, and Osquery integration. + +**[Validate and test rules](/solutions/security/detect-and-alert/validate-and-test-rules.md)** +: Test rule logic against historical data and assess alert volume before enabling in production. diff --git a/solutions/security/detect-and-alert/custom-rules.md b/solutions/security/detect-and-alert/custom-rules.md index d25ff662d8..5f4105d19a 100644 --- a/solutions/security/detect-and-alert/custom-rules.md +++ b/solutions/security/detect-and-alert/custom-rules.md @@ -1,4 +1,6 @@ --- +mapped_pages: + - https://www.elastic.co/guide/en/security/current/rules-ui-create.html applies_to: stack: all serverless: @@ -6,30 +8,33 @@ applies_to: products: - id: security - id: cloud-serverless -description: Overview and workflow for building custom detection rules tailored to your threat model. +description: Create and configure detection rules tailored to your environment and threat model. --- -# Custom rules +# Author custom rules [rules-ui-create] -Build custom detection rules tailored to your environment and threat model. The pages in this section walk you through selecting a rule type, writing rule logic, and configuring rule settings. +Create detection rules tailored to your environment and threat model. Whether you're writing rules from scratch or customizing prebuilt rules, the pages in this section guide you through selecting a rule type, writing rule logic, and configuring settings. -**[Select the right rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md)** -: Start here if you're not sure which rule type fits your use case. Compares all rule types side by side and explains how building block rules fit into detection chains. +**[Choose the right rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md)** +: Start here if you're not sure which rule type fits your use case. Compares all rule types side by side. **[Rule types](/solutions/security/detect-and-alert/rule-types.md)** -: Go here once you've selected a rule type. Each rule type page covers when to use it, how to write effective queries, real-world examples, and the field configuration specific to that type. +: Detailed guidance for each rule type, including when to use it and field configuration specific to that type. **[Using the rule builder](/solutions/security/detect-and-alert/using-the-rule-builder.md)** -: The step-by-step workflow for creating rules in the {{kib}} UI. Covers the creation steps and links to rule settings and rule type guides. +: Step-by-step workflow for creating rules in the {{kib}} UI. **[Using the API](/solutions/security/detect-and-alert/using-the-api.md)** -: Relevant if you need to create or manage rules programmatically, integrate rule management into CI/CD pipelines, or bulk-import rules. +: Create or manage rules programmatically, integrate with CI/CD pipelines, or bulk-import rules. + +**[Common rule settings](/solutions/security/detect-and-alert/common-rule-settings.md)** +: Reference for all shared rule settings: severity, risk score, schedule, actions, and notification variables. **[Set rule data sources](/solutions/security/detect-and-alert/set-rule-data-sources.md)** -: Relevant if you need to override the default index patterns for a specific rule, target a narrower set of indices, or exclude cold and frozen data tiers. +: Override default index patterns, target specific indices, or exclude cold and frozen data tiers. **[Write investigation guides](/solutions/security/detect-and-alert/write-investigation-guides.md)** -: Use this when you want to add triage guidance to a rule. Covers Markdown syntax, Timeline query buttons, and Osquery integration for investigation guides. +: Add triage guidance to rules using Markdown, Timeline query buttons, and Osquery integration. **[Validate and test rules](/solutions/security/detect-and-alert/validate-and-test-rules.md)** -: Relevant before enabling a new rule in production. Covers how to test rule logic against historical data and assess alert volume. +: Test rule logic against historical data and assess alert volume before enabling in production. diff --git a/solutions/toc.yml b/solutions/toc.yml index 5004014e7f..7ca99e6620 100644 --- a/solutions/toc.yml +++ b/solutions/toc.yml @@ -584,7 +584,7 @@ toc: - file: security/detect-and-alert/install-manage-prebuilt-rules.md - file: security/detect-and-alert/update-prebuilt-rules.md - file: security/detect-and-alert/mitre-attack-coverage.md - - file: security/detect-and-alert/custom-rules.md + - file: security/detect-and-alert/author-rules.md children: - file: security/detect-and-alert/choose-the-right-rule-type.md children: @@ -600,11 +600,10 @@ toc: - file: security/detect-and-alert/machine-learning.md - file: security/detect-and-alert/using-the-rule-builder.md - file: security/detect-and-alert/using-the-api.md + - file: security/detect-and-alert/common-rule-settings.md - file: security/detect-and-alert/set-rule-data-sources.md - file: security/detect-and-alert/write-investigation-guides.md - file: security/detect-and-alert/validate-and-test-rules.md - - file: security/detect-and-alert/common-rule-settings.md - - file: security/detect-and-alert/query-alert-indices.md - file: security/detect-and-alert/manage-detection-rules.md - file: security/detect-and-alert/monitor-rule-executions.md - file: security/detect-and-alert/reduce-noise-and-false-positives.md @@ -621,6 +620,7 @@ toc: - file: security/detect-and-alert/visualize-detection-alerts.md - file: security/detect-and-alert/view-detection-alert-details.md - file: security/detect-and-alert/add-detection-alerts-to-cases.md + - file: security/detect-and-alert/query-alert-indices.md - file: security/configure-elastic-defend.md children: - file: security/configure-elastic-defend/elastic-defend-requirements.md From d6397711f12b7cc3ba909b15c7cbb9170628a541 Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Mon, 23 Feb 2026 16:49:31 -0500 Subject: [PATCH 09/24] deleted old file --- .../security/detect-and-alert/custom-rules.md | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 solutions/security/detect-and-alert/custom-rules.md diff --git a/solutions/security/detect-and-alert/custom-rules.md b/solutions/security/detect-and-alert/custom-rules.md deleted file mode 100644 index 5f4105d19a..0000000000 --- a/solutions/security/detect-and-alert/custom-rules.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/security/current/rules-ui-create.html -applies_to: - stack: all - serverless: - security: all -products: - - id: security - - id: cloud-serverless -description: Create and configure detection rules tailored to your environment and threat model. ---- - -# Author custom rules [rules-ui-create] - -Create detection rules tailored to your environment and threat model. Whether you're writing rules from scratch or customizing prebuilt rules, the pages in this section guide you through selecting a rule type, writing rule logic, and configuring settings. - -**[Choose the right rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md)** -: Start here if you're not sure which rule type fits your use case. Compares all rule types side by side. - -**[Rule types](/solutions/security/detect-and-alert/rule-types.md)** -: Detailed guidance for each rule type, including when to use it and field configuration specific to that type. - -**[Using the rule builder](/solutions/security/detect-and-alert/using-the-rule-builder.md)** -: Step-by-step workflow for creating rules in the {{kib}} UI. - -**[Using the API](/solutions/security/detect-and-alert/using-the-api.md)** -: Create or manage rules programmatically, integrate with CI/CD pipelines, or bulk-import rules. - -**[Common rule settings](/solutions/security/detect-and-alert/common-rule-settings.md)** -: Reference for all shared rule settings: severity, risk score, schedule, actions, and notification variables. - -**[Set rule data sources](/solutions/security/detect-and-alert/set-rule-data-sources.md)** -: Override default index patterns, target specific indices, or exclude cold and frozen data tiers. - -**[Write investigation guides](/solutions/security/detect-and-alert/write-investigation-guides.md)** -: Add triage guidance to rules using Markdown, Timeline query buttons, and Osquery integration. - -**[Validate and test rules](/solutions/security/detect-and-alert/validate-and-test-rules.md)** -: Test rule logic against historical data and assess alert volume before enabling in production. From 7d4fba1767ab4863fa6e1ff670d0240c37cc241f Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Mon, 23 Feb 2026 18:21:15 -0500 Subject: [PATCH 10/24] endpoint refs --- solutions/security/detect-and-alert.md | 2 ++ .../install-manage-prebuilt-rules.md | 23 ++++++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/solutions/security/detect-and-alert.md b/solutions/security/detect-and-alert.md index f62e626392..8296773c66 100644 --- a/solutions/security/detect-and-alert.md +++ b/solutions/security/detect-and-alert.md @@ -15,6 +15,8 @@ products: {{elastic-sec}}'s detection engine evaluates your data against detection rules and generates alerts when rule criteria are met. Rules can correlate events across all connected data sources to surface threats that no single data stream would reveal on its own. {{elastic-sec}} provides several [rule types](/solutions/security/detect-and-alert/choose-the-right-rule-type.md), from field-value matches to event correlation, {{ml}} anomaly detection, and more. +The detection engine also surfaces alerts from [{{elastic-defend}}'s endpoint protection](/solutions/security/manage-elastic-defend/endpoint-protection-rules.md) (malware, ransomware, memory threats, and malicious behavior) and [external alerts](https://www.elastic.co/docs/reference/security/prebuilt-rules/rules/promotions/external_alerts) from third-party tools like Suricata, giving you a unified view of threats across your security stack. + ## Where to start | Your goal | Start here | diff --git a/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md b/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md index 1b336965b6..4540a4e55e 100644 --- a/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md +++ b/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md @@ -17,29 +17,22 @@ description: Learn how to install, enable, update, and manage Elastic prebuilt d Follow these guidelines to start using the {{security-app}}'s [prebuilt rules](detection-rules://index.md), keep them updated, and make sure they have the data needed to run successfully. -There are several special prebuilt rules you should know about: - -* **Endpoint protection rules**: Create alerts based on {{elastic-defend}}'s threat monitoring and prevention. -* **External Alerts**: Create an alert for all incoming third-party system alerts (for example, Suricata alerts). - * [Install and enable Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#load-prebuilt-rules) * [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#prebuilt-rule-tags) * [Select and duplicate all prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#select-all-prebuilt-rules) * [Update Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#update-prebuilt-rules) * [Confirm rule prerequisites](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites) -::::{note} -* Most prebuilt rules don’t start running by default. You can use the **Install and enable** option to start running rules as you install them, or first install the rules, then enable them manually. After installation, only a few prebuilt rules will be enabled by default, such as the Endpoint Security rule. - -* Without an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can't modify most settings on Elastic prebuilt rules. You must first duplicate them, then make your changes to the duplicated rules. Refer to [Select and duplicate all prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#select-all-prebuilt-rules) to learn more. -* On {{stack}}, automatic updates of Elastic prebuilt rules are supported for the current {{elastic-sec}} version and the latest three previous minor releases. For example, if you’re on {{elastic-sec}} 9.0, you’ll be able to use the Rules UI to update your prebuilt rules until {{elastic-sec}} 9.4 is released. After that point, you can still manually download and install updated prebuilt rules, but you must upgrade to the latest {{elastic-sec}} version to receive automatic updates. - -:::: - +Without an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can't modify most prebuilt rule settings. You must [duplicate them](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#select-all-prebuilt-rules) first, then customize the copies. +:::{admonition} Endpoint protection rules +Some prebuilt rules serve special purposes: [Endpoint protection rules](/solutions/security/manage-elastic-defend/endpoint-protection-rules.md) generate alerts from {{elastic-defend}}'s threat monitoring and prevention, while the [External Alerts](detection-rules://rules/promotions/external_alerts.md) rule creates alerts for incoming third-party system alerts (for example, Suricata alerts). +::: ## Install and enable Elastic prebuilt rules [load-prebuilt-rules] +Most prebuilt rules don't start running by default. Use **Install and enable** to start rules immediately, or install first and enable later. + 1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then go to the Rules table. The badge next to **Add Elastic rules** shows the number of prebuilt rules available for installation. @@ -130,6 +123,10 @@ If you have an Enterprise subscription on {{stack}} or a Security Analytics Comp Elastic regularly updates prebuilt rules to optimize their performance and ensure they detect the latest threats and techniques. When updated versions are available for your installed prebuilt rules, the **Rule Updates** tab appears on the **Rules** page, allowing you to update your installed rules with the latest versions. +::::{note} +On {{stack}}, automatic updates are supported for the current {{elastic-sec}} version and the latest three previous minor releases. For example, if you're on version 9.0, you can use the Rules UI to update prebuilt rules until version 9.4 is released. After that, you can still manually download and install updates, but must upgrade {{elastic-sec}} to receive automatic updates again. +:::: + 1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). 2. In the **Rules** table, select the **Rule Updates** tab. From 67d4b8df6276c91db46c1dea410acb6c219d8724 Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Mon, 23 Feb 2026 18:27:18 -0500 Subject: [PATCH 11/24] updates tuning page --- .../detect-and-alert/tune-detection-rules.md | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/solutions/security/detect-and-alert/tune-detection-rules.md b/solutions/security/detect-and-alert/tune-detection-rules.md index e1fc203145..a846ba555c 100644 --- a/solutions/security/detect-and-alert/tune-detection-rules.md +++ b/solutions/security/detect-and-alert/tune-detection-rules.md @@ -9,30 +9,21 @@ applies_to: products: - id: security - id: cloud-serverless -description: Optimize detection rules to reduce noise by adding exceptions, disabling rules, cloning rules, or enabling suppression. +description: Modify detection rule queries, thresholds, and schedules to improve detection precision. --- # Tune detection rules [security-tune-detection-signals] -Using the {{security-app}}, you can tune prebuilt and custom detection rules to optimize alert generation. To reduce noise, you can: +Tuning means modifying a rule's query, threshold, or schedule so it only matches events that are genuinely suspicious. Use tuning when the rule itself is too broad and catches normal behavior in any environment, not just yours. -* Add [exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md) to detection rules. +If a rule is correctly written but fires on known-safe activity specific to your environment, use [exceptions](/solutions/security/detect-and-alert/rule-exceptions.md) instead. If you're unsure which approach fits your situation, refer to [Reduce noise and false positives](/solutions/security/detect-and-alert/reduce-noise-and-false-positives.md) to compare all available mechanisms. - ::::{tip} - Using exceptions is recommended as this ensure excluded source event values persist even after prebuilt rules are updated. - :::: +This page covers tuning guidance for specific rule categories: -* Disable detection rules that rarely produce actionable alerts because they match expected local behavior, workflows, or policy exceptions. -* [Clone and modify](/solutions/security/detect-and-alert/manage-detection-rules.md#duplicate-rules) detection rule queries so they are aligned with local policy exceptions. This reduces noise while retaining actionable alerts. -* Clone and modify detection rule risk scores, and use branching logic to map higher risk scores to higher priority workflows. -* Enable [alert suppression](/solutions/security/detect-and-alert/alert-suppression.md) for custom query rules to reduce the number of repeated or duplicate alerts. - -For details about tuning rules for specific categories: - -* [Tune rules detecting authorized processes](/solutions/security/detect-and-alert/tune-detection-rules.md#tune-authorized-processes) -* [Tune Windows child process and PowerShell rules](/solutions/security/detect-and-alert/tune-detection-rules.md#tune-windows-rules) -* [Tune network rules](/solutions/security/detect-and-alert/tune-detection-rules.md#tune-network-rules) -* [Tune indicator match rules](/solutions/security/detect-and-alert/tune-detection-rules.md#tune-indicator-rules) +* [Tune rules detecting authorized processes](#tune-authorized-processes) +* [Tune Windows child process and PowerShell rules](#tune-windows-rules) +* [Tune network rules](#tune-network-rules) +* [Tune indicator match rules](#tune-indicator-rules) ## Filter out uncommon application alerts [filter-rule-process] From acc73dd2044bf666ae6be48ba339f1643a5a4c95 Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Wed, 25 Feb 2026 19:22:34 -0500 Subject: [PATCH 12/24] improvements to rules management and suppression --- get-started/_snippets/security-overview.md | 2 +- redirects.yml | 12 +- .../anomaly-detection.md | 2 +- solutions/security/detect-and-alert.md | 2 +- .../detect-and-alert/alert-suppression.md | 206 ++++++++++-------- .../detect-and-alert/before-you-begin.md | 1 + .../choose-the-right-rule-type.md | 35 --- .../security/detect-and-alert/custom-query.md | 2 +- .../customize-prebuilt-rules.md | 124 +++++++++++ .../detection-rule-concepts.md | 83 +++++++ solutions/security/detect-and-alert/eql.md | 2 +- solutions/security/detect-and-alert/esql.md | 2 +- .../detect-and-alert/fill-rule-gaps.md | 166 ++++++++++++++ .../detect-and-alert/indicator-match.md | 2 +- .../install-manage-prebuilt-rules.md | 162 -------------- .../install-prebuilt-rules.md | 92 ++++++++ .../detect-and-alert/machine-learning.md | 2 +- .../manage-detection-rules.md | 133 ++++------- .../detect-and-alert/mitre-attack-coverage.md | 2 +- .../monitor-rule-executions.md | 202 ++++------------- .../security/detect-and-alert/new-terms.md | 2 +- .../detect-and-alert/prebuilt-rules.md | 67 +++++- .../requirements-privileges.md | 2 +- .../security/detect-and-alert/threshold.md | 2 +- .../detect-and-alert/update-prebuilt-rules.md | 156 ++++++++----- .../get-started-detect-with-siem.md | 2 +- .../get-started-endpoint-security.md | 2 +- .../endpoint-protection-rules.md | 4 +- solutions/toc.yml | 5 +- 29 files changed, 861 insertions(+), 615 deletions(-) create mode 100644 solutions/security/detect-and-alert/customize-prebuilt-rules.md create mode 100644 solutions/security/detect-and-alert/detection-rule-concepts.md create mode 100644 solutions/security/detect-and-alert/fill-rule-gaps.md delete mode 100644 solutions/security/detect-and-alert/install-manage-prebuilt-rules.md create mode 100644 solutions/security/detect-and-alert/install-prebuilt-rules.md diff --git a/get-started/_snippets/security-overview.md b/get-started/_snippets/security-overview.md index 28b58d887f..d3a5f129a3 100644 --- a/get-started/_snippets/security-overview.md +++ b/get-started/_snippets/security-overview.md @@ -13,7 +13,7 @@ Use {{elastic-sec}} to protect your systems from security threats. * [**SIEM:**](https://www.elastic.co/security/siem): {{elastic-sec}}'s modern SIEM provides a centralized platform for ingesting, analyzing, and managing security data from various sources. * [**Third-party integration support**](/solutions/security/get-started/ingest-data-to-elastic-security.md): Ingest data from a various tools and data sources so you can centralize your security data. -* [**Threat detection and analytics:**](/solutions/security/detect-and-alert.md): Identify threats by using [prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md) with the ability to customize or create custom detection rules, automatically detect anomalous activity with built-in machine learning jobs, or proactively search for threats using our powerful [threat hunting and interactive visualization tools](/solutions/security/investigate.md). +* [**Threat detection and analytics:**](/solutions/security/detect-and-alert.md): Identify threats by using [prebuilt rules](/solutions/security/detect-and-alert/install-prebuilt-rules.md) with the ability to customize or create custom detection rules, automatically detect anomalous activity with built-in machine learning jobs, or proactively search for threats using our powerful [threat hunting and interactive visualization tools](/solutions/security/investigate.md). * [**Automatic migration**](/solutions/security/get-started/automatic-migration.md): Migrate SIEM rules from other platforms to {{elastic-sec}}. * [**Endpoint protection and threat prevention**](/solutions/security/configure-elastic-defend.md): Automatically stop cybersecurity attacks—such as malware and ransomware—before damage and loss can occur. * [**AI-powered features**](/solutions/security/ai.md): Leverage generative AI to help enhance threat detection, assist with incident response, and improve day-to-day security operations. diff --git a/redirects.yml b/redirects.yml index a40da4daa9..6adefbcdb3 100644 --- a/redirects.yml +++ b/redirects.yml @@ -705,7 +705,7 @@ redirects: # Related to Detect & Alert section restructure (issue #1210) # Renamed files 'solutions/security/detect-and-alert/detections-requirements.md': 'solutions/security/detect-and-alert/requirements-privileges.md' - 'solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md': 'solutions/security/detect-and-alert/install-manage-prebuilt-rules.md' + 'solutions/security/detect-and-alert/install-manage-elastic-prebuilt-rules.md': 'solutions/security/detect-and-alert/install-prebuilt-rules.md' 'solutions/security/detect-and-alert/prebuilt-rules-update-modified-unmodified.md': 'solutions/security/detect-and-alert/update-prebuilt-rules.md' 'solutions/security/detect-and-alert/mitre-attandckr-coverage.md': 'solutions/security/detect-and-alert/mitre-attack-coverage.md' 'solutions/security/detect-and-alert/about-detection-rules.md': 'solutions/security/detect-and-alert/choose-the-right-rule-type.md' @@ -725,3 +725,13 @@ redirects: 'solutions/security/detect-and-alert/detections-reference.md': 'solutions/security/detect-and-alert/common-rule-settings.md' 'solutions/security/detect-and-alert/rule-settings-reference.md': 'solutions/security/detect-and-alert/common-rule-settings.md' 'solutions/security/detect-and-alert/custom-rules.md': 'solutions/security/detect-and-alert/author-rules.md' + + # Prebuilt rules restructure + 'solutions/security/detect-and-alert/install-manage-prebuilt-rules.md': + to: 'solutions/security/detect-and-alert/install-prebuilt-rules.md' + anchors: + 'load-prebuilt-rules': 'load-prebuilt-rules' + 'prebuilt-rule-tags': 'solutions/security/detect-and-alert/prebuilt-rules.md#prebuilt-rule-tags' + 'rule-prerequisites': 'solutions/security/detect-and-alert/prebuilt-rules.md#rule-prerequisites' + 'select-all-prebuilt-rules': 'solutions/security/detect-and-alert/customize-prebuilt-rules.md#duplicate-prebuilt-rules' + diff --git a/solutions/security/advanced-entity-analytics/anomaly-detection.md b/solutions/security/advanced-entity-analytics/anomaly-detection.md index f51604a40f..c7e5b62291 100644 --- a/solutions/security/advanced-entity-analytics/anomaly-detection.md +++ b/solutions/security/advanced-entity-analytics/anomaly-detection.md @@ -41,7 +41,7 @@ If you have the appropriate role, you can use the **ML job settings** interface You can also check the status of {{ml}} detection rules, and start or stop their associated {{ml}} jobs: -* On the **Rules** page, the **Last response** column displays the rule’s current [status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-status). An indicator icon (![Error icon from rules table](/solutions/images/security-rules-table-error-icon.png "title =20x20")) also appears if a required {{ml}} job isn’t running. Click the icon to list the affected jobs, then click **Visit rule details page to investigate** to open the rule’s details page. +* On the **Rules** page, the **Last response** column displays the rule’s current [status](/solutions/security/detect-and-alert/monitor-rule-executions.md#rule-status). An indicator icon (![Error icon from rules table](/solutions/images/security-rules-table-error-icon.png "title =20x20")) also appears if a required {{ml}} job isn’t running. Click the icon to list the affected jobs, then click **Visit rule details page to investigate** to open the rule’s details page. :::{image} /solutions/images/security-rules-table-ml-job-error.png :alt: Rules table {{ml}} job error diff --git a/solutions/security/detect-and-alert.md b/solutions/security/detect-and-alert.md index 8296773c66..bf6a688ab6 100644 --- a/solutions/security/detect-and-alert.md +++ b/solutions/security/detect-and-alert.md @@ -21,7 +21,7 @@ The detection engine also surfaces alerts from [{{elastic-defend}}'s endpoint pr | Your goal | Start here | |---|---| -| Set up detection for the first time | [Requirements and privileges](/solutions/security/detect-and-alert/requirements-privileges.md) → [Install prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md) | +| Set up detection for the first time | [Requirements and privileges](/solutions/security/detect-and-alert/requirements-privileges.md) → [Install prebuilt rules](/solutions/security/detect-and-alert/install-prebuilt-rules.md) | | Take over an existing deployment | [MITRE ATT&CK coverage](/solutions/security/detect-and-alert/mitre-attack-coverage.md) → [Monitor rule executions](/solutions/security/detect-and-alert/monitor-rule-executions.md) | | Build coverage for a specific threat | [Choose the right rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md) → [Rule builder](/solutions/security/detect-and-alert/using-the-rule-builder.md) | | Reduce noise from existing rules | [Tune detection rules](/solutions/security/detect-and-alert/tune-detection-rules.md) → [Exceptions](/solutions/security/detect-and-alert/rule-exceptions.md), [Suppression](/solutions/security/detect-and-alert/alert-suppression.md), or [Snooze](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) | diff --git a/solutions/security/detect-and-alert/alert-suppression.md b/solutions/security/detect-and-alert/alert-suppression.md index 3fc9ecf42b..e0c24243db 100644 --- a/solutions/security/detect-and-alert/alert-suppression.md +++ b/solutions/security/detect-and-alert/alert-suppression.md @@ -14,156 +14,182 @@ description: Use alert suppression to reduce duplicate detection alerts by group # Suppress detection alerts [security-alert-suppression] -Alert suppression allows you to reduce the number of repeated or duplicate detection alerts created by [detection rules](/solutions/security/detect-and-alert/choose-the-right-rule-type.md). Normally, when a rule meets its criteria repeatedly, it creates multiple alerts, one for each time the rule’s criteria are met. When alert suppression is configured, alerts for duplicate events are not created. Instead, the qualifying events are grouped, and only one alert is created for each group. +When a detection rule runs, it can generate many alerts for similar events—sometimes hundreds of near-identical alerts for the same threat. Alert suppression helps you cut through this noise by grouping related events and creating a single representative alert instead of one alert per event. -Depending on the rule type, you can configure alert suppression to create alerts each time the rule runs, or once within a specified time window. You can also specify multiple fields to group events by unique combinations of values. +::::{admonition} Requirements +* Alert suppression requires the appropriate [subscription](https://www.elastic.co/pricing) for {{stack}} and {{serverless-short}}. +* {{ml-cap}} rules have [additional requirements](/solutions/security/advanced-entity-analytics/machine-learning-job-rule-requirements.md) for alert suppression. +:::: -The {{security-app}} displays several indicators in the Alerts table and the alert details flyout when a detection alert is created with alert suppression enabled. You can view the original events associated with suppressed alerts by investigating the alert in Timeline. +## When to use alert suppression -## Configure alert suppression [security-alert-suppression-configure-alert-suppression] +Alert suppression is useful when: -::::{admonition} Requirements and notices -* To use alert suppression in {{stack}} and {{serverless-short}}, you must have the appropriate [subscription](https://www.elastic.co/pricing). -* {{ml-cap}} rules have [additional requirements](/solutions/security/advanced-entity-analytics/machine-learning-job-rule-requirements.md) for alert suppression. +* A rule generates too many alerts for the same activity (for example, repeated failed login attempts from the same IP address) +* You want to focus analyst attention on unique threats rather than duplicates +* You need to reduce alert volume without weakening your detection coverage + +Alert suppression doesn't ignore events—it groups them. You can still investigate all the original events associated with a suppressed alert. -:::: -You can configure alert suppression when [creating](/solutions/security/detect-and-alert/using-the-rule-builder.md) or editing a rule. +## How alert suppression works -1. When configuring the rule (the **Define rule** step for a new rule, or the **Definition** tab for an existing rule), specify how you want to group alerts for alert suppression: +Without suppression, a rule creates one alert for every event that matches its criteria. With suppression enabled: - * **All rule types except the threshold rule:** In **Suppress alerts by**, enter 1 or more field names to group alerts by the fields' values. The maximum limit of fields that you can enter is as follows: - * {applies_to}`serverless:` {applies_to}`stack: ga 9.2+` Enter up to 5 fields. - * {applies_to}`stack: ga 9.0-9.1` Enter up to 3 fields. +1. You specify one or more fields to group events by (for example, `host.name` or `source.ip`). +2. When multiple events share the same field values, they're grouped together. +3. Instead of creating separate alerts for each event, the rule creates one alert per group. +4. For some rule types, you can also control *how often* alerts are created: + * **Per rule execution**: A new alert is created each time the rule runs (if matching events exist). + * **Per time period**: One alert is created for all matching events within a time window you specify. - ::::{note} - For {{esql}} rules, fields created in the {{esql}} query are available to select in **Suppress alerts by**. For example, fields created with the `EVAL` command can be selected when choosing how to group alerts for alert suppression. - :::: - * **Threshold rule only:** In **Group by**, enter up to 3 field names to group events by the fields' values, or leave the setting empty to group all qualifying events together. +## Configure alert suppression [security-alert-suppression-configure-alert-suppression] - ::::{tip} - - Refer to [Suppression for fields with an array of values](/solutions/security/detect-and-alert/alert-suppression.md#security-alert-suppression-fields-with-multiple-values) to learn how fields with multiple values are handled. +You can configure alert suppression when [creating](/solutions/security/detect-and-alert/using-the-rule-builder.md) or editing a rule. - :::: +::::::{stepper} +::::{step} Choose fields to group by +When configuring the rule (the **Define rule** step for a new rule, or the **Definition** tab for an existing rule), specify how you want to group alerts: -2. Choose how often to create alerts for qualifying events: +:::{dropdown} For all rule types except threshold rules +In **Suppress alerts by**, enter one or more field names to group alerts by. Events with the same values for these fields are grouped together. - * **Per rule execution**: Create an alert each time the rule runs and an event meets its criteria. - * **Per time period**: Create one alert for all qualifying events that occur within a specified time window, beginning from when an event first meets the rule criteria and creates the alert. This is the only option available when configuring alert suppression for threshold rules. +* {applies_to}`serverless:` {applies_to}`stack: ga 9.2+` You can enter up to 5 fields. +* {applies_to}`stack: ga 9.0-9.1` You can enter up to 3 fields. - For example, if a rule runs every 5 minutes but you don’t need alerts that frequently, you can set the suppression time period to a longer time, such as 1 hour. If the rule meets its criteria, it creates an alert at that time, and for the next hour, it’ll suppress any subsequent qualifying events. +For {{esql}} rules, fields created in the {{esql}} query (for example, with the `EVAL` command) are available to select in **Suppress alerts by**. +::: - :::{image} /solutions/images/security-alert-suppression-options.png - :alt: Alert suppression options - :width: 450px - ::: +:::{dropdown} For threshold rules only +In **Group by**, enter up to 3 field names to group events by, or leave the setting empty to group all qualifying events together. +::: -3. Under **If a suppression field is missing**, choose how to handle events with missing suppression fields (events in which one or more of the **Suppress alerts by** fields don’t exist): +:::{tip} +If you're suppressing by fields that contain arrays, refer to [Suppression for fields with an array of values](#security-alert-suppression-fields-with-multiple-values) for details on how different rule types handle them. +::: +:::: - ::::{note} - These options are available for all rule types except threshold rules. - :::: +::::{step} Choose suppression frequency +Choose how often to create alerts for qualifying events: - * **Suppress and group alerts for events with missing fields**: Create one alert for each group of events with missing fields. Missing fields get a `null` value, which is used to group and suppress alerts. - * **Do not suppress alerts for events with missing fields**: Create a separate alert for each matching event. This basically falls back to normal alert creation for events with missing suppression fields. +* **Per rule execution**: Create an alert each time the rule runs and finds matching events. +* **Per time period**: Create one alert for all matching events that occur within a specified time window. The window starts when an event first matches and creates an alert. -4. Configure other rule settings, then save and enable the rule. + For example, if a rule runs every 5 minutes but you don't need alerts that frequently, you can set the suppression time period to 1 hour. The rule creates an alert when it first matches, then suppresses any subsequent matching events for the next hour. -::::{tip} -* Use the **Rule preview** before saving the rule to visualize how alert suppression will affect the alerts created, based on historical data. -* If a rule times out while suppression is turned on, try shortening the rule’s [look-back](/solutions/security/detect-and-alert/common-rule-settings.md#rule-schedule) time or turn off suppression to improve the rule’s performance. + :::{note} + **Per time period** is the only option available for threshold rules. + ::: :::: +::::{step} Handle missing fields +Under **If a suppression field is missing**, choose how to handle events where one or more suppression fields don't exist: + +* **Suppress and group alerts for events with missing fields**: Treat missing fields as having a `null` value. Events with missing fields are grouped together and suppressed. +* **Do not suppress alerts for events with missing fields**: Create a separate alert for each event with missing fields. This falls back to normal alert behavior for those events. -### Suppression for fields with an array of values [security-alert-suppression-fields-with-multiple-values] +:::{note} +These options are available for all rule types except threshold rules. +::: +:::: -When specifying fields to suppress alerts by, you can select fields that have multiple values. When alerts for those fields are generated, they're handled as follows: +::::{step} Save and enable the rule +Configure any other rule settings, then save and enable the rule. -* **Custom query or threshold rules:** Alerts are grouped by each unique value and an alert is created for each group. For example, if you suppress alerts by `destination.ip` of `[127.0.0.1, 127.0.0.2, 127.0.0.3]`, alerts are grouped separately for each value of `127.0.0.1`, `127.0.0.2`, and `127.0.0.3` and an alert is created for each group. +:::{tip} +* Use the **Rule preview** before saving to visualize how alert suppression will affect alerts based on historical data. +* If a rule times out while suppression is enabled, try shortening the rule's [look-back](/solutions/security/detect-and-alert/common-rule-settings.md#rule-schedule) time or turning off suppression to improve performance. +::: +:::: + +:::::: + + +## Suppression for fields with an array of values [security-alert-suppression-fields-with-multiple-values] + +When you suppress alerts by fields that contain multiple values (arrays), the behavior depends on the rule type: + +| Rule type | Behavior | +|-----------|----------| +| Custom query or threshold | Alerts are grouped by each unique value separately. For example, if `destination.ip` contains `[127.0.0.1, 127.0.0.2, 127.0.0.3]`, three separate alert groups are created—one for each IP address. | +| Indicator match, event correlation (non-sequence), new terms, {{esql}}, or {{ml}} | Alerts with identical arrays are grouped together. The entire array must match exactly. | +| Event correlation (sequence queries) | Alerts are grouped only if arrays are an exact match *and* in the same order. For example, `[1.1.1.1, 0.0.0.0]` and `[1.1.1.1, 192.168.0.1]` are not grouped together, even though they share an element. | -* **Indicator match, event correlation (non-sequence queries only), new terms, {{esql}}, or {{ml}} rules:** Alerts with identical array values are grouped together. For example, if you suppress alerts by `destination.ip` of `[127.0.0.1, 127.0.0.2, 127.0.0.3]`, alerts with the entire array are grouped and only one alert is created for the group. - -* **Event correlation (sequence queries only) rules:** Alerts that are an exact match are grouped. To be an exact match, array values must be identical and in the same order. For example, if you specify the field `myips` and one sequence alert has `[1.1.1.1, 0.0.0.0]` and another sequence alert has `[1.1.1.1, 192.168.0.1]`, neither of those alerts is suppressed, despite sharing an array element. ## Confirm suppressed alerts [security-alert-suppression-confirm-suppressed-alerts] -The {{security-app}} displays several indicators of whether a detection alert was created with alert suppression enabled, and how many qualifying alerts were suppressed. +The {{security-app}} shows several indicators when an alert was created with suppression enabled. ::::{important} -Changing an alert's status to `Closed` can affect suppression. Refer to [Impact of closing suppressed alerts](/solutions/security/detect-and-alert/alert-suppression.md#security-alert-suppression-impact-close-alerts) to learn more. +Closing a suppressed alert can affect suppression behavior. Refer to [Impact of closing suppressed alerts](#security-alert-suppression-impact-close-alerts) for details. :::: +### Alerts table -* **Alerts** table — Icon in the **Rule** column. Hover to display the number of suppressed alerts: - - :::{image} /solutions/images/security-suppressed-alerts-table.png - :alt: Suppressed alerts icon and tooltip in Alerts table - :screenshot: - :width: 650px - ::: +- **Icon in the Rule column**: Hover over the icon to see the number of suppressed alerts. -* **Alerts** table — Column for suppressed alerts count. Select **Fields** to open the fields browser, then add `kibana.alert.suppression.docs_count` to the table. + :::{image} /solutions/images/security-suppressed-alerts-table.png + :alt: Suppressed alerts icon and tooltip in Alerts table + :screenshot: + :width: 650px + ::: - :::{image} /solutions/images/security-suppressed-alerts-table-column.png - :alt: Suppressed alerts count field column in Alerts table - :screenshot: - :width: 750px - ::: +- **Suppressed alerts count column**: Select **Fields** to open the fields browser, then add `kibana.alert.suppression.docs_count` to the table. -* Alert details flyout — **Insights** → **Correlations** section: + :::{image} /solutions/images/security-suppressed-alerts-table-column.png + :alt: Suppressed alerts count field column in Alerts table + :screenshot: + :width: 750px + ::: - :::{image} /solutions/images/security-suppressed-alerts-details.png - :alt: Suppressed alerts in the Correlations section within the alert details flyout - :screenshot: - :width: 450px - ::: +### Alert details flyout +Open the **Insights** → **Correlations** section to see suppression details. -## Investigate events for suppressed alerts [security-alert-suppression-investigate-events-for-suppressed-alerts] +:::{image} /solutions/images/security-suppressed-alerts-details.png +:alt: Suppressed alerts in the Correlations section within the alert details flyout +:screenshot: +:width: 450px +::: -With alert suppression, detection alerts aren’t created for the grouped source events, but you can still retrieve the events for further analysis or investigation. Do one of the following to open Timeline with the original events associated with both the created alert and the suppressed alerts: -* **Alerts** table — Select **Investigate in timeline** in the **Actions** column. +## Investigate events for suppressed alerts [security-alert-suppression-investigate-events-for-suppressed-alerts] - :::{image} /solutions/images/security-timeline-button.png - :alt: Investigate in timeline button - :width: 250px - :screenshot: - ::: +Even though suppressed events don't generate their own alerts, you can still access the original events for analysis. Open Timeline with all the events associated with a suppressed alert using one of these methods: -* Alert details flyout — Select **Take action** → **Investigate in timeline**. +* Alerts table— select **Investigate in timeline** in the **Actions** column. +* Alert details flyout— select **Take action > Investigate in timeline**. ## Impact of closing suppressed alerts [security-alert-suppression-impact-close-alerts] -By default, if you close a suppressed alert while a suppression window is still active, suppression resets. Subsequently, any new qualifying alerts are suppressed and added to a new alert for suppression. +By default, closing a suppressed alert while the suppression window is still active resets suppression. The next qualifying event starts a new suppression window and creates a new alert. -For example, say you set the suppression time period to 5 minutes and specify to group alerts by the `host.name` field. The first time an event meets the rule's criteria, an alert is created. Over the next 5 minutes, any subsequent qualifying alerts are suppressed and grouped by unique `host.name` value. If you close that first alert before the active suppression window ends (the 5 minute suppression time period), alert suppression stops and restarts when the next qualifying alert meets the suppression criteria. +For example, say you set suppression to 5 minutes, grouped by `host.name`. When an event matches, an alert is created. For the next 5 minutes, matching events are suppressed and grouped with that alert. If you close the alert before the 5-minute window ends, suppression stops. The next matching event creates a new alert and starts a new 5-minute window. +{applies_to}`stack: ga 9.2` You can change this default behavior to continue suppressing alerts until the suppression window ends, even after you close the alert. To do this, change the `securitySolution:suppressionBehaviorOnAlertClosure` [advanced setting](/solutions/security/get-started/configure-advanced-settings.md#suppression-window-behavior) to **Continue until suppression window ends**. -:::{image} /solutions/images/security-alert-suppression-close-alert-example.png -:alt: Example of suppression configuration for a rule -:screenshot: -:width: 450px -::: -{applies_to}`stack: ga 9.2` You can change the default behavior and continue suppressing alerts until the end of suppression window after you close an investigated alert. To do this, change the `securitySolution:suppressionBehaviorOnAlertClosure` [advanced setting](/solutions/security/get-started/configure-advanced-settings.md#suppression-window-behavior) to **Continue until suppression window ends**. +## Alert suppression limits [security-alert-suppression-alert-suppression-limit-by-rule-type] -## Alert suppression limit by rule type [security-alert-suppression-alert-suppression-limit-by-rule-type] +Some rule types limit the number of alerts that can be suppressed. Custom query rules have no suppression limit. -Some rule types have a maximum number of alerts that can be suppressed (custom query rules don’t have a suppression limit): +| Rule type | Maximum suppressed alerts | +|-----------|---------------------------| +| Threshold, event correlation, {{esql}}, and {{ml}} | Equal to the rule's **Max alerts per run** [advanced setting](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params) (default: `100`) | +| Indicator match and new terms | Five times the rule's **Max alerts per run** setting (default: `500`) | -* **Threshold, event correlation, {{esql}}, and {{ml}}:** The maximum number of alerts is the value you choose for the rule’s **Max alerts per run** [advanced setting](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params), which is `100` by default. -* **Indicator match and new terms:** The maximum number is five times the value you choose for the rule’s **Max alerts per run** [advanced setting](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params). The default value is `100`, which means the default maximum limit for indicator match rules and new terms rules is `500`. -## Bulk apply and remove alert suppression [security-alert-suppression-bulk-apply] +## Bulk apply or remove alert suppression [security-alert-suppression-bulk-apply] ```{applies_to} - stack: ga 9.1 +stack: ga 9.1 ``` -From the Rules table, use the **Bulk actions** menu to apply or remove alert suppression to multiple rules. The **Apply alert suppression** option can be used for all rules types except for the threshold rule type. To bulk-apply alert suppression to threshold rules, use the bulk menu option that's labeled for threshold rules only. \ No newline at end of file +You can apply or remove alert suppression from multiple rules at once using the **Bulk actions** menu in the Rules table. + +* For most rule types, use the **Apply alert suppression** option. +* For threshold rules, use the bulk menu option labeled specifically for threshold rules. \ No newline at end of file diff --git a/solutions/security/detect-and-alert/before-you-begin.md b/solutions/security/detect-and-alert/before-you-begin.md index 531bf3821a..dbce52f6b4 100644 --- a/solutions/security/detect-and-alert/before-you-begin.md +++ b/solutions/security/detect-and-alert/before-you-begin.md @@ -19,6 +19,7 @@ These tasks are typically completed once when you first configure detection capa - [**Turn on detections**](/solutions/security/detect-and-alert/requirements-privileges.md): Enable the Detections feature for your deployment type. On {{serverless-short}}, detections are on by default. - [**Detections privileges**](/solutions/security/detect-and-alert/detections-privileges.md): Understand the cluster, index, and {{kib}} privileges required for detection features, and review predefined roles and the authorization model. +- [**Detection rule concepts**](/solutions/security/detect-and-alert/detection-rule-concepts.md): Learn foundational concepts—data sources, rule authorization, exceptions, and notifications—that apply across all rule types. ## Revisit as your environment changes diff --git a/solutions/security/detect-and-alert/choose-the-right-rule-type.md b/solutions/security/detect-and-alert/choose-the-right-rule-type.md index 00f3d25c22..c6d6ea8c05 100644 --- a/solutions/security/detect-and-alert/choose-the-right-rule-type.md +++ b/solutions/security/detect-and-alert/choose-the-right-rule-type.md @@ -76,38 +76,3 @@ On a building block rule's details page, the rule's alerts are always displayed. Select the **Building block** option in the rule's [advanced settings](/solutions/security/detect-and-alert/common-rule-settings.md#rule-ui-advanced-params) when creating or editing any rule type. END BUILDING BLOCK SECTION --> -## Shared concepts [shared-rule-concepts] - -The following concepts apply to all rule types. For the full settings reference, see [Rule settings reference](/solutions/security/detect-and-alert/common-rule-settings.md). - -### Data sources [views-index-patterns] - -When you create a rule, you must specify either {{es}} index patterns or a {{data-source}} as the data source (except for {{ml}} rules, which do not use queries). If you select a {{data-source}}, you can use [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md) associated with that {{data-source}} in your rule query. - -::::{note} -To access {{data-source}}s in {{stack}}, you must have the [required permissions](/explore-analyze/find-and-organize/data-views.md#data-views-read-only-access). In {{serverless-short}}, you must have the appropriate [predefined Security user role](/deploy-manage/users-roles/cloud-organization/user-roles.md#general-assign-user-roles) or a [custom role](../../../deploy-manage/users-roles/cloud-organization/user-roles.md) with the right privileges. -:::: - -::::{important} -System indices, such as the alert indices, contain important configuration and internal data. Do not change their mappings. Changes can lead to rule execution and alert indexing failures. Use [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md) instead to add fields to existing alert and event documents. -:::: - -### Authorization [alerting-authorization-model] - -Rules are authorized using an [API key](/deploy-manage/api-keys/elasticsearch-api-keys.md) associated with the last user to edit the rule. When you create or modify a rule, an API key is generated that captures a snapshot of your privileges. This API key is used for all background tasks, including detection checks and action execution. - -::::{important} -If a user without the required privileges (such as index privileges) updates a rule, the rule stops functioning. Ensure that only users with appropriate access edit rules. -:::: - -### Exceptions [about-exceptions] - -You can [add exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md) to rules to prevent them from generating alerts even when their criteria are met. This is useful for reducing noise from trusted processes, internal IP addresses, or known-benign activity. - -::::{note} -Exceptions are supported for custom query, {{ml}}, event correlation, and indicator match rule types. -:::: - -### Notifications [about-notifications] - -For both prebuilt and custom rules, you can send notifications when alerts are created. Notifications can be sent through {{jira}}, Microsoft Teams, PagerDuty, Slack, and other connectors. Configure actions when you [create or edit a rule](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications). diff --git a/solutions/security/detect-and-alert/custom-query.md b/solutions/security/detect-and-alert/custom-query.md index 6c479b5ead..99fef8744d 100644 --- a/solutions/security/detect-and-alert/custom-query.md +++ b/solutions/security/detect-and-alert/custom-query.md @@ -106,4 +106,4 @@ The following settings are specific to custom query rules. For settings shared a : An informational list of fields the rule needs to function. This does not affect rule execution. It helps other users understand the rule's data dependencies. **Related integrations** (optional) -: Associate the rule with one or more [Elastic integrations](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). +: Associate the rule with one or more [Elastic integrations](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/prebuilt-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/customize-prebuilt-rules.md b/solutions/security/detect-and-alert/customize-prebuilt-rules.md new file mode 100644 index 0000000000..809bee087d --- /dev/null +++ b/solutions/security/detect-and-alert/customize-prebuilt-rules.md @@ -0,0 +1,124 @@ +--- +navigation_title: Customize prebuilt rules +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Customize Elastic Security prebuilt detection rules by editing directly (Enterprise), duplicating, adding exceptions, or reverting to the original Elastic version. +--- + +# Customize Elastic prebuilt rules [customize-prebuilt-rules] + +Prebuilt rules provide a starting point for threat detection, but you may need to adapt them to your environment. This page explains how to customize prebuilt rules based on your subscription level. + +## What you can customize by subscription [customize-subscription-capabilities] + +Your subscription determines how you can customize prebuilt rules: + +| Capability | Basic–Platinum | Enterprise | +|---|:---:|:---:| +| Add exceptions to rules | ✓ | ✓ | +| Configure rule actions | ✓ | ✓ | +| Duplicate and modify copies | ✓ | ✓ | +| Edit prebuilt rules directly | — | ✓ | +| Revert to Elastic version | — | ✓ | + +For {{serverless-short}}, Security Analytics Essentials corresponds to Basic–Platinum, and Security Analytics Complete corresponds to Enterprise. + + +## Edit prebuilt rules directly [edit-prebuilt-rules] + +::::{admonition} Requirements +This feature requires an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}. +:::: + +With an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can edit most prebuilt rule settings directly (except **Author** and **License**). + +1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). +2. In the Rules table, find the prebuilt rule you want to edit. +3. Do one of the following: + * Select the **All actions** menu {icon}`boxes_horizontal` on a rule, then select **Edit rule settings**. + * Select a rule's name to open its details page, then select **Edit rule settings**. +4. Modify the [rule's settings](/solutions/security/detect-and-alert/using-the-rule-builder.md). +5. Select **Save changes**. + +::::{admonition} Tracking modifications +:applies_to: { stack: ga 9.1+ } + +After saving changes to a prebuilt rule, modified fields are marked with the **Modified** badge. From the rule's details page, select the badge to view a side-by-side comparison of the original Elastic version and your modified version. Deleted characters are highlighted in red; added characters are highlighted in green. You can also access this comparison by clicking the **Modified Elastic rule** badge under the rule's name. +:::: + +### Considerations for editing prebuilt rules + +* **Updates may cause conflicts**: When Elastic releases an update that changes the same fields you modified, you need to resolve the conflict. Refer to [Resolve update conflicts](/solutions/security/detect-and-alert/update-prebuilt-rules.md#resolve-reduce-rule-conflicts). +* **Revert if needed**: You can restore the original Elastic version at any time. Refer to [Revert to Elastic version](#revert-prebuilt-rules). + + +## Duplicate and modify prebuilt rules [duplicate-prebuilt-rules] + +If you can't edit prebuilt rules directly, or if you want to preserve the original rule while creating a customized version, duplicate the rule first. + +::::{note} +Duplicated rules are entirely separate from the original prebuilt rule. They don't receive Elastic updates when the prebuilt rule is updated. +:::: + +1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). +2. In the **Rules** table, select the **Elastic rules** filter. +3. Do one of the following: + * Duplicate a single rule: Select the **All actions** menu {icon}`boxes_horizontal` on the rule, then select **Duplicate**. + * Duplicate multiple rules: Select one or more rules (or select **Select all *x* rules**), then select **Bulk actions** → **Duplicate**. +4. If the rule has exceptions, select how to handle them: + * Duplicate the rule and its exceptions (active and expired) + * Duplicate the rule and active exceptions only + * Duplicate only the rule +5. Select **Duplicate**. + +::::{note} +If you duplicate a rule and its exceptions, copies of the exceptions are created and added to the duplicated rule's [default rule list](/solutions/security/detect-and-alert/rule-exceptions.md). If the original rule used exceptions from a shared exception list, the duplicated rule references the same shared exception list. +:::: + +After duplicating, you can: + +* Modify the duplicated rule's settings as needed. +* Turn off the original prebuilt rule if you don't want both rules running. +* Delete the original prebuilt rule if you no longer need it. + + +## Add exceptions without duplicating [add-exceptions-prebuilt] + +All subscriptions allow you to add exceptions to prebuilt rules without duplicating them. Exceptions prevent rules from generating alerts for specific conditions. + +1. Open the prebuilt rule's details page. +2. Go to the **Rule exceptions** tab. +3. Select **Add rule exception** and configure the exception conditions. + +For detailed guidance, refer to [Add and manage exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md). + + +## Configure rule actions without duplicating [configure-actions-prebuilt] + +All subscriptions allow you to configure rule actions (notifications) on prebuilt rules without duplicating them. + +1. Find **Detection rules (SIEM)** in the navigation menu. +2. Select the **All actions** menu {icon}`boxes_horizontal` on a rule, then select **Edit rule settings**. +3. Go to the **Actions** tab and configure the desired actions. +4. Select **Save changes**. + +For detailed guidance, refer to [Rule actions](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications). + + +## Revert to Elastic version [revert-prebuilt-rules] + +With an Enterprise subscription (or Security Analytics Complete), you can [edit prebuilt rules directly](#edit-prebuilt-rules). If you've modified a prebuilt rule and want to restore the original Elastic version: + +1. Open the rule's details page. +2. Select the **All actions** menu {icon}`boxes_horizontal`, then select **Revert to Elastic version**. +3. In the flyout, review the modified fields. Deleted characters are highlighted in red; added characters are highlighted in green. +4. Select **Revert** to restore the modified fields to their original versions. + +::::{note} +If you haven't updated the rule in a while, its original version might be unavailable for comparison. You can avoid this by regularly [updating prebuilt rules](/solutions/security/detect-and-alert/update-prebuilt-rules.md). +:::: diff --git a/solutions/security/detect-and-alert/detection-rule-concepts.md b/solutions/security/detect-and-alert/detection-rule-concepts.md new file mode 100644 index 0000000000..4860f037ec --- /dev/null +++ b/solutions/security/detect-and-alert/detection-rule-concepts.md @@ -0,0 +1,83 @@ +--- +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Understand key concepts that apply to all detection rules, including data sources, authorization, exceptions, and notifications. +--- + +# Detection rule concepts [detection-rule-concepts] + +Before creating detection rules, familiarize yourself with the foundational concepts that apply across all rule types. Understanding these concepts helps you design effective rules and troubleshoot issues when they arise. + +## Data sources [data-sources-concept] + +Detection rules query data from {{es}} indices. When you create a rule (except for {{ml}} rules, which use anomaly jobs), you specify either: + +* **Index patterns**: Wildcards like `logs-*` or `filebeat-*` that match one or more indices. +* **{{data-source-cap}}s**: Named references to index patterns that can include [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md) for computed values at query time. + +{{data-source-cap}}s are useful when you need consistent field definitions across multiple rules or want to add fields without reindexing data. + +::::{note} +To use {{data-source}}s in {{stack}}, you must have the [required permissions](/explore-analyze/find-and-organize/data-views.md#data-views-read-only-access). In {{serverless-short}}, you must have the appropriate [predefined Security user role](/deploy-manage/users-roles/cloud-organization/user-roles.md#general-assign-user-roles) or a [custom role](../../../deploy-manage/users-roles/cloud-organization/user-roles.md) with the right privileges. +:::: + +For guidance on configuring rule data sources, refer to [Set rule data sources](/solutions/security/detect-and-alert/set-rule-data-sources.md). + +::::{important} +System indices, such as the alert indices, contain important configuration and internal data. Do not change their mappings. Changes can lead to rule execution and alert indexing failures. Use [runtime fields](/solutions/security/get-started/create-runtime-fields-in-elastic-security.md) instead to add fields to existing alert and event documents. +:::: + +## Rule authorization [rule-authorization-concept] + +Rules execute using an [API key](/deploy-manage/api-keys/elasticsearch-api-keys.md) associated with the last user who edited the rule. When you create or modify a rule, {{elastic-sec}} generates an API key that captures a snapshot of your current privileges. The rule uses this API key for all background tasks, including: + +* Executing detection queries on the configured schedule +* Writing alerts to the alerts index +* Executing actions (sending notifications) + +This authorization model means that rules continue to run with the privileges of their last editor, even when that user is not logged in. + +::::{important} +If a user without the required privileges (such as index read access) updates a rule, the rule stops functioning. Ensure that only users with appropriate access edit rules. For required privileges, refer to [Detections privileges](/solutions/security/detect-and-alert/detections-privileges.md). +:::: + +## Exceptions [exceptions-concept] + +Exceptions prevent rules from generating alerts for specific conditions, even when the rule's query criteria are met. Use exceptions to: + +* Exclude trusted processes, IP addresses, or user accounts +* Filter out known-benign activity specific to your environment +* Reduce alert noise without modifying the rule's query logic + +Exceptions can be scoped to a single rule or shared across multiple rules using [shared exception lists](/solutions/security/detect-and-alert/create-manage-shared-exception-lists.md). + +::::{note} +Exceptions are supported for custom query, {{ml}}, event correlation (EQL), indicator match, new terms, and {{esql}} rule types. Threshold rules do not support exceptions. +:::: + +For detailed guidance, refer to [Rule exceptions](/solutions/security/detect-and-alert/rule-exceptions.md). + +## Notifications and actions [notifications-concept] + +Rules can trigger actions when they generate alerts. Actions send notifications or integrate with external systems through connectors. Common use cases include: + +* Sending Slack or Microsoft Teams messages when high-severity alerts fire +* Creating tickets in {{jira}} or {{sn}} for analyst triage +* Triggering PagerDuty incidents for critical detections + +You can configure actions to run for every alert, on a schedule, or when alerts meet specific conditions. Actions are configured in the rule's [Actions settings](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications). + +::::{tip} +To temporarily stop notifications without disabling a rule, use [snooze](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions). +:::: + +## Related resources + +* [Common rule settings](/solutions/security/detect-and-alert/common-rule-settings.md): Full reference for all rule configuration options +* [Select the right rule type](/solutions/security/detect-and-alert/choose-the-right-rule-type.md): Compare rule types and find the best fit for your use case +* [Detections privileges](/solutions/security/detect-and-alert/detections-privileges.md): Required permissions for creating and managing rules diff --git a/solutions/security/detect-and-alert/eql.md b/solutions/security/detect-and-alert/eql.md index 19384e9973..87191b8f05 100644 --- a/solutions/security/detect-and-alert/eql.md +++ b/solutions/security/detect-and-alert/eql.md @@ -119,4 +119,4 @@ The following settings are specific to EQL rules. For settings shared across all : An informational list of fields the rule needs to function. This does not affect rule execution. **Related integrations** (optional) -: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/prebuilt-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/esql.md b/solutions/security/detect-and-alert/esql.md index aa65a2fe31..03d164620c 100644 --- a/solutions/security/detect-and-alert/esql.md +++ b/solutions/security/detect-and-alert/esql.md @@ -117,4 +117,4 @@ The following settings are specific to {{esql}} rules. For settings shared acros : An informational list of fields the rule needs to function. This does not affect rule execution. **Related integrations** (optional) -: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/prebuilt-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/fill-rule-gaps.md b/solutions/security/detect-and-alert/fill-rule-gaps.md new file mode 100644 index 0000000000..0f2d7994a5 --- /dev/null +++ b/solutions/security/detect-and-alert/fill-rule-gaps.md @@ -0,0 +1,166 @@ +--- +navigation_title: Fill rule execution gaps +applies_to: + stack: ga + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Find and fill gaps in Elastic Security detection rule executions manually or automatically to ensure continuous threat monitoring and avoid missed alerts. +--- + +# Fill rule execution gaps [fill-rule-gaps] + +Gaps are periods when a detection rule didn't run as scheduled. They can be caused by various disruptions, including system updates, resource constraints, or simply turning off a rule. Addressing gaps is essential for maintaining consistent detection coverage and avoiding missed alerts. + +This page explains how to find gaps and fill them, either manually or automatically. + +## Find gaps [find-gaps] + +You can find gaps from two locations: + +* **Rule Monitoring tab**: Provides an overview of gaps across all rules +* **Gaps table**: Shows detailed gap information for a specific rule + +### Rule Monitoring tab [rule-monitoring-tab-gaps] + +From the **Rule Monitoring** tab on the {{rules-ui}} page, you can get an overview of existing gaps and their status. The total number of rules with gaps is tracked in the panel above the Rules table. + +::::{applies-switch} + +:::{applies-item} { "stack": "ga 9.3+"} +The panel displays: +* **Rules with gaps**: The number of rules with gaps (left metric) and the number of rules with all gaps filled (right metric). Shows data from the last 90 days. +::: + +:::{applies-item} { "stack": "ga 9.1-9.2" } +The panel displays: +* **Time filter**: Select a time range for viewing gap data. +* **Total rules with gaps**: The number of rules with unfilled gaps (left metric) and rules with gaps being filled (right metric) within the selected time range. +* **Only rules with unfilled gaps**: Filters the Rules table to only display rules with unfilled gaps (excludes rules with gaps currently being filled). +::: + +:::{applies-item} { "stack": "ga =9.0" } +The panel displays: +* **Time filter**: Select a time range for viewing gap data. +* **Total rules with gaps**: The number of rules with unfilled or partially filled gaps within the selected time range. +* **Only rules with gaps**: Filters the Rules table to only display rules with unfilled or partially filled gaps. +::: + +:::: + +Within the Rules table, several columns provide additional gap data: + +* **Last Gap (if any)**: Shows how long the most recent gap for a particular rule lasted. +* **Unfilled gaps duration**: Shows whether a rule still has gaps and provides a total sum of the remaining unfilled or partially filled gaps. The total can change based on the time range you select (data on gaps older than 90 days is not retained). If a rule has no gaps, the column displays a dash (`––`). +* {applies_to}`stack: ga 9.3+` **Gap fill status**: Shows the status of the rule's gaps (`Unfilled`, `In progress`, or `Filled`). + + ::::{tip} + :applies_to:{stack: ga 9.3+} + Use the **Gap fill status** filter in the Rules table to find rules with a specific gap status. + :::: + +### Gaps table [gaps-table] + +```{applies_to} +stack: preview =9.0, ga 9.1+ +``` + +The Gaps table on a rule's **Execution results** tab provides detailed information about that rule's gaps. Use it to assess the scope and severity of gaps for a specific rule. + +To access the Gaps table, select a rule's name to open its details, then scroll to the **Execution results** tab. + +{applies_to}`stack: ga 9.3+` To fill all gaps for the rule at once, select **Fill all gaps**. + +:::{image} /solutions/images/security-gaps-table.png +:alt: Gaps table on the rule execution results tab +:screenshot: +::: + +The Gaps table has the following columns: + +| Column | Description | +|---|---| +| **Status** | The current state of the gap: `Filled`, `Partially filled`, or `Unfilled`. | +| **Detected at** | When the gap was first discovered. | +| **Manual fill tasks** | The status of the manual run filling the gap. For details, refer to the [Manual runs table](/solutions/security/detect-and-alert/manage-detection-rules.md#manual-runs-table). | +| **Event time covered** | How much progress the manual run has made filling the gap. | +| **Range** | When the gap started and ended. | +| **Total gap duration** | How long the gap lasted. | +| **Actions** | Available actions: **Fill gap** (starts a manual run) or **Fill remaining gap** (fills the leftover portion of a partially filled gap). | + +::::{note} +If you stop a manual run before it finishes filling a gap, the gap's status changes to `Partially filled`. To fill the remaining gap, select **Fill remaining gap** or [manually run](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules) the rule over the gap's time frame. +:::: + + +## Fill gaps manually [fill-gaps-manually] + +You can manually fill gaps in two ways: + +* **For a specific rule**: Use the **Fill gap** action in the rule's [Gaps table](#gaps-table). +* **For multiple rules**: Use the **Fill gaps** bulk action from the Rules table. + +### Fill gaps for multiple rules [bulk-fill-gaps] + +```{applies_to} +stack: ga 9.1+ +``` + +From the Rules table, fill gaps for multiple rules using the **Fill gaps** bulk action. + +1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). +2. In the Rules table, select the **Rule Monitoring** tab, then do one of the following: + + * Fill rules with unfilled or partially filled gaps: Select the appropriate rules or all rules on the page, then select **Bulk actions** → **Fill gaps**. + * Only fill rules with unfilled gaps: In the panel above the table, select the **Only rules with unfilled gaps** filter to show only rules with unfilled gaps (excludes rules with gaps currently being filled). Select the appropriate rules, then select **Bulk actions** → **Fill gaps**. + +3. Specify when to start and end the manual run that fills the gaps. +4. Select **Schedule gap fills**. The rule runs over unfilled gaps in the selected time range. + +After scheduling the manual run, track gap fill progress by checking the **Total rules with gaps** field in the panel above the Rules table. The left metric shows remaining rules with unfilled gaps; the right metric shows rules currently having their gaps filled. + +You can also check gap fill progress for individual rules by opening their details page and viewing the [Gaps table](#gaps-table) on the **Execution results** tab. + + +## Fill gaps automatically [fill-gaps-automatically] + +```{applies_to} +stack: ga 9.3+ +``` + +::::{note} +Automatic gap fill requires the appropriate subscription. Refer to the subscription page for [{{ecloud}}](https://www.elastic.co/subscriptions/cloud) and [{{stack}}/self-managed](https://www.elastic.co/subscriptions) for feature availability. +:::: + +When enabled, the automatic gap fill feature runs a job every two minutes to check for new and existing gaps, then schedules tasks to fill them. + +### Enable automatic gap fill + +1. On the {{rules-ui}} page, select **Settings** (above the Rules table). +2. In the **Auto gap fill settings** section, turn on the toggle. + +::::{tip} +The **Auto gap fill status** field (in the panel above the Rules table) shows whether automatic gap fill is on or off. Select the field value to access the settings. +:::: + +### Monitor automatic gap fill + +Details about the automatic gap fill job and scheduled tasks are captured in the gap fill scheduler logs. Access the logs by selecting **Gap fill scheduler** in the **Auto gap fill settings** section. + +In the scheduler logs table, expand rows to learn more about gaps discovered and tasks scheduled each time the job ran. Key details include: + +* When each job ran to check for gaps +* The number of gaps detected and rules affected +* The number of gap fill tasks scheduled +* Task status: + * **Success**: Gap fill tasks were successfully scheduled. Check log details to see if any rules were not processed. + * **Error**: Some gap fill tasks were not scheduled or failed to run. Check log details for more information. + * **Task skipped**: Gap fill tasks were scheduled but did not run (rules may have been disabled or the maximum task limit was reached). + * **No gaps**: No gaps were detected; no tasks were scheduled. + + +## Troubleshoot gaps + +Refer to the [Troubleshoot gaps](../../../troubleshoot/security/detection-rules.md#troubleshoot-gaps) section for strategies on avoiding and resolving gaps. diff --git a/solutions/security/detect-and-alert/indicator-match.md b/solutions/security/detect-and-alert/indicator-match.md index 182cfe3a96..5fae777569 100644 --- a/solutions/security/detect-and-alert/indicator-match.md +++ b/solutions/security/detect-and-alert/indicator-match.md @@ -118,4 +118,4 @@ The following settings are specific to indicator match rules. For settings share : An informational list of fields the rule needs to function. This does not affect rule execution. **Related integrations** (optional) -: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/prebuilt-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md b/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md deleted file mode 100644 index 4540a4e55e..0000000000 --- a/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md +++ /dev/null @@ -1,162 +0,0 @@ ---- -navigation_title: Use Elastic prebuilt rules -mapped_pages: - - https://www.elastic.co/guide/en/security/current/prebuilt-rules-management.html - - https://www.elastic.co/guide/en/serverless/current/security-prebuilt-rules-management.html -applies_to: - stack: all - serverless: - security: all -products: - - id: security - - id: cloud-serverless -description: Learn how to install, enable, update, and manage Elastic prebuilt detection rules. ---- - -# Install and manage Elastic prebuilt rules [security-prebuilt-rules-management] - -Follow these guidelines to start using the {{security-app}}'s [prebuilt rules](detection-rules://index.md), keep them updated, and make sure they have the data needed to run successfully. - -* [Install and enable Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#load-prebuilt-rules) -* [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#prebuilt-rule-tags) -* [Select and duplicate all prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#select-all-prebuilt-rules) -* [Update Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#update-prebuilt-rules) -* [Confirm rule prerequisites](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites) - -Without an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can't modify most prebuilt rule settings. You must [duplicate them](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#select-all-prebuilt-rules) first, then customize the copies. - -:::{admonition} Endpoint protection rules -Some prebuilt rules serve special purposes: [Endpoint protection rules](/solutions/security/manage-elastic-defend/endpoint-protection-rules.md) generate alerts from {{elastic-defend}}'s threat monitoring and prevention, while the [External Alerts](detection-rules://rules/promotions/external_alerts.md) rule creates alerts for incoming third-party system alerts (for example, Suricata alerts). -::: - -## Install and enable Elastic prebuilt rules [load-prebuilt-rules] - -Most prebuilt rules don't start running by default. Use **Install and enable** to start rules immediately, or install first and enable later. - -1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then go to the Rules table. - - The badge next to **Add Elastic rules** shows the number of prebuilt rules available for installation. - - :::{image} /solutions/images/security-prebuilt-rules-add-badge.png - :alt: The Add Elastic Rules page - :screenshot: - ::: - -2. Click **Add Elastic rules**. - - ::::{tip} - To examine the details of a rule before you install it, select the rule name. This opens the rule details flyout. - :::: - -3. Do one of the following: - - * Install all available rules: Click **Install all** at the top of the page. (This doesn’t enable the rules; you still need to do that manually.) - * Install a single rule: In the rules table, either click **Install** to install a rule without enabling it, or click ![Vertical boxes button](/solutions/images/security-boxesVertical.svg "") → **Install and enable** to start running the rule once it’s installed. - * Install multiple rules: Select the rules, and then at the top of the page either click **Install *x* selected rule(s)** to install without enabling the rules, or click ![Vertical boxes button](/solutions/images/serverless-boxesVertical.svg "") → **Install and enable** to install and start running the rules. - - ::::{tip} - Use the search bar and **Tags** filter to find the rules you want to install. For example, filter by `OS: Windows` if your environment only includes Windows endpoints. For more on tag categories, refer to [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#prebuilt-rule-tags). - :::: - - - :::{image} /solutions/images/security-prebuilt-rules-add.png - :alt: The Add Elastic Rules page - :screenshot: - ::: - -4. For any rules you haven’t already enabled, go back to the **Rules** page, search or filter for the rules you want to run, and do either of the following: - - * Enable a single rule: Turn on the rule’s **Enabled** switch. - * Enable multiple rules: Select the rules, then click **Bulk actions** → **Enable**. - - -Once you enable a rule, it starts running on its configured schedule. To confirm that it’s running successfully, check its **Last response** status in the rules table, or open the rule’s details page and check the [**Execution results**](/solutions/security/detect-and-alert/monitor-rule-executions.md#rule-execution-logs) tab. - -If you have an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can also [edit the prebuilt rules](/solutions/security/detect-and-alert/manage-detection-rules.md#edit-rules-settings) that you've installed. - -## Prebuilt rule tags [prebuilt-rule-tags] - -Each prebuilt rule includes several tags identifying the rule’s purpose, detection method, associated resources, and other information to help categorize your rules. These tags are category-value pairs; for example, `OS: Windows` indicates rules designed for Windows endpoints. Categories include: - -* `Data Source`: The application, cloud provider, data shipper, or Elastic integration providing data for the rule. -* `Domain`: A general category of data source types (such as cloud, endpoint, or network). -* `OS`: The host operating system, which could be considered another data source type. -* `Resources`: Additional rule resources such as investigation guides. -* `Rule Type`: Identifies if the rule depends on specialized resources (such as machine learning jobs or threat intelligence indicators), or if it’s a higher-order rule built from other rules' alerts. -* `Tactic`: MITRE ATT&CK tactics that the rule addresses. -* `Threat`: Specific threats the rule detects (such as Cobalt Strike or BPFDoor). -* `Use Case`: The type of activity the rule detects and its purpose. Use cases include: - - * `Active Directory Monitoring`: Detects changes related to Active Directory. - * `Asset Visibility`: Detects changes to specified asset types. - * `Configuration Audit`: Detects undesirable configuration changes. - * `Guided Onboarding`: Example rule, used for {{elastic-sec}}'s guided onboarding tour. - * `Identity and Access Audit`: Detects activity related to identity and access management (IAM). - * `Log Auditing`: Detects activity on log configurations or storage. - * `Network Security Monitoring`: Detects network security configuration activity. - * `Threat Detection`: Detects threats. - * `Vulnerability`: Detects exploitation of specific vulnerabilities. - - - -## Select and duplicate prebuilt rules [select-all-prebuilt-rules] - -Without an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can't modify most settings on Elastic prebuilt rules. You can only edit [rule actions](/solutions/security/detect-and-alert/common-rule-settings.md#rule-schedule) and [add exceptions](/solutions/security/detect-and-alert/add-manage-exceptions.md). If you want to modify other settings on a prebuilt rule, you must first duplicate it, then make your changes to the duplicated rule. Note that your customized rule is entirely separate from the original prebuilt rule, and will not get updates from Elastic if the prebuilt rule is updated. - -1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). -2. In the **Rules** table, select the **Elastic rules** filter. -3. Select one or more rules, or click **Select all *x* rules** above the Rules table. -4. Click **Bulk actions** → **Duplicate**. -5. (Optional) Select whether to duplicate the rules' exceptions, then click **Duplicate**. - -You can then modify the duplicated rules and, if required, delete the prebuilt ones. - - -## Update Elastic prebuilt rules [update-prebuilt-rules] - -::::{important} - -The following steps are only applicable if you have a [Platinum](https://www.elastic.co/pricing) subscription or lower on {{stack}} or a [Security Analytics Essentials project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}. - -If you have an Enterprise subscription on {{stack}} or a Security Analytics Complete project on {{serverless-short}}, follow the guidelines in [Update modified and unmodified Elastic prebuilt rules](/solutions/security/detect-and-alert/update-prebuilt-rules.md) instead. -:::: - -Elastic regularly updates prebuilt rules to optimize their performance and ensure they detect the latest threats and techniques. When updated versions are available for your installed prebuilt rules, the **Rule Updates** tab appears on the **Rules** page, allowing you to update your installed rules with the latest versions. - -::::{note} -On {{stack}}, automatic updates are supported for the current {{elastic-sec}} version and the latest three previous minor releases. For example, if you're on version 9.0, you can use the Rules UI to update prebuilt rules until version 9.4 is released. After that, you can still manually download and install updates, but must upgrade {{elastic-sec}} to receive automatic updates again. -:::: - -1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). -2. In the **Rules** table, select the **Rule Updates** tab. - - ::::{note} - The **Rule Updates** tab doesn’t appear if all your installed prebuilt rules are up to date. - :::: - - - :::{image} /solutions/images/security-prebuilt-rules-update.png - :alt: The Rule Updates tab on the Rules page - :screenshot: - ::: - -3. (Optional) To examine the details of a rule’s latest version before you update it, select the rule name. This opens the rule details flyout. - - Select the **Elastic update overview** tab to view rule changes field by field, or the **JSON view** tab to view changes for the entire rule in JSON format. Both tabs display side-by-side comparisons of the **Current rule** (what you currently have installed) and the **Elastic update** version (what you can choose to install). Deleted characters are highlighted in red; added characters are highlighted in green. - - To accept the changes and install the updated version, select **Update rule**. - - :::{image} /solutions/images/security-prebuilt-rules-update-diff-basic.png - :alt: Prebuilt rule comparison - :screenshot: - ::: - -4. Do one of the following to update prebuilt rules on the **Rules** page: - - * Update all available rules: Click **Update all**. - * Update a single rule: Click **Update rule** for that rule. - * Update multiple rules: Select the rules and click **Update *x* selected rule(s)**. - - ::::{tip} - Use the search bar and **Tags** filter to find the rules you want to update. For example, filter by `OS: Windows` if your environment only includes Windows endpoints. For more on tag categories, refer to [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#prebuilt-rule-tags). - :::: \ No newline at end of file diff --git a/solutions/security/detect-and-alert/install-prebuilt-rules.md b/solutions/security/detect-and-alert/install-prebuilt-rules.md new file mode 100644 index 0000000000..cfee7b56f5 --- /dev/null +++ b/solutions/security/detect-and-alert/install-prebuilt-rules.md @@ -0,0 +1,92 @@ +--- +navigation_title: Install prebuilt rules +mapped_pages: + - https://www.elastic.co/guide/en/security/current/prebuilt-rules-management.html + - https://www.elastic.co/guide/en/serverless/current/security-prebuilt-rules-management.html +applies_to: + stack: ga all + serverless: + security: ga all +products: + - id: security + - id: cloud-serverless +description: Install and enable Elastic Security prebuilt detection rules to quickly gain threat detection coverage across your environment. +--- + +# Install Elastic prebuilt rules [install-prebuilt-rules] + +Elastic provides hundreds of prebuilt detection rules that cover common attack techniques across multiple platforms. This page explains how to install and enable prebuilt rules so they start generating alerts. + +## What you can do by subscription [prebuilt-subscription-capabilities] + +Your subscription determines which prebuilt rule features are available: + +| Capability | Basic–Platinum | Enterprise | +|---|:---:|:---:| +| Install and enable rules | ✓ | ✓ | +| View prerequisites and tags | ✓ | ✓ | +| Add exceptions | ✓ | ✓ | +| Configure rule actions | ✓ | ✓ | +| Update rules (accept Elastic version) | ✓ | ✓ | +| Duplicate and customize | ✓ | ✓ | +| Edit prebuilt rules directly | — | ✓ | +| Review field-level update changes | — | ✓ | +| Resolve update conflicts | — | ✓ | +| Revert to Elastic version | — | ✓ (9.1+) | + +:::{note} +For {{serverless-short}}, Security Analytics Essentials corresponds to Basic–Platinum, and Security Analytics Complete corresponds to Enterprise. +::: + +## Install and enable rules [load-prebuilt-rules] + +Most prebuilt rules don't start running by default. Use **Install and enable** to start rules immediately, or install first and enable later. + +1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then go to the Rules table. + + The badge next to **Add Elastic rules** shows the number of prebuilt rules available for installation. + + :::{image} /solutions/images/security-prebuilt-rules-add-badge.png + :alt: The Add Elastic Rules page + :screenshot: + ::: + +2. Select **Add Elastic rules**. + + ::::{tip} + To examine the details of a rule before you install it, select the rule name. This opens the rule details flyout. + :::: + +3. Do one of the following: + + * Install all available rules: Select **Install all** at the top of the page. (This doesn't enable the rules; you still need to do that manually.) + * Install a single rule: In the rules table, either select **Install** to install a rule without enabling it, or select ![Vertical boxes button](/solutions/images/security-boxesVertical.svg "") → **Install and enable** to start running the rule once it's installed. + * Install multiple rules: Select the rules, and then at the top of the page either select **Install *x* selected rule(s)** to install without enabling the rules, or select ![Vertical boxes button](/solutions/images/serverless-boxesVertical.svg "") → **Install and enable** to install and start running the rules. + + ::::{tip} + Use the search bar and **Tags** filter to find the rules you want to install. For example, filter by `OS: Windows` if your environment only includes Windows endpoints. For more on tag categories, refer to [Prebuilt rule tags](#prebuilt-rule-tags). + :::: + + :::{image} /solutions/images/security-prebuilt-rules-add.png + :alt: The Add Elastic Rules page + :screenshot: + ::: + +4. For any rules you haven't already enabled, go back to the {{rules-ui}} page, search or filter for the rules you want to run, and do either of the following: + + * Enable a single rule: Turn on the rule's **Enabled** switch. + * Enable multiple rules: Select the rules, then select **Bulk actions** → **Enable**. + +Once you enable a rule, it starts running on its configured schedule. To confirm that it's running successfully, check its **Last response** status in the rules table, or open the rule's details page and check the [**Execution results**](/solutions/security/detect-and-alert/monitor-rule-executions.md#rule-execution-logs) tab. + +:::{admonition} Endpoint protection rules +Some prebuilt rules serve special purposes: [Endpoint protection rules](/solutions/security/manage-elastic-defend/endpoint-protection-rules.md) generate alerts from {{elastic-defend}}'s threat monitoring and prevention, while the [External Alerts](detection-rules://rules/promotions/external_alerts.md) rule creates alerts for incoming third-party system alerts (for example, Suricata alerts). +::: + +## Next steps + +After installing prebuilt rules: + +* **Keep rules current**: Elastic regularly updates prebuilt rules to detect new threats. Refer to [Update Elastic prebuilt rules](/solutions/security/detect-and-alert/update-prebuilt-rules.md) to learn how to apply updates. +* **Customize rules**: Adapt prebuilt rules to your environment by editing them directly (Enterprise) or duplicating and modifying copies. Refer to [Customize Elastic prebuilt rules](/solutions/security/detect-and-alert/customize-prebuilt-rules.md). +* **Build custom rules**: Create detection logic tailored to your infrastructure. Refer to [Author rules](/solutions/security/detect-and-alert/author-rules.md). diff --git a/solutions/security/detect-and-alert/machine-learning.md b/solutions/security/detect-and-alert/machine-learning.md index 33ceafb68d..6bb311f515 100644 --- a/solutions/security/detect-and-alert/machine-learning.md +++ b/solutions/security/detect-and-alert/machine-learning.md @@ -97,4 +97,4 @@ The following settings are specific to {{ml}} rules. For settings shared across : Reduce repeated or duplicate alerts by grouping them on one or more fields. Only anomaly fields are available for suppression because {{ml}} alerts do not contain source event fields. For details, refer to [Alert suppression](/solutions/security/detect-and-alert/alert-suppression.md). **Related integrations** (optional) -: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/prebuilt-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/manage-detection-rules.md b/solutions/security/detect-and-alert/manage-detection-rules.md index 5f1b8b432b..d3192164d1 100644 --- a/solutions/security/detect-and-alert/manage-detection-rules.md +++ b/solutions/security/detect-and-alert/manage-detection-rules.md @@ -23,12 +23,12 @@ The Rules page allows you to view and manage all prebuilt and custom detection r On the Rules page, you can: -* [Sort and filter the rules list](/solutions/security/detect-and-alert/manage-detection-rules.md#sort-filter-rules) and [check rule status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-status) +* [Sort and filter the rules list](/solutions/security/detect-and-alert/manage-detection-rules.md#sort-filter-rules) and [check rule status](/solutions/security/detect-and-alert/monitor-rule-executions.md#rule-status) * [Edit](/solutions/security/detect-and-alert/manage-detection-rules.md#edit-rules-settings), [enable/disable](/solutions/security/detect-and-alert/manage-detection-rules.md#enable-disable-rules), [duplicate](/solutions/security/detect-and-alert/manage-detection-rules.md#duplicate-rules), and [delete](/solutions/security/detect-and-alert/manage-detection-rules.md#delete-rules) rules -* [Run rules manually](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules) and [fill gaps](/solutions/security/detect-and-alert/manage-detection-rules.md#bulk-fill-gaps-multiple-rules) +* [Run rules manually](#manually-run-rules) and [fill gaps](/solutions/security/detect-and-alert/fill-rule-gaps.md) * [Snooze rule actions](/solutions/security/detect-and-alert/manage-detection-rules.md#snooze-rule-actions) * [Export and import rules](/solutions/security/detect-and-alert/manage-detection-rules.md#import-export-rules-ui) -* [Confirm rule prerequisites](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites) and [troubleshoot missing alerts](/troubleshoot/security/detection-rules.md#troubleshoot-signals) +* [Troubleshoot missing alerts](/troubleshoot/security/detection-rules.md#troubleshoot-signals) * Perform actions on multiple rules with [bulk actions](/solutions/security/detect-and-alert/manage-detection-rules.md#bulk-actions-reference) :::{note} @@ -56,16 +56,6 @@ You can also filter the rules list by selecting the **Tags**, **Last response**, The rules list retains your sorting and filtering settings when you navigate away and return to the page. These settings are also preserved when you copy the page’s URL and paste into another browser. Select **Clear filters** above the table to revert to the default view. -## Check the current status of rules [rule-status] - -The **Last response** column displays the current status of each rule, based on the most recent attempt to run the rule: - -* **Succeeded**: The rule completed its defined search. This doesn’t necessarily mean it generated an alert, just that it ran without error. -* **Failed**: The rule encountered an error that prevented it from running. For example, a {{ml}} rule whose corresponding {{ml}} job wasn’t running. -* **Warning**: Nothing prevented the rule from running, but it might have returned unexpected results. For example, a custom query rule tried to search an index pattern that couldn’t be found in {{es}}. - -For {{ml}} rules, an indicator icon {icon}`warning` also appears in this column if a required {{ml}} job isn’t running. Click the icon to list the affected jobs, then click **Visit rule details page to investigate** to open the rule’s details page, where you can start the {{ml}} job. - ## Edit rule settings [edit-rules-settings] @@ -109,23 +99,6 @@ Use bulk editing to update settings on multiple rules simultaneously. Rules that 5. If available, select **Overwrite all selected _x_** to overwrite the settings on the rules. For example, if you're adding tags to multiple rules, selecting **Overwrite all selected rules tags** removes all the rules' original tags and replaces them with the tags you specify. 6. Click **Save**. -## Revert modifications to prebuilt rules [revert-rule-changes] - -```{applies_to} - stack: ga 9.1 -``` - -After modifying a prebuilt rule, you can restore it's original version. To do this: - -1. Open the rule's details page, click the **All actions** menu {icon}`boxes_horizontal`, then **Revert to Elastic version**. -2. In the flyout, review the modified fields. Deleted characters are highlighted in red; added characters are highlighted in green. -3. Click **Revert** to restore the modified fields to their original versions. - -::::{note} -If you haven’t updated the rule in a while, its original version might be unavailable for comparison. You can avoid this by regularly updating prebuilt rules. -:::: - - ## Enable and disable rules [enable-disable-rules] Enable rules to activate them so they run on their defined schedules and generate alerts. Disable rules to stop them from running without deleting them. @@ -193,71 +166,75 @@ Delete rules to permanently remove them from your system. This action cannot be + +## Snooze rule actions [snooze-rule-actions] + +Instead of turning rules off to stop alert notifications, you can snooze rule actions for a specified time period. When you snooze rule actions, the rule continues to run on its defined schedule, but won’t perform any actions or send alert notifications. + +You can snooze notifications temporarily or indefinitely. When actions are snoozed, you can cancel or change the duration of the snoozed state. You can also schedule and manage recurring downtime for actions. + +You can snooze rule notifications from the **Installed Rules** tab, the rule details page, or the **Actions** tab when editing a rule. + +:::{image} /solutions/images/security-rule-snoozing.png +:alt: Rules snooze options +:width: 75% +:screenshot: +::: + + ## Run rules manually [manually-run-rules] -Manually run enabled rules for a specified period of time to deliberately test them, provide additional rule coverage, or fill gaps in rule executions. +Manually run enabled rules for a specified time period to deliberately test them, provide additional rule coverage, or fill gaps in rule executions. ::::{important} Before manually running rules, make sure you properly understand and plan for rule dependencies. Incorrect scheduling can lead to inconsistent rule results. :::: - 1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). -2. In the **Rules** table, do one of the following: +2. In the Rules table, do one of the following: * Select the **All actions** menu {icon}`boxes_horizontal` on a rule, then select **Manual run**. * Select all the rules you want to manually run, select the **Bulk actions** menu, then select **Manual run**. -3. Specify when the manual run starts and ends. The default selection is the current day starting three hours in the past. The rule will search for events during the selected time range. -4. Click **Run** to manually run the rule. - -The rule will run over the time range that you selected. Note that all [rule actions](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications) will also be activated, except for **Summary of alerts** actions that run at a custom frequency. +3. Specify when the manual run starts and ends. The default selection is the current day starting three hours in the past. The rule searches for events during the selected time range. +4. Select **Run** to manually run the rule. -Go to the [Manual runs table](/solutions/security/detect-and-alert/monitor-rule-executions.md#manual-runs-table) on the **Execution results** tab to track the manual rule executions. If you manually ran the rule over a gap, you can also monitor the gap fill's progress from the [Gaps table](/solutions/security/detect-and-alert/monitor-rule-executions.md#gaps-table). +The rule runs over the time range that you selected. All [rule actions](/solutions/security/detect-and-alert/common-rule-settings.md#rule-notifications) are also activated, except for **Summary of alerts** actions that run at a custom frequency. ::::{note} Be mindful of the following: -* Any changes that you make to the manual run or rule settings will display in the Manual runs table after the current run completes. +* Any changes that you make to the manual run or rule settings display in the Manual runs table after the current run completes. * Except for threshold rules, duplicate alerts aren't created if you manually run a rule during a time range that was already covered by a scheduled run. * Manually running a custom query rule with suppression may incorrectly inflate the number of suppressed alerts. :::: -## Fill gaps for multiple rules [bulk-fill-gaps-multiple-rules] - -```{applies_to} - stack: ga 9.1 -``` - -From the Rules table, fill gaps for multiple rules by using the **Fill gaps** bulk action. - -1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). -2. In the Rules table, click the **Rule Monitoring** tab, then do one of the following: - - * Fill rules with unfilled or partially filled gaps: Select the appropriate rules or all rules on the page, then click **Bulk actions → Fill gaps**. - - * Only fill rules with unfilled gaps: In the panel above the table, click the **Only rules with unfilled gaps** filter to only show rules with unfilled gaps (rules with gaps that are being filled are excluded). Select the appropriate rules or all of them, then click **Bulk actions → Fill gaps**. +### Manual runs table [manual-runs-table] -3. Specify when to start and end the manual run that will fill the gaps. -4. Click **Schedule gap fills**. The rule will manually run over unfilled gaps in the selected time range. +Each manual run can produce multiple rule executions, depending on the time range of the run and the rule's execution schedule. -After scheduling the manual run, you can track gap fill progress by checking the **Total rules with gaps:** field in the panel above the Rules table. The field displays two metrics separated by a forward slash. The metric on the left tells you the remaining number of rules with unfilled gaps. The metric on the right tells you the number of rules that are currently having their gaps filled. +::::{note} +Manual runs are executed with low priority and limited concurrency, meaning they might take longer to complete. This can be especially apparent for rules requiring multiple executions. +:::: -Alternatively, you can check gap fill progress for individual rules by going to their details page, clicking the **Execution results** tab, then going to the [Gaps table](/solutions/security/detect-and-alert/monitor-rule-executions.md#gaps-table). - +The Manual runs table (found on a rule's **Execution results** tab) tracks manual rule executions and provides important details such as: -## Snooze rule actions [snooze-rule-actions] +* The total number of rule executions that the manual run will produce and how many are failing, pending, running, and completed. +* When the manual run started and the time range that it will cover. -Instead of turning rules off to stop alert notifications, you can snooze rule actions for a specified time period. When you snooze rule actions, the rule continues to run on its defined schedule, but won’t perform any actions or send alert notifications. + ::::{note} + To stop an active run, go to the appropriate row in the table and select **Stop run** in the **Actions** column. Completed rule executions for each manual run are logged in the Execution log table. + :::: -You can snooze notifications temporarily or indefinitely. When actions are snoozed, you can cancel or change the duration of the snoozed state. You can also schedule and manage recurring downtime for actions. +* The status of each manual run: -You can snooze rule notifications from the **Installed Rules** tab, the rule details page, or the **Actions** tab when editing a rule. + * `Pending`: The rule is not yet running. + * `Running`: The rule is executing during the time range you specified. Some rule types, such as indicator match rules, can take longer to run. + * `Error`: The rule's configuration is preventing it from running correctly. For example, the rule's conditions cannot be validated. -:::{image} /solutions/images/security-rule-snoozing.png -:alt: Rules snooze options -:width: 75% +:::{image} /solutions/images/security-manual-rule-run-table.png +:alt: Manual rule runs table on the rule execution results tab :screenshot: ::: @@ -321,8 +298,8 @@ The following table summarizes bulk actions that are available from the **Bulk a | Duplicate | Create copies of selected rules. Refer to [Duplicate rules](/solutions/security/detect-and-alert/manage-detection-rules.md#duplicate-rules). | | Delete | Permanently remove selected rules. Refer to [Delete rules](/solutions/security/detect-and-alert/manage-detection-rules.md#delete-rules). | | Export | Export selected rules to an `.ndjson` file. Refer to [Export rules](/solutions/security/detect-and-alert/manage-detection-rules.md#export-rules-ui). | -| Manual run | Run selected rules for a specified time period. Refer to [Run rules manually](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules). | -| Fill gaps | Fill gaps for selected rules. Refer to [Fill gaps for multiple rules](/solutions/security/detect-and-alert/manage-detection-rules.md#bulk-fill-gaps-multiple-rules). | +| Manual run | Run selected rules for a specified time period. Refer to [Run rules manually](#manually-run-rules). | +| Fill gaps | Fill gaps for selected rules. Refer to [Fill gaps for multiple rules](/solutions/security/detect-and-alert/fill-rule-gaps.md#bulk-fill-gaps). | | Index patterns | Add or delete index patterns on selected rules. Refer to [Bulk edit rule settings](/solutions/security/detect-and-alert/manage-detection-rules.md#bulk-edit-rules). | | Tags | Add or delete tags on selected rules. Refer to [Bulk edit rule settings](/solutions/security/detect-and-alert/manage-detection-rules.md#bulk-edit-rules). | | Custom highlighted fields | Add custom highlighted fields on selected rules. Refer to [Bulk edit rule settings](/solutions/security/detect-and-alert/manage-detection-rules.md#bulk-edit-rules). | @@ -330,25 +307,3 @@ The following table summarizes bulk actions that are available from the **Bulk a | Update rule schedules | Update schedules and look-back times on selected rules. Refer to [Bulk edit rule settings](/solutions/security/detect-and-alert/manage-detection-rules.md#bulk-edit-rules). | | Apply Timeline template | Apply a Timeline template to selected rules. Refer to [Bulk edit rule settings](/solutions/security/detect-and-alert/manage-detection-rules.md#bulk-edit-rules). | - -## Confirm rule prerequisites [rule-prerequisites] - -Many detection rules are designed to work with specific [Elastic integrations](https://docs.elastic.co/en/integrations) and data fields. These prerequisites are identified in **Related integrations** and **Required fields** on a rule’s details page. **Related integrations** also displays each integration’s installation status and includes links for installing and configuring the listed integrations. - -Additionally, the **Setup guide** section provides guidance on setting up the rule’s requirements. - -:::{image} /solutions/images/security-rule-details-prerequisites.png -:alt: Rule details page with Related integrations -:screenshot: -::: - -You can also check rules' related integrations in the **Installed Rules** and **Rule Monitoring** tables. Click the **integrations** badge to display the related integrations in a popup. - -:::{image} /solutions/images/security-rules-table-related-integrations.png -:alt: Rules table with related integrations popup -:screenshot: -::: - -::::{tip} -You can hide the **integrations** badge in the Rules tables by turning off the `securitySolution:showRelatedIntegrations` [advanced setting](/solutions/security/get-started/configure-advanced-settings.md#show-related-integrations). -:::: \ No newline at end of file diff --git a/solutions/security/detect-and-alert/mitre-attack-coverage.md b/solutions/security/detect-and-alert/mitre-attack-coverage.md index fb6761e8ec..f6d802fa13 100644 --- a/solutions/security/detect-and-alert/mitre-attack-coverage.md +++ b/solutions/security/detect-and-alert/mitre-attack-coverage.md @@ -18,7 +18,7 @@ The **MITRE ATT&CK coverage** page shows which [MITRE ATT&CK](https://attack.mit How you use this page depends on your goal: -- **Identifying coverage gaps**: Filter to view only enabled rules, then look for light or empty cells to find tactics and techniques where your detection coverage is thin. Use this to prioritize which [prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md) to install or which [custom rules](/solutions/security/detect-and-alert/using-the-rule-builder.md) to build next. +- **Identifying coverage gaps**: Filter to view only enabled rules, then look for light or empty cells to find tactics and techniques where your detection coverage is thin. Use this to prioritize which [prebuilt rules](/solutions/security/detect-and-alert/install-prebuilt-rules.md) to install or which [custom rules](/solutions/security/detect-and-alert/using-the-rule-builder.md) to build next. - **Activating coverage quickly**: Find techniques you care about and [enable installed rules](#security-rules-coverage-enable-rules) directly from the coverage grid, without leaving the page. ## Access the page [access-mitre-page] diff --git a/solutions/security/detect-and-alert/monitor-rule-executions.md b/solutions/security/detect-and-alert/monitor-rule-executions.md index 18d3db496b..668d209f32 100644 --- a/solutions/security/detect-and-alert/monitor-rule-executions.md +++ b/solutions/security/detect-and-alert/monitor-rule-executions.md @@ -3,218 +3,104 @@ mapped_pages: - https://www.elastic.co/guide/en/security/current/alerts-ui-monitor.html - https://www.elastic.co/guide/en/serverless/current/security-alerts-ui-monitor.html applies_to: - stack: all + stack: ga all serverless: - security: all + security: ga all products: - id: security - id: cloud-serverless -description: Monitor rule executions, execution logs, gaps, and manual runs for detection rules. +description: Monitor Elastic Security detection rule executions, view execution logs, check rule status, and troubleshoot performance issues using the Rule Monitoring tab. --- # Monitor rule executions [alerts-ui-monitor] -Several tools can help you gain insight into the performance of your detection rules: +After enabling detection rules, you'll want to verify they're running successfully and generating the expected alerts. This page explains how to check rule status, review execution history, and identify performance issues. -* [Rule Monitoring tab](#rule-monitoring-tab) — The current state of all detection rules and their most recent executions. Go to the **Rule Monitoring** tab to get an overview of which rules are running, how long they’re taking, and if they’re having any trouble. -* [Execution results](#rule-execution-logs) — Historical data for a single detection rule’s executions over time. Consult the execution results to understand how a particular rule is running and whether it’s creating the alerts you expect. -* [Detection rule monitoring dashboard](../dashboards/detection-rule-monitoring-dashboard.md) — Visualizations to help you monitor the overall health and performance of {{elastic-sec}}'s detection rules. Consult this dashboard for a high-level view of whether your rules are running successfully and how long they’re taking to run, search data, and create alerts. +Start by checking the [rule execution status](#rule-status) in the Rules table to see if rules are succeeding, failing, or returning warnings. For a broader view across all your rules, use the [Rule Monitoring tab](#rule-monitoring-tab). To dig into a specific rule's history, open its [Execution results](#rule-execution-logs) tab. -Refer to the [Troubleshoot missing alerts](../../../troubleshoot/security/detection-rules.md#troubleshoot-signals) section for strategies on adjusting rules if they aren’t creating the expected alerts. +If you notice gaps where rules didn't run, refer to [Fill rule execution gaps](/solutions/security/detect-and-alert/fill-rule-gaps.md). To test rules or cover specific time ranges, you can [run rules manually](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules). For a high-level dashboard view, see the [Detection rule monitoring dashboard](../dashboards/detection-rule-monitoring-dashboard.md). +If rules aren't creating expected alerts, refer to [Troubleshoot missing alerts](../../../troubleshoot/security/detection-rules.md#troubleshoot-signals). -## Rule Monitoring tab [rule-monitoring-tab] -To view a summary of all rule executions (including the most recent failures, execution times, and gaps in rule executions), select the **Rule Monitoring** tab on the **Rules** page. To access the tab, find **Detection rules (SIEM)** in the navigation menu or look for “Detection rules (SIEM)” using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then go to the **Rule Monitoring** tab. +## Rule execution status [rule-status] -:::{image} /solutions/images/security-monitor-table.png -:alt: monitor table -:screenshot: -::: +The **Last response** column in the Rules table displays the current status of each rule, based on the most recent attempt to run: -On the **Rule Monitoring** tab, you can [sort and filter rules](../detect-and-alert/manage-detection-rules.md#sort-filter-rules) just like you can on the **Installed Rules** tab. +* **Succeeded**: The rule completed its defined search. This doesn't necessarily mean it generated an alert, just that it ran without error. +* **Failed**: The rule encountered an error that prevented it from running. For example, a {{ml}} rule whose corresponding {{ml}} job wasn't running. +* **Warning**: Nothing prevented the rule from running, but it might have returned unexpected results. For example, a custom query rule tried to search an index pattern that couldn't be found in {{es}}. -::::{tip} -To sort the rules list, click any column header. To sort in descending order, click the column header again. -:::: +For {{ml}} rules, an indicator icon {icon}`warning` also appears in this column if a required {{ml}} job isn't running. Select the icon to list the affected jobs, then select **Visit rule details page to investigate** to open the rule's details page, where you can start the {{ml}} job. -For detailed information on a rule, the alerts it generated, and associated errors, click on its name in the table. This also allows you to perform the same actions that are available on the [**Installed Rules** tab](manage-detection-rules.md), such as modifying or deleting rules, activating or deactivating rules, exporting or importing rules, and duplicating prebuilt rules. -### Find and resolve rule execution gaps [rule-monitoring-tab-gaps] - -The **Rule Monitoring** tab provides a starting point for understanding and remediating gaps in rule executions, which are periods of time where a rule didn’t run. Gaps can be caused by various disruptions, including system updates or simply turning off a rule. Addressing gaps is essential for maintaining consistent coverage and avoiding missed alerts. - -#### Find gaps [rule-monitoring-tab-find-gaps] - -From the **Rule Monitoring** tab, you can get an overview of existing gaps and their status. The total number of rules with gaps is tracked in the panel above the Rules table. The information and functionality in the panel depends on the version of {{elastic-sec}} that you're using. - -::::{applies-switch} +## Rule Monitoring tab [rule-monitoring-tab] -:::{applies-item} { "stack": "ga 9.3+"} -The panel has the following: -* **Rules with gaps:** Tells you the number of rules with gaps (left metric) and the number of rules with all gaps being filled (right metric). The metric shows data from the last 90 days. -::: +To view a summary of all rule executions (including the most recent failures, execution times, and gaps), select the **Rule Monitoring** tab on the {{rules-ui}} page. To access the tab, find **Detection rules (SIEM)** in the navigation menu or search for "Detection rules (SIEM)" using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md), then go to the **Rule Monitoring** tab. -:::{applies-item} { "stack": "ga 9.1-9.2" } -The panel has the following: -* **Time filter**: Allows you to select a time range for viewing gap data. -* **Total rules with gaps:** Tells you the number of rules with unfilled gaps (left metric) and the number of rules with gaps being filled (right metric) within the selected time range. -* **Only rules with unfilled gaps**: Filters the Rules table to only display rules with unfilled gaps. Note that the filter excludes rules with gaps that are being filled. +:::{image} /solutions/images/security-monitor-table.png +:alt: monitor table +:screenshot: ::: -:::{applies-item} { "stack": "ga =9.0" } -The panel has the following: -* **Time filter**: Allows you to select a time range for viewing gap data. -* **Total rules with gaps:** Tells you how many rules have unfilled or partially filled gaps within the selected time range. -* **Only rules with gaps**: Filters the Rules table to only display rules with unfilled or partially filled gaps. -::: +On the **Rule Monitoring** tab, you can [sort and filter rules](../detect-and-alert/manage-detection-rules.md#sort-filter-rules) just like you can on the **Installed Rules** tab. +::::{tip} +To sort the rules list, select any column header. To sort in descending order, select the column header again. :::: -Within the Rules table, several columns provide additional gap data: - -* **Last Gap (if any)**: Shows how long the most recent gap for a particular rule lasted. -* **Unfilled gaps duration**: Shows whether a rule still has gaps and provides a total sum of the remaining unfilled or partially filled gaps. The total sum can change based on the time range that you select during the past 90 days (data on gaps that occurred past the last 90 days is not retained). If a rule has no gaps, the columns display a dash (`––`). -* {applies_to}`stack: ga 9.3+` **Gap fill status**: Shows the status of the rule's gaps. If any gaps are unfilled, the gap status is `Unfilled`. If any gaps are being are being filled, the status is `In progress`. If all gaps are filled, the status is `Filled`. - - ::::{tip} - :applies_to:{stack: ga 9.3+} - Use the **Gap fill status** filter in the Rules table to find rules with the specified gap status. - :::: +For detailed information on a rule, the alerts it generated, and associated errors, select its name in the table. This also allows you to perform the same actions available on the [**Installed Rules** tab](manage-detection-rules.md), such as modifying or deleting rules, activating or deactivating rules, exporting or importing rules, and duplicating prebuilt rules. -#### Manually fill gaps [rule-monitoring-tab-manually-fill-gaps] +### Gap information -You can manually fill gaps in the following ways: +The **Rule Monitoring** tab also displays information about gaps in rule executions. Gaps are periods when a rule didn't run as scheduled. -- **For a specific rule**: Use the **Fill gaps** action in the rule's Gaps table, which lets you assess and respond to existing gaps. Refer to [](#gaps-table) to learn more. +Several columns provide gap data: -- **For multiple rules**: Use the **Fill gaps** bulk action in the Rules table. Refer to [](/solutions/security/detect-and-alert/manage-detection-rules.md#bulk-fill-gaps-multiple-rules) to learn more. +* **Last Gap (if any)**: How long the most recent gap lasted. +* **Unfilled gaps duration**: Total duration of remaining unfilled or partially filled gaps. If a rule has no gaps, the column displays a dash (`––`). +* {applies_to}`stack: ga 9.3+` **Gap fill status**: The status of the rule's gaps (`Unfilled`, `In progress`, or `Filled`). -#### Automatically fill gaps [rule-monitoring-tab-auto-fill-gaps] +To learn how to find and fill gaps, refer to [Fill rule execution gaps](/solutions/security/detect-and-alert/fill-rule-gaps.md). -```{applies_to} -stack: ga 9.3+ -``` - -::::{note} -You must have the appropriate subscription to use the automatic gap fill feature. Refer to the subscription page for [Elastic Cloud](https://www.elastic.co/subscriptions/cloud) and [Elastic Stack/self-managed](https://www.elastic.co/subscriptions) for the breakdown of available features and their associated subscription tiers. -:::: - -To automate gap remediation for your rules, enable the automatic gap fill feature. When turned on, a job runs every two minutes to check for new and existing gaps, then schedules tasks to fill them. - -You can find the setting that controls the automatic gap fill feature by clicking **Settings** (above the Rules table), then going to the **Auto gap fill settings** section. Use the toggle to turn the feature on and off. - -::::{tip} -The **Auto gap fill status:** field (located in the panel above the Rules table) shows whether the automatic gap fill feature is on or off. Click the field value to access the automatic gap fill settings. -:::: - -Details about the job and scheduled gap fill tasks are captured in the gap fill scheduler logs. You access the gap fill scheduler logs by clicking **Gap fill scheduler** in the **Auto gap fill settings** section description. - -In the scheduler logs table, expand the rows to learn more about gaps that were discovered and gap fill tasks that were scheduled each time the automatic gap fill job ran. You can also find key details such as: - -- When each job ran to check for gaps -- The number of gaps each job detected and the number of rules that had gaps -- The number of gap fill tasks each job scheduled -- The status of the scheduled gap fill tasks. They can be: - - **Success**: Confirms that gap fill tasks were successfully scheduled for rules assessed by the auto gap fill job. Refer to the log details to learn if any rules were not processed. - - **Error**: Indicates that some gap fill tasks were not scheduled or scheduled gap fill tasks failed to run. Refer to the log details for more information. - - **Task skipped**: Indicates that gap fill tasks were scheduled, but did not run. This can happen if rules with gaps were disabled or the maximum limit for scheduled gap fill tasks was reached. - - **No gaps**: Indicates that gaps weren't detected, and no gap fill tasks were scheduled. ## Execution results tab [rule-execution-logs] -From the **Execution results** tab, you can access the rule’s execution log, monitor and address gaps in a rule's execution schedule, and check manual runs for the rule. To find the tab, click the rule's name to open its details, then scroll down. +From the **Execution results** tab on a rule's details page, you can access the rule's execution log, monitor gaps, and check manual runs. To find this tab, select the rule's name to open its details, then scroll down. ### Execution log table [execution-log-table] -Each detection rule execution is logged, including the execution type, the execution’s success or failure, any warning or error messages, how long it took to search for data, create alerts, and complete. This can help you troubleshoot a particular rule if it isn’t behaving as expected (for example, if it isn’t creating alerts or takes a long time to run). +Each detection rule execution is logged, including the execution type, success or failure status, any warning or error messages, how long it took to search for data, create alerts, and complete. This can help you troubleshoot a rule if it isn't behaving as expected (for example, if it isn't creating alerts or takes a long time to run). :::{image} /solutions/images/security-rule-execution-logs.png :alt: Execution log table on the rule execution results tab :screenshot: ::: -You can hover over each column heading to display a tooltip about that column’s data. Click a column heading to sort the table by that column. Within the Execution log table, you can click the arrow at the end of a row to expand a long warning or error message. +You can hover over each column heading to display a tooltip about that column's data. Select a column heading to sort the table by that column. Within the Execution log table, you can select the arrow at the end of a row to expand a long warning or error message. -Use these controls to filter what’s included in the logs table: +Use these controls to filter what's included in the logs table: -* The **Run type** drop-down filters the table by rule execution type: +* The **Run type** drop-down filters by rule execution type: * **Scheduled**: Automatic, scheduled rule executions. - * **Manual**: Rule executions that were [started manually](manage-detection-rules.md#manually-run-rules). + * **Manual**: Rule executions that were [started manually](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules). -* The **Status** drop-down filters the table by rule execution status: +* The **Status** drop-down filters by rule execution status: - * **Succeeded**: The rule completed its defined search. This doesn’t necessarily mean it generated an alert, just that it ran without error. - * **Failed**: The rule encountered an error that prevented it from running. For example, a {{ml}} rule whose corresponding {{ml}} job wasn’t running. - * **Warning**: Nothing prevented the rule from running, but it might have returned unexpected results. For example, a custom query rule tried to search an index pattern that couldn’t be found in {{es}}. + * **Succeeded**: The rule completed its defined search. + * **Failed**: The rule encountered an error that prevented it from running. + * **Warning**: The rule ran but might have returned unexpected results. * The date and time picker sets the time range of rule executions included in the table. This is separate from the global date and time picker at the top of the rule details page. * The **Source event time range** button toggles the display of data pertaining to the time range of manual runs. * The **Show metrics columns** toggle includes more or less data in the table, pertaining to the timing of each rule execution. -* The **Actions** column allows you to show alerts generated from a given rule execution. Click the filter icon (![Filter icon](/solutions/images/security-filter-icon.png "title =20x20")) to create a global search filter based on the rule execution’s ID value. This replaces any previously applied filters, changes the global date and time range to 24 hours before and after the rule execution, and displays a confirmation notification. You can revert this action by clicking **Restore previous filters** in the notification. - - -### Gaps table [gaps-table] -```{applies_to} -stack: preview =9.0, ga 9.1+ -``` - -Use the information in the Gaps table to assess the scope and severity of rule execution gaps. To control what's shown in the table, you can filter the table by gap status, select a time range for viewing gap data, and sort multiple columns. - -{applies_to}`stack: ga 9.3+` Fill all gaps for the rule by clicking **Fill all gaps**. - -::::{tip} -Refer to the [Troubleshoot gaps](../../../troubleshoot/security/detection-rules.md#troubleshoot-gaps) section for strategies for avoiding gaps. -:::: - -:::{image} /solutions/images/security-gaps-table.png -:alt: Gaps table on the rule execution results tab -:screenshot: -::: - -The Gaps table has the following columns: +* The **Actions** column allows you to show alerts generated from a given rule execution. Select the filter icon (![Filter icon](/solutions/images/security-filter-icon.png "title =20x20")) to create a global search filter based on the rule execution's ID value. This replaces any previously applied filters, changes the global date and time range to 24 hours before and after the rule execution, and displays a confirmation notification. You can revert this action by selecting **Restore previous filters** in the notification. -* **Status**: The current state of the gap. It can be `Filled`, `Partially filled`, or `Unfilled`. -* **Detected at**: The date and time the gap was first discovered. -* **Manual fill tasks**: The status of the manual run that’s filling the gap. For more details about the manual run, refer to its entry in the [Manual runs table](/solutions/security/detect-and-alert/monitor-rule-executions.md#manual-runs-table). -* **Event time covered**: How much progress the manual run has made filling the gap. +### Gaps table - ::::{note} - If you stop a manual run that's hasn't finished filling a gap, the gap’s status will be set to `Partially filled`. To fill the remaining gap, you can select the **Fill remaining gap** action or [manually run](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules) the rule over the gap's time frame. - :::: - -* **Range**: When the gap started and ended. -* **Total gap duration**: How long the gap lasted. -* **Actions**: The actions that you can take for the gap. They can be **Fill gap** (starts a manual run to fill the gap) or **Fill remaining gap** (starts a manual run that fills the leftover portion of the gap). - - -### Manual runs table [manual-runs-table] - -You can [manually run](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules) enabled rules for a specified period of time to deliberately test them, provide additional rule coverage, or fill gaps in rule executions. Each manual run can produce multiple rule executions, depending on the time range of the run and the rule's execution schedule. - -::::{note} -Manual runs are executed with low priority and limited concurrency, meaning they might take longer to complete. This can be especially apparent for rules requiring multiple executions. -:::: - -The Manual runs table tracks manual rule executions and provides important details such as: - -* The total number of rule executions that the manual run will produce and how many are failing, pending, running, and completed. -* When the manual run started and the time range that it will cover. - - ::::{note} - To stop an active run, go to the appropriate row in the table and click **Stop run** in the **Actions** column. Completed rule executions for each manual run are logged in the Execution log table. - :::: - -* The status of each manual run: - - * `Pending`: The rule is not yet running. - * `Running`: The rule is executing during the time range you specified. Some rule types, such as indicator match rules, can take longer to run. - * `Error`: The rule's configuration is preventing it from running correctly. For example, the rule's conditions cannot be validated. - -:::{image} /solutions/images/security-manual-rule-run-table.png -:alt: Manual rule runs table on the rule execution results tab -:screenshot: -::: +The **Execution results** tab also includes a Gaps table that shows detailed gap information for the specific rule. To learn how to use the Gaps table to find and fill gaps, refer to [Fill rule execution gaps](/solutions/security/detect-and-alert/fill-rule-gaps.md#gaps-table). +### Manual runs table +The **Execution results** tab includes a Manual runs table that tracks manual rule executions. To learn more about running rules manually and monitoring manual runs, refer to [Run rules manually](/solutions/security/detect-and-alert/manage-detection-rules.md#manually-run-rules). diff --git a/solutions/security/detect-and-alert/new-terms.md b/solutions/security/detect-and-alert/new-terms.md index 574ab16fa7..3e506709b1 100644 --- a/solutions/security/detect-and-alert/new-terms.md +++ b/solutions/security/detect-and-alert/new-terms.md @@ -99,4 +99,4 @@ The following settings are specific to new terms rules. For settings shared acro : An informational list of fields the rule needs to function. This does not affect rule execution. **Related integrations** (optional) -: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/prebuilt-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/prebuilt-rules.md b/solutions/security/detect-and-alert/prebuilt-rules.md index 25d40da887..122198afb4 100644 --- a/solutions/security/detect-and-alert/prebuilt-rules.md +++ b/solutions/security/detect-and-alert/prebuilt-rules.md @@ -1,8 +1,8 @@ --- applies_to: - stack: all + stack: ga all serverless: - security: all + security: ga all products: - id: security - id: cloud-serverless @@ -13,11 +13,66 @@ description: Overview of Elastic's prebuilt detection rules library mapped to MI Elastic maintains a library of prebuilt detection rules mapped to the MITRE ATT&CK framework. Enabling prebuilt rules is the fastest path to detection coverage and the recommended starting point before building custom rules. You can browse the full [prebuilt rule catalog](detection-rules://index.md) to see what's available. -**[Install and manage prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md)** -: Start here if you're setting up prebuilt rules for the first time, or if you need to install new rules, enable or turn off existing ones, or review what data sources each rule requires. Also covers rule tags, customization options, and how to export or duplicate prebuilt rules. +**[Install prebuilt rules](/solutions/security/detect-and-alert/install-prebuilt-rules.md)** +: Start here to install and enable prebuilt rules. Includes a subscription capability matrix showing which features are available at each tier. **[Update prebuilt rules](/solutions/security/detect-and-alert/update-prebuilt-rules.md)** -: Relevant when Elastic releases rule updates and you need to decide how to apply them. Explains how the update process works for rules you haven't modified versus rules you've customized, and how to resolve conflicts between your changes and Elastic's updates. Requires an Enterprise subscription on {{stack}} or a Security Analytics Complete project on {{serverless-short}}. +: Apply Elastic's rule updates to keep your detection coverage current. Explains how to review updates, handle modified rules, and resolve conflicts (Enterprise only). + +**[Customize prebuilt rules](/solutions/security/detect-and-alert/customize-prebuilt-rules.md)** +: Adapt prebuilt rules to your environment. Edit rules directly (Enterprise), duplicate and modify copies (all subscriptions), add exceptions, configure actions, or revert to the original Elastic version. **[MITRE ATT&CK coverage](/solutions/security/detect-and-alert/mitre-attack-coverage.md)** -: Use this page to visualize which MITRE ATT&CK tactics and techniques your installed rules cover. Helpful for identifying gaps in your detection posture and deciding which additional prebuilt rules to enable or where to build custom rules. +: Visualize which MITRE ATT&CK tactics and techniques your installed rules cover. Identify gaps in your detection posture and prioritize which rules to enable or where to build custom rules. + + +## Prebuilt rule tags [prebuilt-rule-tags] + +Each prebuilt rule includes several tags identifying the rule's purpose, detection method, associated resources, and other information to help categorize your rules. These tags are category-value pairs; for example, `OS: Windows` indicates rules designed for Windows endpoints. Categories include: + +* `Data Source`: The application, cloud provider, data shipper, or Elastic integration providing data for the rule. +* `Domain`: A general category of data source types (such as cloud, endpoint, or network). +* `OS`: The host operating system, which could be considered another data source type. +* `Resources`: Additional rule resources such as investigation guides. +* `Rule Type`: Identifies if the rule depends on specialized resources (such as {{ml}} jobs or threat intelligence indicators), or if it's a higher-order rule built from other rules' alerts. +* `Tactic`: MITRE ATT&CK tactics that the rule addresses. +* `Threat`: Specific threats the rule detects (such as Cobalt Strike or BPFDoor). +* `Use Case`: The type of activity the rule detects and its purpose. Use cases include: + + * `Active Directory Monitoring`: Detects changes related to Active Directory. + * `Asset Visibility`: Detects changes to specified asset types. + * `Configuration Audit`: Detects undesirable configuration changes. + * `Guided Onboarding`: Example rule, used for {{elastic-sec}}'s guided onboarding tour. + * `Identity and Access Audit`: Detects activity related to identity and access management (IAM). + * `Log Auditing`: Detects activity on log configurations or storage. + * `Network Security Monitoring`: Detects network security configuration activity. + * `Threat Detection`: Detects threats. + * `Vulnerability`: Detects exploitation of specific vulnerabilities. + + +## Rule prerequisites [rule-prerequisites] + +Many prebuilt rules are designed to work with specific [{{product.integrations}}](https://docs.elastic.co/en/integrations) and data fields. These prerequisites are identified in **Related integrations** and **Required fields** on a rule's details page. **Related integrations** also displays each integration's installation status and includes links for installing and configuring the listed integrations. + +Additionally, the **Setup guide** section provides guidance on setting up the rule's requirements. + +:::{image} /solutions/images/security-rule-details-prerequisites.png +:alt: Rule details page with Related integrations +:screenshot: +::: + +You can also check rules' related integrations in the **Installed Rules** and **Rule Monitoring** tables. Select the **integrations** badge to display the related integrations in a popup. + +:::{image} /solutions/images/security-rules-table-related-integrations.png +:alt: Rules table with related integrations popup +:screenshot: +::: + +::::{tip} +You can hide the **integrations** badge in the {{rules-ui}} tables by turning off the `securitySolution:showRelatedIntegrations` [advanced setting](/solutions/security/get-started/configure-advanced-settings.md#show-related-integrations). +:::: + + +## Build custom rules + +Prebuilt rules cover common threats, but your environment has unique risks. When you need detection logic tailored to your infrastructure, applications, or threat model, refer to [Author rules](/solutions/security/detect-and-alert/author-rules.md) for guidance on selecting a rule type, writing queries, and configuring rule settings. \ No newline at end of file diff --git a/solutions/security/detect-and-alert/requirements-privileges.md b/solutions/security/detect-and-alert/requirements-privileges.md index f0a306affa..63b2db83f7 100644 --- a/solutions/security/detect-and-alert/requirements-privileges.md +++ b/solutions/security/detect-and-alert/requirements-privileges.md @@ -90,7 +90,7 @@ To enable detections in multiple spaces, visit the **Rules** page in each space. With detections enabled, you're ready to create rules and start responding to threats. Do the following: - **Add detection rules** - - [Install Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md): Get started quickly with hundreds of rules that detect common threats + - [Install Elastic prebuilt rules](/solutions/security/detect-and-alert/install-prebuilt-rules.md): Get started quickly with hundreds of rules that detect common threats - [Create custom rules](/solutions/security/detect-and-alert/using-the-rule-builder.md): Write rules tailored to your environment - [Configure {{anomaly-detect}} rules](/solutions/security/advanced-entity-analytics/machine-learning-job-rule-requirements.md): Use {{ml}} to detect unusual behavior diff --git a/solutions/security/detect-and-alert/threshold.md b/solutions/security/detect-and-alert/threshold.md index 188b33aa95..5e614284d8 100644 --- a/solutions/security/detect-and-alert/threshold.md +++ b/solutions/security/detect-and-alert/threshold.md @@ -102,4 +102,4 @@ The following settings are specific to threshold rules. For settings shared acro : An informational list of fields the rule needs to function. This does not affect rule execution. **Related integrations** (optional) -: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/manage-detection-rules.md#rule-prerequisites). +: Associate the rule with one or more [{{product.integrations}}](https://docs.elastic.co/en/integrations) to indicate data dependencies and allow users to verify each integration's [installation status](/solutions/security/detect-and-alert/prebuilt-rules.md#rule-prerequisites). diff --git a/solutions/security/detect-and-alert/update-prebuilt-rules.md b/solutions/security/detect-and-alert/update-prebuilt-rules.md index 106cc05da5..7c21a70c6c 100644 --- a/solutions/security/detect-and-alert/update-prebuilt-rules.md +++ b/solutions/security/detect-and-alert/update-prebuilt-rules.md @@ -1,31 +1,30 @@ --- +navigation_title: Update prebuilt rules applies_to: - stack: all + stack: ga all serverless: - security: all + security: ga all products: - id: security - id: cloud-serverless - - id: cloud-hosted - - id: cloud-enterprise - - id: cloud-kubernetes - - id: elastic-stack -description: Update modified and unmodified Elastic prebuilt rules, resolve conflicts, and understand rule field update statuses. +description: Update Elastic Security prebuilt detection rules to stay current with the latest threat intelligence, resolve conflicts, and preserve your customizations. --- -# Update modified and unmodified Elastic prebuilt rules [prebuilt-rules-update-modified-unmodified] +# Update Elastic prebuilt rules [update-prebuilt-rules] -::::{admonition} Requirements +Elastic regularly updates prebuilt rules to optimize their performance and ensure they detect the latest threats and techniques. This page explains how to review and apply updates to your installed prebuilt rules. -You must have an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}} to access this feature. +## Update availability [update-availability] -If you have a Platinum subscription or lower on {{stack}} or a Security Analytics Essentials project on {{serverless-short}}, follow the guidelines in [Update Elastic prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#update-prebuilt-rules) instead. +When updated versions are available for your installed prebuilt rules, the **Rule Updates** tab appears on the **Rules** page. +::::{note} +On {{stack}}, automatic updates are supported for the current {{elastic-sec}} version and the latest three previous minor releases. For example, if you're on version 9.0, you can use the Rules UI to update prebuilt rules until version 9.4 is released. After that, you can still manually download and install updates, but must upgrade {{elastic-sec}} to receive automatic updates again. :::: -This page provides instructions for updating modified and unmodified prebuilt rules. You can also find information about [statuses](/solutions/security/detect-and-alert/update-prebuilt-rules.md#rule-field-update-statuses) or [conflicts](/solutions/security/detect-and-alert/update-prebuilt-rules.md#resolve-reduce-rule-conflicts) that you might encounter when updating rules. +## Review updates [review-updates] -To update rules: +Before applying updates, you can examine what's changing in each rule. 1. Find **Detection rules (SIEM)** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). 2. In the **Rules** table, select the **Rule Updates** tab. @@ -34,74 +33,117 @@ To update rules: The **Rule Updates** tab doesn't appear if all your installed prebuilt rules are up to date. :::: - :::{image} /solutions/images/security-prebuilt-rules-update-advanced.png - :alt: The Rule Updates tab on the Rules page - :screenshot: - ::: +3. Select a rule name to open the rule update flyout and review the changes. -3. (Optional) To examine the details of a rule's latest version before you update it, select the rule name. This opens the rule update flyout, where you can: +All subscriptions can preview incoming updates by selecting the **Elastic update overview** tab (field-by-field view) or the **JSON view** tab (full rule comparison). Both tabs display side-by-side comparisons of the **Current rule** (what you have installed) and the **Elastic update** version (what you can install). Deleted characters are highlighted in red; added characters are highlighted in green. - * Preview incoming updates: Select the **Elastic update overview** tab to view rule changes field by field, or the **JSON view** tab to view changes for the entire rule in JSON format. +:::{image} /solutions/images/security-prebuilt-rules-update-diff-basic.png +:alt: Prebuilt rule comparison +:screenshot: +::: - * Compare different versions of a rule field: Use the **Diff view** drop-down menu to compare different versions of a rule field. For example, compare the changes that you made to the current version of the field with changes that will be applied from the incoming Elastic update. +::::{dropdown} Additional options with Enterprise subscription +:name: enterprise-review-options + +With an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you also have access to: + +* **Compare different versions**: Use the **Diff view** drop-down menu to compare different versions of a rule field. For example, compare changes you made to the current version with changes from the incoming Elastic update. ::::{note} - If you haven't updated the rule in a while, its original version might be unavailable for comparison. Instead, you will only have access to the rule's current version and the incoming Elastic update. You can avoid this by updating prebuilt rules more often. + If you haven't updated the rule in a while, its original version might be unavailable for comparison. You can avoid this by updating prebuilt rules regularly. :::: - * Check the update status: View the status of the entire rule update and for [each field that's being changed](/solutions/security/detect-and-alert/update-prebuilt-rules.md#rule-field-update-statuses). +* **Check update status**: View the status of the entire rule update and for each field being changed. Refer to [Field update statuses](#rule-field-update-statuses) for status definitions. - * Address update conflicts: Find and address conflicts that [need additional attention](/solutions/security/detect-and-alert/update-prebuilt-rules.md#resolve-reduce-rule-conflicts). - - * Edit the final update: Change the update that will be applied to the field when you update the rule. To change the update, go to the **Final update** section, make your changes, and then save them. +* **Edit the final update**: Change the update that will be applied to any field. Go to the **Final update** section, make your changes, and save them. ::::{important} - Elastic updates containing a rule type change cannot be edited. Before updating the rule, duplicate it if you need to record changes that you made to the rule fields. + Elastic updates containing a rule type change cannot be edited. Duplicate the rule before updating if you need to preserve your modifications. :::: - :::{image} /solutions/images/security-prebuilt-rules-update-diff-advanced.png - :alt: Prebuilt rule comparison - :screenshot: - ::: +:::{image} /solutions/images/security-prebuilt-rules-update-diff-advanced.png +:alt: Prebuilt rule comparison with advanced options +:screenshot: +::: +:::: -4. From the **Rule Updates** tab, do one of the following to update prebuilt rules: - * Update all available rules: Click **Update all**. If any rules have conflicts, you will be prompted to take [additional action](/solutions/security/detect-and-alert/update-prebuilt-rules.md#resolve-reduce-rule-conflicts). - * Update a single rule without conflicts: Click **Update rule** for that rule. - * Update multiple rules: Select the rules and click **Update _x_ selected rule(s)**. If any rules have conflicts, you will be prompted to take additional action. +## Preserve customizations during updates [preserve-customizations] +If you've customized prebuilt rules and want to preserve your changes when applying updates, review the guidance for your subscription level below. The update process differs based on your subscription. - ::::{tip} +::::{dropdown} Enterprise / Security Analytics Complete +:name: enterprise-modified-rules - To find specific rules to update: +With an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, {{elastic-sec}} attempts to merge your changes with the Elastic update. If conflicts arise: - * Use the **Modified/Unmodified** drop-down menu to only display modified or unmodified prebuilt rules. - * Use the search bar and **Tags** filter to find the rules you want to update. For example, filter by `OS: Windows` if your environment only includes Windows endpoints. For more on tag categories, refer to [Prebuilt rule tags](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#prebuilt-rule-tags). +* **Auto-resolved conflicts**: {{elastic-sec}} suggests a resolution for your review. The field displays a `Review required` status. +* **Unresolved conflicts**: You must manually select how to resolve the conflict. The field displays an `Action required` status. - :::: +Refer to [Resolve update conflicts](#resolve-reduce-rule-conflicts) for guidance on handling conflicts. + +::::{tip} +Use the **Modified/Unmodified** drop-down menu in the **Rule Updates** tab to filter for modified rules that may need attention. +:::: + +:::: + +::::{dropdown} Basic–Platinum / Security Analytics Essentials +:name: basic-modified-rules + +With a [Basic–Platinum subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Essentials project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, updates overwrite your modifications with the Elastic version. To preserve your changes: + +1. [Duplicate the rule](/solutions/security/detect-and-alert/customize-prebuilt-rules.md#duplicate-prebuilt-rules) before updating. +2. Apply the update to the original prebuilt rule. +3. Continue using your duplicated rule with your customizations. + +:::: + + +## Apply updates [apply-updates] + +From the **Rule Updates** tab, do one of the following: + +* **Update all available rules**: Select **Update all**. If any rules have conflicts (Enterprise only), you are prompted to resolve them first. +* **Update a single rule**: Select **Update rule** for that rule. +* **Update multiple rules**: Select the rules and select **Update *x* selected rule(s)**. + +::::{tip} +Use the search bar and **Tags** filter to find specific rules. For example, filter by `OS: Windows` if your environment only includes Windows endpoints. For more on tag categories, refer to [Prebuilt rule tags](/solutions/security/detect-and-alert/prebuilt-rules.md#prebuilt-rule-tags). +:::: + +:::{image} /solutions/images/security-prebuilt-rules-update.png +:alt: The Rule Updates tab on the Rules page +:screenshot: +::: + + +## Field update statuses [rule-field-update-statuses] + +With an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}, you can [edit prebuilt rules directly](/solutions/security/detect-and-alert/customize-prebuilt-rules.md#edit-prebuilt-rules). When you update a rule you've customized, each field displays a status indicating whether conflicts exist between your changes and the incoming Elastic update: -## Understand rule field update statuses [rule-field-update-statuses] +| Status | Description | Action required | +|---|---|---| +| **Ready for update** | No conflicts. The field can be updated. | None | +| **No update** | The field isn't being updated by Elastic, but your current value differs from the original. | None (you can still edit the final value if needed) | +| **Review required** | Elastic auto-resolved a conflict between your changes and the Elastic update. | Review the suggested resolution and accept or edit it. | +| **Action required** | Elastic couldn't auto-resolve the conflict. | Manually set the field's final value. | -This table describes statuses that might appear for rule fields being updated. -| Status | Description | -| --- | --- | -| **Ready for update** | Displays when there are no conflicts to resolve.

Further action is not required for the field. It is ready to be updated.
| -| **No update** | Displays when the field is not being updated by Elastic, but the current field value differs from the original one. This typically happens when the field's value was changed after the prebuilt rule was initially installed.

Further action is not required for the field. It is ready to be updated.

**Tip**: You can still change the final field update, if needed. To do so, make your changes in the **Final update** section and save them.

| -| **Review required** | Displays when Elastic auto-resolves a conflict between the current field value and the value from the incoming Elastic update.

You must accept or edit the field's final update and save the changes. Refer to [Resolve and reduce update conflicts](/solutions/security/detect-and-alert/update-prebuilt-rules.md#resolve-reduce-rule-conflicts) to learn more about auto-resolved conflicts and how to reduce future conflicts.
| -| **Action required** | Displays when Elastic could not auto-resolve the conflict between the current field value and the value from the incoming Elastic update.

You must manually set and save the field's final update. Refer to [Resolve and reduce update conflicts](/solutions/security/detect-and-alert/update-prebuilt-rules.md#resolve-reduce-rule-conflicts) to learn more about conflicts that need manual fixes and how to reduce future conflicts.
| +## Resolve update conflicts [resolve-reduce-rule-conflicts] +When you [edit prebuilt rules directly](/solutions/security/detect-and-alert/customize-prebuilt-rules.md#edit-prebuilt-rules) (available with an [Enterprise subscription](https://www.elastic.co/pricing) on {{stack}} or a [Security Analytics Complete project](../../../deploy-manage/deploy/elastic-cloud/project-settings.md) on {{serverless-short}}), conflicts can arise if Elastic updates the same fields you modified. Keeping prebuilt rules up to date helps minimize the frequency and complexity of these conflicts. -## Resolve and reduce update conflicts [resolve-reduce-rule-conflicts] +### Auto-resolved conflicts -Keeping prebuilt rules up to date might help you minimize the frequency and complexity of conflicts that occur during rule updates. +When {{elastic-sec}} can suggest a resolution, the field displays `Review required`. You can still update rules with auto-resolved conflicts, but review each rule individually rather than bulk-updating to avoid unintended changes. -When a conflict does happen, Elastic attempts to resolve it and will suggest a fix for your review. This is called an _auto-resolved conflict_. You can still update rules with auto-resolved conflicts, but we advise against bulk-updating multiple rules as it's risky and can sometimes lead to lost rule modifications and other issues. Instead, we recommend carefully reviewing each rule with auto-resolved conflicts from the rule update flyout. +### Unresolved conflicts -If Elastic can't resolve a conflict, you must manually fix it before updating the rule. This is called an _unresolved conflict_. To fix unresolved conflicts in a rule, do the following: +When {{elastic-sec}} can't resolve a conflict, the field displays `Action required`. To fix unresolved conflicts: -1. From the **Rule update** tab, click on the rule name or click **Review**. This opens the rule update flyout, where you can find rule fields with unresolved conflicts. +1. From the **Rule Updates** tab, select the rule name or select **Review** to open the rule update flyout. ::::{tip} Fields with unresolved conflicts have the `Action required` status. @@ -109,10 +151,10 @@ If Elastic can't resolve a conflict, you must manually fix it before updating th 2. Go to the **Final update** section and do any of the following: - * Keep the current value instead of accepting the Elastic update. - * Accept the Elastic update and overwrite the current value. - * Edit the final field value by combining the current value with the Elastic update or making the appropriate changes. + * Keep your current value instead of accepting the Elastic update. + * Accept the Elastic update and overwrite your current value. + * Combine your changes with the Elastic update. -3. Click **Save and accept** to apply your changes. The field's status changes to `Ready for update`. +3. Select **Save and accept** to apply your changes. The field's status changes to `Ready for update`. -After you've resolved the remaining conflicts, click **Update rule** to accept the changes and install the updated version. +After resolving all conflicts, select **Update rule** to apply the update. diff --git a/solutions/security/get-started/get-started-detect-with-siem.md b/solutions/security/get-started/get-started-detect-with-siem.md index bb333315f1..c954026ebd 100644 --- a/solutions/security/get-started/get-started-detect-with-siem.md +++ b/solutions/security/get-started/get-started-detect-with-siem.md @@ -110,7 +110,7 @@ Detection rules allow you to monitor your environment by searching for source ev ::: ::::{tip} -{{elastic-sec}} regularly updates prebuilt rules to ensure they detect the latest threats. However, you must manually update these rules to the latest version. To learn how to do this, refer to [Update prebuilt rules](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#update-prebuilt-rules). To learn how to view and manage all detection rules, refer to [Manage detection rules](/solutions/security/detect-and-alert/manage-detection-rules.md). +{{elastic-sec}} regularly updates prebuilt rules to ensure they detect the latest threats. However, you must manually update these rules to the latest version. To learn how to do this, refer to [Update prebuilt rules](/solutions/security/detect-and-alert/update-prebuilt-rules.md). To learn how to view and manage all detection rules, refer to [Manage detection rules](/solutions/security/detect-and-alert/manage-detection-rules.md). :::: ::: diff --git a/solutions/security/get-started/get-started-endpoint-security.md b/solutions/security/get-started/get-started-endpoint-security.md index 064d7f1ca7..12d31054f6 100644 --- a/solutions/security/get-started/get-started-endpoint-security.md +++ b/solutions/security/get-started/get-started-endpoint-security.md @@ -120,5 +120,5 @@ You can apply trusted applications, blocklist entries, and host isolation except After your hosts are secure and your environment has all the appropriate security settings configured, we recommend taking these next steps: * Check out the [Hosts page](/solutions/security/explore/hosts-page.md) for a comprehensive overview of all hosts and host-related security events. This page is also useful to identify uncommon processes and anomalies discovered by {{ml}} jobs. -* Install and turn on prebuilt detection rules. You're already set to receive endpoint threat alerts from {{elastic-defend}}, but did you know {{elastic-sec}} ships with several out-of-the-box rules you can turn on? Check out our [SIEM quick start guide](/solutions/security/get-started/get-started-detect-with-siem.md#add-elastic-prebuilt-detection-rules) or our [documentation](/solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#load-prebuilt-rules). +* Install and turn on prebuilt detection rules. You're already set to receive endpoint threat alerts from {{elastic-defend}}, but did you know {{elastic-sec}} ships with several out-of-the-box rules you can turn on? Check out our [SIEM quick start guide](/solutions/security/get-started/get-started-detect-with-siem.md#add-elastic-prebuilt-detection-rules) or our [documentation](/solutions/security/detect-and-alert/install-prebuilt-rules.md#load-prebuilt-rules). * Discover all the other tools available to [manage {{elastic-defend}}](/solutions/security/manage-elastic-defend.md). diff --git a/solutions/security/manage-elastic-defend/endpoint-protection-rules.md b/solutions/security/manage-elastic-defend/endpoint-protection-rules.md index 6e48f530dc..6408ff4a26 100644 --- a/solutions/security/manage-elastic-defend/endpoint-protection-rules.md +++ b/solutions/security/manage-elastic-defend/endpoint-protection-rules.md @@ -13,7 +13,7 @@ products: # Endpoint protection rules [endpoint-protection-rules] -Endpoint protection rules are [prebuilt rules](../detect-and-alert/install-manage-prebuilt-rules.md) designed to help you manage and respond to alerts generated by {{elastic-endpoint}}, the installed component that performs {{elastic-defend}}'s threat monitoring and prevention. These rules include the Endpoint Security ({{elastic-defend}}) rule as well as additional detection and prevention rules for different {{elastic-defend}} protection features. +Endpoint protection rules are [prebuilt rules](../detect-and-alert/install-prebuilt-rules.md) designed to help you manage and respond to alerts generated by {{elastic-endpoint}}, the installed component that performs {{elastic-defend}}'s threat monitoring and prevention. These rules include the Endpoint Security ({{elastic-defend}}) rule as well as additional detection and prevention rules for different {{elastic-defend}} protection features. ::::{important} To receive {{elastic-endpoint}} alerts, you must install {{agent}} and the {{elastic-defend}} integration on your hosts (refer to [Install {{elastic-defend}}](../configure-elastic-defend/install-elastic-defend.md)). @@ -54,7 +54,7 @@ If you choose to use the feature-specific protection rules, we recommend that yo :::: -To use these rules, you need to manually enable them from the **Rules** page in the {{security-app}}. Follow the instructions for [installing and enabling Elastic prebuilt rules](../detect-and-alert/install-manage-prebuilt-rules.md#load-prebuilt-rules). +To use these rules, you need to manually enable them from the **Rules** page in the {{security-app}}. Follow the instructions for [installing and enabling Elastic prebuilt rules](../detect-and-alert/install-prebuilt-rules.md#load-prebuilt-rules). ## Endpoint security exception handling [_endpoint_security_exception_handling] diff --git a/solutions/toc.yml b/solutions/toc.yml index 77a9948aa0..66243b3cbc 100644 --- a/solutions/toc.yml +++ b/solutions/toc.yml @@ -576,14 +576,16 @@ toc: children: - file: security/detect-and-alert/requirements-privileges.md - file: security/detect-and-alert/detections-privileges.md + - file: security/detect-and-alert/detection-rule-concepts.md - file: security/detect-and-alert/advanced-data-source-configuration.md children: - file: security/detect-and-alert/cross-cluster-search-detection-rules.md - file: security/detect-and-alert/using-logsdb-index-mode-with-elastic-security.md - file: security/detect-and-alert/prebuilt-rules.md children: - - file: security/detect-and-alert/install-manage-prebuilt-rules.md + - file: security/detect-and-alert/install-prebuilt-rules.md - file: security/detect-and-alert/update-prebuilt-rules.md + - file: security/detect-and-alert/customize-prebuilt-rules.md - file: security/detect-and-alert/mitre-attack-coverage.md - file: security/detect-and-alert/author-rules.md children: @@ -607,6 +609,7 @@ toc: - file: security/detect-and-alert/validate-and-test-rules.md - file: security/detect-and-alert/manage-detection-rules.md - file: security/detect-and-alert/monitor-rule-executions.md + - file: security/detect-and-alert/fill-rule-gaps.md - file: security/detect-and-alert/reduce-noise-and-false-positives.md children: - file: security/detect-and-alert/tune-detection-rules.md From 07d8c82916437a6864f96bddfa09c8aa51740181 Mon Sep 17 00:00:00 2001 From: Nastasha Solomon Date: Wed, 25 Feb 2026 19:27:57 -0500 Subject: [PATCH 13/24] Fixed errors in docs build --- redirects.yml | 8 +++++--- solutions/security/detect-and-alert/custom-query.md | 2 +- .../security/detect-and-alert/install-prebuilt-rules.md | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/redirects.yml b/redirects.yml index 6adefbcdb3..5f8a5b4fd8 100644 --- a/redirects.yml +++ b/redirects.yml @@ -731,7 +731,9 @@ redirects: to: 'solutions/security/detect-and-alert/install-prebuilt-rules.md' anchors: 'load-prebuilt-rules': 'load-prebuilt-rules' - 'prebuilt-rule-tags': 'solutions/security/detect-and-alert/prebuilt-rules.md#prebuilt-rule-tags' - 'rule-prerequisites': 'solutions/security/detect-and-alert/prebuilt-rules.md#rule-prerequisites' - 'select-all-prebuilt-rules': 'solutions/security/detect-and-alert/customize-prebuilt-rules.md#duplicate-prebuilt-rules' + + # Anchors moved to different files (separate redirects needed) + 'solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#prebuilt-rule-tags': 'solutions/security/detect-and-alert/prebuilt-rules.md#prebuilt-rule-tags' + 'solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#rule-prerequisites': 'solutions/security/detect-and-alert/prebuilt-rules.md#rule-prerequisites' + 'solutions/security/detect-and-alert/install-manage-prebuilt-rules.md#select-all-prebuilt-rules': 'solutions/security/detect-and-alert/customize-prebuilt-rules.md#duplicate-prebuilt-rules' diff --git a/solutions/security/detect-and-alert/custom-query.md b/solutions/security/detect-and-alert/custom-query.md index 99fef8744d..130cc8b547 100644 --- a/solutions/security/detect-and-alert/custom-query.md +++ b/solutions/security/detect-and-alert/custom-query.md @@ -32,7 +32,7 @@ Custom query rules are **not** the best fit when you need to: ### Data requirements -Custom query rules require at least one {{es}} index pattern or [data view](/solutions/security/get-started/data-views-elastic-security.md) that contains the events you want to match. The indices must be accessible to the user who creates or last edits the rule, because the rule executes with that user's [API key privileges](/solutions/security/detect-and-alert/choose-the-right-rule-type.md#alerting-authorization-model). +Custom query rules require at least one {{es}} index pattern or [data view](/solutions/security/get-started/data-views-elastic-security.md) that contains the events you want to match. The indices must be accessible to the user who creates or last edits the rule, because the rule executes with that user's [API key privileges](/solutions/security/detect-and-alert/detection-rule-concepts.md#rule-authorization-concept).