[Creator] [design-spec] elasticsearch-generic-log-query

codebundles/elasticsearch-generic-log-query/.runwhen/generation-rules/elasticsearch-generic-log-query.yaml

@@ -0,0 +1,24 @@
+apiVersion: runwhen.com/v1
+kind: GenerationRules
+spec:
+  platform: elasticsearch
+  generationRules:
+    - resourceTypes:
+        - elasticsearch_cluster
+      matchRules:
+        - type: pattern
+          pattern: ".+"
+          properties: ["name"]
+          mode: substring
+      slxs:
+        - baseName: elasticsearch-generic-log-query
+          qualifiers:
+            - ELASTICSEARCH_BASE_URL
+            - ELASTICSEARCH_INDEX_PATTERN
+          baseTemplateName: elasticsearch-generic-log-query
+          levelOfDetail: basic
+          outputItems:
+            - type: slx
+            - type: sli
+            - type: runbook
+              templateName: elasticsearch-generic-log-query-taskset.yaml

codebundles/elasticsearch-generic-log-query/.runwhen/templates/elasticsearch-generic-log-query-sli.yaml

@@ -0,0 +1,44 @@
+apiVersion: runwhen.com/v1
+kind: ServiceLevelIndicator
+metadata:
+  name: {{slx_name}}
+  labels:
+    {% include "common-labels.yaml" %}
+  annotations:
+    {% include "common-annotations.yaml" %}
+spec:
+  displayUnitsLong: Health Score
+  displayUnitsShort: score
+  locations:
+    - {{default_location}}
+  description: Probes Elasticsearch _cluster/health for reachability and maps a 2xx response to a 0-1 score.
+  codeBundle:
+    {% if repo_url %}
+    repoUrl: {{repo_url}}
+    {% else %}
+    repoUrl: https://github.com/runwhen-contrib/rw-cli-codecollection.git
+    {% endif %}
+    {% if ref %}
+    ref: {{ref}}
+    {% else %}
+    ref: main
+    {% endif %}
+    pathToRobot: codebundles/elasticsearch-generic-log-query/sli.robot
+  intervalStrategy: intermezzo
+  intervalSeconds: 300
+  configProvided:
+    - name: ELASTICSEARCH_BASE_URL
+      value: "{{ custom.elasticsearch_base_url | default(match_resource.elasticsearch_base_url | default('')) }}"
+    - name: REQUEST_TIMEOUT_SECONDS
+      value: "{{ custom.sli_request_timeout_seconds | default('15') }}"
+  secretsProvided:
+    {% if wb_version %}
+    {% include "elasticsearch-auth.yaml" ignore missing %}
+    {% else %}
+    - name: elasticsearch_credentials
+      workspaceKey: AUTH DETAILS NOT FOUND
+    {% endif %}
+  alertConfig:
+    tasks:
+      persona: eager-edgar
+      sessionTTL: 10m

codebundles/elasticsearch-generic-log-query/.runwhen/templates/elasticsearch-generic-log-query-slx.yaml

@@ -0,0 +1,25 @@
+apiVersion: runwhen.com/v1
+kind: ServiceLevelX
+metadata:
+  name: {{ slx_name }}
+  labels:
+    {% include "common-labels.yaml" %}
+  annotations:
+    {% include "common-annotations.yaml" %}
+spec:
+  imageURL: https://www.vectorlogo.zone/logos/elastic_elasticsearch/elastic_elasticsearch-icon.svg
+  alias: Elasticsearch Generic Log Search
+  asMeasuredBy: Successful HTTP access to the Elasticsearch API and search results within optional hit thresholds.
+  configProvided:
+    - name: SLX_PLACEHOLDER
+      value: SLX_PLACEHOLDER
+  owners:
+    - {{ workspace.owner_email }}
+  statement: Elasticsearch should accept searches against the configured index pattern with the supplied query body.
+  additionalContext:
+    qualified_name: "{{ match_resource.qualified_name | default('elasticsearch') }}"
+  tags:
+    - name: access
+      value: read-only
+    - name: service
+      value: elasticsearch

codebundles/elasticsearch-generic-log-query/.runwhen/templates/elasticsearch-generic-log-query-taskset.yaml

@@ -0,0 +1,43 @@
+apiVersion: runwhen.com/v1
+kind: Runbook
+metadata:
+  name: {{slx_name}}
+  labels:
+    {% include "common-labels.yaml" %}
+  annotations:
+    {% include "common-annotations.yaml" %}
+spec:
+  location: {{default_location}}
+  description: Runs configurable Elasticsearch _search requests with the base URL separated from the query JSON body.
+  codeBundle:
+    {% if repo_url %}
+    repoUrl: {{repo_url}}
+    {% else %}
+    repoUrl: https://github.com/runwhen-contrib/rw-cli-codecollection.git
+    {% endif %}
+    {% if ref %}
+    ref: {{ref}}
+    {% else %}
+    ref: main
+    {% endif %}
+    pathToRobot: codebundles/elasticsearch-generic-log-query/runbook.robot
+  configProvided:
+    - name: ELASTICSEARCH_BASE_URL
+      value: "{{ custom.elasticsearch_base_url | default(match_resource.elasticsearch_base_url | default('')) }}"
+    - name: ELASTICSEARCH_INDEX_PATTERN
+      value: "{{ custom.elasticsearch_index_pattern | default(match_resource.elasticsearch_index_pattern | default('logs-*')) }}"
+    - name: ELASTICSEARCH_QUERY_BODY
+      value: "{{ custom.elasticsearch_query_body | default('{\"size\": 0, \"query\": {\"match_all\": {}}}') }}"
+    - name: SEARCH_THRESHOLD_MAX_HITS
+      value: "{{ custom.search_threshold_max_hits | default('') }}"
+    - name: SEARCH_THRESHOLD_MIN_HITS
+      value: "{{ custom.search_threshold_min_hits | default('') }}"
+    - name: REQUEST_TIMEOUT_SECONDS
+      value: "{{ custom.request_timeout_seconds | default('60') }}"
+  secretsProvided:
+    {% if wb_version %}
+    {% include "elasticsearch-auth.yaml" ignore missing %}
+    {% else %}
+    - name: elasticsearch_credentials
+      workspaceKey: AUTH DETAILS NOT FOUND
+    {% endif %}

codebundles/elasticsearch-generic-log-query/.test/Taskfile.yaml

@@ -0,0 +1,23 @@
+version: "3"
+
+tasks:
+  default:
+    desc: "Validate CodeBundle scripts and structure"
+    cmds:
+      - task: validate-bundle
+
+  clean:
+    desc: "Remove local test artifacts"
+    cmds:
+      - rm -rf output workspaceInfo.yaml || true
+
+  build-infra:
+    desc: "Static validation only (Elasticsearch cluster is user-supplied)"
+    cmds:
+      - task: validate-bundle
+
+  validate-bundle:
+    desc: "Run bash syntax checks on CodeBundle shell scripts"
+    dir: .
+    cmds:
+      - ./validate-bundle.sh

codebundles/elasticsearch-generic-log-query/.test/validate-bundle.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# Static validation for the CodeBundle (no live Elasticsearch required).
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+echo "Validating shell scripts under ${ROOT}"
+for f in "${ROOT}"/*.sh; do
+  if [[ -f "${f}" ]]; then
+    bash -n "${f}"
+    echo "OK ${f}"
+  fi
+done
+echo "All shell scripts passed bash -n."

codebundles/elasticsearch-generic-log-query/README.md

@@ -0,0 +1,66 @@
+# Elasticsearch Generic Log Search
+
+This CodeBundle runs configurable Elasticsearch log searches using the HTTP Search API. The cluster base URL (`ELASTICSEARCH_BASE_URL`) is kept separate from the JSON query body (`ELASTICSEARCH_QUERY_BODY`) so you can reuse the same query across environments or swap endpoints without duplicating bundle logic.
+
+## Overview
+
+- **Endpoint reachability**: GET the configured base URL with optional basic auth or API key to confirm the HTTP API responds before running searches.
+- **Generic search**: POST `ELASTICSEARCH_QUERY_BODY` to `/${ELASTICSEARCH_INDEX_PATTERN}/_search` and report total hits plus a bounded sample of hits.
+- **Thresholds**: Optionally compare total hits to `SEARCH_THRESHOLD_MAX_HITS` and `SEARCH_THRESHOLD_MIN_HITS` and raise issues when out of range.
+- **SLI**: Lightweight `GET /_cluster/health` probe producing a 0–1 score for periodic monitoring.
+
+## Configuration
+
+### Required Variables
+
+- `ELASTICSEARCH_BASE_URL`: Base URL for the Elasticsearch HTTP API, without path (for example `https://es.example.com:9200` or `http://localhost:9200`).
+- `ELASTICSEARCH_INDEX_PATTERN`: Index name or pattern used in the Search path (for example `logs-*`, `filebeat-*`, or a concrete index name).
+- `ELASTICSEARCH_QUERY_BODY`: JSON body for `POST .../_search` (query, `size`, `sort`, aggregations). Do not embed the cluster base URL in this JSON.
+
+### Optional Variables
+
+- `SEARCH_THRESHOLD_MAX_HITS`: When set to a non-empty integer string, an issue is raised if total hits exceed this value.
+- `SEARCH_THRESHOLD_MIN_HITS`: When set to a non-empty integer string, an issue is raised if total hits are below this value.
+- `REQUEST_TIMEOUT_SECONDS`: HTTP timeout in seconds for probes and search (default: `60`).
+
+### Secrets
+
+- `elasticsearch_credentials` (optional): JSON or key-value material containing any of:
+  - `ELASTICSEARCH_USERNAME` / `ELASTICSEARCH_PASSWORD` for HTTP Basic auth, and/or
+  - `ELASTICSEARCH_API_KEY` for `Authorization: ApiKey ...` (Elasticsearch API key header).
+
+Unauthenticated clusters are supported for lab use only.
+
+## Tasks Overview
+
+### Check Elasticsearch Endpoint Reachability
+
+Verifies the base URL is reachable over HTTP(S) and returns a 2xx response from a lightweight GET. Issues typically indicate connection failures (severity 2) or non-success HTTP (severity 3).
+
+### Run Generic Log Search and Summarize Results
+
+Executes `POST ${ELASTICSEARCH_BASE_URL}/${ELASTICSEARCH_INDEX_PATTERN}/_search` with `Content-Type: application/json` and surfaces total hits plus a sample of up to 20 hits in the report. Issues surface invalid query JSON, transport errors, or non-2xx HTTP from Elasticsearch.
+
+### Evaluate Search Result Thresholds
+
+Reads `search_summary.json` from the previous task and compares `total_hits` to optional min/max thresholds when configured.
+
+## Examples
+
+Reuse the same `ELASTICSEARCH_QUERY_BODY` against two clusters by changing only the base URL:
+
+```text
+# Staging
+ELASTICSEARCH_BASE_URL=https://staging-es.example.com:9200
+ELASTICSEARCH_INDEX_PATTERN=logs-*
+ELASTICSEARCH_QUERY_BODY={"size":5,"query":{"match_all":{}},"sort":[{"@timestamp":"desc"}]}
+
+# Production (same query body)
+ELASTICSEARCH_BASE_URL=https://prod-es.example.com:9200
+ELASTICSEARCH_INDEX_PATTERN=logs-*
+ELASTICSEARCH_QUERY_BODY={"size":5,"query":{"match_all":{}},"sort":[{"@timestamp":"desc"}]}
+```
+
+## API Reference
+
+Search API: [Elasticsearch search](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html).

codebundles/elasticsearch-generic-log-query/check-elasticsearch-endpoint.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+# -----------------------------------------------------------------------------
+# Verifies HTTP(S) reachability of ELASTICSEARCH_BASE_URL (optional auth).
+# Writes JSON array of issues to endpoint_check_issues.json
+# -----------------------------------------------------------------------------
+: "${ELASTICSEARCH_BASE_URL:?Must set ELASTICSEARCH_BASE_URL}"
+: "${REQUEST_TIMEOUT_SECONDS:=60}"
+
+OUTPUT_FILE="endpoint_check_issues.json"
+issues_json='[]'
+
+# shellcheck source=/dev/null
+load_credentials() {
+  if [[ -n "${elasticsearch_credentials:-}" ]]; then
+    local creds_raw
+    if [[ -f "${elasticsearch_credentials}" ]]; then
+      creds_raw=$(cat "${elasticsearch_credentials}")
+    else
+      creds_raw="${elasticsearch_credentials}"
+    fi
+    if echo "${creds_raw}" | jq -e . >/dev/null 2>&1; then
+      export ELASTICSEARCH_USERNAME
+      ELASTICSEARCH_USERNAME=$(echo "${creds_raw}" | jq -r '.ELASTICSEARCH_USERNAME // empty')
+      export ELASTICSEARCH_PASSWORD
+      ELASTICSEARCH_PASSWORD=$(echo "${creds_raw}" | jq -r '.ELASTICSEARCH_PASSWORD // empty')
+      export ELASTICSEARCH_API_KEY
+      ELASTICSEARCH_API_KEY=$(echo "${creds_raw}" | jq -r '.ELASTICSEARCH_API_KEY // empty')
+    fi
+  fi
+}
+
+load_credentials
+
+BASE="${ELASTICSEARCH_BASE_URL%/}"
+PROBE_URL="${BASE}/"
+
+curl_args=(
+  -sS
+  --max-time "${REQUEST_TIMEOUT_SECONDS}"
+  -o /tmp/es_probe_body.$$
+  -w "%{http_code}"
+)
+
+if [[ -n "${ELASTICSEARCH_API_KEY:-}" ]]; then
+  curl_args+=(-H "Authorization: ApiKey ${ELASTICSEARCH_API_KEY}")
+elif [[ -n "${ELASTICSEARCH_USERNAME:-}" && -n "${ELASTICSEARCH_PASSWORD:-}" ]]; then
+  curl_args+=(-u "${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}")
+fi
+
+http_code="000"
+curl_err=""
+if ! http_code=$(curl "${curl_args[@]}" "${PROBE_URL}" 2>/tmp/es_probe_err.$$); then
+  curl_err=$(cat /tmp/es_probe_err.$$ || true)
+fi
+rm -f /tmp/es_probe_err.$$
+
+body_preview=""
+if [[ -f /tmp/es_probe_body.$$ ]]; then
+  body_preview=$(head -c 2048 /tmp/es_probe_body.$$ | tr -d '\0' || true)
+fi
+rm -f /tmp/es_probe_body.$$
+
+echo "Elasticsearch endpoint probe: ${PROBE_URL}"
+echo "HTTP status: ${http_code}"
+if [[ -n "${body_preview}" ]]; then
+  echo "Response preview (first 2KB):"
+  echo "${body_preview}"
+fi
+
+if [[ "${http_code}" == "000" ]]; then
+  issues_json=$(echo "${issues_json}" | jq \
+    --arg title "Elasticsearch Endpoint Unreachable at \`${BASE}\`" \
+    --arg hc "${http_code}" \
+    --arg cerr "${curl_err}" \
+    --arg severity "2" \
+    --arg next_steps "Verify ELASTICSEARCH_BASE_URL, network paths, TLS, and firewall rules. Test with curl -v against the base URL." \
+    '. += [{
+       "title": $title,
+       "details": ("curl failed to complete (HTTP " + $hc + "). " + $cerr),
+       "severity": ($severity | tonumber),
+       "next_steps": $next_steps
+     }]')
+elif [[ "${http_code}" -lt 200 || "${http_code}" -ge 300 ]]; then
+  issues_json=$(echo "${issues_json}" | jq \
+    --arg title "Elasticsearch Endpoint Returned Non-Success HTTP ${http_code} for \`${BASE}\`" \
+    --arg url "${PROBE_URL}" \
+    --arg preview "${body_preview}" \
+    --arg severity "3" \
+    --arg next_steps "Confirm credentials, Elasticsearch is running, and the base URL targets the HTTP API (port 9200 for OSS/Elastic Cloud)." \
+    '. += [{
+       "title": $title,
+       "details": ("Expected 2xx from GET " + $url + ". Body preview: " + $preview),
+       "severity": ($severity | tonumber),
+       "next_steps": $next_steps
+     }]')
+fi
+
+echo "${issues_json}" > "${OUTPUT_FILE}"
+echo "Wrote ${OUTPUT_FILE}"
+exit 0