diff --git a/README.md b/README.md index 55fba6d..17f1562 100644 --- a/README.md +++ b/README.md @@ -130,8 +130,19 @@ playbooks/ Tier 2 — on demand (→ target .contex safe-refactor.md, extract-module.md, dependency-upgrade.md docs/ Developer-facing documentation generation (1) gitbook.md + setup/ Setup and tooling playbooks (4) + create-local-otel-stack.md + create-local-otel-stack/ (companion scripts and configs — deployed alongside the playbook) + discover-local-otel-stack.md + use-local-otel-stack.md + instrument-dotnet-otel.md ``` +> Scripts in `playbooks/setup/create-local-otel-stack/` are deployed to +> `.context/playbooks/setup/create-local-otel-stack/` alongside the playbook. +> Playbook content references scripts at this path so agents can run them from +> the repository root. + ## Playbook Format Playbooks use a universal markdown format with YAML frontmatter: diff --git a/core/.context/index.md b/core/.context/index.md index 0d788bc..89aea49 100644 --- a/core/.context/index.md +++ b/core/.context/index.md @@ -108,6 +108,17 @@ Combine multiple matches when a task spans domains. --- +## Playbooks — Setup (operational setup procedures) + +| Keywords | File | Summary | +|----------|------|---------| +| create otel stack, local otel, set up opentelemetry, local telemetry, opentelemetry local | `.context/playbooks/setup/create-local-otel-stack.md` | Deploy a local OTel collector stack for development and testing | +| discover otel stack, find otel stack, otel stack running | `.context/playbooks/setup/discover-local-otel-stack.md` | Discover and validate a running local OTel stack | +| use otel stack, connect otel, send telemetry, otlp endpoint | `.context/playbooks/setup/use-local-otel-stack.md` | Configure services to emit to a local OTel stack | +| instrument dotnet, dotnet otel, opentelemetry dotnet, dotnet sdk otel | `.context/playbooks/setup/instrument-dotnet-otel.md` | Instrument a .NET service with OpenTelemetry SDK | + +--- + ## Conventions (style and workflow guidance) | Keywords | File | Summary | diff --git a/deploy.ps1 b/deploy.ps1 index c2e94d4..609614b 100644 --- a/deploy.ps1 +++ b/deploy.ps1 @@ -594,6 +594,7 @@ if ((Test-AgentEnabled 'claude') -or (Test-AgentEnabled 'copilot')) { @{ Dir = 'plan'; Tools = $null } @{ Dir = 'refactor'; Tools = $null } @{ Dir = 'docs'; Tools = $null } + @{ Dir = 'setup'; Tools = 'Read, Grep, Glob, Bash, Write, Edit, Agent' } ) foreach ($category in $playbookCategories) { diff --git a/deploy.sh b/deploy.sh index edd982d..7e12f3b 100755 --- a/deploy.sh +++ b/deploy.sh @@ -606,10 +606,22 @@ if agent_enabled claude || agent_enabled copilot; then generate_skills_for_selected_agents "$playbook" "refactor/$filename" done - for playbook in "$SCRIPT_DIR"/playbooks/docs/*.md; do - filename=$(basename "$playbook") - generate_skills_for_selected_agents "$playbook" "docs/$filename" - done + if [[ -d "$SCRIPT_DIR/playbooks/docs" ]]; then + for playbook in "$SCRIPT_DIR"/playbooks/docs/*.md; do + [[ -f "$playbook" ]] || continue + filename=$(basename "$playbook") + generate_skills_for_selected_agents "$playbook" "docs/$filename" + done + fi + + if [[ -d "$SCRIPT_DIR/playbooks/setup" ]]; then + for playbook in "$SCRIPT_DIR"/playbooks/setup/*.md; do + [[ -f "$playbook" ]] || continue + filename=$(basename "$playbook") + generate_skills_for_selected_agents "$playbook" "setup/$filename" \ + "Read, Grep, Glob, Bash, Write, Edit, Agent" + done + fi else echo " Skipping skill wrapper generation (no selected agent uses skills)." fi diff --git a/playbooks/setup/create-local-otel-stack.md b/playbooks/setup/create-local-otel-stack.md new file mode 100644 index 0000000..d68fbd5 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack.md @@ -0,0 +1,320 @@ +--- +name: setup-create-local-otel-stack +# Ported from devopsin@9fa20ff0 (feature/split-telemetry-skills) — not automatically synced. +description: "Create and start a local OpenTelemetry observability stack (OTel Collector, VictoriaMetrics, VictoriaLogs, VictoriaTraces) for development and testing" +keywords: [create otel stack, local otel, set up opentelemetry, local telemetry, opentelemetry local] +--- + +# Create Local OTel Stack + +> **Local development and testing only.** Do not use these configs in shared, staging, or production environments. + +Create and start a local OpenTelemetry observability stack for development and testing. This skill provides container runtime-agnostic instructions for deploying metrics, logs, and traces backends. + +> **Prerequisites:** Run `discover-local-otel-stack` first to confirm no stack is already running. +> Once the stack is up, use `use-local-otel-stack` to send telemetry to it. + +## Architecture + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ Local OTel Stack │ +│ │ +│ ┌─────────────────┐ ┌──────────────────────────────────────┐ │ +│ │ OTel Collector │ │ Victoria* Backends │ │ +│ │ :4317 (gRPC) │───►│ VictoriaMetrics :8428 (PromQL) │ │ +│ │ :4318 (HTTP) │ │ VictoriaLogs :9428 (LogsQL) │ │ +│ └─────────────────┘ │ VictoriaTraces :10428 (Jaeger) │ │ +│ └──────────────────────────────────────┘ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +| Component | Image | Port | Purpose | +|-----------|-------|------|---------| +| OTel Collector | `ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.133.0` | 4317 (gRPC), 4318 (HTTP) | Receives OTLP, routes to backends | +| VictoriaMetrics | `victoriametrics/victoria-metrics:v1.130.0` | 8428 | Metrics, queryable via PromQL | +| VictoriaLogs | `victoriametrics/victoria-logs:v1.47.0` | 9428 | Logs, queryable via LogsQL | +| VictoriaTraces | `victoriametrics/victoria-traces:v0.7.1` | 10428 | Traces, queryable via Jaeger API | + +## Deployment Scenarios + +### Scenario 1: Stack on host, agents in containers + +In this scenario: +- The stack runs directly on the host (Podman pod or Docker network) +- AI agents run in containers on the same host +- Each agent container runs a lightweight OTel Collector **sidecar** that forwards telemetry to the host via `host.containers.internal:4318` + +Agent instrumentation: +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +``` +(pointing at the in-container sidecar, which forwards to `host.containers.internal:4318`) + +Sidecar config (`otel-collector-sidecar-config.yaml`): +- Receives on `0.0.0.0:4317` and `0.0.0.0:4318` +- Exports via `otlphttp` to `http://host.containers.internal:4318` + +### Scenario 2: Stack and agents on the same host + +No sidecar required. Agents connect directly: +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +``` + +## Container Runtime Examples + +### Podman (pod-based, no compose) + +```bash +# Start the stack +.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh + +# Stop the stack +.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh --stop + +# Force recreate if already running +.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh --force +``` + +Uses shared network namespace: containers communicate via `localhost`. + +### Docker (user-defined bridge network) + +```bash +# Create network +docker network create otel-stack + +# Start VictoriaMetrics +docker run -d --network otel-stack --name victoriametrics \ + -p 127.0.0.1:8428:8428 \ + victoriametrics/victoria-metrics:v1.130.0 \ + --storageDataPath=/storage + +# Start VictoriaLogs +docker run -d --network otel-stack --name victorialogs \ + -p 127.0.0.1:9428:9428 \ + victoriametrics/victoria-logs:v1.47.0 \ + --storageDataPath=/vlogs + +# Start VictoriaTraces +docker run -d --network otel-stack --name victoriatraces \ + -p 127.0.0.1:10428:10428 \ + victoriametrics/victoria-traces:v0.7.1 \ + --storageDataPath=/vtraces --servicegraph.enableTask=true + +# Start OTel Collector (using compose config) +docker run -d --network otel-stack --name otel-collector \ + -p 127.0.0.1:4317:4317 -p 127.0.0.1:4318:4318 \ + -v ./otel-collector-config-compose.yaml:/etc/otel-collector-config.yml:ro \ + ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.133.0 \ + --config=/etc/otel-collector-config.yml +``` + +Uses service names for inter-container communication (e.g., `victoriametrics:8428`). + +### Docker Compose / Podman Compose + +Image versions are read from `versions.env` via variable substitution. Pass `--env-file` so Docker Compose can resolve the `${IMAGE_*}` variables: + +```bash +COMPOSE=".context/playbooks/setup/create-local-otel-stack" + +# Start all services (versions read from versions.env) +docker-compose -f "$COMPOSE/docker-compose.yaml" --env-file "$COMPOSE/versions.env" up -d + +# Stop all services +docker-compose -f "$COMPOSE/docker-compose.yaml" --env-file "$COMPOSE/versions.env" down + +# View logs +docker-compose -f "$COMPOSE/docker-compose.yaml" --env-file "$COMPOSE/versions.env" logs -f + +# Restart specific service +docker-compose -f "$COMPOSE/docker-compose.yaml" --env-file "$COMPOSE/versions.env" restart otel-collector +``` + +Automatically creates network; uses service names for communication. + +### Rancher Desktop (nerdctl) + +Same syntax as Docker, but use `nerdctl` instead of `docker`: +```bash +nerdctl network create otel-stack +nerdctl run -d --network otel-stack --name victoriametrics ... +``` + +## Configuration Files + +### versions.env + +Single source of truth for container image versions. Update versions here to propagate to all scripts automatically. + +### otel-collector-config.yaml + +Host-side collector configuration: +- Receives OTLP on `:4317` (gRPC) and `:4318` (HTTP) +- Includes `hostmetrics` scraper (CPU, memory, disk, network) +- Routes metrics → VictoriaMetrics, logs → VictoriaLogs, traces → VictoriaTraces +- Uses `localhost` throughout since all containers share the pod/network namespace + +### otel-collector-sidecar-config.yaml + +In-container forwarder configuration: +- Receives OTLP on `:4317` and `:4318` (in-container) +- Exports via `otlphttp` to `http://host.containers.internal:4318` +- Includes health check extension on `:13133` +- No `hostmetrics` (that's the host-side collector's responsibility) + +### otel-collector-config-compose.yaml + +Docker Compose-specific collector configuration: +- Same as host-side config but uses service names (`victoriametrics`, `victorialogs`, `victoriatraces`) instead of `localhost` +- Required because Docker Compose creates a bridge network where services communicate via service names + +## Testing + +### Lightweight validation (no container runtime required) + +`validate-config.sh` checks that all required files are present, scripts are executable, YAML is valid, and `docker-compose.yaml` references the correct `${IMAGE_*}` placeholders. It does not require Podman or Docker and is suitable for CI: + +```bash +.context/playbooks/setup/create-local-otel-stack/validate-config.sh +``` + +### Full smoke test (requires Podman) + +Run the smoke test to verify the stack works end-to-end: + +```bash +# Run from the skill directory on Linux (requires podman): +.context/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh +``` + +The script will: +1. Force-clean any leftover containers from a previous run (idempotent pre-flight) +2. Start the stack +3. Wait for all services (including the OTel Collector health endpoint) to be ready before sending telemetry +4. Send sample telemetry via `telemetrygen` +5. Poll each backend for ingested data (up to 30s each, 2s intervals) +6. Assert non-empty results +7. Check that vmui endpoints are reachable +8. Tear down and exit 0 (success) or 1 (failure) + +**CI usage:** Wrap with `timeout 120 .context/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh` to ensure SIGTERM (not SIGKILL) fires on timeout, which allows the trap-based cleanup to run. If the runner may be killed with SIGKILL, add a post-job step: `.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh --force-cleanup`. + +## Platform-Specific Notes + +### Windows (PowerShell) + +Use the PowerShell script for Windows environments: +```powershell +# Start the stack +.\.context\playbooks\setup\create-local-otel-stack\Start-LocalOtelStack.ps1 + +# Stop the stack +.\.context\playbooks\setup\create-local-otel-stack\Start-LocalOtelStack.ps1 -Stop + +# Force recreate +.\.context\playbooks\setup\create-local-otel-stack\Start-LocalOtelStack.ps1 -Force +``` + +### macOS + +Same commands as Linux, but you may need to use `docker` instead of `podman` if Podman is not installed. + +### Networking Differences + +**Podman pods**: Share a network namespace, so containers refer to each other via `localhost`. + +**Docker networks**: Use a named network, so containers use service names (e.g., `victoriametrics:8428`). + +The OTel Collector configurations differ accordingly between the two approaches. + +## Troubleshooting + +### Orphaned containers / port conflicts after a failed run + +If the start script or smoke test exited uncleanly (e.g. killed with SIGKILL in CI), containers may remain running and ports 4317, 4318, 8428, 9428, 10428 may still be bound. Use `--force-cleanup` to unconditionally remove all named containers and the pod: + +```bash +# Podman +.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh --force-cleanup + +# PowerShell +.\.context\playbooks\setup\create-local-otel-stack\Start-LocalOtelStack.ps1 -ForceCleanup +``` + +If the scripts themselves are broken, clean up manually: + +```bash +# Podman +podman pod rm -f local-otel-stack +podman rm -f otel-collector victoriametrics victorialogs victoriatraces + +# Docker +docker rm -f otel-collector victoriametrics victorialogs victoriatraces +docker network rm otel-stack +``` + +### Port conflicts (other services) + +If ports are already in use by a different service, the start script will fail. You can either: +- Stop the conflicting services +- Modify the port mappings in the scripts +- Use `--force` to recreate the stack (if the previous stack is still running) + +### Containers not starting + +Check container logs: +```bash +# Podman +podman logs victoriametrics +podman logs otel-collector + +# Docker +docker logs victoriametrics +docker logs otel-collector +``` + +### Health check failures + +If backends don't become healthy within 30 seconds: +- Check system resources (memory, disk space) +- Verify no firewall blocks are preventing communication +- Review container logs for error messages + +### Telemetry not appearing + +1. Verify your application is configured with the correct OTLP endpoint +2. Check the OTel Collector logs for ingestion errors +3. Query the backends directly to verify they're receiving data +4. Run the smoke test to validate the full pipeline + +## Version Updates + +`versions.env` is the single source of truth for image versions. The Bash and PowerShell start scripts source it directly. `docker-compose.yaml` references the same variables via `${IMAGE_*}` substitution (pass `-f docker-compose.yaml --env-file versions.env` when running `docker-compose`). + +To update component versions: + +1. Edit `versions.env` with new image tags +2. Run `.context/playbooks/setup/create-local-otel-stack/validate-config.sh` to confirm `docker-compose.yaml` still references the variables correctly +3. Test with `.context/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh` +4. Update the architecture table in the `## Architecture` section of this file if version numbers are shown there + +**Note:** The architecture table in this document contains version strings for reference. Update them alongside `versions.env` when upgrading. The SKILL.md Docker examples in `## Container Runtime Examples` also contain full image references — update those too. + +Always test after version updates as APIs may change between major versions. + +## Related Skills + +- [discover-local-otel-stack](discover-local-otel-stack.md) — Check whether a local OTel stack is running. +- [use-local-otel-stack](use-local-otel-stack.md) — Configure OTLP endpoint and query the local stack. +- [instrument-dotnet-otel](instrument-dotnet-otel.md) — Instrument a .NET app with the OTel SDK. + +## External References + +- Observability standard — see `.context/standards/observability.md` for production OTel setup +- [OpenTelemetry Documentation](https://opentelemetry.io/docs/) +- [VictoriaMetrics Documentation](https://docs.victoriametrics.com/) +- [VictoriaLogs Documentation](https://docs.victoriametrics.com/VictoriaLogs/) +- [VictoriaTraces Documentation](https://docs.victoriametrics.com/VictoriaTraces/) diff --git a/playbooks/setup/create-local-otel-stack/Start-LocalOtelStack.ps1 b/playbooks/setup/create-local-otel-stack/Start-LocalOtelStack.ps1 new file mode 100644 index 0000000..7f95a45 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/Start-LocalOtelStack.ps1 @@ -0,0 +1,180 @@ +#!/usr/bin/env pwsh +# Start or stop the local OpenTelemetry observability stack. +# +# Usage: +# ./Start-LocalOtelStack.ps1 # start the stack +# ./Start-LocalOtelStack.ps1 -Stop # tear down the stack +# ./Start-LocalOtelStack.ps1 -Force # recreate if already running +# ./Start-LocalOtelStack.ps1 -ForceCleanup # remove containers/pod by name unconditionally +# +# Manual cleanup (if scripts fail completely): +# podman pod rm -f local-otel-stack +# docker rm -f otel-collector victoriametrics victorialogs victoriatraces + +param( + [switch]$Stop, + [switch]$Force, + [switch]$ForceCleanup +) + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $PSCommandPath +$VersionsFile = Join-Path $ScriptDir "versions.env" + +# Image versions — single source of truth +$Versions = @{} +Get-Content $VersionsFile | ForEach-Object { + if ($_ -match '^IMAGE_(\w+)=(.+)$') { + $Versions[$Matches[1]] = $Matches[2] + } +} + +$PodName = "local-otel-stack" + +function Invoke-ForceCleanup { + Write-Host "Force-cleaning up local OTel stack (removing by name, ignoring errors)..." + podman pod rm -f $PodName 2>$null; $LASTEXITCODE = 0 + foreach ($cname in @('otel-collector', 'victoriametrics', 'victorialogs', 'victoriatraces')) { + podman rm -f $cname 2>$null; $LASTEXITCODE = 0 + } + Write-Host "Force-cleanup complete." +} + +function Stop-Stack { + Write-Host "Stopping local OTel stack..." + $pod = podman pod exists $PodName 2>$null + if ($LASTEXITCODE -eq 0) { + podman pod rm -f $PodName *>$null + Write-Host "Pod '$PodName' removed." + } else { + Write-Host "Pod '$PodName' does not exist." + } +} + +function Wait-ForHealth { + param( + [string]$Url, + [string]$Name, + [int]$Timeout = 30 + ) + + $deadline = (Get-Date).AddSeconds($Timeout) + while ((Get-Date) -lt $deadline) { + try { + $response = Invoke-WebRequest -Uri $Url -TimeoutSec 2 -UseBasicParsing -ErrorAction Stop + if ($response.StatusCode -eq 200) { + Write-Host " $Name is healthy" + return $true + } + } catch { + # Continue trying + } + Start-Sleep 1 + } + Write-Host " WARNING: $Name did not become healthy within ${Timeout}s" + return $false +} + +function Start-Stack { + $pod = podman pod exists $PodName 2>$null + if ($LASTEXITCODE -eq 0) { + if ($Force) { + Write-Host "Pod '$PodName' already exists. Recreating (-Force)..." + Stop-Stack + } else { + Write-Error "ERROR: Pod '$PodName' already exists. Use -Force to recreate, or -Stop to tear down." + } + } + + $ConfigPath = Join-Path $ScriptDir "otel-collector-config.yaml" + if (-not (Test-Path $ConfigPath)) { + Write-Error "ERROR: OTel Collector config not found at: $ConfigPath" + } + + Write-Host "Creating pod '$PodName'..." + podman pod create --name $PodName ` + -p 127.0.0.1:4317:4317 ` + -p 127.0.0.1:4318:4318 ` + -p 127.0.0.1:8428:8428 ` + -p 127.0.0.1:9428:9428 ` + -p 127.0.0.1:10428:10428 ` + -p 127.0.0.1:13133:13133 + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to create pod" + } + + Write-Host "Starting VictoriaMetrics..." + podman run -d --pod $PodName --name victoriametrics ` + $Versions["VM"] ` + --storageDataPath=/storage + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to start VictoriaMetrics" + } + + Write-Host "Starting VictoriaLogs..." + podman run -d --pod $PodName --name victorialogs ` + $Versions["VL"] ` + --storageDataPath=/vlogs + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to start VictoriaLogs" + } + + Write-Host "Starting VictoriaTraces..." + podman run -d --pod $PodName --name victoriatraces ` + $Versions["VT"] ` + --storageDataPath=/vtraces ` + --servicegraph.enableTask=true + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to start VictoriaTraces" + } + + Write-Host "Starting OTel Collector..." + podman run -d --pod $PodName --name otel-collector ` + -v "${ConfigPath}:/etc/otel-collector-config.yml:ro" ` + $Versions["OTEL"] ` + --config=/etc/otel-collector-config.yml + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to start OTel Collector" + } + + Write-Host "" + Write-Host "Waiting for backends to become healthy..." + $vmOk = Wait-ForHealth "http://localhost:8428/health" "VictoriaMetrics" + $vlOk = Wait-ForHealth "http://localhost:9428/health" "VictoriaLogs" + # VictoriaTraces has no /health endpoint; use Jaeger services API as readiness probe + $vtOk = Wait-ForHealth "http://localhost:10428/select/jaeger/api/services" "VictoriaTraces" + + if (-not ($vmOk -and $vlOk -and $vtOk)) { + Write-Host "WARNING: Some backends did not become healthy. Check 'podman pod ps' and container logs." + } + + Write-Host "" + Write-Host "--- Local OTel Stack Ready ---" + Write-Host "Metrics UI (vmui): http://localhost:8428/vmui" + Write-Host "Logs UI (vmui): http://localhost:9428/select/vmui/" + Write-Host "Traces UI (vmui): http://localhost:10428/select/vmui" + Write-Host "OTLP HTTP: http://localhost:4318" + Write-Host "OTLP gRPC: localhost:4317" + Write-Host "Metrics (PromQL): http://localhost:8428/api/v1/query" + Write-Host "Logs (LogsQL): http://localhost:9428/select/logsql/query" + Write-Host "Traces (Jaeger): http://localhost:10428/select/jaeger/api/traces" + Write-Host "" + Write-Host "Example queries:" + Write-Host " curl 'http://localhost:8428/api/v1/query?query=up'" + Write-Host " curl 'http://localhost:9428/select/logsql/query?query=*'" + Write-Host " curl 'http://localhost:10428/select/jaeger/api/services'" +} + +if ($ForceCleanup) { + Invoke-ForceCleanup +} elseif ($Stop) { + Stop-Stack +} else { + Start-Stack +} diff --git a/playbooks/setup/create-local-otel-stack/docker-compose.yaml b/playbooks/setup/create-local-otel-stack/docker-compose.yaml new file mode 100644 index 0000000..25bd7dd --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/docker-compose.yaml @@ -0,0 +1,82 @@ +version: '3.8' +# Image versions are read from versions.env (the single source of truth). +# Run with: docker-compose --env-file versions.env up -d +# Or set the IMAGE_* variables in your environment before running. +# The ${IMAGE_VM} syntax is native Docker Compose variable substitution. + +services: + victoriametrics: + image: ${IMAGE_VM} + container_name: victoriametrics + ports: + - "127.0.0.1:8428:8428" + command: + - --storageDataPath=/storage + volumes: + - vm_data:/storage + networks: + - otel-stack + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8428/health"] + interval: 10s + timeout: 5s + retries: 5 + + victorialogs: + image: ${IMAGE_VL} + container_name: victorialogs + ports: + - "127.0.0.1:9428:9428" + command: + - --storageDataPath=/vlogs + volumes: + - vl_data:/vlogs + networks: + - otel-stack + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9428/health"] + interval: 10s + timeout: 5s + retries: 5 + + victoriatraces: + image: ${IMAGE_VT} + container_name: victoriatraces + ports: + - "127.0.0.1:10428:10428" + command: + - --storageDataPath=/vtraces + - --servicegraph.enableTask=true + volumes: + - vt_data:/vtraces + networks: + - otel-stack + + otel-collector: + image: ${IMAGE_OTEL} + container_name: otel-collector + ports: + - "127.0.0.1:4317:4317" # OTLP gRPC receiver + - "127.0.0.1:4318:4318" # OTLP HTTP receiver + volumes: + - ./otel-collector-config-compose.yaml:/etc/otel-collector-config.yml:ro + command: + - --config=/etc/otel-collector-config.yml + networks: + - otel-stack + depends_on: + victoriametrics: + condition: service_healthy + victorialogs: + condition: service_healthy + victoriatraces: + condition: service_started + +volumes: + vm_data: + vl_data: + vt_data: + +networks: + otel-stack: + driver: bridge diff --git a/playbooks/setup/create-local-otel-stack/otel-collector-config-compose.yaml b/playbooks/setup/create-local-otel-stack/otel-collector-config-compose.yaml new file mode 100644 index 0000000..808db83 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/otel-collector-config-compose.yaml @@ -0,0 +1,67 @@ +# Docker Compose OTel Collector — backend router +# Runs in Docker Compose, using service names for inter-container communication. +# Receives OTLP telemetry on :4318 (HTTP) / :4317 (gRPC) and routes it +# to the Victoria* backends (VictoriaMetrics, VictoriaLogs, VictoriaTraces). +# +# This config uses service names (e.g., victoriametrics:8428) instead of localhost +# because Docker Compose creates a bridge network where services communicate via +# service names, not shared namespaces. +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + cors: + allowed_origins: ["http://localhost:*", "http://127.0.0.1:*"] + grpc: + endpoint: "0.0.0.0:4317" + hostmetrics: + collection_interval: 15s + scrapers: + cpu: + memory: + disk: + network: + +exporters: + otlphttp/victoriametrics: + metrics_endpoint: "http://victoriametrics:8428/opentelemetry/v1/metrics" + tls: + insecure: true + otlphttp/victorialogs: + logs_endpoint: "http://victorialogs:9428/insert/opentelemetry/v1/logs" + tls: + insecure: true + otlphttp/victoriatraces: + traces_endpoint: "http://victoriatraces:10428/insert/opentelemetry/v1/traces" + tls: + insecure: true + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + timeout: 5s + send_batch_size: 1024 + +extensions: + health_check: + endpoint: "0.0.0.0:13133" + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victoriatraces] + metrics: + receivers: [otlp, hostmetrics] + processors: [memory_limiter, batch] + exporters: [otlphttp/victoriametrics] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victorialogs] diff --git a/playbooks/setup/create-local-otel-stack/otel-collector-config.yaml b/playbooks/setup/create-local-otel-stack/otel-collector-config.yaml new file mode 100644 index 0000000..17b0c07 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/otel-collector-config.yaml @@ -0,0 +1,66 @@ +# Host Local OTel Stack Collector — backend router +# Runs on the developer's host as part of the local observability stack. +# Receives OTLP telemetry on :4318 (HTTP) / :4317 (gRPC) and routes it +# to the Victoria* backends (VictoriaMetrics, VictoriaLogs, VictoriaTraces). +# +# See also: otel-collector-sidecar-config.yaml (the in-container forwarder +# that sends telemetry from agent containers to this collector). +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + cors: + allowed_origins: ["http://localhost:*", "http://127.0.0.1:*"] + grpc: + endpoint: "0.0.0.0:4317" + hostmetrics: + collection_interval: 15s + scrapers: + cpu: + memory: + disk: + network: + +exporters: + otlphttp/victoriametrics: + metrics_endpoint: "http://localhost:8428/opentelemetry/v1/metrics" + tls: + insecure: true + otlphttp/victorialogs: + logs_endpoint: "http://localhost:9428/insert/opentelemetry/v1/logs" + tls: + insecure: true + otlphttp/victoriatraces: + traces_endpoint: "http://localhost:10428/insert/opentelemetry/v1/traces" + tls: + insecure: true + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + timeout: 5s + send_batch_size: 1024 + +extensions: + health_check: + endpoint: "0.0.0.0:13133" + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victoriatraces] + metrics: + receivers: [otlp, hostmetrics] + processors: [memory_limiter, batch] + exporters: [otlphttp/victoriametrics] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victorialogs] diff --git a/playbooks/setup/create-local-otel-stack/otel-collector-sidecar-config.yaml b/playbooks/setup/create-local-otel-stack/otel-collector-sidecar-config.yaml new file mode 100644 index 0000000..d26d089 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/otel-collector-sidecar-config.yaml @@ -0,0 +1,49 @@ +# Container OTel Collector — lightweight forwarder +# Runs inside agent containers as a sidecar, forwarding OTLP telemetry +# from in-container processes to the host local-otel-stack collector via +# host.containers.internal:4318. +# +# See also: otel-collector-config.yaml (the host-side collector +# that receives forwarded telemetry and routes it to Victoria* backends). +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + grpc: + endpoint: "0.0.0.0:4317" + +exporters: + otlphttp: + endpoint: "http://host.containers.internal:4318" + tls: + insecure: true + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + timeout: 5s + send_batch_size: 1024 + +extensions: + health_check: + endpoint: "0.0.0.0:13133" + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp] diff --git a/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh b/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh new file mode 100755 index 0000000..5818906 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# Start or stop the local OpenTelemetry observability stack. +# +# Usage: +# ./start-local-otel-stack.sh # start the stack +# ./start-local-otel-stack.sh --stop # tear down the stack +# ./start-local-otel-stack.sh --force # recreate if already running +# ./start-local-otel-stack.sh --force-cleanup # remove containers/pod by name unconditionally +# +# Manual cleanup (if scripts fail completely): +# podman pod rm -f local-otel-stack +# docker rm -f otel-collector victoriametrics victorialogs victoriatraces + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Image versions — single source of truth +# shellcheck source=versions.env +source "$SCRIPT_DIR/versions.env" + +POD_NAME="local-otel-stack" + +STOP=false +FORCE=false +FORCE_CLEANUP=false + +for arg in "$@"; do + case "$arg" in + --stop) STOP=true ;; + --force) FORCE=true ;; + --force-cleanup) FORCE_CLEANUP=true ;; + *) echo "Unknown argument: $arg"; exit 1 ;; + esac +done + +# force_cleanup removes the pod and all named containers unconditionally, +# regardless of whether they are running or even exist. Safe to call multiple +# times (idempotent). Use this when --stop may have failed or was never reached. +force_cleanup() { + echo "Force-cleaning up local OTel stack (removing by name, ignoring errors)..." + podman pod rm -f "$POD_NAME" 2>/dev/null || true + # Also remove any stray named containers that may have outlived the pod + for cname in otel-collector victoriametrics victorialogs victoriatraces; do + podman rm -f "$cname" 2>/dev/null || true + done + echo "Force-cleanup complete." +} + +stop_stack() { + echo "Stopping local OTel stack..." + if podman pod exists "$POD_NAME" 2>/dev/null; then + podman pod rm -f "$POD_NAME" >/dev/null 2>&1 + echo "Pod '$POD_NAME' removed." + else + echo "Pod '$POD_NAME' does not exist." + fi +} + +wait_for_health() { + local url="$1" + local name="$2" + local timeout="${3:-30}" + local deadline=$((SECONDS + timeout)) + + while [ $SECONDS -lt $deadline ]; do + if curl -sf --max-time 2 "$url" >/dev/null 2>&1; then + echo " $name is healthy" + return 0 + fi + sleep 1 + done + echo " WARNING: $name did not become healthy within ${timeout}s" + return 1 +} + +start_stack() { + if podman pod exists "$POD_NAME" 2>/dev/null; then + if [ "$FORCE" = true ]; then + echo "Pod '$POD_NAME' already exists. Recreating (--force)..." + stop_stack + else + echo "ERROR: Pod '$POD_NAME' already exists. Use --force to recreate, or --stop to tear down." >&2 + exit 1 + fi + fi + + CONFIG_PATH="$SCRIPT_DIR/otel-collector-config.yaml" + + if [ ! -f "$CONFIG_PATH" ]; then + echo "ERROR: OTel Collector config not found at: $CONFIG_PATH" >&2 + exit 1 + fi + + # On Git Bash (MINGW), MSYS_NO_PATHCONV prevents automatic POSIX-to-Windows + # path translation on volume mounts, which would mangle the container-side path. + export MSYS_NO_PATHCONV=1 + + echo "Creating pod '$POD_NAME'..." + podman pod create --name "$POD_NAME" \ + -p 127.0.0.1:4317:4317 \ + -p 127.0.0.1:4318:4318 \ + -p 127.0.0.1:8428:8428 \ + -p 127.0.0.1:9428:9428 \ + -p 127.0.0.1:10428:10428 \ + -p 127.0.0.1:13133:13133 + + echo "Starting VictoriaMetrics..." + # --opentelemetry.usePrometheusNaming was removed: VictoriaMetrics 1.100+ + # defaults to Prometheus-compatible naming for OTel metrics, making the + # explicit flag unnecessary. + podman run -d --pod "$POD_NAME" --name victoriametrics \ + "$IMAGE_VM" \ + --storageDataPath=/storage + + echo "Starting VictoriaLogs..." + podman run -d --pod "$POD_NAME" --name victorialogs \ + "$IMAGE_VL" \ + --storageDataPath=/vlogs + + echo "Starting VictoriaTraces..." + podman run -d --pod "$POD_NAME" --name victoriatraces \ + "$IMAGE_VT" \ + --storageDataPath=/vtraces \ + --servicegraph.enableTask=true + + echo "Starting OTel Collector..." + podman run -d --pod "$POD_NAME" --name otel-collector \ + -v "$CONFIG_PATH:/etc/otel-collector-config.yml:ro" \ + "$IMAGE_OTEL" \ + --config=/etc/otel-collector-config.yml + + echo "" + echo "Waiting for backends to become healthy..." + vm_ok=true + vl_ok=true + vt_ok=true + wait_for_health "http://localhost:8428/health" "VictoriaMetrics" || vm_ok=false + wait_for_health "http://localhost:9428/health" "VictoriaLogs" || vl_ok=false + # VictoriaTraces has no /health endpoint; use Jaeger services API as readiness probe + wait_for_health "http://localhost:10428/select/jaeger/api/services" "VictoriaTraces" || vt_ok=false + + if [ "$vm_ok" = false ] || [ "$vl_ok" = false ] || [ "$vt_ok" = false ]; then + echo "WARNING: Some backends did not become healthy. Check 'podman pod ps' and container logs." + fi + + echo "" + echo "--- Local OTel Stack Ready ---" + echo "Metrics UI (vmui): http://localhost:8428/vmui" + echo "Logs UI (vmui): http://localhost:9428/select/vmui/" + echo "Traces UI (vmui): http://localhost:10428/select/vmui" + echo "OTLP HTTP: http://localhost:4318" + echo "OTLP gRPC: localhost:4317" + echo "Metrics (PromQL): http://localhost:8428/api/v1/query" + echo "Logs (LogsQL): http://localhost:9428/select/logsql/query" + echo "Traces (Jaeger): http://localhost:10428/select/jaeger/api/traces" + echo "" + echo "Example queries:" + echo " curl 'http://localhost:8428/api/v1/query?query=up'" + echo " curl 'http://localhost:9428/select/logsql/query?query=*'" + echo " curl 'http://localhost:10428/select/jaeger/api/services'" +} + +if [ "$FORCE_CLEANUP" = true ]; then + force_cleanup +elif [ "$STOP" = true ]; then + stop_stack +else + start_stack +fi diff --git a/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh b/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh new file mode 100755 index 0000000..52494d9 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# Smoke test for the local OpenTelemetry observability stack. +# +# Starts the stack, sends sample telemetry via telemetrygen, queries each +# backend API, asserts non-empty results, and tears down. +# +# Usage: +# ./test-local-otel-stack.sh +# +# CI note: register trap before any fallible commands so cleanup fires even +# on early failures. If the CI runner may be killed with SIGKILL (not +# SIGTERM), add a post-job step: ./start-local-otel-stack.sh --force-cleanup +# or run with: timeout 120 ./test-local-otel-stack.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +START_SCRIPT="$SCRIPT_DIR/start-local-otel-stack.sh" + +# Image versions — single source of truth +# shellcheck source=versions.env +source "$SCRIPT_DIR/versions.env" + +POD_NAME="local-otel-stack" + +PASSED=0 +FAILED=0 + +assert_non_empty() { + local test_name="$1" + local content="$2" + local trimmed + trimmed="$(echo "$content" | tr -d '[:space:]')" + + if [ -z "$trimmed" ] || [ "$trimmed" = "{}" ] || [ "$trimmed" = "[]" ] || [ "$trimmed" = '{"status":"success","data":[]}' ]; then + echo " FAIL: $test_name - empty response" + FAILED=$((FAILED + 1)) + return 1 + fi + echo " PASS: $test_name" + PASSED=$((PASSED + 1)) + return 0 +} + +assert_contains() { + local test_name="$1" + local content="$2" + local expected="$3" + + if echo "$content" | grep -qi "$expected"; then + echo " PASS: $test_name (contains '$expected')" + PASSED=$((PASSED + 1)) + return 0 + fi + echo " FAIL: $test_name - response does not contain '$expected'" + FAILED=$((FAILED + 1)) + return 1 +} + +# poll_url