From 62357c2f8820bdf153feb67cb6d6654d7e23db39 Mon Sep 17 00:00:00 2001 From: Devin Container Agent Date: Tue, 19 May 2026 08:24:10 +0000 Subject: [PATCH 1/4] test: add failing tests for OTel setup playbooks (Red) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capture regression fixture from unmodified codebase and write test suites (bash + PowerShell) covering setup/ playbook deployment, agent-scoped wrappers, idempotency, and validate-config execution. All new test cases fail (Red) as expected — setup/ content does not exist yet. Generated with [Devin](https://cli.devin.ai/docs) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/fixtures/assess-observability-skill.md | 7 + tests/test-deploy.ps1 | 217 ++++++++++++++ tests/test-deploy.sh | 283 +++++++++++++++++++ 3 files changed, 507 insertions(+) create mode 100644 tests/fixtures/assess-observability-skill.md create mode 100644 tests/test-deploy.ps1 create mode 100644 tests/test-deploy.sh diff --git a/tests/fixtures/assess-observability-skill.md b/tests/fixtures/assess-observability-skill.md new file mode 100644 index 0000000..40f4aab --- /dev/null +++ b/tests/fixtures/assess-observability-skill.md @@ -0,0 +1,7 @@ +--- +name: assess-observability +description: "Run observability maturity assessment covering distributed tracing, structured logging, metrics, health checks, and OpenTelemetry compliance" +allowed-tools: "Read, Grep, Glob, Bash(git *), Write, Edit, Agent" +--- + +Read and follow `.context/playbooks/assess/observability.md` in full. diff --git a/tests/test-deploy.ps1 b/tests/test-deploy.ps1 new file mode 100644 index 0000000..42cc6fe --- /dev/null +++ b/tests/test-deploy.ps1 @@ -0,0 +1,217 @@ +#!/usr/bin/env pwsh +# Test suite for deploy.ps1 — verifies setup/ playbook deployment and regressions. +# +# Usage: +# pwsh ./tests/test-deploy.ps1 +# +# Exit codes: +# 0 All tests passed +# 1 One or more tests failed + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $PSCommandPath +$RepoDir = Split-Path -Parent $ScriptDir + +$script:Passed = 0 +$script:Failed = 0 + +function Pass { + param([string]$Label) + Write-Host " PASS: $Label" + $script:Passed++ +} + +function Fail { + param([string]$Label) + Write-Host " FAIL: $Label" + $script:Failed++ +} + +function Assert-FileExists { + param([string]$Label, [string]$Path) + if (Test-Path $Path -PathType Leaf) { + Pass "$Label exists" + } else { + Fail "$Label does not exist: $Path" + } +} + +function Assert-FileNotExists { + param([string]$Label, [string]$Path) + if (-not (Test-Path $Path -PathType Leaf)) { + Pass "$Label does not exist (expected)" + } else { + Fail "$Label unexpectedly exists: $Path" + } +} + +function Assert-DirNotExists { + param([string]$Label, [string]$Path) + if (-not (Test-Path $Path -PathType Container)) { + Pass "$Label directory does not exist (expected)" + } else { + Fail "$Label directory unexpectedly exists: $Path" + } +} + +function Assert-Executable { + param([string]$Label, [string]$Path) + if ($IsLinux -or $IsMacOS) { + if (Test-Path $Path) { + $mode = (Get-Item $Path).UnixMode + if ($mode -match 'x') { + Pass "$Label is executable" + } else { + Fail "$Label is not executable: $Path" + } + } else { + Fail "$Label does not exist: $Path" + } + } else { + # On Windows, skip execute bit check + Pass "$Label executable check skipped (Windows)" + } +} + +function Assert-Contains { + param([string]$Label, [string]$Path, [string]$Expected) + if (Test-Path $Path) { + $content = Get-Content $Path -Raw + if ($content -match [regex]::Escape($Expected)) { + Pass "$Label contains '$Expected'" + } else { + Fail "$Label does not contain '$Expected'" + } + } else { + Fail "$Label file not found: $Path" + } +} + +function Assert-NotContains { + param([string]$Label, [string]$Path, [string]$Unexpected) + if (Test-Path $Path) { + $content = Get-Content $Path -Raw + if ($content -notmatch [regex]::Escape($Unexpected)) { + Pass "$Label does not contain '$Unexpected'" + } else { + Fail "$Label unexpectedly contains '$Unexpected'" + } + } else { + Fail "$Label file not found: $Path" + } +} + +# ═══════════════════════════════════════════════════════════════════════ +# TC1: Fresh deploy — all agents +# ═══════════════════════════════════════════════════════════════════════ +Write-Host "" +Write-Host "=== TC1: Fresh deploy — all agents ===" +$tc1Dir = Join-Path ([System.IO.Path]::GetTempPath()) "tc1-$([guid]::NewGuid().ToString('N').Substring(0,8))" +New-Item -ItemType Directory -Path $tc1Dir -Force | Out-Null +& "$RepoDir/deploy.ps1" -Agents all -Overwrite -Target $tc1Dir *>$null + +Write-Host " --- Playbook files ---" +Assert-FileExists "create-local-otel-stack.md" "$tc1Dir/.context/playbooks/setup/create-local-otel-stack.md" +Assert-FileExists "discover-local-otel-stack.md" "$tc1Dir/.context/playbooks/setup/discover-local-otel-stack.md" +Assert-FileExists "use-local-otel-stack.md" "$tc1Dir/.context/playbooks/setup/use-local-otel-stack.md" +Assert-FileExists "instrument-dotnet-otel.md" "$tc1Dir/.context/playbooks/setup/instrument-dotnet-otel.md" + +Write-Host " --- Companion scripts ---" +Assert-FileExists "start-local-otel-stack.sh" "$tc1Dir/.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh" +Assert-Executable "start-local-otel-stack.sh" "$tc1Dir/.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh" +Assert-FileExists "test-local-otel-stack.sh" "$tc1Dir/.context/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh" +Assert-Executable "test-local-otel-stack.sh" "$tc1Dir/.context/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh" +Assert-FileExists "validate-config.sh" "$tc1Dir/.context/playbooks/setup/create-local-otel-stack/validate-config.sh" +Assert-Executable "validate-config.sh" "$tc1Dir/.context/playbooks/setup/create-local-otel-stack/validate-config.sh" + +Write-Host " --- Non-executable files ---" +Assert-FileExists "Start-LocalOtelStack.ps1" "$tc1Dir/.context/playbooks/setup/create-local-otel-stack/Start-LocalOtelStack.ps1" +Assert-FileExists "versions.env" "$tc1Dir/.context/playbooks/setup/create-local-otel-stack/versions.env" + +Write-Host " --- Claude thin wrappers ---" +Assert-FileExists "claude/setup-create-local-otel-stack" "$tc1Dir/.claude/skills/setup-create-local-otel-stack/SKILL.md" +Assert-FileExists "claude/setup-discover-local-otel-stack" "$tc1Dir/.claude/skills/setup-discover-local-otel-stack/SKILL.md" +Assert-FileExists "claude/setup-use-local-otel-stack" "$tc1Dir/.claude/skills/setup-use-local-otel-stack/SKILL.md" +Assert-FileExists "claude/setup-instrument-dotnet-otel" "$tc1Dir/.claude/skills/setup-instrument-dotnet-otel/SKILL.md" + +Write-Host " --- Copilot thin wrappers ---" +Assert-FileExists "copilot/setup-create-local-otel-stack" "$tc1Dir/.github/skills/setup-create-local-otel-stack/SKILL.md" +Assert-FileExists "copilot/setup-discover-local-otel-stack" "$tc1Dir/.github/skills/setup-discover-local-otel-stack/SKILL.md" +Assert-FileExists "copilot/setup-use-local-otel-stack" "$tc1Dir/.github/skills/setup-use-local-otel-stack/SKILL.md" +Assert-FileExists "copilot/setup-instrument-dotnet-otel" "$tc1Dir/.github/skills/setup-instrument-dotnet-otel/SKILL.md" + +Write-Host " --- allowed-tools check ---" +Assert-Contains "claude wrapper allowed-tools" "$tc1Dir/.claude/skills/setup-create-local-otel-stack/SKILL.md" "allowed-tools:" +Assert-NotContains "claude wrapper no git-only bash" "$tc1Dir/.claude/skills/setup-create-local-otel-stack/SKILL.md" "Bash(git *)" + +Write-Host " --- Safety and provenance ---" +Assert-Contains "local-dev-only warning" "$tc1Dir/.context/playbooks/setup/create-local-otel-stack.md" "Local development and testing only" +Assert-Contains "provenance comment" "$tc1Dir/.context/playbooks/setup/create-local-otel-stack.md" "Ported from devopsin" + +Write-Host " --- Index routing ---" +Assert-Contains "index has setup playbooks" "$tc1Dir/.context/index.md" "playbooks/setup/" + +Write-Host " --- Negative ---" +Assert-FileNotExists "local-otel-stack.md" "$tc1Dir/.context/playbooks/setup/local-otel-stack.md" + +Remove-Item -Recurse -Force $tc1Dir + +# ═══════════════════════════════════════════════════════════════════════ +# TC2: Agent-scoped deploy — Claude only +# ═══════════════════════════════════════════════════════════════════════ +Write-Host "" +Write-Host "=== TC2: Agent-scoped deploy — Claude only ===" +$tc2Dir = Join-Path ([System.IO.Path]::GetTempPath()) "tc2-$([guid]::NewGuid().ToString('N').Substring(0,8))" +New-Item -ItemType Directory -Path $tc2Dir -Force | Out-Null +& "$RepoDir/deploy.ps1" -Agents claude -Overwrite -Target $tc2Dir *>$null + +Assert-FileExists "claude wrapper present" "$tc2Dir/.claude/skills/setup-create-local-otel-stack/SKILL.md" +Assert-DirNotExists "copilot dir absent" "$tc2Dir/.github/skills/setup-create-local-otel-stack" + +Remove-Item -Recurse -Force $tc2Dir + +# ═══════════════════════════════════════════════════════════════════════ +# TC3: Agent-scoped deploy — Copilot only +# ═══════════════════════════════════════════════════════════════════════ +Write-Host "" +Write-Host "=== TC3: Agent-scoped deploy — Copilot only ===" +$tc3Dir = Join-Path ([System.IO.Path]::GetTempPath()) "tc3-$([guid]::NewGuid().ToString('N').Substring(0,8))" +New-Item -ItemType Directory -Path $tc3Dir -Force | Out-Null +& "$RepoDir/deploy.ps1" -Agents copilot -Overwrite -Target $tc3Dir *>$null + +Assert-FileExists "copilot wrapper present" "$tc3Dir/.github/skills/setup-create-local-otel-stack/SKILL.md" +Assert-DirNotExists "claude dir absent" "$tc3Dir/.claude/skills/setup-create-local-otel-stack" + +Remove-Item -Recurse -Force $tc3Dir + +# ═══════════════════════════════════════════════════════════════════════ +# TC4: No regressions — existing thin wrappers +# ═══════════════════════════════════════════════════════════════════════ +Write-Host "" +Write-Host "=== TC4: No regressions — existing thin wrappers ===" +$tc4Dir = Join-Path ([System.IO.Path]::GetTempPath()) "tc4-$([guid]::NewGuid().ToString('N').Substring(0,8))" +New-Item -ItemType Directory -Path $tc4Dir -Force | Out-Null +& "$RepoDir/deploy.ps1" -Agents claude -Overwrite -Target $tc4Dir *>$null + +Assert-FileExists "assess-observability" "$tc4Dir/.claude/skills/assess-observability/SKILL.md" + +Remove-Item -Recurse -Force $tc4Dir + +# ═══════════════════════════════════════════════════════════════════════ +# Summary +# ═══════════════════════════════════════════════════════════════════════ +Write-Host "" +Write-Host "=== Results ===" +Write-Host " Passed: $($script:Passed)" +Write-Host " Failed: $($script:Failed)" + +if ($script:Failed -gt 0) { + Write-Host "" + Write-Host "TEST SUITE FAILED" + exit 1 +} else { + Write-Host "" + Write-Host "TEST SUITE PASSED" + exit 0 +} diff --git a/tests/test-deploy.sh b/tests/test-deploy.sh new file mode 100644 index 0000000..2bfea53 --- /dev/null +++ b/tests/test-deploy.sh @@ -0,0 +1,283 @@ +#!/usr/bin/env bash +# Test suite for deploy.sh — verifies setup/ playbook deployment and regressions. +# +# Usage: +# ./tests/test-deploy.sh +# +# Exit codes: +# 0 All tests passed +# 1 One or more tests failed + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +PASSED=0 +FAILED=0 + +pass() { + echo " PASS: $1" + PASSED=$((PASSED + 1)) +} + +fail() { + echo " FAIL: $1" + FAILED=$((FAILED + 1)) +} + +assert_file_exists() { + local label="$1" + local path="$2" + if [ -f "$path" ]; then + pass "$label exists" + else + fail "$label does not exist: $path" + fi +} + +assert_file_not_exists() { + local label="$1" + local path="$2" + if [ ! -f "$path" ]; then + pass "$label does not exist (expected)" + else + fail "$label unexpectedly exists: $path" + fi +} + +assert_dir_not_exists() { + local label="$1" + local path="$2" + if [ ! -d "$path" ]; then + pass "$label directory does not exist (expected)" + else + fail "$label directory unexpectedly exists: $path" + fi +} + +assert_executable() { + local label="$1" + local path="$2" + if [ -x "$path" ]; then + pass "$label is executable" + else + fail "$label is not executable: $path" + fi +} + +assert_not_executable() { + local label="$1" + local path="$2" + if [ ! -x "$path" ]; then + pass "$label is not executable (expected)" + else + fail "$label is unexpectedly executable: $path" + fi +} + +assert_contains() { + local label="$1" + local path="$2" + local expected="$3" + if grep -qF "$expected" "$path" 2>/dev/null; then + pass "$label contains '$expected'" + else + fail "$label does not contain '$expected'" + fi +} + +assert_not_contains() { + local label="$1" + local path="$2" + local unexpected="$3" + if ! grep -qF "$unexpected" "$path" 2>/dev/null; then + pass "$label does not contain '$unexpected'" + else + fail "$label unexpectedly contains '$unexpected'" + fi +} + +assert_files_identical() { + local label="$1" + local file_a="$2" + local file_b="$3" + if diff -q "$file_a" "$file_b" >/dev/null 2>&1; then + pass "$label files are identical" + else + fail "$label files differ" + diff "$file_a" "$file_b" || true + fi +} + +# ═══════════════════════════════════════════════════════════════════════ +# TC1: Fresh deploy — all agents +# ═══════════════════════════════════════════════════════════════════════ +echo "" +echo "=== TC1: Fresh deploy — all agents ===" +TC1_DIR=$(mktemp -d) +"$REPO_DIR/deploy.sh" --agents all --overwrite "$TC1_DIR" >/dev/null 2>&1 + +echo " --- Playbook files ---" +assert_file_exists "create-local-otel-stack.md" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack.md" +assert_file_exists "discover-local-otel-stack.md" "$TC1_DIR/.context/playbooks/setup/discover-local-otel-stack.md" +assert_file_exists "use-local-otel-stack.md" "$TC1_DIR/.context/playbooks/setup/use-local-otel-stack.md" +assert_file_exists "instrument-dotnet-otel.md" "$TC1_DIR/.context/playbooks/setup/instrument-dotnet-otel.md" + +echo " --- Companion scripts (executable) ---" +assert_file_exists "start-local-otel-stack.sh" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh" +assert_executable "start-local-otel-stack.sh" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh" +assert_file_exists "test-local-otel-stack.sh" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh" +assert_executable "test-local-otel-stack.sh" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh" +assert_file_exists "validate-config.sh" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack/validate-config.sh" +assert_executable "validate-config.sh" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack/validate-config.sh" + +echo " --- Non-executable files ---" +assert_file_exists "Start-LocalOtelStack.ps1" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack/Start-LocalOtelStack.ps1" +assert_file_exists "versions.env" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack/versions.env" +assert_not_executable "versions.env" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack/versions.env" + +echo " --- Claude thin wrappers ---" +assert_file_exists "claude/setup-create-local-otel-stack" "$TC1_DIR/.claude/skills/setup-create-local-otel-stack/SKILL.md" +assert_file_exists "claude/setup-discover-local-otel-stack" "$TC1_DIR/.claude/skills/setup-discover-local-otel-stack/SKILL.md" +assert_file_exists "claude/setup-use-local-otel-stack" "$TC1_DIR/.claude/skills/setup-use-local-otel-stack/SKILL.md" +assert_file_exists "claude/setup-instrument-dotnet-otel" "$TC1_DIR/.claude/skills/setup-instrument-dotnet-otel/SKILL.md" + +echo " --- Copilot thin wrappers ---" +assert_file_exists "copilot/setup-create-local-otel-stack" "$TC1_DIR/.github/skills/setup-create-local-otel-stack/SKILL.md" +assert_file_exists "copilot/setup-discover-local-otel-stack" "$TC1_DIR/.github/skills/setup-discover-local-otel-stack/SKILL.md" +assert_file_exists "copilot/setup-use-local-otel-stack" "$TC1_DIR/.github/skills/setup-use-local-otel-stack/SKILL.md" +assert_file_exists "copilot/setup-instrument-dotnet-otel" "$TC1_DIR/.github/skills/setup-instrument-dotnet-otel/SKILL.md" + +echo " --- allowed-tools check ---" +assert_contains "claude wrapper allowed-tools" "$TC1_DIR/.claude/skills/setup-create-local-otel-stack/SKILL.md" "allowed-tools:" +assert_not_contains "claude wrapper no git-only bash" "$TC1_DIR/.claude/skills/setup-create-local-otel-stack/SKILL.md" "Bash(git *)" + +echo " --- Safety and provenance ---" +assert_contains "local-dev-only warning" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack.md" "Local development and testing only" +assert_contains "provenance comment" "$TC1_DIR/.context/playbooks/setup/create-local-otel-stack.md" "Ported from devopsin" + +echo " --- Index routing ---" +assert_contains "index has setup playbooks" "$TC1_DIR/.context/index.md" "playbooks/setup/" + +echo " --- Negative: monolithic skill not ported ---" +assert_file_not_exists "local-otel-stack.md" "$TC1_DIR/.context/playbooks/setup/local-otel-stack.md" + +rm -rf "$TC1_DIR" + +# ═══════════════════════════════════════════════════════════════════════ +# TC2: Agent-scoped deploy — Claude only +# ═══════════════════════════════════════════════════════════════════════ +echo "" +echo "=== TC2: Agent-scoped deploy — Claude only ===" +TC2_DIR=$(mktemp -d) +"$REPO_DIR/deploy.sh" --agents claude --overwrite "$TC2_DIR" >/dev/null 2>&1 + +assert_file_exists "claude wrapper present" "$TC2_DIR/.claude/skills/setup-create-local-otel-stack/SKILL.md" +assert_dir_not_exists "copilot dir absent" "$TC2_DIR/.github/skills/setup-create-local-otel-stack" + +rm -rf "$TC2_DIR" + +# ═══════════════════════════════════════════════════════════════════════ +# TC3: Agent-scoped deploy — Copilot only +# ═══════════════════════════════════════════════════════════════════════ +echo "" +echo "=== TC3: Agent-scoped deploy — Copilot only ===" +TC3_DIR=$(mktemp -d) +"$REPO_DIR/deploy.sh" --agents copilot --overwrite "$TC3_DIR" >/dev/null 2>&1 + +assert_file_exists "copilot wrapper present" "$TC3_DIR/.github/skills/setup-create-local-otel-stack/SKILL.md" +assert_dir_not_exists "claude dir absent" "$TC3_DIR/.claude/skills/setup-create-local-otel-stack" + +rm -rf "$TC3_DIR" + +# ═══════════════════════════════════════════════════════════════════════ +# TC4: No regressions — existing thin-wrapper generation +# ═══════════════════════════════════════════════════════════════════════ +echo "" +echo "=== TC4: No regressions — existing thin wrappers ===" +TC4_DIR=$(mktemp -d) +"$REPO_DIR/deploy.sh" --agents claude --overwrite "$TC4_DIR" >/dev/null 2>&1 + +assert_file_exists "assess-observability" "$TC4_DIR/.claude/skills/assess-observability/SKILL.md" +assert_file_exists "review-security" "$TC4_DIR/.claude/skills/review-security/SKILL.md" +assert_file_exists "plan-adr" "$TC4_DIR/.claude/skills/plan-adr/SKILL.md" +assert_file_exists "refactor-safe-refactor" "$TC4_DIR/.claude/skills/safe-refactor/SKILL.md" + +echo " --- Regression content check ---" +assert_files_identical "assess-observability fixture" \ + "$TC4_DIR/.claude/skills/assess-observability/SKILL.md" \ + "$SCRIPT_DIR/fixtures/assess-observability-skill.md" + +rm -rf "$TC4_DIR" + +# ═══════════════════════════════════════════════════════════════════════ +# TC5: Idempotency +# ═══════════════════════════════════════════════════════════════════════ +echo "" +echo "=== TC5: Idempotency ===" +TC5_DIR=$(mktemp -d) + +"$REPO_DIR/deploy.sh" --agents all --overwrite "$TC5_DIR" >/dev/null 2>&1 +find "$TC5_DIR" -type f | sort | xargs sha256sum > /tmp/tc5_run1.txt +find "$TC5_DIR" -type f -perm /111 | sort > /tmp/tc5_perms1.txt + +"$REPO_DIR/deploy.sh" --agents all --overwrite "$TC5_DIR" >/dev/null 2>&1 +find "$TC5_DIR" -type f | sort | xargs sha256sum > /tmp/tc5_run2.txt +find "$TC5_DIR" -type f -perm /111 | sort > /tmp/tc5_perms2.txt + +if diff -q /tmp/tc5_run1.txt /tmp/tc5_run2.txt >/dev/null 2>&1; then + pass "File checksums identical across both runs" +else + fail "File checksums differ between runs" + diff /tmp/tc5_run1.txt /tmp/tc5_run2.txt || true +fi + +if diff -q /tmp/tc5_perms1.txt /tmp/tc5_perms2.txt >/dev/null 2>&1; then + pass "Executable permissions identical across both runs" +else + fail "Executable permissions differ between runs" + diff /tmp/tc5_perms1.txt /tmp/tc5_perms2.txt || true +fi + +rm -rf "$TC5_DIR" /tmp/tc5_run1.txt /tmp/tc5_run2.txt /tmp/tc5_perms1.txt /tmp/tc5_perms2.txt + +# ═══════════════════════════════════════════════════════════════════════ +# TC6: validate-config passes — deployed copy +# ═══════════════════════════════════════════════════════════════════════ +echo "" +echo "=== TC6: validate-config passes ===" +TC6_DIR=$(mktemp -d) +"$REPO_DIR/deploy.sh" --agents all --overwrite "$TC6_DIR" >/dev/null 2>&1 + +if "$TC6_DIR/.context/playbooks/setup/create-local-otel-stack/validate-config.sh" >/dev/null 2>&1; then + pass "Deployed validate-config.sh exits 0" +else + fail "Deployed validate-config.sh exited non-zero" +fi + +if "$REPO_DIR/playbooks/setup/create-local-otel-stack/validate-config.sh" >/dev/null 2>&1; then + pass "Source validate-config.sh exits 0" +else + fail "Source validate-config.sh exited non-zero" +fi + +rm -rf "$TC6_DIR" + +# ═══════════════════════════════════════════════════════════════════════ +# Summary +# ═══════════════════════════════════════════════════════════════════════ +echo "" +echo "=== Results ===" +echo " Passed: $PASSED" +echo " Failed: $FAILED" + +if [ "$FAILED" -gt 0 ]; then + echo "" + echo "TEST SUITE FAILED" + exit 1 +else + echo "" + echo "TEST SUITE PASSED" + exit 0 +fi From a0d91bef59021e0f5f7f1618c962c98fdd829e44 Mon Sep 17 00:00:00 2001 From: Devin Container Agent Date: Tue, 19 May 2026 08:37:37 +0000 Subject: [PATCH 2/4] feat: add OTel setup playbooks from devopsin Port four OpenTelemetry operational skills from devopsin@9fa20ff0 (feature/split-telemetry-skills) into playbooks/setup/: - create-local-otel-stack.md + companion scripts/configs - discover-local-otel-stack.md - use-local-otel-stack.md - instrument-dotnet-otel.md Key changes: - New playbooks/setup/ category with keywords frontmatter - deploy.sh: setup/ loop with unrestricted Bash allowed-tools - deploy.ps1: setup entry in $playbookCategories - core/.context/index.md: setup playbook routing entries - README.md: setup/ in repository structure - docker-compose.yaml: all ports bound to 127.0.0.1 - validate-config.sh: per-port localhost binding assertions - Related Skills links updated to sibling-file format - Provenance comments and local-dev-only warnings added Generated with [Devin](https://cli.devin.ai/docs) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- README.md | 11 + core/.context/index.md | 11 + deploy.ps1 | 1 + deploy.sh | 6 + playbooks/setup/create-local-otel-stack.md | 322 ++++++++++++++++++ .../Start-LocalOtelStack.ps1 | 179 ++++++++++ .../docker-compose.yaml | 82 +++++ .../otel-collector-config-compose.yaml | 62 ++++ .../otel-collector-config.yaml | 61 ++++ .../otel-collector-sidecar-config.yaml | 49 +++ .../start-local-otel-stack.sh | 169 +++++++++ .../test-local-otel-stack.sh | 225 ++++++++++++ .../validate-config.sh | 191 +++++++++++ .../create-local-otel-stack/versions.env | 9 + playbooks/setup/discover-local-otel-stack.md | 95 ++++++ playbooks/setup/instrument-dotnet-otel.md | 226 ++++++++++++ playbooks/setup/use-local-otel-stack.md | 151 ++++++++ tests/test-deploy.sh | 13 + 18 files changed, 1863 insertions(+) create mode 100644 playbooks/setup/create-local-otel-stack.md create mode 100644 playbooks/setup/create-local-otel-stack/Start-LocalOtelStack.ps1 create mode 100644 playbooks/setup/create-local-otel-stack/docker-compose.yaml create mode 100644 playbooks/setup/create-local-otel-stack/otel-collector-config-compose.yaml create mode 100644 playbooks/setup/create-local-otel-stack/otel-collector-config.yaml create mode 100644 playbooks/setup/create-local-otel-stack/otel-collector-sidecar-config.yaml create mode 100755 playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh create mode 100755 playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh create mode 100755 playbooks/setup/create-local-otel-stack/validate-config.sh create mode 100644 playbooks/setup/create-local-otel-stack/versions.env create mode 100644 playbooks/setup/discover-local-otel-stack.md create mode 100644 playbooks/setup/instrument-dotnet-otel.md create mode 100644 playbooks/setup/use-local-otel-stack.md diff --git a/README.md b/README.md index 55fba6d..17f1562 100644 --- a/README.md +++ b/README.md @@ -130,8 +130,19 @@ playbooks/ Tier 2 — on demand (→ target .contex safe-refactor.md, extract-module.md, dependency-upgrade.md docs/ Developer-facing documentation generation (1) gitbook.md + setup/ Setup and tooling playbooks (4) + create-local-otel-stack.md + create-local-otel-stack/ (companion scripts and configs — deployed alongside the playbook) + discover-local-otel-stack.md + use-local-otel-stack.md + instrument-dotnet-otel.md ``` +> Scripts in `playbooks/setup/create-local-otel-stack/` are deployed to +> `.context/playbooks/setup/create-local-otel-stack/` alongside the playbook. +> Playbook content references scripts at this path so agents can run them from +> the repository root. + ## Playbook Format Playbooks use a universal markdown format with YAML frontmatter: diff --git a/core/.context/index.md b/core/.context/index.md index 0d788bc..89aea49 100644 --- a/core/.context/index.md +++ b/core/.context/index.md @@ -108,6 +108,17 @@ Combine multiple matches when a task spans domains. --- +## Playbooks — Setup (operational setup procedures) + +| Keywords | File | Summary | +|----------|------|---------| +| create otel stack, local otel, set up opentelemetry, local telemetry, opentelemetry local | `.context/playbooks/setup/create-local-otel-stack.md` | Deploy a local OTel collector stack for development and testing | +| discover otel stack, find otel stack, otel stack running | `.context/playbooks/setup/discover-local-otel-stack.md` | Discover and validate a running local OTel stack | +| use otel stack, connect otel, send telemetry, otlp endpoint | `.context/playbooks/setup/use-local-otel-stack.md` | Configure services to emit to a local OTel stack | +| instrument dotnet, dotnet otel, opentelemetry dotnet, dotnet sdk otel | `.context/playbooks/setup/instrument-dotnet-otel.md` | Instrument a .NET service with OpenTelemetry SDK | + +--- + ## Conventions (style and workflow guidance) | Keywords | File | Summary | diff --git a/deploy.ps1 b/deploy.ps1 index c2e94d4..609614b 100644 --- a/deploy.ps1 +++ b/deploy.ps1 @@ -594,6 +594,7 @@ if ((Test-AgentEnabled 'claude') -or (Test-AgentEnabled 'copilot')) { @{ Dir = 'plan'; Tools = $null } @{ Dir = 'refactor'; Tools = $null } @{ Dir = 'docs'; Tools = $null } + @{ Dir = 'setup'; Tools = 'Read, Grep, Glob, Bash, Write, Edit, Agent' } ) foreach ($category in $playbookCategories) { diff --git a/deploy.sh b/deploy.sh index edd982d..c8605af 100755 --- a/deploy.sh +++ b/deploy.sh @@ -610,6 +610,12 @@ if agent_enabled claude || agent_enabled copilot; then filename=$(basename "$playbook") generate_skills_for_selected_agents "$playbook" "docs/$filename" done + + for playbook in "$SCRIPT_DIR"/playbooks/setup/*.md; do + filename=$(basename "$playbook") + generate_skills_for_selected_agents "$playbook" "setup/$filename" \ + "Read, Grep, Glob, Bash, Write, Edit, Agent" + done else echo " Skipping skill wrapper generation (no selected agent uses skills)." fi diff --git a/playbooks/setup/create-local-otel-stack.md b/playbooks/setup/create-local-otel-stack.md new file mode 100644 index 0000000..606ce7b --- /dev/null +++ b/playbooks/setup/create-local-otel-stack.md @@ -0,0 +1,322 @@ + +--- +name: setup-create-local-otel-stack +description: > + Use to create and start a local OpenTelemetry observability stack + (OTel Collector, VictoriaMetrics, VictoriaLogs, VictoriaTraces). + Only use this after running discover-local-otel-stack and confirming that no stack is currently running. + To send telemetry to a running stack, use use-local-otel-stack instead. +keywords: [create otel stack, local otel, set up opentelemetry, local telemetry, opentelemetry local] +--- + +# Create Local OTel Stack + +> **Local development and testing only.** Do not use these configs in shared, staging, or production environments. + +Create and start a local OpenTelemetry observability stack for development and testing. This skill provides container runtime-agnostic instructions for deploying metrics, logs, and traces backends. + +> **Prerequisites:** Run `discover-local-otel-stack` first to confirm no stack is already running. +> Once the stack is up, use `use-local-otel-stack` to send telemetry to it. + +## Architecture + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ Local OTel Stack │ +│ │ +│ ┌─────────────────┐ ┌──────────────────────────────────────┐ │ +│ │ OTel Collector │ │ Victoria* Backends │ │ +│ │ :4317 (gRPC) │───►│ VictoriaMetrics :8428 (PromQL) │ │ +│ │ :4318 (HTTP) │ │ VictoriaLogs :9428 (LogsQL) │ │ +│ └─────────────────┘ │ VictoriaTraces :10428 (Jaeger) │ │ +│ └──────────────────────────────────────┘ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +| Component | Image | Port | Purpose | +|-----------|-------|------|---------| +| OTel Collector | `ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.133.0` | 4317 (gRPC), 4318 (HTTP) | Receives OTLP, routes to backends | +| VictoriaMetrics | `victoriametrics/victoria-metrics:v1.130.0` | 8428 | Metrics, queryable via PromQL | +| VictoriaLogs | `victoriametrics/victoria-logs:v1.47.0` | 9428 | Logs, queryable via LogsQL | +| VictoriaTraces | `victoriametrics/victoria-traces:v0.7.1` | 10428 | Traces, queryable via Jaeger API | + +## Deployment Scenarios + +### Scenario 1: Stack on host, agents in containers + +In this scenario: +- The stack runs directly on the host (Podman pod or Docker network) +- AI agents run in containers on the same host +- Each agent container runs a lightweight OTel Collector **sidecar** that forwards telemetry to the host via `host.containers.internal:4318` + +Agent instrumentation: +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +``` +(pointing at the in-container sidecar, which forwards to `host.containers.internal:4318`) + +Sidecar config (`otel-collector-sidecar-config.yaml`): +- Receives on `0.0.0.0:4317` and `0.0.0.0:4318` +- Exports via `otlphttp` to `http://host.containers.internal:4318` + +### Scenario 2: Stack and agents on the same host + +No sidecar required. Agents connect directly: +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +``` + +## Container Runtime Examples + +### Podman (pod-based, no compose) + +```bash +# Start the stack +.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh + +# Stop the stack +.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh --stop + +# Force recreate if already running +.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh --force +``` + +Uses shared network namespace: containers communicate via `localhost`. + +### Docker (user-defined bridge network) + +```bash +# Create network +docker network create otel-stack + +# Start VictoriaMetrics +docker run -d --network otel-stack --name victoriametrics \ + -p 8428:8428 \ + victoriametrics/victoria-metrics:v1.130.0 \ + --storageDataPath=/storage + +# Start VictoriaLogs +docker run -d --network otel-stack --name victorialogs \ + -p 9428:9428 \ + victoriametrics/victoria-logs:v1.47.0 \ + --storageDataPath=/vlogs + +# Start VictoriaTraces +docker run -d --network otel-stack --name victoriatraces \ + -p 10428:10428 \ + victoriametrics/victoria-traces:v0.7.1 \ + --storageDataPath=/vtraces --servicegraph.enableTask=true + +# Start OTel Collector (using compose config) +docker run -d --network otel-stack --name otel-collector \ + -p 4317:4317 -p 4318:4318 \ + -v ./otel-collector-config-compose.yaml:/etc/otel-collector-config.yml:ro \ + ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.133.0 \ + --config=/etc/otel-collector-config.yml +``` + +Uses service names for inter-container communication (e.g., `victoriametrics:8428`). + +### Docker Compose / Podman Compose + +Image versions are read from `versions.env` via variable substitution. Pass `--env-file` so Docker Compose can resolve the `${IMAGE_*}` variables: + +```bash +# Start all services (versions read from versions.env) +docker-compose --env-file .context/playbooks/setup/create-local-otel-stack/versions.env up -d + +# Stop all services +docker-compose --env-file .context/playbooks/setup/create-local-otel-stack/versions.env down + +# View logs +docker-compose --env-file .context/playbooks/setup/create-local-otel-stack/versions.env logs -f + +# Restart specific service +docker-compose --env-file .context/playbooks/setup/create-local-otel-stack/versions.env restart otel-collector +``` + +Automatically creates network; uses service names for communication. + +### Rancher Desktop (nerdctl) + +Same syntax as Docker, but use `nerdctl` instead of `docker`: +```bash +nerdctl network create otel-stack +nerdctl run -d --network otel-stack --name victoriametrics ... +``` + +## Configuration Files + +### versions.env + +Single source of truth for container image versions. Update versions here to propagate to all scripts automatically. + +### otel-collector-config.yaml + +Host-side collector configuration: +- Receives OTLP on `:4317` (gRPC) and `:4318` (HTTP) +- Includes `hostmetrics` scraper (CPU, memory, disk, network) +- Routes metrics → VictoriaMetrics, logs → VictoriaLogs, traces → VictoriaTraces +- Uses `localhost` throughout since all containers share the pod/network namespace + +### otel-collector-sidecar-config.yaml + +In-container forwarder configuration: +- Receives OTLP on `:4317` and `:4318` (in-container) +- Exports via `otlphttp` to `http://host.containers.internal:4318` +- Includes health check extension on `:13133` +- No `hostmetrics` (that's the host-side collector's responsibility) + +### otel-collector-config-compose.yaml + +Docker Compose-specific collector configuration: +- Same as host-side config but uses service names (`victoriametrics`, `victorialogs`, `victoriatraces`) instead of `localhost` +- Required because Docker Compose creates a bridge network where services communicate via service names + +## Testing + +### Lightweight validation (no container runtime required) + +`validate-config.sh` checks that all required files are present, scripts are executable, YAML is valid, and `docker-compose.yaml` references the correct `${IMAGE_*}` placeholders. It does not require Podman or Docker and is suitable for CI: + +```bash +.context/playbooks/setup/create-local-otel-stack/validate-config.sh +``` + +### Full smoke test (requires Podman) + +Run the smoke test to verify the stack works end-to-end: + +```bash +# Run from the skill directory on Linux (requires podman): +.context/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh +``` + +The script will: +1. Force-clean any leftover containers from a previous run (idempotent pre-flight) +2. Start the stack +3. Wait for all services (including the OTel Collector health endpoint) to be ready before sending telemetry +4. Send sample telemetry via `telemetrygen` +5. Poll each backend for ingested data (up to 30s each, 2s intervals) +6. Assert non-empty results +7. Check that vmui endpoints are reachable +8. Tear down and exit 0 (success) or 1 (failure) + +**CI usage:** Wrap with `timeout 120 .context/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh` to ensure SIGTERM (not SIGKILL) fires on timeout, which allows the trap-based cleanup to run. If the runner may be killed with SIGKILL, add a post-job step: `.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh --force-cleanup`. + +## Platform-Specific Notes + +### Windows (PowerShell) + +Use the PowerShell script for Windows environments: +```powershell +# Start the stack +.\.context\playbooks\setup\create-local-otel-stack\Start-LocalOtelStack.ps1 + +# Stop the stack +.\.context\playbooks\setup\create-local-otel-stack\Start-LocalOtelStack.ps1 -Stop + +# Force recreate +.\.context\playbooks\setup\create-local-otel-stack\Start-LocalOtelStack.ps1 -Force +``` + +### macOS + +Same commands as Linux, but you may need to use `docker` instead of `podman` if Podman is not installed. + +### Networking Differences + +**Podman pods**: Share a network namespace, so containers refer to each other via `localhost`. + +**Docker networks**: Use a named network, so containers use service names (e.g., `victoriametrics:8428`). + +The OTel Collector configurations differ accordingly between the two approaches. + +## Troubleshooting + +### Orphaned containers / port conflicts after a failed run + +If the start script or smoke test exited uncleanly (e.g. killed with SIGKILL in CI), containers may remain running and ports 4317, 4318, 8428, 9428, 10428 may still be bound. Use `--force-cleanup` to unconditionally remove all named containers and the pod: + +```bash +# Podman +.context/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh --force-cleanup + +# PowerShell +.\.context\playbooks\setup\create-local-otel-stack\Start-LocalOtelStack.ps1 -ForceCleanup +``` + +If the scripts themselves are broken, clean up manually: + +```bash +# Podman +podman pod rm -f local-otel-stack +podman rm -f otel-collector victoriametrics victorialogs victoriatraces + +# Docker +docker rm -f otel-collector victoriametrics victorialogs victoriatraces +docker network rm otel-stack +``` + +### Port conflicts (other services) + +If ports are already in use by a different service, the start script will fail. You can either: +- Stop the conflicting services +- Modify the port mappings in the scripts +- Use `--force` to recreate the stack (if the previous stack is still running) + +### Containers not starting + +Check container logs: +```bash +# Podman +podman logs victoriametrics +podman logs otel-collector + +# Docker +docker logs victoriametrics +docker logs otel-collector +``` + +### Health check failures + +If backends don't become healthy within 30 seconds: +- Check system resources (memory, disk space) +- Verify no firewall blocks are preventing communication +- Review container logs for error messages + +### Telemetry not appearing + +1. Verify your application is configured with the correct OTLP endpoint +2. Check the OTel Collector logs for ingestion errors +3. Query the backends directly to verify they're receiving data +4. Run the smoke test to validate the full pipeline + +## Version Updates + +`versions.env` is the single source of truth for image versions. The Bash and PowerShell start scripts source it directly. `docker-compose.yaml` references the same variables via `${IMAGE_*}` substitution (pass `--env-file versions.env` when running `docker-compose`). + +To update component versions: + +1. Edit `versions.env` with new image tags +2. Run `.context/playbooks/setup/create-local-otel-stack/validate-config.sh` to confirm `docker-compose.yaml` still references the variables correctly +3. Test with `.context/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh` +4. Update the architecture table in the `## Architecture` section of this file if version numbers are shown there + +**Note:** The architecture table in this document contains version strings for reference. Update them alongside `versions.env` when upgrading. The SKILL.md Docker examples in `## Container Runtime Examples` also contain full image references — update those too. + +Always test after version updates as APIs may change between major versions. + +## Related Skills + +- [discover-local-otel-stack](discover-local-otel-stack.md) — Check whether a local OTel stack is running. +- [use-local-otel-stack](use-local-otel-stack.md) — Configure OTLP endpoint and query the local stack. +- [instrument-dotnet-otel](instrument-dotnet-otel.md) — Instrument a .NET app with the OTel SDK. + +## External References + +- Observability standard — see `.context/standards/observability.md` for production OTel setup +- [OpenTelemetry Documentation](https://opentelemetry.io/docs/) +- [VictoriaMetrics Documentation](https://docs.victoriametrics.com/) +- [VictoriaLogs Documentation](https://docs.victoriametrics.com/VictoriaLogs/) +- [VictoriaTraces Documentation](https://docs.victoriametrics.com/VictoriaTraces/) diff --git a/playbooks/setup/create-local-otel-stack/Start-LocalOtelStack.ps1 b/playbooks/setup/create-local-otel-stack/Start-LocalOtelStack.ps1 new file mode 100644 index 0000000..8698767 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/Start-LocalOtelStack.ps1 @@ -0,0 +1,179 @@ +#!/usr/bin/env pwsh +# Start or stop the local OpenTelemetry observability stack. +# +# Usage: +# ./Start-LocalOtelStack.ps1 # start the stack +# ./Start-LocalOtelStack.ps1 -Stop # tear down the stack +# ./Start-LocalOtelStack.ps1 -Force # recreate if already running +# ./Start-LocalOtelStack.ps1 -ForceCleanup # remove containers/pod by name unconditionally +# +# Manual cleanup (if scripts fail completely): +# podman pod rm -f local-otel-stack +# docker rm -f otel-collector victoriametrics victorialogs victoriatraces + +param( + [switch]$Stop, + [switch]$Force, + [switch]$ForceCleanup +) + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $PSCommandPath +$VersionsFile = Join-Path $ScriptDir "versions.env" + +# Image versions — single source of truth +$Versions = @{} +Get-Content $VersionsFile | ForEach-Object { + if ($_ -match '^IMAGE_(\w+)=(.+)$') { + $Versions[$Matches[1]] = $Matches[2] + } +} + +$PodName = "local-otel-stack" + +function Invoke-ForceCleanup { + Write-Host "Force-cleaning up local OTel stack (removing by name, ignoring errors)..." + podman pod rm -f $PodName 2>$null; $LASTEXITCODE = 0 + foreach ($cname in @('otel-collector', 'victoriametrics', 'victorialogs', 'victoriatraces')) { + podman rm -f $cname 2>$null; $LASTEXITCODE = 0 + } + Write-Host "Force-cleanup complete." +} + +function Stop-Stack { + Write-Host "Stopping local OTel stack..." + $pod = podman pod exists $PodName 2>$null + if ($LASTEXITCODE -eq 0) { + podman pod rm -f $PodName *>$null + Write-Host "Pod '$PodName' removed." + } else { + Write-Host "Pod '$PodName' does not exist." + } +} + +function Wait-ForHealth { + param( + [string]$Url, + [string]$Name, + [int]$Timeout = 30 + ) + + $deadline = (Get-Date).AddSeconds($Timeout) + while ((Get-Date) -lt $deadline) { + try { + $response = Invoke-WebRequest -Uri $Url -TimeoutSec 2 -UseBasicParsing *>$null + if ($response.StatusCode -eq 200) { + Write-Host " $Name is healthy" + return $true + } + } catch { + # Continue trying + } + Start-Sleep 1 + } + Write-Host " WARNING: $Name did not become healthy within ${Timeout}s" + return $false +} + +function Start-Stack { + $pod = podman pod exists $PodName 2>$null + if ($LASTEXITCODE -eq 0) { + if ($Force) { + Write-Host "Pod '$PodName' already exists. Recreating (-Force)..." + Stop-Stack + } else { + Write-Error "ERROR: Pod '$PodName' already exists. Use -Force to recreate, or -Stop to tear down." + } + } + + $ConfigPath = Join-Path $ScriptDir "otel-collector-config.yaml" + if (-not (Test-Path $ConfigPath)) { + Write-Error "ERROR: OTel Collector config not found at: $ConfigPath" + } + + Write-Host "Creating pod '$PodName'..." + podman pod create --name $PodName ` + -p 4317:4317 ` + -p 4318:4318 ` + -p 8428:8428 ` + -p 9428:9428 ` + -p 10428:10428 + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to create pod" + } + + Write-Host "Starting VictoriaMetrics..." + podman run -d --pod $PodName --name victoriametrics ` + $Versions["VM"] ` + --storageDataPath=/storage + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to start VictoriaMetrics" + } + + Write-Host "Starting VictoriaLogs..." + podman run -d --pod $PodName --name victorialogs ` + $Versions["VL"] ` + --storageDataPath=/vlogs + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to start VictoriaLogs" + } + + Write-Host "Starting VictoriaTraces..." + podman run -d --pod $PodName --name victoriatraces ` + $Versions["VT"] ` + --storageDataPath=/vtraces ` + --servicegraph.enableTask=true + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to start VictoriaTraces" + } + + Write-Host "Starting OTel Collector..." + podman run -d --pod $PodName --name otel-collector ` + -v "${ConfigPath}:/etc/otel-collector-config.yml:ro" ` + $Versions["OTEL"] ` + --config=/etc/otel-collector-config.yml + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to start OTel Collector" + } + + Write-Host "" + Write-Host "Waiting for backends to become healthy..." + $vmOk = Wait-ForHealth "http://localhost:8428/health" "VictoriaMetrics" + $vlOk = Wait-ForHealth "http://localhost:9428/health" "VictoriaLogs" + # VictoriaTraces has no /health endpoint; use Jaeger services API as readiness probe + $vtOk = Wait-ForHealth "http://localhost:10428/select/jaeger/api/services" "VictoriaTraces" + + if (-not ($vmOk -and $vlOk -and $vtOk)) { + Write-Host "WARNING: Some backends did not become healthy. Check 'podman pod ps' and container logs." + } + + Write-Host "" + Write-Host "--- Local OTel Stack Ready ---" + Write-Host "Metrics UI (vmui): http://localhost:8428/vmui" + Write-Host "Logs UI (vmui): http://localhost:9428/select/vmui/" + Write-Host "Traces UI (vmui): http://localhost:10428/select/vmui" + Write-Host "OTLP HTTP: http://localhost:4318" + Write-Host "OTLP gRPC: localhost:4317" + Write-Host "Metrics (PromQL): http://localhost:8428/api/v1/query" + Write-Host "Logs (LogsQL): http://localhost:9428/select/logsql/query" + Write-Host "Traces (Jaeger): http://localhost:10428/select/jaeger/api/traces" + Write-Host "" + Write-Host "Example queries:" + Write-Host " curl 'http://localhost:8428/api/v1/query?query=up'" + Write-Host " curl 'http://localhost:9428/select/logsql/query?query=*'" + Write-Host " curl 'http://localhost:10428/select/jaeger/api/services'" +} + +if ($ForceCleanup) { + Invoke-ForceCleanup +} elseif ($Stop) { + Stop-Stack +} else { + Start-Stack +} diff --git a/playbooks/setup/create-local-otel-stack/docker-compose.yaml b/playbooks/setup/create-local-otel-stack/docker-compose.yaml new file mode 100644 index 0000000..25bd7dd --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/docker-compose.yaml @@ -0,0 +1,82 @@ +version: '3.8' +# Image versions are read from versions.env (the single source of truth). +# Run with: docker-compose --env-file versions.env up -d +# Or set the IMAGE_* variables in your environment before running. +# The ${IMAGE_VM} syntax is native Docker Compose variable substitution. + +services: + victoriametrics: + image: ${IMAGE_VM} + container_name: victoriametrics + ports: + - "127.0.0.1:8428:8428" + command: + - --storageDataPath=/storage + volumes: + - vm_data:/storage + networks: + - otel-stack + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8428/health"] + interval: 10s + timeout: 5s + retries: 5 + + victorialogs: + image: ${IMAGE_VL} + container_name: victorialogs + ports: + - "127.0.0.1:9428:9428" + command: + - --storageDataPath=/vlogs + volumes: + - vl_data:/vlogs + networks: + - otel-stack + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9428/health"] + interval: 10s + timeout: 5s + retries: 5 + + victoriatraces: + image: ${IMAGE_VT} + container_name: victoriatraces + ports: + - "127.0.0.1:10428:10428" + command: + - --storageDataPath=/vtraces + - --servicegraph.enableTask=true + volumes: + - vt_data:/vtraces + networks: + - otel-stack + + otel-collector: + image: ${IMAGE_OTEL} + container_name: otel-collector + ports: + - "127.0.0.1:4317:4317" # OTLP gRPC receiver + - "127.0.0.1:4318:4318" # OTLP HTTP receiver + volumes: + - ./otel-collector-config-compose.yaml:/etc/otel-collector-config.yml:ro + command: + - --config=/etc/otel-collector-config.yml + networks: + - otel-stack + depends_on: + victoriametrics: + condition: service_healthy + victorialogs: + condition: service_healthy + victoriatraces: + condition: service_started + +volumes: + vm_data: + vl_data: + vt_data: + +networks: + otel-stack: + driver: bridge diff --git a/playbooks/setup/create-local-otel-stack/otel-collector-config-compose.yaml b/playbooks/setup/create-local-otel-stack/otel-collector-config-compose.yaml new file mode 100644 index 0000000..40253f0 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/otel-collector-config-compose.yaml @@ -0,0 +1,62 @@ +# Docker Compose OTel Collector — backend router +# Runs in Docker Compose, using service names for inter-container communication. +# Receives OTLP telemetry on :4318 (HTTP) / :4317 (gRPC) and routes it +# to the Victoria* backends (VictoriaMetrics, VictoriaLogs, VictoriaTraces). +# +# This config uses service names (e.g., victoriametrics:8428) instead of localhost +# because Docker Compose creates a bridge network where services communicate via +# service names, not shared namespaces. +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + cors: + allowed_origins: ["http://*", "https://*"] + grpc: + endpoint: "0.0.0.0:4317" + hostmetrics: + collection_interval: 15s + scrapers: + cpu: + memory: + disk: + network: + +exporters: + otlphttp/victoriametrics: + metrics_endpoint: "http://victoriametrics:8428/opentelemetry/v1/metrics" + tls: + insecure: true + otlphttp/victorialogs: + logs_endpoint: "http://victorialogs:9428/insert/opentelemetry/v1/logs" + tls: + insecure: true + otlphttp/victoriatraces: + traces_endpoint: "http://victoriatraces:10428/insert/opentelemetry/v1/traces" + tls: + insecure: true + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + timeout: 5s + send_batch_size: 1024 + +service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victoriatraces] + metrics: + receivers: [otlp, hostmetrics] + processors: [memory_limiter, batch] + exporters: [otlphttp/victoriametrics] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victorialogs] diff --git a/playbooks/setup/create-local-otel-stack/otel-collector-config.yaml b/playbooks/setup/create-local-otel-stack/otel-collector-config.yaml new file mode 100644 index 0000000..f2fcf26 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/otel-collector-config.yaml @@ -0,0 +1,61 @@ +# Host Local OTel Stack Collector — backend router +# Runs on the developer's host as part of the local observability stack. +# Receives OTLP telemetry on :4318 (HTTP) / :4317 (gRPC) and routes it +# to the Victoria* backends (VictoriaMetrics, VictoriaLogs, VictoriaTraces). +# +# See also: otel-collector-sidecar-config.yaml (the in-container forwarder +# that sends telemetry from agent containers to this collector). +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + cors: + allowed_origins: ["http://*", "https://*"] + grpc: + endpoint: "0.0.0.0:4317" + hostmetrics: + collection_interval: 15s + scrapers: + cpu: + memory: + disk: + network: + +exporters: + otlphttp/victoriametrics: + metrics_endpoint: "http://localhost:8428/opentelemetry/v1/metrics" + tls: + insecure: true + otlphttp/victorialogs: + logs_endpoint: "http://localhost:9428/insert/opentelemetry/v1/logs" + tls: + insecure: true + otlphttp/victoriatraces: + traces_endpoint: "http://localhost:10428/insert/opentelemetry/v1/traces" + tls: + insecure: true + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + timeout: 5s + send_batch_size: 1024 + +service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victoriatraces] + metrics: + receivers: [otlp, hostmetrics] + processors: [memory_limiter, batch] + exporters: [otlphttp/victoriametrics] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/victorialogs] diff --git a/playbooks/setup/create-local-otel-stack/otel-collector-sidecar-config.yaml b/playbooks/setup/create-local-otel-stack/otel-collector-sidecar-config.yaml new file mode 100644 index 0000000..d26d089 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/otel-collector-sidecar-config.yaml @@ -0,0 +1,49 @@ +# Container OTel Collector — lightweight forwarder +# Runs inside agent containers as a sidecar, forwarding OTLP telemetry +# from in-container processes to the host local-otel-stack collector via +# host.containers.internal:4318. +# +# See also: otel-collector-config.yaml (the host-side collector +# that receives forwarded telemetry and routes it to Victoria* backends). +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + grpc: + endpoint: "0.0.0.0:4317" + +exporters: + otlphttp: + endpoint: "http://host.containers.internal:4318" + tls: + insecure: true + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + timeout: 5s + send_batch_size: 1024 + +extensions: + health_check: + endpoint: "0.0.0.0:13133" + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp] diff --git a/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh b/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh new file mode 100755 index 0000000..581ac53 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/start-local-otel-stack.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Start or stop the local OpenTelemetry observability stack. +# +# Usage: +# ./start-local-otel-stack.sh # start the stack +# ./start-local-otel-stack.sh --stop # tear down the stack +# ./start-local-otel-stack.sh --force # recreate if already running +# ./start-local-otel-stack.sh --force-cleanup # remove containers/pod by name unconditionally +# +# Manual cleanup (if scripts fail completely): +# podman pod rm -f local-otel-stack +# docker rm -f otel-collector victoriametrics victorialogs victoriatraces + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Image versions — single source of truth +# shellcheck source=versions.env +source "$SCRIPT_DIR/versions.env" + +POD_NAME="local-otel-stack" + +STOP=false +FORCE=false +FORCE_CLEANUP=false + +for arg in "$@"; do + case "$arg" in + --stop) STOP=true ;; + --force) FORCE=true ;; + --force-cleanup) FORCE_CLEANUP=true ;; + *) echo "Unknown argument: $arg"; exit 1 ;; + esac +done + +# force_cleanup removes the pod and all named containers unconditionally, +# regardless of whether they are running or even exist. Safe to call multiple +# times (idempotent). Use this when --stop may have failed or was never reached. +force_cleanup() { + echo "Force-cleaning up local OTel stack (removing by name, ignoring errors)..." + podman pod rm -f "$POD_NAME" 2>/dev/null || true + # Also remove any stray named containers that may have outlived the pod + for cname in otel-collector victoriametrics victorialogs victoriatraces; do + podman rm -f "$cname" 2>/dev/null || true + done + echo "Force-cleanup complete." +} + +stop_stack() { + echo "Stopping local OTel stack..." + if podman pod exists "$POD_NAME" 2>/dev/null; then + podman pod rm -f "$POD_NAME" >/dev/null 2>&1 + echo "Pod '$POD_NAME' removed." + else + echo "Pod '$POD_NAME' does not exist." + fi +} + +wait_for_health() { + local url="$1" + local name="$2" + local timeout="${3:-30}" + local deadline=$((SECONDS + timeout)) + + while [ $SECONDS -lt $deadline ]; do + if curl -sf --max-time 2 "$url" >/dev/null 2>&1; then + echo " $name is healthy" + return 0 + fi + sleep 1 + done + echo " WARNING: $name did not become healthy within ${timeout}s" + return 1 +} + +start_stack() { + if podman pod exists "$POD_NAME" 2>/dev/null; then + if [ "$FORCE" = true ]; then + echo "Pod '$POD_NAME' already exists. Recreating (--force)..." + stop_stack + else + echo "ERROR: Pod '$POD_NAME' already exists. Use --force to recreate, or --stop to tear down." >&2 + exit 1 + fi + fi + + CONFIG_PATH="$SCRIPT_DIR/otel-collector-config.yaml" + + if [ ! -f "$CONFIG_PATH" ]; then + echo "ERROR: OTel Collector config not found at: $CONFIG_PATH" >&2 + exit 1 + fi + + # On Git Bash (MINGW), MSYS_NO_PATHCONV prevents automatic POSIX-to-Windows + # path translation on volume mounts, which would mangle the container-side path. + export MSYS_NO_PATHCONV=1 + + echo "Creating pod '$POD_NAME'..." + podman pod create --name "$POD_NAME" \ + -p 4317:4317 \ + -p 4318:4318 \ + -p 8428:8428 \ + -p 9428:9428 \ + -p 10428:10428 + + echo "Starting VictoriaMetrics..." + # --opentelemetry.usePrometheusNaming was removed: VictoriaMetrics 1.100+ + # defaults to Prometheus-compatible naming for OTel metrics, making the + # explicit flag unnecessary. + podman run -d --pod "$POD_NAME" --name victoriametrics \ + "$IMAGE_VM" \ + --storageDataPath=/storage + + echo "Starting VictoriaLogs..." + podman run -d --pod "$POD_NAME" --name victorialogs \ + "$IMAGE_VL" \ + --storageDataPath=/vlogs + + echo "Starting VictoriaTraces..." + podman run -d --pod "$POD_NAME" --name victoriatraces \ + "$IMAGE_VT" \ + --storageDataPath=/vtraces \ + --servicegraph.enableTask=true + + echo "Starting OTel Collector..." + podman run -d --pod "$POD_NAME" --name otel-collector \ + -v "$CONFIG_PATH:/etc/otel-collector-config.yml:ro" \ + "$IMAGE_OTEL" \ + --config=/etc/otel-collector-config.yml + + echo "" + echo "Waiting for backends to become healthy..." + vm_ok=true + vl_ok=true + vt_ok=true + wait_for_health "http://localhost:8428/health" "VictoriaMetrics" || vm_ok=false + wait_for_health "http://localhost:9428/health" "VictoriaLogs" || vl_ok=false + # VictoriaTraces has no /health endpoint; use Jaeger services API as readiness probe + wait_for_health "http://localhost:10428/select/jaeger/api/services" "VictoriaTraces" || vt_ok=false + + if [ "$vm_ok" = false ] || [ "$vl_ok" = false ] || [ "$vt_ok" = false ]; then + echo "WARNING: Some backends did not become healthy. Check 'podman pod ps' and container logs." + fi + + echo "" + echo "--- Local OTel Stack Ready ---" + echo "Metrics UI (vmui): http://localhost:8428/vmui" + echo "Logs UI (vmui): http://localhost:9428/select/vmui/" + echo "Traces UI (vmui): http://localhost:10428/select/vmui" + echo "OTLP HTTP: http://localhost:4318" + echo "OTLP gRPC: localhost:4317" + echo "Metrics (PromQL): http://localhost:8428/api/v1/query" + echo "Logs (LogsQL): http://localhost:9428/select/logsql/query" + echo "Traces (Jaeger): http://localhost:10428/select/jaeger/api/traces" + echo "" + echo "Example queries:" + echo " curl 'http://localhost:8428/api/v1/query?query=up'" + echo " curl 'http://localhost:9428/select/logsql/query?query=*'" + echo " curl 'http://localhost:10428/select/jaeger/api/services'" +} + +if [ "$FORCE_CLEANUP" = true ]; then + force_cleanup +elif [ "$STOP" = true ]; then + stop_stack +else + start_stack +fi diff --git a/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh b/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh new file mode 100755 index 0000000..52494d9 --- /dev/null +++ b/playbooks/setup/create-local-otel-stack/test-local-otel-stack.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# Smoke test for the local OpenTelemetry observability stack. +# +# Starts the stack, sends sample telemetry via telemetrygen, queries each +# backend API, asserts non-empty results, and tears down. +# +# Usage: +# ./test-local-otel-stack.sh +# +# CI note: register trap before any fallible commands so cleanup fires even +# on early failures. If the CI runner may be killed with SIGKILL (not +# SIGTERM), add a post-job step: ./start-local-otel-stack.sh --force-cleanup +# or run with: timeout 120 ./test-local-otel-stack.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +START_SCRIPT="$SCRIPT_DIR/start-local-otel-stack.sh" + +# Image versions — single source of truth +# shellcheck source=versions.env +source "$SCRIPT_DIR/versions.env" + +POD_NAME="local-otel-stack" + +PASSED=0 +FAILED=0 + +assert_non_empty() { + local test_name="$1" + local content="$2" + local trimmed + trimmed="$(echo "$content" | tr -d '[:space:]')" + + if [ -z "$trimmed" ] || [ "$trimmed" = "{}" ] || [ "$trimmed" = "[]" ] || [ "$trimmed" = '{"status":"success","data":[]}' ]; then + echo " FAIL: $test_name - empty response" + FAILED=$((FAILED + 1)) + return 1 + fi + echo " PASS: $test_name" + PASSED=$((PASSED + 1)) + return 0 +} + +assert_contains() { + local test_name="$1" + local content="$2" + local expected="$3" + + if echo "$content" | grep -qi "$expected"; then + echo " PASS: $test_name (contains '$expected')" + PASSED=$((PASSED + 1)) + return 0 + fi + echo " FAIL: $test_name - response does not contain '$expected'" + FAILED=$((FAILED + 1)) + return 1 +} + +# poll_url