From ac51f0ae4f5d47715ca2916852482402361a9377 Mon Sep 17 00:00:00 2001 From: Onur Yilmaz Date: Thu, 12 Mar 2026 14:04:05 +0300 Subject: [PATCH 1/2] Run benchmarks weekly basis --- .github/workflows/benchmark.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5fc85bf..f32e5cf 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,11 +2,8 @@ name: Run HTTP Input Benchmark on: workflow_dispatch: - inputs: - tag: - description: "Tag to apply to the release" - required: true - type: string + schedule: + - cron: '0 8 * * 1' # Every Monday at 8:00 AM UTC env: CLAUDE_CODE_MAX_OUTPUT_TOKENS: 8192 @@ -21,6 +18,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v6 + - name: Generate tag + id: tag + run: echo "value=$(date -u +'%Y%m%d-%H%M%S')" >> $GITHUB_OUTPUT - uses: hashicorp/setup-terraform@v3 with: terraform_version: "1.5.7" @@ -62,4 +62,5 @@ jobs: with: artifacts: benchmark_results/*/*.log bodyFile: benchmark_results/report.md - tag: ${{ inputs.tag }} + tag: ${{ steps.tag.outputs.value }} + name: "Benchmark Results - ${{ steps.tag.outputs.value }}" From 470177d1195bfdb588a4654c7a5f3f026601cd96 Mon Sep 17 00:00:00 2001 From: Onur Yilmaz Date: Thu, 12 Mar 2026 15:50:15 +0300 Subject: [PATCH 2/2] Refactor skill --- .claude/skills/analyse-benchmark/SKILL.md | 23 +++++++++++------------ .github/workflows/benchmark.yml | 2 +- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.claude/skills/analyse-benchmark/SKILL.md b/.claude/skills/analyse-benchmark/SKILL.md index dd705a9..5005f28 100644 --- a/.claude/skills/analyse-benchmark/SKILL.md +++ b/.claude/skills/analyse-benchmark/SKILL.md @@ -39,22 +39,21 @@ You will receive benchmark logs in benchmark_results folder from multiple vendor | Vendor | Avg Throughput | Peak Throughput | Total Logs | Avg CPU | Peak Memory | Rank | |--------|----------------|-----------------|------------|---------|-------------|------| -| Vendor A | X.XX logs/sec 🥇 | Y.YY logs/sec | N | X.X% | XXX MB | 1 | -| Vendor B | X.XX logs/sec | Y.YY logs/sec 🥇 | N | X.X% 🥇 | XXX MB 🥇 | 2 | -| Vendor C | X.XX logs/sec | Y.YY logs/sec | N | X.X% | XXX MB | 3 | +| Edge Delta | X.XX logs/sec | Y.YY logs/sec | N | X.X% | XXX MB | 1 | +| Bindplane | X.XX logs/sec | Y.YY logs/sec | N | X.X% | XXX MB | 2 | +| Cribl | X.XX logs/sec | Y.YY logs/sec | N | X.X% | XXX MB | 3 | ### Reliability Comparison | Vendor | Total Errors | Error Rate | Backpressure (429/503) | Backpressure % | Status | |--------|--------------|------------|------------------------|----------------|--------| -| Vendor A | 0 | 0.00% | 0 | 0.0% | ✅ | -| Vendor B | 5 | 0.05% | 0 | 0.0% | ✅ | -| Vendor C | 0 | 0.00% | 120 | 12.5% | ⚠️ | +| Edge Delta | 0 | 0.00% | 0 | 0.0% | ✅ | +| Bindplane | 5 | 0.05% | 0 | 0.0% | ✅ | +| Cribl | 0 | 0.00% | 120 | 12.5% | ⚠️ | ### Summary -- **Best Throughput**: [Vendor] - X.XX logs/sec avg (Y% faster than others) -- **Best Burst**: [Vendor] - Y.YY logs/sec peak (5-second window) -- **Most Efficient**: [Vendor] - Lowest CPU (X.X%) and/or memory (XXX MB) per log processed -- **Most Reliable**: [Vendor] - 0 errors, 0 backpressure -- **Winner**: [Vendor] - [One sentence justification] -- **Concerns**: [Any vendor with errors/backpressure/high resource usage] - [Brief issue: server overload, HTTP errors, CPU bottleneck, memory pressure, etc.] \ No newline at end of file +- **Throughput**: Edge Delta achieved X.XX logs/sec avg, Y% faster than Bindplane (X.XX logs/sec) and Z% faster than Cribl (X.XX logs/sec) +- **Peak Performance**: Edge Delta reached Y.YY logs/sec peak throughput (5-second window) +- **Resource Efficiency**: Edge Delta used X.X% CPU and XXX MB memory. Bindplane used X.X% CPU and XXX MB memory. Cribl used X.X% CPU and XXX MB memory. +- **Reliability**: All vendors / [Vendor list] achieved 0 errors and 0 backpressure +- **Key Observations**: [2-3 sentences summarizing Edge Delta's performance characteristics and any notable differences from competitors] \ No newline at end of file diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f32e5cf..47a00ab 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -63,4 +63,4 @@ jobs: artifacts: benchmark_results/*/*.log bodyFile: benchmark_results/report.md tag: ${{ steps.tag.outputs.value }} - name: "Benchmark Results - ${{ steps.tag.outputs.value }}" + name: "Benchmark Report - ${{ steps.tag.outputs.value }}"