diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml
index 62e2f89..bb531d6 100644
--- a/.github/workflows/container.yml
+++ b/.github/workflows/container.yml
@@ -1,47 +1,47 @@
-name: Create and publish a Docker image then deploy it
-on:
- workflow_dispatch:
- push:
- paths:
- - '**.go'
-env:
- REGISTRY: ghcr.io
- IMAGE_NAME: '${{ github.repository }}'
-
-jobs:
- build-and-push-image:
- runs-on: ubuntu-latest
- permissions:
- contents: read
- packages: write
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4.1.1
- - name: Log in to the Container registry
- uses: docker/login-action@v3.1.0
- with:
- registry: '${{ env.REGISTRY }}'
- username: '${{ github.actor }}'
- password: '${{ secrets.GITHUB_TOKEN }}'
- - name: 'Extract metadata (tags, labels) for Docker'
- id: meta
- uses: docker/metadata-action@v5.5.1
- with:
- images: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}'
- - name: Build and push Docker image
- uses: docker/build-push-action@v5.3.0
- with:
- context: .
- push: true
- tags: '${{ steps.meta.outputs.tags }}'
- labels: '${{ steps.meta.outputs.labels }}'
- deploy:
- needs: build-and-push-image
- runs-on: ubuntu-latest
- steps:
- - name: sleep for 5s
- run: sleep 5s
- shell: bash
- - name: redeploy
- run: 'curl -s ${{secrets.DEPLOY_HOOK}} > /dev/null'
- shell: bash
+name: Create and publish a Docker image then deploy it
+on:
+ workflow_dispatch:
+ push:
+ paths:
+ - "**.go"
+env:
+ REGISTRY: ghcr.io
+ IMAGE_NAME: "${{ github.repository }}"
+
+jobs:
+ build-and-push-image:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4.1.1
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3.1.0
+ with:
+ registry: "${{ env.REGISTRY }}"
+ username: "${{ github.actor }}"
+ password: "${{ secrets.GITHUB_TOKEN }}"
+ - name: "Extract metadata (tags, labels) for Docker"
+ id: meta
+ uses: docker/metadata-action@v5.5.1
+ with:
+ images: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
+ - name: Build and push Docker image
+ uses: docker/build-push-action@v5.3.0
+ with:
+ context: .
+ push: true
+ tags: "${{ steps.meta.outputs.tags }}"
+ labels: "${{ steps.meta.outputs.labels }}"
+ deploy:
+ needs: build-and-push-image
+ runs-on: ubuntu-latest
+ steps:
+ - name: sleep for 5s
+ run: sleep 5s
+ shell: bash
+ - name: redeploy
+ run: "curl -s ${{secrets.DEPLOY_HOOK}} > /dev/null"
+ shell: bash
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 7d4b796..4e6efef 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -1,86 +1,86 @@
-name: Test Abacus
-on:
- push:
- paths:
- - '**.go'
- pull_request:
- paths:
- - '**.go'
-
-jobs:
- test:
- runs-on: ubuntu-latest
-
- services:
- redis:
- image: redis
- options: >-
- --health-cmd "redis-cli ping"
- --health-interval 10s
- --health-timeout 5s
- --health-retries 5
- ports:
- - 6379:6379
-
- steps:
- - name: Checkout
- uses: actions/checkout@v4
-
- - name: Setup Go
- uses: actions/setup-go@v5
- with:
- go-version: '^1.23.4'
-
- - name: Install dependencies
- run: go mod download
-
- - name: Setup K6
- uses: grafana/setup-k6-action@v1
-
- - name: Run Unit tests
- run: go test ./...
-
- - name: Start Abacus server
- run: |
- # Set required environment variables
- export PORT=8080
- export REDIS_HOST=localhost
- export REDIS_PORT=6379
-
- # Start server in background
- nohup go run . > server.log 2>&1 &
- echo $! > server.pid
-
- # Wait for the server to be ready (max 30 seconds)
- echo "Waiting for Abacus server to start..."
- timeout=30
- while ! curl -s http://localhost:8080/healthcheck > /dev/null; do
- if [[ $timeout -eq 0 ]]; then
- echo "Timed out waiting for server to start"
- cat server.log
- exit 1
- fi
- echo "Server not ready yet, waiting..."
- sleep 1
- ((timeout--))
- done
- echo "Server is running!"
-
- - name: Run k6 tests
- run: |
- # Run the performance tests
- k6 run --vus 10 --duration 30s ./tests/performance.js
-
- - name: Check server logs
- if: always()
- run: |
- echo "Server logs:"
- cat server.log
-
- - name: Stop Abacus server
- if: always()
- run: |
- if [ -f server.pid ]; then
- kill $(cat server.pid) || true
- echo "Server stopped"
- fi
+name: Test Abacus
+on:
+ push:
+ paths:
+ - "**.go"
+ pull_request:
+ paths:
+ - "**.go"
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+
+ services:
+ redis:
+ image: redis
+ options: >-
+ --health-cmd "redis-cli ping"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+ ports:
+ - 6379:6379
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Setup Go
+ uses: actions/setup-go@v5
+ with:
+ go-version: "^1.23.4"
+
+ - name: Install dependencies
+ run: go mod download
+
+ - name: Setup K6
+ uses: grafana/setup-k6-action@v1
+
+ - name: Run Unit tests
+ run: go test ./...
+
+ - name: Start Abacus server
+ run: |
+ # Set required environment variables
+ export PORT=8080
+ export REDIS_HOST=localhost
+ export REDIS_PORT=6379
+
+ # Start server in background
+ nohup go run . > server.log 2>&1 &
+ echo $! > server.pid
+
+ # Wait for the server to be ready (max 30 seconds)
+ echo "Waiting for Abacus server to start..."
+ timeout=30
+ while ! curl -s http://localhost:8080/healthcheck > /dev/null; do
+ if [[ $timeout -eq 0 ]]; then
+ echo "Timed out waiting for server to start"
+ cat server.log
+ exit 1
+ fi
+ echo "Server not ready yet, waiting..."
+ sleep 1
+ ((timeout--))
+ done
+ echo "Server is running!"
+
+ - name: Run k6 tests
+ run: |
+ # Run the performance tests
+ k6 run --vus 10 --duration 30s ./tests/performance.js
+
+ - name: Check server logs
+ if: always()
+ run: |
+ echo "Server logs:"
+ cat server.log
+
+ - name: Stop Abacus server
+ if: always()
+ run: |
+ if [ -f server.pid ]; then
+ kill $(cat server.pid) || true
+ echo "Server stopped"
+ fi
diff --git a/Dockerfile b/Dockerfile
index 8522a95..f38748c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,22 +1,23 @@
+
# Build stage
-FROM golang:1.24 as builder
+FROM golang:1.24 AS builder
WORKDIR /src
COPY . .
RUN go mod download
RUN CGO_ENABLED=0 GOOS=linux go build -o ./abacus -tags=jsoniter
# Run stage
-FROM alpine:latest
+
+FROM scratch
+
+LABEL maintainer="Jason Cameron abacus@jasoncameron.dev"
+LABEL version="1.5.5"
+LABEL description="This is a simple countAPI service written in Go."
+
+
COPY --from=builder /src/abacus /abacus
COPY assets /assets
EXPOSE 8080
ENV GIN_MODE=release
-#USER nonroot:nonroot
CMD ["/abacus"]
-# note: curl is not installed by default in alpine so we use wget
-HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 CMD wget -S -O - http://0.0.0.0:8080/healthcheck || exit 1
-
-LABEL maintainer="Jason Cameron abacus@jasoncameron.dev"
-LABEL version="1.5.4"
-LABEL description="This is a simple countAPI service written in Go."
diff --git a/README.md b/README.md
index 2836772..ed89ea1 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
# Abacus—A highly scalable and stateless counting API
-
+
> Note: Abacus was designed as a direct replacement/upgrade for [CountAPI](https://countapi.xyz/) as it got taken down
In order to get started, please visit the docs at https://abacus.jsn.cam
@@ -10,23 +10,20 @@ In order to get started, please visit the docs at https://abacus.jsn.cam
- JSONP Support: Seamlessly integrate Abacus into your web applications with cross-origin resource sharing (CORS) support.
-->
-
---
-
+
### Development
1. Install Golang & Redis
2. Run `go mod install` to install the dependencies
-
3. Add a `.env` file to the root of the project (or set the environment variables manually) following the format specified in .env.example
4. Run `air` (if installed) or `go run .` to build and run the API locally.
5. The API will be running on `http://0.0.0.0:8080` by default.
-
## Todos
@@ -35,11 +32,10 @@ In order to get started, please visit the docs at https://abacus.jsn.cam
- [x] JSONP Support (https://gin-gonic.com/docs/examples/jsonp/)
- [x] impl /create endpoint which creates a new counter initialized to 0 and returns a secret key that can be used to modify the counter via the following endpoints
- [x] /delete endpoint
- - [x] /set endpoint
+ - [x] /set endpoint
- [x] /reset (alias to /set 0)
- [x] /update endpoint (updates the counter x)
- [x] SSE Stream for the counters? Low priority.
- [x] Tests
- [x] Rate limiting (max 30 requests per 3 second per IP address)
- [ ] Create [Python](https://github.com/BenJetson/py-countapi), [JS Wrappers](https://github.com/mlomb/countapi-js) & Go client libraries
-
diff --git a/cmd/loadtest/main.go b/cmd/loadtest/main.go
new file mode 100644
index 0000000..1728686
--- /dev/null
+++ b/cmd/loadtest/main.go
@@ -0,0 +1,294 @@
+package main
+
+import (
+ "bufio"
+ "context"
+ "flag"
+ "fmt"
+ "log"
+ "net/http"
+ "runtime"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+var (
+ targetConnections = flag.Int("connections", 10000, "Number of concurrent connections")
+ serverURL = flag.String("url", "http://localhost:8080", "Server URL")
+ testKey = flag.String("key", "loadtest/10k", "Test key path")
+ duration = flag.Duration("duration", 30*time.Second, "Test duration")
+ rampUpRate = flag.Int("ramp", 500, "Connections per second during ramp-up")
+)
+
+func main() {
+ flag.Parse()
+
+ log.Printf("Starting load test: %d connections to %s", *targetConnections, *serverURL)
+
+ // Create test counter first
+ createCounter()
+
+ // Metrics
+ var (
+ activeConnections int32
+ successfulConnects int32
+ failedConnects int32
+ messagesReceived int64
+ connectionErrors int32
+ readErrors int32
+ totalLatency int64
+ latencyCount int32
+ )
+
+ ctx, cancel := context.WithTimeout(context.Background(), *duration+10*time.Second)
+ defer cancel()
+
+ var wg sync.WaitGroup
+ connections := make(chan *http.Response, *targetConnections)
+
+ // Start metrics reporter
+ go func() {
+ ticker := time.NewTicker(5 * time.Second)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ active := atomic.LoadInt32(&activeConnections)
+ successful := atomic.LoadInt32(&successfulConnects)
+ failed := atomic.LoadInt32(&failedConnects)
+ messages := atomic.LoadInt64(&messagesReceived)
+
+ avgLatency := int64(0)
+ if count := atomic.LoadInt32(&latencyCount); count > 0 {
+ avgLatency = atomic.LoadInt64(&totalLatency) / int64(count)
+ }
+
+ log.Printf("Connections: active=%d, successful=%d, failed=%d | Messages=%d | AvgLatency=%dms",
+ active, successful, failed, messages, avgLatency)
+
+ // Memory stats
+ var m runtime.MemStats
+ runtime.ReadMemStats(&m)
+ log.Printf("Memory: Alloc=%dMB, Sys=%dMB, NumGC=%d, Goroutines=%d",
+ m.Alloc/1024/1024, m.Sys/1024/1024, m.NumGC, runtime.NumGoroutine())
+ }
+ }
+ }()
+
+ // Connection establishment phase
+ log.Println("Starting connection ramp-up...")
+ connectionBatch := *targetConnections / (*rampUpRate / 100)
+ if connectionBatch < 1 {
+ connectionBatch = 1
+ }
+
+ for i := 0; i < *targetConnections; i += connectionBatch {
+ select {
+ case <-ctx.Done():
+ break
+ default:
+ }
+
+ batchSize := connectionBatch
+ if i+batchSize > *targetConnections {
+ batchSize = *targetConnections - i
+ }
+
+ for j := 0; j < batchSize; j++ {
+ wg.Add(1)
+ go func(connID int) {
+ defer wg.Done()
+
+ client := &http.Client{
+ // No timeout for SSE connections - they're long-lived streams
+ Transport: &http.Transport{
+ MaxIdleConns: 100,
+ MaxIdleConnsPerHost: 10,
+ },
+ }
+
+ req, err := http.NewRequest("GET", fmt.Sprintf("%s/stream/%s", *serverURL, *testKey), nil)
+ if err != nil {
+ atomic.AddInt32(&failedConnects, 1)
+ return
+ }
+
+ req.Header.Set("Accept", "text/event-stream")
+
+ start := time.Now()
+ resp, err := client.Do(req)
+ if err != nil {
+ atomic.AddInt32(&failedConnects, 1)
+ atomic.AddInt32(&connectionErrors, 1)
+ log.Printf("Connection error: %v", err)
+ return
+ }
+
+ latency := time.Since(start).Milliseconds()
+ atomic.AddInt64(&totalLatency, latency)
+ atomic.AddInt32(&latencyCount, 1)
+
+ atomic.AddInt32(&successfulConnects, 1)
+ atomic.AddInt32(&activeConnections, 1)
+
+ // Store connection for cleanup later (check if context is done first)
+ select {
+ case <-ctx.Done():
+ resp.Body.Close()
+ return
+ case connections <- resp:
+ // Successfully stored connection
+ }
+
+ // Read events in a separate goroutine - keep connection alive
+ go func(response *http.Response) {
+ defer func() {
+ atomic.AddInt32(&activeConnections, -1)
+ }()
+
+ scanner := bufio.NewScanner(response.Body)
+ for scanner.Scan() {
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ }
+
+ line := scanner.Text()
+ if strings.HasPrefix(line, "data: ") {
+ atomic.AddInt64(&messagesReceived, 1)
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ if err.Error() != "EOF" && !strings.Contains(err.Error(), "closed") {
+ atomic.AddInt32(&readErrors, 1)
+ }
+ }
+ }(resp)
+ }(i + j)
+ }
+
+ time.Sleep(100 * time.Millisecond) // Control ramp-up rate
+ }
+
+ // Wait for connections to establish
+ log.Println("Waiting for connections to establish...")
+ time.Sleep(5 * time.Second)
+
+ // Sustain phase - send updates
+ log.Println("Starting sustained load phase...")
+ sustainCtx, sustainCancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer sustainCancel()
+
+ go func() {
+ ticker := time.NewTicker(1 * time.Second)
+ defer ticker.Stop()
+
+ updateCount := 0
+ for {
+ select {
+ case <-sustainCtx.Done():
+ return
+ case <-ticker.C:
+ // Hit the counter to generate SSE events
+ client := &http.Client{Timeout: 5 * time.Second}
+ resp, err := client.Get(fmt.Sprintf("%s/hit/%s", *serverURL, *testKey))
+ if err == nil {
+ resp.Body.Close()
+ updateCount++
+ log.Printf("Sent update #%d", updateCount)
+ } else {
+ log.Printf("Failed to send update: %v", err)
+ }
+ }
+ }
+ }()
+
+ // Wait for sustain phase
+ <-sustainCtx.Done()
+
+ // Graceful shutdown
+ log.Println("Starting graceful shutdown...")
+ cancel()
+
+ // Close all connections
+ close(connections)
+ for resp := range connections {
+ resp.Body.Close()
+ }
+
+ // Wait for all goroutines
+ done := make(chan struct{})
+ go func() {
+ wg.Wait()
+ close(done)
+ }()
+
+ select {
+ case <-done:
+ log.Println("All connections closed gracefully")
+ case <-time.After(10 * time.Second):
+ log.Println("Timeout waiting for connections to close")
+ }
+
+ // Final metrics
+ fmt.Println("\n=== Final Metrics ===")
+ fmt.Printf("Target Connections: %d\n", *targetConnections)
+ fmt.Printf("Successful Connections: %d\n", atomic.LoadInt32(&successfulConnects))
+ fmt.Printf("Failed Connections: %d\n", atomic.LoadInt32(&failedConnects))
+ fmt.Printf("Connection Errors: %d\n", atomic.LoadInt32(&connectionErrors))
+ fmt.Printf("Read Errors: %d\n", atomic.LoadInt32(&readErrors))
+ fmt.Printf("Total Messages Received: %d\n", atomic.LoadInt64(&messagesReceived))
+
+ avgLatency := int64(0)
+ if count := atomic.LoadInt32(&latencyCount); count > 0 {
+ avgLatency = atomic.LoadInt64(&totalLatency) / int64(count)
+ }
+ fmt.Printf("Average Connection Latency: %dms\n", avgLatency)
+
+ // Memory stats
+ var m runtime.MemStats
+ runtime.ReadMemStats(&m)
+ fmt.Printf("Final Memory - Alloc: %dMB, Sys: %dMB, NumGC: %d\n",
+ m.Alloc/1024/1024, m.Sys/1024/1024, m.NumGC)
+
+ // Get SSE stats from server
+ resp, err := http.Get(fmt.Sprintf("%s/stats", *serverURL))
+ if err == nil {
+ defer resp.Body.Close()
+ // You could parse and display SSE stats here
+ fmt.Println("\nCheck /stats endpoint for SSE server statistics")
+ }
+
+ successRate := float64(atomic.LoadInt32(&successfulConnects)) / float64(*targetConnections)
+ fmt.Printf("\nSuccess Rate: %.2f%%\n", successRate*100)
+
+ if successRate < 0.95 {
+ log.Fatal("Failed to achieve 95% connection success rate")
+ }
+}
+
+func createCounter() {
+ client := &http.Client{Timeout: 5 * time.Second}
+
+ // Try to create the counter
+ resp, err := client.Post(fmt.Sprintf("%s/create/%s", *serverURL, *testKey), "", nil)
+ if err != nil {
+ log.Fatalf("Failed to create counter: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode == http.StatusConflict {
+ log.Println("Counter already exists, continuing...")
+ } else if resp.StatusCode != http.StatusCreated {
+ log.Fatalf("Failed to create counter: status %d", resp.StatusCode)
+ } else {
+ log.Println("Counter created successfully")
+ }
+}
diff --git a/docs/DB.md b/docs/DB.md
index 233aca0..bd83a1b 100644
--- a/docs/DB.md
+++ b/docs/DB.md
@@ -1,11 +1,11 @@
# Scheme
# Standard Keys
+
`K:{namespace}:{key}` = INT64
-if `namespace` is not specified, it is assumed to be `default`.
+if `namespace` is not specified, it is assumed to be `default`.
# Admin Keys
`A:{namespace}:{key}` = 16 byte UUID
-
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 8368bf7..5e70516 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -3,6 +3,7 @@ All Benchmarks are conducted with go run locally, and the redis being ran on the
These numbers were taken using tests/avg.py
## /hit with different TTL implementations (in relation to the data return)
+
### With expire before
AVG: 66.62284ms, MIN: 53.67ms, MAX: 311.721ms, COUNT: 100
@@ -18,5 +19,3 @@ AVG: 43.408190000000005ms, MIN: 28.913999999999998ms, MAX: 167.731ms, COUNT: 100
### With no expire
AVG: 38.95288ms, MIN: 22.008ms, MAX: 165.397ms, COUNT: 100
-
-
diff --git a/docs/bugs/GHSA-vh64-54px-qgf8/results.md b/docs/bugs/GHSA-vh64-54px-qgf8/results.md
index 7a873bd..055c2e7 100644
--- a/docs/bugs/GHSA-vh64-54px-qgf8/results.md
+++ b/docs/bugs/GHSA-vh64-54px-qgf8/results.md
@@ -1,5 +1,5 @@
-
# After malicious script
+
```bash
❯ curl localhost:8080/stream/stream/eee/ -vvv
23:01:20.486586 [0-x] == Info: [READ] client_reset, clear readers
@@ -27,6 +27,7 @@ curl: (56) Recv failure: Connection was reset
```
# Before malicious script
+
```bash
❯ curl localhost:8080/stream/stream/eee/ -vvv
23:01:48.679494 [0-x] == Info: [READ] client_reset, clear readers
diff --git a/docs/index.html b/docs/index.html
index 59ca10d..9b7cb10 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -725,7 +725,7 @@