feat: add prometheus metrics for workload manager and router observability

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/gin-gonic/gin v1.10.0
 	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/google/uuid v1.6.0
+	github.com/prometheus/client_golang v1.23.2
 	github.com/redis/go-redis/v9 v9.17.1
 	github.com/stretchr/testify v1.11.1
 	github.com/valkey-io/valkey-go v1.0.69
@@ -67,14 +68,14 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.1 // indirect
 	github.com/prometheus/procfs v0.17.0 // indirect

manifests/charts/base/templates/agentcube-router.yaml

@@ -101,6 +101,10 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    prometheus.io/path: /metrics
+    prometheus.io/port: {{ .Values.router.service.port | quote }}
+    prometheus.io/scrape: "true"
   name: agentcube-router
   namespace: {{ .Release.Namespace }}
   labels:

manifests/charts/base/templates/workloadmanager.yaml

@@ -97,6 +97,10 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    prometheus.io/path: /metrics
+    prometheus.io/port: {{ .Values.workloadmanager.service.port | quote }}
+    prometheus.io/scrape: "true"
   name: workloadmanager
   namespace: {{ .Release.Namespace }}
   labels:

pkg/router/handlers.go

@@ -56,6 +56,11 @@ func (s *Server) handleHealthReady(c *gin.Context) {
 
 // handleInvoke is a private helper function that handles invocation requests for both agents and code interpreters
 func (s *Server) handleInvoke(c *gin.Context, namespace, name, path, kind string) {
+	start := time.Now()
+	defer func() {
+		routerRequestDuration.WithLabelValues(kind).Observe(time.Since(start).Seconds())
+	}()
+
 	klog.V(4).Infof("%s invoke request: namespace=%s, name=%s, path=%s", kind, namespace, name, path)
 
 	// Extract session ID from header
@@ -201,14 +206,24 @@ func (s *Server) waitForUpstreamReachable(ctx context.Context, targetURL *url.UR
 
 func upstreamUnavailableResponse(err error) (int, gin.H) {
 	errText := strings.ToLower(err.Error())
+	var category string
+	var code int
+	var resp gin.H
+
 	switch {
 	case connectionRefusedRetryable(err):
-		return http.StatusBadGateway, gin.H{"error": "sandbox unreachable"}
+		category = "connection_refused"
+		code, resp = http.StatusBadGateway, gin.H{"error": "sandbox unreachable"}
 	case strings.Contains(errText, "deadline exceeded") || strings.Contains(errText, "timeout"):
-		return http.StatusGatewayTimeout, gin.H{"error": "sandbox timeout"}
+		category = "timeout"
+		code, resp = http.StatusGatewayTimeout, gin.H{"error": "sandbox timeout"}
 	default:
-		return http.StatusServiceUnavailable, gin.H{"error": "sandbox unreachable"}
+		category = "other"
+		code, resp = http.StatusServiceUnavailable, gin.H{"error": "sandbox unreachable"}
 	}
+
+	routerProxyErrorsTotal.WithLabelValues(category).Inc()
+	return code, resp
 }
 
 func (s *Server) resolveSandboxTarget(c *gin.Context, sandbox *types.SandboxInfo, path string) (*url.URL, bool) {

pkg/router/metrics.go

@@ -0,0 +1,55 @@
+/*
+Copyright The Volcano Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	routerRequestDuration = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "agentcube_router_request_duration_seconds",
+			Help:    "Duration of proxy requests through the router in seconds.",
+			Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30},
+		},
+		[]string{"kind"},
+	)
+
+	routerProxyErrorsTotal = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "agentcube_router_proxy_errors_total",
+			Help: "Total number of proxy errors encountered by the router.",
+		},
+		[]string{"error_category"},
+	)
+
+	routerConcurrentRequests = promauto.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "agentcube_router_concurrent_requests",
+			Help: "Current number of concurrent requests being processed by the router.",
+		},
+	)
+
+	routerSessionCreateTotal = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "agentcube_router_session_create_total",
+			Help: "Total number of sandbox sessions implicitly created via the router.",
+		},
+	)
+)

pkg/router/metrics_test.go

@@ -0,0 +1,64 @@
+/*
+Copyright The Volcano Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRouterMetricsRegistrationAndCollection(t *testing.T) {
+	// Reset/initialize counters to know starting points (metrics are global)
+	initialSessionCreates := testutil.ToFloat64(routerSessionCreateTotal)
+	routerSessionCreateTotal.Inc()
+	assert.Equal(t, initialSessionCreates+1, testutil.ToFloat64(routerSessionCreateTotal))
+
+	initialProxyErrors := testutil.ToFloat64(routerProxyErrorsTotal.WithLabelValues("connection_refused"))
+	routerProxyErrorsTotal.WithLabelValues("connection_refused").Inc()
+	assert.Equal(t, initialProxyErrors+1, testutil.ToFloat64(routerProxyErrorsTotal.WithLabelValues("connection_refused")))
+
+	// Gauge test
+	routerConcurrentRequests.Set(5.0)
+	assert.Equal(t, float64(5), testutil.ToFloat64(routerConcurrentRequests))
+	routerConcurrentRequests.Set(0.0)
+
+	// Histogram test
+	routerRequestDuration.WithLabelValues("AgentRuntime").Observe(0.12)
+}
+
+func TestRouterMetricsRoute(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	server := &Server{config: &Config{MaxConcurrentRequests: 10}}
+	server.setupRoutes()
+	routerProxyErrorsTotal.WithLabelValues("other").Add(0)
+
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest(http.MethodGet, "/metrics", nil)
+	server.engine.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusOK, w.Code)
+	body := w.Body.String()
+	assert.True(t, strings.Contains(body, "agentcube_router_session_create_total"))
+	assert.True(t, strings.Contains(body, "agentcube_router_proxy_errors_total"))
+	assert.True(t, strings.Contains(body, "agentcube_router_concurrent_requests"))
+}

pkg/router/server.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"k8s.io/klog/v2"
 
 	"github.com/volcano-sh/agentcube/pkg/store"
@@ -111,10 +112,12 @@ func (s *Server) concurrencyLimitMiddleware() gin.HandlerFunc {
 		// Try to acquire a slot in the semaphore
 		select {
 		case concurrency <- struct{}{}:
+			routerConcurrentRequests.Inc()
 			// Successfully acquired a slot, continue processing
 			defer func() {
 				// Release the slot when done
 				<-concurrency
+				routerConcurrentRequests.Dec()
 			}()
 			c.Next()
 		default:
@@ -135,6 +138,7 @@ func (s *Server) setupRoutes() {
 	// Health check endpoints (no authentication required, no concurrency limit)
 	s.engine.GET("/health/live", s.handleHealthLive)
 	s.engine.GET("/health/ready", s.handleHealthReady)
+	s.engine.GET("/metrics", gin.WrapH(promhttp.Handler()))
 
 	// API v1 routes with concurrency limiting
 	v1 := s.engine.Group("/v1")

pkg/router/session_manager.go

@@ -183,6 +183,8 @@ func (m *manager) createSandbox(ctx context.Context, namespace string, name stri
 		return nil, api.NewInternalError(fmt.Errorf("response with empty session id from workload manager"))
 	}
 
+	routerSessionCreateTotal.Inc()
+
 	// Construct Sandbox Info from response
 	sandbox := &types.SandboxInfo{
 		Kind:             res.Kind,

pkg/workloadmanager/garbage_collection.go

@@ -74,6 +74,11 @@ func (gc *garbageCollector) run(stopCh <-chan struct{}) {
 }
 
 func (gc *garbageCollector) once() {
+	start := time.Now()
+	defer func() {
+		gcCycleDuration.Observe(time.Since(start).Seconds())
+	}()
+
 	ctx, cancel := context.WithTimeout(context.Background(), gcOnceTimeout)
 	defer cancel()
 	now := time.Now()
@@ -84,11 +89,13 @@ func (gc *garbageCollector) once() {
 	candidates, err := gc.storeClient.ListInactiveSandboxes(ctx, now.Add(-gcMinInactiveLookback), gcCandidateLimit)
 	if err != nil {
 		klog.Errorf("garbage collector error listing inactive sandboxes: %v", err)
+		gcErrorsTotal.Inc()
 	}
 
 	// Apply per-sandbox idle timeout: only include sandboxes whose own IdleTimeout
 	// (stored in the session JSON) has actually elapsed since LastActivityAt.
 	inactiveSandboxes := make([]*types.SandboxInfo, 0, len(candidates))
+	reclaimReason := make(map[string]string)
 	for _, s := range candidates {
 		activityAt := s.LastActivityAt
 		if activityAt.IsZero() {
@@ -101,14 +108,20 @@ func (gc *garbageCollector) once() {
 		}
 		if activityAt.Add(idleTimeout).Before(now) {
 			inactiveSandboxes = append(inactiveSandboxes, s)
+			reclaimReason[s.SessionID] = "inactive"
 		}
 	}
 
 	// List sandboxes that have reached their expiry deadline
 	expiredSandboxes, err := gc.storeClient.ListExpiredSandboxes(ctx, now, gcCandidateLimit)
 	if err != nil {
 		klog.Errorf("garbage collector error listing expired sandboxes: %v", err)
+		gcErrorsTotal.Inc()
+	}
+	for _, s := range expiredSandboxes {
+		reclaimReason[s.SessionID] = "expired"
 	}
+
 	// Merge and deduplicate: a sandbox may appear in both lists when it is
 	// simultaneously idle-timed-out and past its TTL.
 	gcSandboxes := deduplicateSandboxes(inactiveSandboxes, expiredSandboxes)
@@ -127,12 +140,20 @@ func (gc *garbageCollector) once() {
 		}
 		if err != nil {
 			errs = append(errs, err)
+			gcErrorsTotal.Inc()
 			continue
 		}
 		klog.Infof("garbage collector %s %s/%s session %s deleted", gcSandbox.Kind, gcSandbox.SandboxNamespace, gcSandbox.Name, gcSandbox.SessionID)
 		err = gc.storeClient.DeleteSandboxBySessionID(ctx, gcSandbox.SessionID)
 		if err != nil {
 			errs = append(errs, err)
+			gcErrorsTotal.Inc()
+		} else {
+			reason := reclaimReason[gcSandbox.SessionID]
+			if reason == "" {
+				reason = "inactive"
+			}
+			gcSandboxesReclaimedTotal.WithLabelValues(reason).Inc()
 		}
 	}
 	err = utilerrors.NewAggregate(errs)