add configurable Prometheus metrics with panic recovery and tests

.coverage

@@ -1,7 +1,7 @@
 mode: set
 github.com/adjoeio/djoemo/dynamo_global_index.go:22.50,24.2 1 0
 github.com/adjoeio/djoemo/dynamo_global_index.go:27.71,29.2 1 0
-github.com/adjoeio/djoemo/dynamo_global_index.go:32.98,36.2 3 0
+github.com/adjoeio/djoemo/dynamo_global_index.go:32.121,36.2 3 1
 github.com/adjoeio/djoemo/dynamo_global_index.go:39.105,43.39 3 1
 github.com/adjoeio/djoemo/dynamo_global_index.go:43.39,45.3 1 1
 github.com/adjoeio/djoemo/dynamo_global_index.go:47.2,48.16 2 1
@@ -52,7 +52,7 @@ github.com/adjoeio/djoemo/dynamo_global_index.go:168.16,170.3 1 1
 github.com/adjoeio/djoemo/dynamo_repository.go:25.80,31.2 1 1
 github.com/adjoeio/djoemo/dynamo_repository.go:34.57,36.2 1 1
 github.com/adjoeio/djoemo/dynamo_repository.go:39.78,41.2 1 1
-github.com/adjoeio/djoemo/dynamo_repository.go:44.104,48.2 3 0
+github.com/adjoeio/djoemo/dynamo_repository.go:44.127,48.2 3 1
 github.com/adjoeio/djoemo/dynamo_repository.go:53.120,57.39 3 1
 github.com/adjoeio/djoemo/dynamo_repository.go:57.39,59.3 1 1
 github.com/adjoeio/djoemo/dynamo_repository.go:61.2,62.16 2 1
@@ -280,32 +280,49 @@ github.com/adjoeio/djoemo/metrics.go:84.124,85.26 1 0
 github.com/adjoeio/djoemo/metrics.go:85.26,87.3 1 0
 github.com/adjoeio/djoemo/metrics_cloudwatch.go:10.126,13.2 0 0
 github.com/adjoeio/djoemo/metrics_cloudwatch.go:16.111,17.2 0 0
-github.com/adjoeio/djoemo/metrics_prometheus.go:24.78,30.53 3 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:30.53,31.61 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:31.61,33.4 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:34.3,34.13 1 0
-github.com/adjoeio/djoemo/metrics_prometheus.go:36.2,36.16 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:39.85,47.55 3 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:47.55,48.61 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:48.61,50.4 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:51.3,51.13 1 0
-github.com/adjoeio/djoemo/metrics_prometheus.go:53.2,53.18 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:63.77,70.2 2 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:72.128,78.32 5 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:78.32,80.34 2 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:80.34,82.4 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:83.3,83.37 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:83.37,85.4 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:86.3,88.16 3 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:91.2,92.13 2 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:92.13,94.3 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:96.2,97.41 2 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:97.41,99.3 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:101.2,106.31 3 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:106.31,108.46 1 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:108.46,112.4 2 1
-github.com/adjoeio/djoemo/metrics_prometheus.go:112.9,114.4 1 0
-github.com/adjoeio/djoemo/metrics_prometheus.go:117.2,118.52 2 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:33.50,40.2 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:52.78,61.53 3 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:61.53,62.61 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:62.61,64.4 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:65.3,65.13 1 0
+github.com/adjoeio/djoemo/metrics_prometheus.go:67.2,67.16 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:70.85,72.23 2 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:72.23,74.3 1 0
+github.com/adjoeio/djoemo/metrics_prometheus.go:75.2,85.55 3 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:85.55,86.61 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:86.61,88.4 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:89.3,89.13 1 0
+github.com/adjoeio/djoemo/metrics_prometheus.go:91.2,91.18 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:102.100,103.16 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:103.16,105.3 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:106.2,112.10 2 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:115.128,116.15 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:116.15,117.31 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:117.31,119.24 2 0
+github.com/adjoeio/djoemo/metrics_prometheus.go:119.24,121.5 1 0
+github.com/adjoeio/djoemo/metrics_prometheus.go:121.10,123.5 1 0
+github.com/adjoeio/djoemo/metrics_prometheus.go:127.2,132.32 5 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:132.32,134.34 2 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:134.34,136.4 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:137.3,137.37 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:137.37,139.4 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:140.3,142.16 3 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:145.2,146.13 2 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:146.13,148.3 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:150.2,151.41 2 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:151.41,153.3 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:155.2,160.31 3 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:160.31,162.3 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:164.2,165.52 2 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:177.13,179.8 2 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:179.8,181.3 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:199.30,206.6 5 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:206.6,208.34 2 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:208.34,211.4 2 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:212.3,212.12 1 1
+github.com/adjoeio/djoemo/metrics_prometheus.go:212.12,213.9 1 0
+github.com/adjoeio/djoemo/metrics_prometheus.go:217.2,217.18 1 0
+github.com/adjoeio/djoemo/metrics_prometheus.go:222.39,224.2 1 1
 github.com/adjoeio/djoemo/model.go:11.35,13.2 1 1
 github.com/adjoeio/djoemo/model.go:16.35,18.2 1 1
 github.com/adjoeio/djoemo/model.go:21.33,22.24 1 1

dynamo_global_index.go

@@ -28,9 +28,9 @@ func (gi *GlobalIndex) WithMetrics(metricsInterface MetricsInterface) {
 	gi.metrics.Add(metricsInterface)
 }
 
-// WithPrometheusMetrics enables prometheus metrics
-func (gi *GlobalIndex) WithPrometheusMetrics(registry *prometheus.Registry) GlobalIndexInterface {
-	prommetrics := NewPrometheusMetrics(registry)
+// WithPrometheusMetrics enables prometheus metrics with the given config
+func (gi *GlobalIndex) WithPrometheusMetrics(registry *prometheus.Registry, cfg *PrometheusConfig) GlobalIndexInterface {
+	prommetrics := NewPrometheusMetrics(registry, cfg)
 	gi.metrics.Add(prommetrics)
 	return gi
 }

dynamo_global_index_interface.go

@@ -15,8 +15,8 @@ type GlobalIndexInterface interface {
 	// WithMetrics enables metrics; it accepts MetricsInterface as metrics publisher
 	WithMetrics(metricsInterface MetricsInterface)
 
-	// WithPrometheusMetrics enables prometheus metrics
-	WithPrometheusMetrics(registry *prometheus.Registry) GlobalIndexInterface
+	// WithPrometheusMetrics enables prometheus metrics with the given config
+	WithPrometheusMetrics(registry *prometheus.Registry, cfg *PrometheusConfig) GlobalIndexInterface
 
 	// GetItemWithContext get item from index; it accepts a key interface that is used to get the table name, hash key and range key if it exists;
 	// context which used to enable log with context; the output will be given in item

dynamo_repository.go

@@ -40,9 +40,9 @@ func (repository *Repository) WithMetrics(metricsInterface MetricsInterface) {
 	repository.metrics.Add(metricsInterface)
 }
 
-// WithPrometheusMetrics enables prometheus metrics
-func (repository *Repository) WithPrometheusMetrics(registry *prometheus.Registry) RepositoryInterface {
-	prommetrics := NewPrometheusMetrics(registry)
+// WithPrometheusMetrics enables prometheus metrics with the given config
+func (repository *Repository) WithPrometheusMetrics(registry *prometheus.Registry, cfg *PrometheusConfig) RepositoryInterface {
+	prommetrics := NewPrometheusMetrics(registry, cfg)
 	repository.metrics.Add(prommetrics)
 	return repository
 }

dynamo_repository_interface.go

@@ -17,8 +17,8 @@ type RepositoryInterface interface {
 	// WithMetrics enables metrics; it accepts MetricsInterface as metrics publisher
 	WithMetrics(metricsInterface MetricsInterface)
 
-	// WithPrometheusMetrics enables prometheus metrics
-	WithPrometheusMetrics(registry *prometheus.Registry) RepositoryInterface
+	// WithPrometheusMetrics enables prometheus metrics with the given config
+	WithPrometheusMetrics(registry *prometheus.Registry, cfg *PrometheusConfig) RepositoryInterface
 
 	// GetItemWithContext get item; it accepts a key interface that is used to get the table name, hash key and range key if it exists; the output will be given in item
 	// returns true if item is found, returns false and nil if no item found, returns false and an error in case of error

metrics_prometheus.go

@@ -2,6 +2,8 @@ package djoemo
 
 import (
 	"context"
+	"fmt"
+	"log"
 	"maps"
 	"path"
 	"runtime"
@@ -12,8 +14,34 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+// PrometheusConfig holds configuration for Prometheus metrics.
+type PrometheusConfig struct {
+	// Namespace is the metric namespace (e.g., "adjoe").
+	Namespace string
+	// Subsystem is the metric subsystem (e.g., "djoemo").
+	Subsystem string
+	// HistogramBuckets defines the histogram bucket boundaries in seconds.
+	// If nil, defaults to ExponentialBuckets(0.001, 2.5, 12) (1ms to ~60s).
+	HistogramBuckets []float64
+	// ConstLabels are labels added to all metrics.
+	ConstLabels prometheus.Labels
+	// Log is an optional logger for panic recovery. If nil, uses standard log.
+	Log LogInterface
+}
+
+// DefaultPrometheusConfig returns a config with sensible defaults.
+func DefaultPrometheusConfig() *PrometheusConfig {
+	return &PrometheusConfig{
+		Namespace:        "adjoe",
+		Subsystem:        "djoemo",
+		HistogramBuckets: prometheus.ExponentialBuckets(0.001, 2.5, 12),
+		Log:              NewNopLog(),
+	}
+}
+
 type prometheusmetrics struct {
 	registry      *prometheus.Registry
+	cfg           *PrometheusConfig
 	mu            sync.RWMutex
 	queryCount    map[string]*prometheus.CounterVec
 	queryDuration map[string]*prometheus.HistogramVec
@@ -23,8 +51,11 @@ var metricLabelNames = []string{statusLabel, tableLabel, sourceLabel}
 
 func (m *prometheusmetrics) newCounter(caller string) *prometheus.CounterVec {
 	opts := prometheus.CounterOpts{
-		Name: strings.ToLower(caller),
-		Help: "counter for function " + caller,
+		Namespace:   m.cfg.Namespace,
+		Subsystem:   m.cfg.Subsystem,
+		Name:        strings.ToLower(caller),
+		Help:        "counter for function " + caller,
+		ConstLabels: m.cfg.ConstLabels,
 	}
 	counter := prometheus.NewCounterVec(opts, metricLabelNames)
 	if err := m.registry.Register(counter); err != nil {
@@ -37,10 +68,17 @@ func (m *prometheusmetrics) newCounter(caller string) *prometheus.CounterVec {
 }
 
 func (m *prometheusmetrics) newHistogramVec(caller string) *prometheus.HistogramVec {
+	buckets := m.cfg.HistogramBuckets
+	if len(buckets) == 0 {
+		buckets = prometheus.ExponentialBuckets(0.001, 2.5, 12)
+	}
 	opts := prometheus.HistogramOpts{
-		Name:    strings.ToLower(caller) + "_duration_seconds",
-		Help:    "histogram duration for function " + caller + " in seconds",
-		Buckets: prometheus.ExponentialBuckets(0.001, 2.5, 5),
+		Namespace:   m.cfg.Namespace,
+		Subsystem:   m.cfg.Subsystem,
+		Name:        strings.ToLower(caller) + "_duration_seconds",
+		Help:        "histogram duration for function " + caller + " in seconds",
+		Buckets:     buckets,
+		ConstLabels: m.cfg.ConstLabels,
 	}
 	// WARNING: add high cardinality labels like sdkhash, etc with caution
 	histogram := prometheus.NewHistogramVec(opts, metricLabelNames)
@@ -60,16 +98,32 @@ const (
 	tableLabel  = "table"
 )
 
-func NewPrometheusMetrics(registry *prometheus.Registry) *prometheusmetrics {
+// NewPrometheusMetrics creates Prometheus metrics with default config.
+func NewPrometheusMetrics(registry *prometheus.Registry, cfg *PrometheusConfig) *prometheusmetrics {
+	if cfg == nil {
+		cfg = DefaultPrometheusConfig()
+	}
 	m := &prometheusmetrics{
 		registry:      registry,
+		cfg:           cfg,
 		queryCount:    make(map[string]*prometheus.CounterVec),
 		queryDuration: make(map[string]*prometheus.HistogramVec),
 	}
 	return m
 }
 
 func (m *prometheusmetrics) Record(ctx context.Context, caller string, key KeyInterface, duration time.Duration, success bool) {
+	defer func() {
+		if r := recover(); r != nil {
+			msg := fmt.Sprintf("prometheus metrics Record panic recovered: caller=%q panic=%v", caller, r)
+			if m.cfg.Log != nil {
+				m.cfg.Log.WithContext(ctx).Error(msg)
+			} else {
+				log.Printf("djoemo: %s", msg)
+			}
+		}
+	}()
+
 	m.mu.RLock()
 	counter, counterOk := m.queryCount[caller]
 	histogram, histogramOk := m.queryDuration[caller]
@@ -104,16 +158,67 @@ func (m *prometheusmetrics) Record(ctx context.Context, caller string, key KeyIn
 	}
 	maps.Copy(labels, GetLabelsFromContext(ctx))
 	if labels[sourceLabel] == "" {
-		// Set to the filename of the calling function rather than the caller string
-		if _, file, _, ok := runtime.Caller(2); ok {
-			// Extract just the file name, not the full path
-			_, filename := path.Split(file)
-			labels[sourceLabel] = filename
-		} else {
-			labels[sourceLabel] = "unknown"
-		}
+		labels[sourceLabel] = externalCaller()
 	}
 
 	counter.With(labels).Inc()
 	histogram.With(labels).Observe(duration.Seconds())
 }
+
+// libraryDir is the directory on disk that contains this library's source files,
+// determined once at init time via runtime.Caller(0). We compare frame file paths
+// against this directory to decide whether a frame belongs to this library.
+//
+// This approach is immune to:
+//   - go.mod replace directives that change the module path. Ex: replace github.com/adjoeio/djoemo => github.com/pm-nilesh-chate/djoemo v0.2.1-0.20260301162703-078983d9e34a
+//   - callers that happen to have a package also named "djoemo". Ex. package djoemo in main backend repo
+var libraryDir string
+
+func init() {
+	_, file, _, ok := runtime.Caller(0)
+	if ok {
+		libraryDir = path.Dir(file)
+	}
+}
+
+// externalCaller walks the call stack and returns the first caller outside this
+// library as "filename:line". This gives the user-facing source location that
+// triggered the DynamoDB operation rather than an internal library frame.
+//
+// Example stack when Repository.GetItemWithContext is called from user code:
+//
+//	frame 0: runtime.Callers                                          (runtime)
+//	frame 1: djoemo.externalCaller                                    (metrics_prometheus.go)
+//	frame 2: djoemo.(*prometheusmetrics).Record                       (metrics_prometheus.go)
+//	frame 3: djoemo.(*Metrics).Record                                 (metrics.go)
+//	frame 4: djoemo.Repository.GetItemWithContext.recordMetrics.func1  (dynamo_repository.go) <- deferred closure
+//	frame 5: djoemo.Repository.GetItemWithContext                      (dynamo_repository.go)
+//	frame 6: main/service.(*UserService).GetUser                      (user_service.go:42)   <- first external caller ✓
+//
+// The returned value would be "user_service.go:42".
+func externalCaller() string {
+	const maxDepth = 15
+	var pcs [maxDepth]uintptr
+	// skip 0=Callers, 1=externalCaller, start from 2
+	n := runtime.Callers(2, pcs[:])
+	frames := runtime.CallersFrames(pcs[:n])
+
+	for {
+		frame, more := frames.Next()
+		if !isLibraryFrame(frame.File) {
+			_, filename := path.Split(frame.File)
+			return fmt.Sprintf("%s:%d", filename, frame.Line)
+		}
+		if !more {
+			break
+		}
+	}
+
+	return "unknown"
+}
+
+// isLibraryFrame reports whether the given file path belongs to this library
+// by checking if it resides in the same directory as this package's source files.
+func isLibraryFrame(file string) bool {
+	return libraryDir != "" && path.Dir(file) == libraryDir
+}