Week 7: prometheus metrics

services/auth/Makefile

@@ -42,4 +42,15 @@ gen-cert:
 	openssl genrsa -out certificate/keys/service.key 4096
 	openssl req -new -key certificate/keys/service.key -out certificate/keys/service.csr -config certificate/certificate.conf
 	openssl x509 -req -in certificate/keys/service.csr -CA certificate/keys/ca.cert -CAkey certificate/keys/ca.key -CAcreateserial \
-    		-out certificate/keys/service.pem -days 365 -sha256 -extfile certificate/certificate.conf -extensions req_ext
+    		-out certificate/keys/service.pem -days 365 -sha256 -extfile certificate/certificate.conf -extensions req_ext
+
+grpc-load-test:
+	ghz \
+		--proto ../../libraries/api/user/v1/user.proto \
+		--import-paths ../../libraries/api/vendor.protogen/ \
+		--call user_v1.UserV1.Get \
+		--data '{"id": 1}' \
+		--rps 100 \
+		--total 3000  \
+		--cacert certificate/keys/service.pem \
+		localhost:50050

services/auth/go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/joho/godotenv v1.5.1
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.20.3
 	github.com/rakyll/statik v0.1.7
 	github.com/rs/cors v1.11.0
 	github.com/stretchr/testify v1.9.0
@@ -42,6 +43,8 @@ require (
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/eapache/go-resiliency v1.6.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
@@ -65,16 +68,20 @@ require (
 	github.com/jcmturner/gofork v1.7.6 // indirect
 	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
-	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a // indirect

services/auth/go.sum

@@ -5,8 +5,12 @@ github.com/IBM/sarama v1.43.2/go.mod h1:Kyo4WkF24Z+1nz7xeVUFWIuKVV8RS3wM8mkvPKMd
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=
 github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/brianvoe/gofakeit/v7 v7.0.4 h1:Mkxwz9jYg8Ad8NvT9HA27pCMZGFQo08MK6jD0QTKEww=
 github.com/brianvoe/gofakeit/v7 v7.0.4/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/cockroachdb/cockroach-go/v2 v2.2.0 h1:/5znzg5n373N/3ESjHF5SMLxiW4RKB05Ql//KWfeTFs=
@@ -142,8 +146,8 @@ github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXL
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -174,6 +178,8 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
@@ -183,6 +189,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4=
+github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
 github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
@@ -290,8 +304,8 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

services/auth/internal/app/app.go

@@ -12,12 +12,14 @@ import (
 	"time"
 
 	"github.com/natefinch/lumberjack"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/cors"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"google.golang.org/grpc/credentials"
 
 	"github.com/Genvekt/cli-chat/libraries/logger/pkg/logger"
+	"github.com/Genvekt/cli-chat/services/auth/internal/metric"
 
 	"github.com/rakyll/statik/fs"
 
@@ -37,12 +39,13 @@ import (
 
 // App is an application starting point
 type App struct {
-	configPath    string
-	logLevel      string
-	provider      *ServiceProvider
-	grpcServer    *grpc.Server
-	httpServer    *http.Server
-	swaggerServer *http.Server
+	configPath       string
+	logLevel         string
+	provider         *ServiceProvider
+	grpcServer       *grpc.Server
+	httpServer       *http.Server
+	swaggerServer    *http.Server
+	prometheusServer *http.Server
 }
 
 // NewApp initialises app and all its dependencies
@@ -62,10 +65,12 @@ func (a *App) initDeps(ctx context.Context) error {
 		a.initArgs,
 		a.initConfig,
 		a.initLogger,
+		a.initMetrics,
 		a.initServiceProvider,
 		a.initGRPCServer,
 		a.initHTTPServer,
 		a.initSwaggerServer,
+		a.initPrometheusServer,
 	}
 
 	for _, dep := range deps {
@@ -103,6 +108,14 @@ func (a *App) initLogger(_ context.Context) error {
 	return nil
 }
 
+func (a *App) initMetrics(ctx context.Context) error {
+	err := metric.Init(ctx)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (a *App) initServiceProvider(_ context.Context) error {
 	a.provider = newServiceProvider()
 
@@ -126,6 +139,7 @@ func (a *App) initGRPCServer(ctx context.Context) error {
 		grpc.Creds(creds),
 		grpc.ChainUnaryInterceptor(
 			interceptor.LogInterceptor,
+			interceptor.MetricsInterceptor,
 			interceptor.ValidateInterceptor,
 		),
 	)
@@ -186,6 +200,19 @@ func (a *App) initSwaggerServer(_ context.Context) error {
 	return nil
 }
 
+func (a *App) initPrometheusServer(_ context.Context) error {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+
+	a.prometheusServer = &http.Server{
+		Addr:              a.provider.PrometheusConfig().Address(),
+		Handler:           mux,
+		ReadHeaderTimeout: time.Second * 5,
+	}
+
+	return nil
+}
+
 func (a *App) runGRPCServer(_ context.Context) error {
 	lis, err := net.Listen("tcp", a.provider.GRPCConfig().Address())
 	if err != nil {
@@ -231,6 +258,19 @@ func (a *App) runSwaggerServer() error {
 	return nil
 }
 
+func (a *App) runPrometheusServer() error {
+	logger.Info("Started Prometheus server",
+		zap.String("address", a.provider.PrometheusConfig().Address()),
+	)
+
+	err := a.prometheusServer.ListenAndServe()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // Run starts application and triggers closer on stop
 func (a *App) Run(ctx context.Context) error {
 	defer func() {
@@ -239,7 +279,7 @@ func (a *App) Run(ctx context.Context) error {
 	}()
 
 	wg := sync.WaitGroup{}
-	wg.Add(4)
+	wg.Add(5)
 
 	go func() {
 		defer wg.Done()
@@ -262,6 +302,13 @@ func (a *App) Run(ctx context.Context) error {
 		}
 	}()
 
+	go func() {
+		defer wg.Done()
+		if err := a.runPrometheusServer(); err != nil {
+			logger.Fatal("Failed to run prometheus server", zap.Error(err))
+		}
+	}()
+
 	go func() {
 		defer wg.Done()
 		if err := a.runConsumers(ctx); err != nil {

services/auth/internal/app/service_provider.go

@@ -44,6 +44,7 @@ type ServiceProvider struct {
 	gRPCConfig        config.GRPCConfig
 	httpConfig        config.HTTPConfig
 	swaggerConfig     config.HTTPConfig
+	prometheusConfig  config.HTTPConfig
 	postgresConfig    config.PostgresConfig
 	redisConfig       cacheConfig.RedisConfig
 	userServiceConfig config.UserServiceConfig
@@ -129,6 +130,20 @@ func (s *ServiceProvider) SwaggerConfig() config.HTTPConfig {
 	return s.swaggerConfig
 }
 
+// PrometheusConfig provides configuration of prometheus server of this application
+func (s *ServiceProvider) PrometheusConfig() config.HTTPConfig {
+	if s.prometheusConfig == nil {
+		prometheusConfig, err := env.NewPrometheusConfigEnv()
+		if err != nil {
+			logger.Fatal("failed to load prometheus config", zap.Error(err))
+		}
+
+		s.prometheusConfig = prometheusConfig
+	}
+
+	return s.prometheusConfig
+}
+
 // PGConfig provides configuration parameters for postgres db
 func (s *ServiceProvider) PGConfig() config.PostgresConfig {
 	if s.postgresConfig == nil {

services/auth/internal/config/env/http.go

@@ -16,6 +16,9 @@ const (
 
 	swaggerHostEnv = "SWAGGER_HOST"
 	swaggerPortEnv = "SWAGGER_PORT"
+
+	prometheusHostEnv = "PROMETHEUS_HOST"
+	prometheusPortEnv = "PROMETHEUS_PORT"
 )
 
 type httpConfigEnv struct {
@@ -53,6 +56,21 @@ func NewSwaggerConfigEnv() (*httpConfigEnv, error) {
 	return &httpConfigEnv{host: host, port: port}, nil
 }
 
+// NewPrometheusConfigEnv retrieves parameters for prometheus http server
+func NewPrometheusConfigEnv() (*httpConfigEnv, error) {
+	host := os.Getenv(prometheusHostEnv)
+	if host == "" {
+		return nil, fmt.Errorf("environment variable %q not set", prometheusHostEnv)
+	}
+
+	port := os.Getenv(prometheusPortEnv)
+	if port == "" {
+		return nil, fmt.Errorf("environment variable %q not set", prometheusPortEnv)
+	}
+
+	return &httpConfigEnv{host: host, port: port}, nil
+}
+
 // Address provides host:port string
 func (e *httpConfigEnv) Address() string {
 	return net.JoinHostPort(e.host, e.port)

services/auth/internal/interceptor/metrics.go

@@ -0,0 +1,29 @@
+package interceptor
+
+import (
+	"context"
+	"time"
+
+	"google.golang.org/grpc"
+
+	"github.com/Genvekt/cli-chat/services/auth/internal/metric"
+)
+
+// MetricsInterceptor records amount and duration of requests
+func MetricsInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+	metric.IncRequestCounter()
+
+	startTime := time.Now()
+	res, err := handler(ctx, req)
+	duration := time.Since(startTime)
+
+	if err != nil {
+		metric.IncResponseCounter("error", info.FullMethod)
+		metric.HistogramResponseTimeObserve("error", duration.Seconds())
+	} else {
+		metric.IncResponseCounter("success", info.FullMethod)
+		metric.HistogramResponseTimeObserve("success", duration.Seconds())
+	}
+
+	return res, err
+}