diff --git a/services/auth/Makefile b/services/auth/Makefile index 06384ca..e682614 100644 --- a/services/auth/Makefile +++ b/services/auth/Makefile @@ -42,4 +42,15 @@ gen-cert: openssl genrsa -out certificate/keys/service.key 4096 openssl req -new -key certificate/keys/service.key -out certificate/keys/service.csr -config certificate/certificate.conf openssl x509 -req -in certificate/keys/service.csr -CA certificate/keys/ca.cert -CAkey certificate/keys/ca.key -CAcreateserial \ - -out certificate/keys/service.pem -days 365 -sha256 -extfile certificate/certificate.conf -extensions req_ext \ No newline at end of file + -out certificate/keys/service.pem -days 365 -sha256 -extfile certificate/certificate.conf -extensions req_ext + +grpc-load-test: + ghz \ + --proto ../../libraries/api/user/v1/user.proto \ + --import-paths ../../libraries/api/vendor.protogen/ \ + --call user_v1.UserV1.Get \ + --data '{"id": 1}' \ + --rps 100 \ + --total 3000 \ + --cacert certificate/keys/service.pem \ + localhost:50050 \ No newline at end of file diff --git a/services/auth/go.mod b/services/auth/go.mod index 70ccec7..98d1834 100644 --- a/services/auth/go.mod +++ b/services/auth/go.mod @@ -32,6 +32,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.20.3 github.com/rakyll/statik v0.1.7 github.com/rs/cors v1.11.0 github.com/stretchr/testify v1.9.0 @@ -42,6 +43,8 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect @@ -65,16 +68,20 @@ require ( github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a // indirect diff --git a/services/auth/go.sum b/services/auth/go.sum index 9c72f11..8adb3d9 100644 --- a/services/auth/go.sum +++ b/services/auth/go.sum @@ -5,8 +5,12 @@ github.com/IBM/sarama v1.43.2/go.mod h1:Kyo4WkF24Z+1nz7xeVUFWIuKVV8RS3wM8mkvPKMd github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/brianvoe/gofakeit/v7 v7.0.4 h1:Mkxwz9jYg8Ad8NvT9HA27pCMZGFQo08MK6jD0QTKEww= github.com/brianvoe/gofakeit/v7 v7.0.4/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go/v2 v2.2.0 h1:/5znzg5n373N/3ESjHF5SMLxiW4RKB05Ql//KWfeTFs= @@ -142,8 +146,8 @@ github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXL github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -174,6 +178,8 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -183,6 +189,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4= +github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= @@ -290,8 +304,8 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/services/auth/internal/app/app.go b/services/auth/internal/app/app.go index 1ed9a91..8a5154a 100644 --- a/services/auth/internal/app/app.go +++ b/services/auth/internal/app/app.go @@ -12,12 +12,14 @@ import ( "time" "github.com/natefinch/lumberjack" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/cors" "go.uber.org/zap" "go.uber.org/zap/zapcore" "google.golang.org/grpc/credentials" "github.com/Genvekt/cli-chat/libraries/logger/pkg/logger" + "github.com/Genvekt/cli-chat/services/auth/internal/metric" "github.com/rakyll/statik/fs" @@ -37,12 +39,13 @@ import ( // App is an application starting point type App struct { - configPath string - logLevel string - provider *ServiceProvider - grpcServer *grpc.Server - httpServer *http.Server - swaggerServer *http.Server + configPath string + logLevel string + provider *ServiceProvider + grpcServer *grpc.Server + httpServer *http.Server + swaggerServer *http.Server + prometheusServer *http.Server } // NewApp initialises app and all its dependencies @@ -62,10 +65,12 @@ func (a *App) initDeps(ctx context.Context) error { a.initArgs, a.initConfig, a.initLogger, + a.initMetrics, a.initServiceProvider, a.initGRPCServer, a.initHTTPServer, a.initSwaggerServer, + a.initPrometheusServer, } for _, dep := range deps { @@ -103,6 +108,14 @@ func (a *App) initLogger(_ context.Context) error { return nil } +func (a *App) initMetrics(ctx context.Context) error { + err := metric.Init(ctx) + if err != nil { + return err + } + return nil +} + func (a *App) initServiceProvider(_ context.Context) error { a.provider = newServiceProvider() @@ -126,6 +139,7 @@ func (a *App) initGRPCServer(ctx context.Context) error { grpc.Creds(creds), grpc.ChainUnaryInterceptor( interceptor.LogInterceptor, + interceptor.MetricsInterceptor, interceptor.ValidateInterceptor, ), ) @@ -186,6 +200,19 @@ func (a *App) initSwaggerServer(_ context.Context) error { return nil } +func (a *App) initPrometheusServer(_ context.Context) error { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + + a.prometheusServer = &http.Server{ + Addr: a.provider.PrometheusConfig().Address(), + Handler: mux, + ReadHeaderTimeout: time.Second * 5, + } + + return nil +} + func (a *App) runGRPCServer(_ context.Context) error { lis, err := net.Listen("tcp", a.provider.GRPCConfig().Address()) if err != nil { @@ -231,6 +258,19 @@ func (a *App) runSwaggerServer() error { return nil } +func (a *App) runPrometheusServer() error { + logger.Info("Started Prometheus server", + zap.String("address", a.provider.PrometheusConfig().Address()), + ) + + err := a.prometheusServer.ListenAndServe() + if err != nil { + return err + } + + return nil +} + // Run starts application and triggers closer on stop func (a *App) Run(ctx context.Context) error { defer func() { @@ -239,7 +279,7 @@ func (a *App) Run(ctx context.Context) error { }() wg := sync.WaitGroup{} - wg.Add(4) + wg.Add(5) go func() { defer wg.Done() @@ -262,6 +302,13 @@ func (a *App) Run(ctx context.Context) error { } }() + go func() { + defer wg.Done() + if err := a.runPrometheusServer(); err != nil { + logger.Fatal("Failed to run prometheus server", zap.Error(err)) + } + }() + go func() { defer wg.Done() if err := a.runConsumers(ctx); err != nil { diff --git a/services/auth/internal/app/service_provider.go b/services/auth/internal/app/service_provider.go index de24ab1..35c0a3f 100644 --- a/services/auth/internal/app/service_provider.go +++ b/services/auth/internal/app/service_provider.go @@ -44,6 +44,7 @@ type ServiceProvider struct { gRPCConfig config.GRPCConfig httpConfig config.HTTPConfig swaggerConfig config.HTTPConfig + prometheusConfig config.HTTPConfig postgresConfig config.PostgresConfig redisConfig cacheConfig.RedisConfig userServiceConfig config.UserServiceConfig @@ -129,6 +130,20 @@ func (s *ServiceProvider) SwaggerConfig() config.HTTPConfig { return s.swaggerConfig } +// PrometheusConfig provides configuration of prometheus server of this application +func (s *ServiceProvider) PrometheusConfig() config.HTTPConfig { + if s.prometheusConfig == nil { + prometheusConfig, err := env.NewPrometheusConfigEnv() + if err != nil { + logger.Fatal("failed to load prometheus config", zap.Error(err)) + } + + s.prometheusConfig = prometheusConfig + } + + return s.prometheusConfig +} + // PGConfig provides configuration parameters for postgres db func (s *ServiceProvider) PGConfig() config.PostgresConfig { if s.postgresConfig == nil { diff --git a/services/auth/internal/config/env/http.go b/services/auth/internal/config/env/http.go index 1e4dc2b..7d81814 100644 --- a/services/auth/internal/config/env/http.go +++ b/services/auth/internal/config/env/http.go @@ -16,6 +16,9 @@ const ( swaggerHostEnv = "SWAGGER_HOST" swaggerPortEnv = "SWAGGER_PORT" + + prometheusHostEnv = "PROMETHEUS_HOST" + prometheusPortEnv = "PROMETHEUS_PORT" ) type httpConfigEnv struct { @@ -53,6 +56,21 @@ func NewSwaggerConfigEnv() (*httpConfigEnv, error) { return &httpConfigEnv{host: host, port: port}, nil } +// NewPrometheusConfigEnv retrieves parameters for prometheus http server +func NewPrometheusConfigEnv() (*httpConfigEnv, error) { + host := os.Getenv(prometheusHostEnv) + if host == "" { + return nil, fmt.Errorf("environment variable %q not set", prometheusHostEnv) + } + + port := os.Getenv(prometheusPortEnv) + if port == "" { + return nil, fmt.Errorf("environment variable %q not set", prometheusPortEnv) + } + + return &httpConfigEnv{host: host, port: port}, nil +} + // Address provides host:port string func (e *httpConfigEnv) Address() string { return net.JoinHostPort(e.host, e.port) diff --git a/services/auth/internal/interceptor/metrics.go b/services/auth/internal/interceptor/metrics.go new file mode 100644 index 0000000..9de6f8e --- /dev/null +++ b/services/auth/internal/interceptor/metrics.go @@ -0,0 +1,29 @@ +package interceptor + +import ( + "context" + "time" + + "google.golang.org/grpc" + + "github.com/Genvekt/cli-chat/services/auth/internal/metric" +) + +// MetricsInterceptor records amount and duration of requests +func MetricsInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + metric.IncRequestCounter() + + startTime := time.Now() + res, err := handler(ctx, req) + duration := time.Since(startTime) + + if err != nil { + metric.IncResponseCounter("error", info.FullMethod) + metric.HistogramResponseTimeObserve("error", duration.Seconds()) + } else { + metric.IncResponseCounter("success", info.FullMethod) + metric.HistogramResponseTimeObserve("success", duration.Seconds()) + } + + return res, err +} diff --git a/services/auth/internal/metric/metrics.go b/services/auth/internal/metric/metrics.go new file mode 100644 index 0000000..d6ab8ca --- /dev/null +++ b/services/auth/internal/metric/metrics.go @@ -0,0 +1,72 @@ +package metric + +import ( + "context" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + namespace = "cli_chat" + appName = "auth" +) + +// Metrics holds all prometheus metrics of this application +type Metrics struct { + requestCounter prometheus.Counter + responseCounter *prometheus.CounterVec + histogramResponseTime *prometheus.HistogramVec +} + +var metrics *Metrics + +// Init initialises prometheus metrics +func Init(_ context.Context) error { + metrics = &Metrics{ + requestCounter: promauto.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "grpc", + Name: appName + "_requests_total", + Help: "Количество запросов к серверу", + }, + ), + responseCounter: promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "grpc", + Name: appName + "_responses_total", + Help: "Количество ответов от сервера", + }, + []string{"status", "method"}, + ), + histogramResponseTime: promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: "grpc", + Name: appName + "_histogram_response_time_seconds", + Help: "Время ответа от сервера", + Buckets: prometheus.ExponentialBuckets(0.0001, 2, 16), + }, + []string{"status"}, + ), + } + + return nil +} + +// IncRequestCounter increments number of incoming grpc requests +func IncRequestCounter() { + metrics.requestCounter.Inc() +} + +// IncResponseCounter increments number of outcoming grpc responces +func IncResponseCounter(status string, method string) { + metrics.responseCounter.WithLabelValues(status, method).Inc() +} + +// HistogramResponseTimeObserve tracks sample of grpc response time +func HistogramResponseTimeObserve(status string, time float64) { + metrics.histogramResponseTime.WithLabelValues(status).Observe(time) +} diff --git a/services/metrics/alerts.yml b/services/metrics/alerts.yml new file mode 100644 index 0000000..e113432 --- /dev/null +++ b/services/metrics/alerts.yml @@ -0,0 +1,11 @@ +groups: + - name: alerts + rules: + - alert: TargetIsDown + expr: up == 0 + for: 30s + labels: + severity: medium + annotations: + summary: "The target {{ $labels.job }} is down" + description: "Instance {{ $labels.instance }} из job {{ $labels.job }} не отвечает в течении 30 секунд." \ No newline at end of file diff --git a/services/metrics/docker-compose.yaml b/services/metrics/docker-compose.yaml new file mode 100644 index 0000000..8b8a41c --- /dev/null +++ b/services/metrics/docker-compose.yaml @@ -0,0 +1,23 @@ +version: '3' + +volumes: + prometheus_data: + grafana_data: + +services: + prometheus: + image: prom/prometheus:v2.37.9 + container_name: prometheus + ports: + - "9090:9090" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./alerts.yml:/etc/prometheus/alerts.yml + - prometheus_data:/prometheus + + grafana: + image: grafana/grafana-oss:10.0.3 + ports: + - "3000:3000" + volumes: + - grafana_data:/var/lib/grafana \ No newline at end of file diff --git a/services/metrics/grafana_dashboard.json b/services/metrics/grafana_dashboard.json new file mode 100644 index 0000000..43ff520 --- /dev/null +++ b/services/metrics/grafana_dashboard.json @@ -0,0 +1,400 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "bf865d35-2eb7-4209-91c4-5ecea9095e44" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bf865d35-2eb7-4209-91c4-5ecea9095e44" + }, + "editorMode": "code", + "expr": "rate(cli_chat_grpc_auth_requests_total[5s])", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Входящий трафик (rps)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bf865d35-2eb7-4209-91c4-5ecea9095e44" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/error/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/success/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bf865d35-2eb7-4209-91c4-5ecea9095e44" + }, + "editorMode": "code", + "expr": "rate(cli_chat_grpc_auth_responses_total[10s])", + "instant": false, + "legendFormat": "{{status}}:{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Ответы от сервера (rps)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bf865d35-2eb7-4209-91c4-5ecea9095e44" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/error/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/success/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bf865d35-2eb7-4209-91c4-5ecea9095e44" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9,\n sum(rate(cli_chat_grpc_auth_histogram_response_time_seconds_bucket[10s])) by (le, status)\n)", + "instant": false, + "legendFormat": "0.9q : {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bf865d35-2eb7-4209-91c4-5ecea9095e44" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,\n sum(rate(cli_chat_grpc_auth_histogram_response_time_seconds_bucket[10s])) by (le, status)\n)", + "hide": false, + "instant": false, + "legendFormat": "0.5q : {{status}}", + "range": true, + "refId": "B" + } + ], + "title": "Квантили времени ответа", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Service overview", + "uid": "b6b656b1-d42c-454d-b033-a0be8019db6f", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/services/metrics/prometheus.yml b/services/metrics/prometheus.yml new file mode 100644 index 0000000..d999d41 --- /dev/null +++ b/services/metrics/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 2s # Как часто собирать метрики + evaluation_interval: 2s # Как часто вычислять правила агрегации и алертинга + +rule_files: + - "alerts.yml" + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: [ "localhost:9090" ] + + - job_name: "app" + static_configs: + - targets: [ "host.docker.internal:2112" ] \ No newline at end of file