Add structured logging and OpenMetrics endpoint

.env.example

@@ -54,13 +54,6 @@ STACKS_CREATE_MAX=1
 LOG_DIR=logs
 LOG_FILE_PREFIX=app
 LOG_MAX_BODY_BYTES=1048576
-LOG_WEBHOOK_QUEUE_SIZE=1000
-LOG_WEBHOOK_TIMEOUT=5s
-LOG_WEBHOOK_BATCH_SIZE=20
-LOG_WEBHOOK_BATCH_WAIT=2s
-LOG_WEBHOOK_MAX_CHARS=1800
-LOG_DISCORD_WEBHOOK_URL=
-LOG_SLACK_WEBHOOK_URL=
 
 # S3 Challenge Files
 S3_ENABLED=false

README.md

@@ -56,8 +56,7 @@ See [SMCTF Docs](https://ctf.null4u.cloud/smctf/) for more details. This README
 - Flag submission with rate limiting and HMAC verification
 - Scoreboard and Timeline (Redis caching support)
 - User profile with statistics (Some implementations are still WIP)
-- Logging middleware with file logging and webhook support (e.g., Discord, Slack, etc.)
-    - Supports queuing and batching for webhooks to prevent rate limiting issues, and splitting long messages.
+- Logging middleware with file logging support
     - Ref Issue: [#9](https://github.com/nullforu/smctf/issues/9), PR: [#10](https://github.com/nullforu/smctf/pull/10)
 - User and Team management (WIP)
     - Ref Issue: [#11](https://github.com/nullforu/smctf/issues/11), [#22](https://github.com/nullforu/smctf/issues/22), PR: [#12](https://github.com/nullforu/smctf/pull/12), [#15](https://github.com/nullforu/smctf/pull/15), [#23](https://github.com/nullforu/smctf/pull/23)
@@ -178,13 +177,6 @@ STACKS_CREATE_MAX=1
 LOG_DIR=logs
 LOG_FILE_PREFIX=app
 LOG_MAX_BODY_BYTES=1048576
-LOG_WEBHOOK_QUEUE_SIZE=1000
-LOG_WEBHOOK_TIMEOUT=5s
-LOG_WEBHOOK_BATCH_SIZE=20
-LOG_WEBHOOK_BATCH_WAIT=2s
-LOG_WEBHOOK_MAX_CHARS=1800
-LOG_DISCORD_WEBHOOK_URL=
-LOG_SLACK_WEBHOOK_URL=
 
 # S3 Challenge Files
 S3_ENABLED=false
@@ -218,6 +210,52 @@ go build -o smctf ./cmd/server
 > [!NOTE]
 >
 > Running in Docker environment will be supported in the future.
+
+**Logging Schema**
+
+```json
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "title": "SMCTF Log Event",
+    "type": "object",
+    "additionalProperties": true,
+    "required": ["ts", "level", "msg", "app"],
+    "properties": {
+        "ts": {
+            "type": "string",
+            "format": "date-time",
+            "description": "RFC3339 timestamp with timezone"
+        },
+        "level": {
+            "type": "string",
+            "enum": ["debug", "info", "warn", "error"]
+        },
+        "msg": { "type": "string" },
+        "app": { "type": "string" },
+        "legacy": { "type": "boolean" },
+        "error": {},
+        "stack": { "type": "string" },
+        "http": {
+            "type": "object",
+            "additionalProperties": true,
+            "properties": {
+                "method": { "type": "string" },
+                "path": { "type": "string" },
+                "status": { "type": "integer" },
+                "latency": { "type": "string" },
+                "ip": { "type": "string" },
+                "query": { "type": "string" },
+                "user_agent": { "type": "string" },
+                "content_type": { "type": "string" },
+                "content_length": { "type": "integer" },
+                "user_id": { "type": "integer" },
+                "body": { "type": "string" }
+            }
+        }
+    }
+}
+```
+
 > Currently, please use local installation for development and testing. Requires Go and NodeJS, NPM installation.
 
 ## Testing
@@ -288,7 +326,7 @@ Defaults live in `./scripts/generate_dummy_sql/defaults/` and can be overridden
 It provides sample challenges, 30 users (including admin), and random submissions data from the last ~48 hours.
 
 > [!WARNING]
-> 
+>
 > **This will TRUNCATE all tables in the database! Use only in development/test environments.**
 
 ## FAQ, Troubleshooting

cmd/server/main.go

@@ -2,8 +2,7 @@ package main
 
 import (
 	"context"
-	"io"
-	"log"
+	"log/slog"
 	nethttp "net/http"
 	"os"
 	"os/signal"
@@ -24,41 +23,54 @@ import (
 func main() {
 	cfg, err := config.Load()
 	if err != nil {
-		log.Fatalf("config error: %v", err)
+		boot := slog.New(slog.NewJSONHandler(os.Stderr, nil))
+		boot.Error("config error", slog.Any("error", err))
+		os.Exit(1)
 	}
 
-	logger, err := logging.New(cfg.Logging)
+	logger, err := logging.New(cfg.Logging, logging.Options{
+		Service:   "smctf",
+		Env:       cfg.AppEnv,
+		AddSource: false,
+	})
 	if err != nil {
-		log.Fatalf("logging init error: %v", err)
+		boot := slog.New(slog.NewJSONHandler(os.Stderr, nil))
+		boot.Error("logging init error", slog.Any("error", err))
+		os.Exit(1)
 	}
 
+	slog.SetDefault(logger.Logger)
+
 	defer func() {
 		if err := logger.Close(); err != nil {
-			log.Printf("log close error: %v", err)
+			logger.Error("log close error", slog.Any("error", err))
 		}
 	}()
 
-	log.SetOutput(io.MultiWriter(os.Stdout, logger))
-	log.Printf("config loaded:\n%s", config.FormatForLog(cfg))
+	logger.Info("config loaded", slog.Any("config", config.FormatForLog(cfg)))
 
 	ctx := context.Background()
 	database, err := db.New(cfg.DB, cfg.AppEnv)
 	if err != nil {
-		log.Fatalf("db init error: %v", err)
+		logger.Error("db init error", slog.Any("error", err))
+		os.Exit(1)
 	}
 
 	if err := database.PingContext(ctx); err != nil {
-		log.Fatalf("db ping error: %v", err)
+		logger.Error("db ping error", slog.Any("error", err))
+		os.Exit(1)
 	}
 
 	redisClient := cache.New(cfg.Redis)
 	if err := redisClient.Ping(ctx).Err(); err != nil {
-		log.Fatalf("redis ping error: %v", err)
+		logger.Error("redis ping error", slog.Any("error", err))
+		os.Exit(1)
 	}
 
 	if cfg.AutoMigrate {
 		if err := db.AutoMigrate(ctx, database); err != nil {
-			log.Fatalf("auto migrate error: %v", err)
+			logger.Error("auto migrate error", slog.Any("error", err))
+			os.Exit(1)
 		}
 	}
 
@@ -75,7 +87,8 @@ func main() {
 	if cfg.S3.Enabled {
 		store, err := storage.NewS3ChallengeFileStore(ctx, cfg.S3)
 		if err != nil {
-			log.Fatalf("s3 init error: %v", err)
+			logger.Error("s3 init error", slog.Any("error", err))
+			os.Exit(1)
 		}
 		fileStore = store
 	}
@@ -90,9 +103,9 @@ func main() {
 	stackSvc := service.NewStackService(cfg.Stack, stackRepo, challengeRepo, submissionRepo, stackClient, redisClient)
 
 	if cfg, _, _, err := appConfigSvc.Get(ctx); err != nil {
-		log.Printf("app config load warning: %v", err)
+		logger.Warn("app config load warning", slog.Any("error", err))
 	} else if cfg.CTFStartAt == "" && cfg.CTFEndAt == "" {
-		log.Printf("warning: ctf_start_at and ctf_end_at not configured; competition will always be active at all times")
+		logger.Warn("ctf window not configured; competition always active")
 	}
 
 	router := httpserver.NewRouter(cfg, authSvc, ctfSvc, appConfigSvc, userSvc, scoreSvc, teamSvc, stackSvc, redisClient, logger)
@@ -109,9 +122,10 @@ func main() {
 	defer stop()
 
 	go func() {
-		log.Printf("server listening on %s", cfg.HTTPAddr)
+		logger.Info("server listening", slog.String("addr", cfg.HTTPAddr))
 		if err := srv.ListenAndServe(); err != nil && err != nethttp.ErrServerClosed {
-			log.Fatalf("server error: %v", err)
+			logger.Error("server error", slog.Any("error", err))
+			os.Exit(1)
 		}
 	}()
 
@@ -120,14 +134,14 @@ func main() {
 	defer cancel()
 
 	if err := srv.Shutdown(shutdownCtx); err != nil {
-		log.Printf("server shutdown error: %v", err)
+		logger.Error("server shutdown error", slog.Any("error", err))
 	}
 
 	if err := redisClient.Close(); err != nil {
-		log.Printf("redis close error: %v", err)
+		logger.Error("redis close error", slog.Any("error", err))
 	}
 
 	if err := database.Close(); err != nil {
-		log.Printf("db close error: %v", err)
+		logger.Error("db close error", slog.Any("error", err))
 	}
 }

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.38.5 // indirect
 	github.com/aws/smithy-go v1.23.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bytedance/sonic v1.14.0 // indirect
 	github.com/bytedance/sonic/loader v0.3.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -87,12 +88,17 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/prometheus/client_golang v1.23.2 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
 	github.com/quic-go/qpack v0.6.0 // indirect
 	github.com/quic-go/quic-go v0.57.0 // indirect
@@ -116,6 +122,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.39.0 // indirect
 	go.opentelemetry.io/otel/trace v1.39.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/arch v0.20.0 // indirect
 	golang.org/x/net v0.48.0 // indirect
 	golang.org/x/sys v0.40.0 // indirect

go.sum

@@ -44,6 +44,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.38.5 h1:+LVB0xBqEgjQoqr9bGZbRzvg212B
 github.com/aws/aws-sdk-go-v2/service/sts v1.38.5/go.mod h1:xoaxeqnnUaZjPjaICgIy5B+MHCSb/ZSOn4MvkFNOUA0=
 github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE=
 github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
@@ -171,6 +173,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
@@ -183,6 +187,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
 github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg=
 github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
 github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
@@ -257,6 +269,8 @@ go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjce
 go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
 go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
 go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
 golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
 golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=