From 1f3bd5a04f0ad103ecd9804feefc3a70ddd425a7 Mon Sep 17 00:00:00 2001 From: tambir-jazz Date: Tue, 4 Mar 2025 12:06:59 +0500 Subject: [PATCH 1/2] chore: add zap logger replacing log --- go.mod | 3 ++- go.sum | 7 ++++++- main.go | 52 ++++++++++++++++++++++++++++++++-------------------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index e05e0a7..e13dd80 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/redis/go-redis/v9 v9.7.0 github.com/stretchr/testify v1.10.0 github.com/tom-draper/api-analytics/analytics/go/gin v0.0.0-20250124150115-22f84a4e90ef + go.uber.org/zap v1.27.0 ) require ( @@ -32,7 +33,6 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect - github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -43,6 +43,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.13.0 // indirect golang.org/x/crypto v0.32.0 // indirect golang.org/x/net v0.34.0 // indirect diff --git a/go.sum b/go.sum index 261d125..cf3ae65 100644 --- a/go.sum +++ b/go.sum @@ -20,7 +20,6 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -100,6 +99,12 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/arch v0.13.0 h1:KCkqVVV1kGg0X87TFysjCJ8MxtZEIU4Ja/yXGeoECdA= golang.org/x/arch v0.13.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= diff --git a/main.go b/main.go index 998ee80..ec51a35 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "log" "net/http" "net/http/pprof" "os" @@ -28,6 +27,8 @@ import ( "github.com/jasonlovesdoggo/abacus/utils" "github.com/gin-gonic/gin" + + "go.uber.org/zap" ) const ( @@ -41,11 +42,18 @@ var ( DbNum = 0 // 0-16 StartTime time.Time Shard string + logger *zap.Logger ) const is32Bit = uint64(^uintptr(0)) != ^uint64(0) func init() { + var logErr error + logger, logErr = zap.NewProduction() + if logErr != nil { + panic(fmt.Sprintf("Failed to initialize logger: %v", logErr)) + } + utils.LoadEnv() if strings.ToLower(os.Getenv("DEBUG")) == "true" { @@ -64,13 +72,14 @@ func init() { Shard = namegen.New().Get() ADDR := os.Getenv("REDIS_HOST") + ":" + os.Getenv("REDIS_PORT") - log.Println("Listening to redis on: " + ADDR) + logger.Info("Listening to Redis", zap.String("address", ADDR)) var err error DbNum, err = strconv.Atoi(os.Getenv("REDIS_DB")) if err != nil { DbNum = 0 // Default to 0 if not set + logger.Warn("Invalid REDIS_DB value, defaulting to 0", zap.Error(err)) } else if DbNum < 0 || DbNum > 16 { - log.Fatalf("Redis DB must be between 0-16: %v", DbNum) + logger.Fatal("Redis DB must be between 0-16", zap.Int("DbNum", DbNum)) } Client = redis.NewClient(&redis.Options{ @@ -91,10 +100,10 @@ func setupMockRedis() { // Used for testing, "miniredis" is a mock Redis server that runs in-memory for testing purposes only (no persistence) mr, err := miniredis.Run() if err != nil { - log.Fatalf("Failed to start miniredis: %v", err) + logger.Fatal("Failed to start miniredis", zap.Error(err)) } - log.Println("Using miniredis for testing") + logger.Info("Using miniredis for testing") // Connect clients to miniredis Client = redis.NewClient(&redis.Options{ @@ -140,13 +149,13 @@ func CreateRouter() *gin.Engine { r.Use(gin.Recovery()) // recover from panics and returns a 500 error if os.Getenv("API_ANALYTICS_ENABLED") == "true" { r.Use(analytics.Analytics(os.Getenv("API_ANALYTICS_KEY"))) // Add middleware - log.Println("Analytics enabled") + logger.Info("Analytics enabled") } route := r.Group("") route.Use(middleware.Stats()) if os.Getenv("RATE_LIMIT_ENABLED") == "true" { route.Use(middleware.RateLimit(RateLimitClient)) - log.Println("Rate limiting enabled") + logger.Info("Rate limiting enabled") } // Define routes r.NoRoute(func(c *gin.Context) { @@ -195,8 +204,10 @@ func CreateRouter() *gin.Engine { } func main() { + defer logger.Sync() + if is32Bit { - log.Fatal("This program is not supported on 32-bit systems, " + + logger.Fatal("This program is not supported on 32-bit systems, " + "please run on a 64-bit system. If you wish for 32-bit support, " + "please open an issue on the GitHub repository.\nexiting...") } @@ -212,37 +223,38 @@ func main() { Addr: ":" + os.Getenv("PORT"), Handler: r, } - fmt.Println("Listening on port " + os.Getenv("PORT")) + port := os.Getenv("PORT") + logger.Info("Listening on port", zap.String("port", port)) go func() { if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Fatalf("listen: %s\n", err) + logger.Fatal("Failed to start server", zap.Error(err)) } }() // Wait for interrupt signal <-ctx.Done() - log.Println("Shutdown signal received") + logger.Info("Shutdown signal received") // Signal StatsManager to save and wait for completion - log.Println("Signaling stats manager to save data...") + logger.Info("Signaling stats manager to save data...") utils.ServerClose <- true // Wait for the response on the same channel - log.Println("Waiting for stats manager to complete...") + logger.Info("Waiting for stats manager to complete...") <-utils.ServerClose - log.Println("Stats saving confirmed complete") + logger.Info("Stats saving confirmed complete") // Now close Redis connections - log.Println("Closing Redis connections...") + logger.Info("Closing Redis connections...") if Client != nil { if err := Client.Close(); err != nil { - log.Printf("Error closing Redis client: %v", err) + logger.Error("Error closing Redis client", zap.Error(err)) } } if RateLimitClient != nil { if err := RateLimitClient.Close(); err != nil { - log.Printf("Error closing Redis rate limit client: %v", err) + logger.Error("Error closing Redis rate limit client", zap.Error(err)) } } @@ -251,12 +263,12 @@ func main() { defer cancel() if err := srv.Shutdown(shutdownCtx); err != nil { - log.Fatal("Server Shutdown:", err) + logger.Fatal("Server shutdown failed", zap.Error(err)) } <-shutdownCtx.Done() if errors.Is(shutdownCtx.Err(), context.DeadlineExceeded) { - log.Println("Shutdown timeout of 5 seconds exceeded") + logger.Warn("Shutdown timeout of 5 seconds exceeded") } - log.Println("Server exiting") + logger.Info("Server exiting") } From 6e8d115dc7d51bba70ded1e2dd3364c643c020a3 Mon Sep 17 00:00:00 2001 From: tambir-jazz Date: Mon, 17 Mar 2025 11:44:57 +0500 Subject: [PATCH 2/2] chore: convert go log to zap log --- main.go | 1 - middleware/stats.go | 6 ++++-- routes.go | 25 ++++++++++++++++++------- utils/sse.go | 28 +++++++++++++++++++++------- utils/stats.go | 15 ++++++--------- 5 files changed, 49 insertions(+), 26 deletions(-) diff --git a/main.go b/main.go index ec51a35..ab688b6 100644 --- a/main.go +++ b/main.go @@ -42,7 +42,6 @@ var ( DbNum = 0 // 0-16 StartTime time.Time Shard string - logger *zap.Logger ) const is32Bit = uint64(^uintptr(0)) != ^uint64(0) diff --git a/middleware/stats.go b/middleware/stats.go index 0163d33..b7eeeaa 100644 --- a/middleware/stats.go +++ b/middleware/stats.go @@ -1,14 +1,16 @@ package middleware import ( - "log" "strings" "sync/atomic" "github.com/gin-gonic/gin" "github.com/jasonlovesdoggo/abacus/utils" + "go.uber.org/zap" ) +var logger, _ = zap.NewProduction() + func formatPath(path string) string { if len(path) == 0 { return "" @@ -23,7 +25,7 @@ func formatPath(path string) string { func Stats() gin.HandlerFunc { return func(c *gin.Context) { if utils.StatsManager == nil { - log.Fatal("StatsManager not initialized. Call InitializeStatsManager first") + logger.Fatal("StatsManager not initialized. Call InitializeStatsManager first") } path := formatPath(c.Request.URL.Path) diff --git a/routes.go b/routes.go index a7a9516..0deb723 100644 --- a/routes.go +++ b/routes.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "log" "math" "net/http" "strconv" @@ -20,8 +19,21 @@ import ( "github.com/jasonlovesdoggo/abacus/utils" "github.com/gin-gonic/gin" + "go.uber.org/zap" ) +var logger *zap.Logger + +func init() { + // Initialize Zap logger + var err error + logger, err = zap.NewProduction() + if err != nil { + panic(fmt.Sprintf("Failed to initialize logger: %v", err)) + } + defer logger.Sync() // Ensures logs are flushed properly +} + func StreamValueView(c *gin.Context) { namespace, key := utils.GetNamespaceKey(c) if namespace == "" || key == "" { @@ -68,8 +80,7 @@ func StreamValueView(c *gin.Context) { case utils.ValueEventServer.ClosedClients <- utils.KeyClientPair{Key: dbKey, Client: clientChan}: // Successfully sent cleanup signal case <-time.After(500 * time.Millisecond): - // Timed out waiting to send cleanup signal - log.Printf("Warning: Timed out sending cleanup signal for %s", dbKey) + logger.Warn("Timed out sending cleanup signal", zap.String("key", dbKey)) } } else { cleanupMutex.Unlock() @@ -85,7 +96,7 @@ func StreamValueView(c *gin.Context) { cleanupDone = true cleanupMutex.Unlock() - log.Printf("Client disconnected for key %s, cleaning up", dbKey) + logger.Info("Client disconnected, cleaning up", zap.String("key", dbKey)) // Signal the event server to remove this client select { @@ -93,7 +104,7 @@ func StreamValueView(c *gin.Context) { // Successfully sent cleanup signal case <-time.After(500 * time.Millisecond): // Timed out waiting to send cleanup signal - log.Printf("Warning: Timed out sending cleanup signal for %s after disconnect", dbKey) + logger.Warn("Timed out sending cleanup signal after disconnect", zap.String("key", dbKey)) } } else { cleanupMutex.Unlock() @@ -106,7 +117,7 @@ func StreamValueView(c *gin.Context) { // Keep your exact format _, err := c.Writer.WriteString(fmt.Sprintf("data: {\"value\":%d}\n\n", count)) if err != nil { - log.Printf("Error writing to client: %v", err) + logger.Error("Error writing to client", zap.Error(err)) return } c.Writer.Flush() @@ -124,7 +135,7 @@ func StreamValueView(c *gin.Context) { // Keep your exact format _, err := c.Writer.WriteString(fmt.Sprintf("data: {\"value\":%d}\n\n", count)) if err != nil { - log.Printf("Error writing to client: %v", err) + logger.Error("Error writing to client", zap.Error(err)) return false } c.Writer.Flush() diff --git a/utils/sse.go b/utils/sse.go index b09d155..f8efe37 100644 --- a/utils/sse.go +++ b/utils/sse.go @@ -1,9 +1,10 @@ package utils import ( - "log" "sync" "time" + + "go.uber.org/zap" ) type ValueEvent struct { @@ -24,6 +25,7 @@ type KeyClientPair struct { Client chan int } + func NewValueEventServer() *ValueEvent { event := &ValueEvent{ // Use buffered channels to prevent blocking @@ -46,7 +48,10 @@ func (v *ValueEvent) listen() { } v.TotalClients[newClient.Key][newClient.Client] = true v.Mu.Unlock() - log.Printf("Client added for key %s. Total clients: %d", newClient.Key, len(v.TotalClients[newClient.Key])) + logger.Info("Client added", + zap.String("key", newClient.Key), + zap.Int("total_clients", len(v.TotalClients[newClient.Key])), + ) case closedClient := <-v.ClosedClients: v.Mu.Lock() @@ -57,12 +62,12 @@ func (v *ValueEvent) listen() { // Close channel safely close(closedClient.Client) - log.Printf("Removed client for key %s", closedClient.Key) + logger.Info("Removed client", zap.String("key", closedClient.Key)) // Clean up key map if no more clients if len(clients) == 0 { delete(v.TotalClients, closedClient.Key) - log.Printf("No more clients for key %s, removed key entry", closedClient.Key) + logger.Info("No more clients, removed key entry", zap.String("key", closedClient.Key)) } } } @@ -110,7 +115,7 @@ func (v *ValueEvent) listen() { case v.ClosedClients <- KeyClientPair{Key: key, Client: client}: // Success on retry default: - log.Printf("Failed to remove client for key %s even after retry", key) + logger.Warn("Failed to remove client even after retry", zap.String("key", key)) } }(keyValue.Key, failedClient) } @@ -133,6 +138,12 @@ func (v *ValueEvent) CountClientsForKey(key string) int { var ValueEventServer *ValueEvent func init() { + var err error + logger, err = zap.NewProduction() + if err != nil { + panic(err) + } + ValueEventServer = NewValueEventServer() } @@ -143,7 +154,7 @@ func SetStream(dbKey string, newValue int) { case ValueEventServer.Message <- KeyValue{Key: dbKey, Value: newValue}: // Message sent successfully default: - log.Printf("Warning: Message channel full, update for key %s dropped", dbKey) + logger.Warn("Message channel full, update dropped", zap.String("key", dbKey)) } } @@ -168,6 +179,9 @@ func CloseStream(dbKey string) { } if len(channelsToClose) > 0 { - log.Printf("Closed all streams for key %s (%d clients)", dbKey, len(channelsToClose)) + logger.Info("Closed all streams for", + zap.String("key", dbKey), + zap.Int("clients", len(channelsToClose)), + ) } } diff --git a/utils/stats.go b/utils/stats.go index bbd5477..c50e322 100644 --- a/utils/stats.go +++ b/utils/stats.go @@ -3,13 +3,13 @@ package utils import ( "context" "fmt" - "log" "sync" "sync/atomic" "time" "github.com/goccy/go-json" "github.com/redis/go-redis/v9" + "go.uber.org/zap" ) const ( @@ -23,6 +23,7 @@ var ( Total int64 = 0 ServerClose = make(chan bool, 1) StatsManager *StatManager + logger, _ = zap.NewProduction() ) // StatManager handles collecting and saving statistics @@ -151,8 +152,7 @@ func (sm *StatManager) saveStats(force bool) { // Execute Redis pipeline _, err := pipe.Exec(ctx) if err != nil { - // On error, restore the values - log.Printf("Error saving stats: %v", err) + logger.Error("Error saving stats", zap.Error(err)) atomic.AddInt64(&Total, totalCopy) for path, count := range pathStats { @@ -161,8 +161,7 @@ func (sm *StatManager) saveStats(force bool) { } } } else { - log.Printf("Saved %d total stats across %d paths", - totalCopy, len(pathStats)) + logger.Info("Stats saved", zap.Int64("total", totalCopy), zap.Int("paths", len(pathStats))) } } @@ -188,12 +187,10 @@ func (sm *StatManager) logStats() { }) stats, _ := json.MarshalIndent(snapshot, "", " ") - log.Printf("Stats Health Check:\n%s", string(stats)) + logger.Info("Stats Health Check", zap.String("snapshot", string(stats))) - // Check if we should save stats based on totals if snapshot.Total >= saveThreshold { - log.Printf("Total count high (%d/%d). Triggering save operation.", - snapshot.Total, saveThreshold) + logger.Info("High total count, triggering save", zap.Int64("total", snapshot.Total), zap.Int("threshold", saveThreshold)) go sm.saveStats(true) } }