feat(logger): add async compression for rotated files

CMakeLists.txt

@@ -3,6 +3,9 @@ project(log-it-cpp VERSION 1.0.0 LANGUAGES CXX)
 
 option(LOG_IT_CPP_BUILD_TESTS "Build log-it-cpp tests" ${PROJECT_IS_TOP_LEVEL})
 option(LOG_IT_CPP_BUILD_EXAMPLES "Build log-it-cpp examples" OFF)
+option(LOGIT_WITH_GZIP "Enable gzip via zlib" OFF)
+option(LOGIT_WITH_ZSTD "Enable zstd" OFF)
+option(LOGIT_USE_SUBMODULES "Allow bundled third_party fallback" OFF)
 
 if(NOT DEFINED CMAKE_CXX_STANDARD)
     set(CMAKE_CXX_STANDARD 11)
@@ -30,6 +33,50 @@ target_include_directories(log-it-cpp INTERFACE
 
 target_link_libraries(log-it-cpp INTERFACE time_shield::time_shield)
 
+# ---------- GZIP (zlib) ----------
+if(LOGIT_WITH_GZIP)
+    if(NOT TARGET ZLIB::ZLIB)
+        find_package(ZLIB QUIET)
+    endif()
+    if(NOT TARGET ZLIB::ZLIB AND LOGIT_USE_SUBMODULES)
+        add_subdirectory(libs/zlib EXCLUDE_FROM_ALL)
+        if(TARGET zlibstatic)
+            add_library(ZLIB::ZLIB ALIAS zlibstatic)
+        elseif(TARGET zlib)
+            add_library(ZLIB::ZLIB ALIAS zlib)
+        endif()
+    endif()
+    if(NOT TARGET ZLIB::ZLIB)
+        message(FATAL_ERROR "ZLIB not found. Enable LOGIT_USE_SUBMODULES or install zlib.")
+    endif()
+    target_compile_definitions(log-it-cpp INTERFACE LOGIT_HAS_ZLIB=1)
+    target_link_libraries(log-it-cpp INTERFACE ZLIB::ZLIB)
+endif()
+
+# ---------- ZSTD ----------
+if(LOGIT_WITH_ZSTD)
+    if(NOT TARGET ZSTD::ZSTD)
+        find_package(ZSTD QUIET)
+    endif()
+    if(NOT TARGET ZSTD::ZSTD AND LOGIT_USE_SUBMODULES)
+        add_subdirectory(libs/zstd/build/cmake EXCLUDE_FROM_ALL)
+        if(TARGET libzstd_static)
+            add_library(ZSTD::ZSTD ALIAS libzstd_static)
+        elseif(TARGET libzstd_shared)
+            add_library(ZSTD::ZSTD ALIAS libzstd_shared)
+        endif()
+    endif()
+    if(NOT TARGET ZSTD::ZSTD)
+        message(FATAL_ERROR "ZSTD not found. Enable LOGIT_USE_SUBMODULES or install zstd.")
+    endif()
+    target_compile_definitions(log-it-cpp INTERFACE LOGIT_HAS_ZSTD=1)
+    target_link_libraries(log-it-cpp INTERFACE ZSTD::ZSTD)
+endif()
+
+if(DEFINED VCPKG_TARGET_TRIPLET)
+    set_property(CACHE LOGIT_USE_SUBMODULES PROPERTY VALUE OFF)
+endif()
+
 if(LOG_IT_CPP_BUILD_TESTS)
     enable_testing()
     add_subdirectory(tests)

README.md

@@ -44,7 +44,11 @@ auto now = std::chrono::system_clock::now();
 LOGIT_PRINT_INFO("TimePoint example: ", now);
 ```
 
-- **Support for Multiple Backends**: 
+- **Rotating File Logs**:
+
+  Automatic file rotation based on size with optional asynchronous compression using gzip or zstd.
+
+- **Support for Multiple Backends**:
 
 Easily configure loggers for console and file output. If necessary, add support for sending messages to servers or databases by creating custom backends.
 

include/logit_cpp/logit/LogMacros.hpp

@@ -937,8 +937,7 @@ static_assert(LOGIT_LEVEL_FATAL == static_cast<int>(logit::LogLevel::LOG_LVL_FAT
     logit::Logger::get_instance().add_logger( \
         std::make_unique<logit::FileLogger>( \
             LOGIT_FILE_LOGGER_PATH, true, LOGIT_FILE_LOGGER_AUTO_DELETE_DAYS, \
-            LOGIT_FILE_LOGGER_MAX_FILE_SIZE_BYTES, LOGIT_FILE_LOGGER_MAX_ROTATED_FILES, \
-            LOGIT_FILE_LOGGER_COMPRESS_ROTATED, LOGIT_FILE_LOGGER_COMPRESS_CMD), \
+            LOGIT_FILE_LOGGER_MAX_FILE_SIZE_BYTES, LOGIT_FILE_LOGGER_MAX_ROTATED_FILES), \
         std::make_unique<logit::SimpleLogFormatter>(LOGIT_FILE_LOGGER_PATTERN))
 
 /// \brief Macro for adding a unique file logger with custom parameters.
@@ -1129,8 +1128,7 @@ static_assert(LOGIT_LEVEL_FATAL == static_cast<int>(logit::LogLevel::LOG_LVL_FAT
     logit::Logger::get_instance().add_logger( \
         std::unique_ptr<logit::FileLogger>(new logit::FileLogger( \
             LOGIT_FILE_LOGGER_PATH, true, LOGIT_FILE_LOGGER_AUTO_DELETE_DAYS, \
-            LOGIT_FILE_LOGGER_MAX_FILE_SIZE_BYTES, LOGIT_FILE_LOGGER_MAX_ROTATED_FILES, \
-            LOGIT_FILE_LOGGER_COMPRESS_ROTATED, LOGIT_FILE_LOGGER_COMPRESS_CMD)), \
+            LOGIT_FILE_LOGGER_MAX_FILE_SIZE_BYTES, LOGIT_FILE_LOGGER_MAX_ROTATED_FILES)), \
         std::unique_ptr<logit::SimpleLogFormatter>(new logit::SimpleLogFormatter(LOGIT_FILE_LOGGER_PATTERN)))
 
 /// \brief Macro for adding a unique file logger with custom parameters.

include/logit_cpp/logit/config.hpp

@@ -117,12 +117,6 @@
 #ifndef LOGIT_FILE_LOGGER_MAX_ROTATED_FILES
     #define LOGIT_FILE_LOGGER_MAX_ROTATED_FILES 0
 #endif
-#ifndef LOGIT_FILE_LOGGER_COMPRESS_ROTATED
-    #define LOGIT_FILE_LOGGER_COMPRESS_ROTATED 0
-#endif
-#ifndef LOGIT_FILE_LOGGER_COMPRESS_CMD
-    #define LOGIT_FILE_LOGGER_COMPRESS_CMD ""
-#endif
 
 /// \brief Defines the default log pattern for unique file-based loggers.
 /// If `LOGIT_UNIQUE_FILE_LOGGER_PATTERN` is not defined, it defaults to "%v".

include/logit_cpp/logit/detail/CompressionWorker.hpp

@@ -0,0 +1,231 @@
+#pragma once
+#ifndef _LOGIT_COMPRESSION_WORKER_HPP_INCLUDED
+#define _LOGIT_COMPRESSION_WORKER_HPP_INCLUDED
+
+/// \file CompressionWorker.hpp
+/// \brief Background worker that compresses rotated log files.
+
+#include <string>
+#include <queue>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+#include <fstream>
+#include <vector>
+#include <cstdio>
+
+#if defined(LOGIT_HAS_ZLIB)
+#   include <zlib.h>
+#endif
+#if defined(LOGIT_HAS_ZSTD)
+#   include <zstd.h>
+#endif
+
+#include "../enums.hpp"
+
+namespace logit { namespace detail {
+
+    /// \brief Compress a file using the specified compression type.
+    /// \param type Compression algorithm to use.
+    /// \param src Path to the source file.
+    /// \param level Compression level.
+    /// \param external_cmd Command template for CompressType::EXTERNAL_CMD.
+    /// \return true on success.
+    bool compress_file(CompressType type,
+                       const std::string& src,
+                       int level,
+                       const std::string& external_cmd);
+
+    /// \class CompressionWorker
+    /// \brief Background worker performing asynchronous compression.
+    class CompressionWorker {
+    public:
+        CompressionWorker(CompressType type,
+                           int level,
+                           std::string external_cmd);
+        ~CompressionWorker();
+
+        /// \brief Enqueue a file for compression.
+        void enqueue(std::string path);
+
+        /// \brief Wait until all queued files are processed.
+        void wait();
+
+    private:
+        void run();
+
+        CompressType m_type;
+        int m_level;
+        std::string m_external_cmd;
+        std::queue<std::string> m_q;
+        std::thread m_thread;
+        std::mutex m_mx;
+        std::condition_variable m_cv;
+        std::condition_variable m_cv_idle;
+        bool m_stop = false;
+        bool m_busy = false;
+    };
+
+    inline int clamp_level(int v, int lo, int hi) {
+        if (v < lo) return lo;
+        if (v > hi) return hi;
+        return v;
+    }
+
+    inline bool compress_file_gzip(const std::string& src,
+                                   const std::string& dst_tmp,
+                                   int level) {
+#       if defined(LOGIT_HAS_ZLIB)
+        std::ifstream in(src.c_str(), std::ios::binary);
+        if (!in) return false;
+
+        char mode[8] = "wb6";
+        mode[2] = char('0' + clamp_level(level, 1, 9));
+        gzFile out = gzopen(dst_tmp.c_str(), mode);
+        if (!out) return false;
+        gzbuffer(out, 256 * 1024);
+
+        std::vector<char> buf(256 * 1024);
+        while (in) {
+            in.read(&buf[0], buf.size());
+            std::streamsize got = in.gcount();
+            if (got > 0) {
+                int written = gzwrite(out, &buf[0], static_cast<unsigned>(got));
+                if (written == 0) { gzclose(out); return false; }
+            }
+        }
+        return gzclose(out) == Z_OK;
+#       else
+        (void)src; (void)dst_tmp; (void)level; return false;
+#       endif
+    }
+
+    inline bool compress_file_zstd(const std::string& src,
+                                   const std::string& dst_tmp,
+                                   int level) {
+#       if defined(LOGIT_HAS_ZSTD)
+        std::ifstream in(src.c_str(), std::ios::binary);
+        std::ofstream out(dst_tmp.c_str(), std::ios::binary | std::ios::trunc);
+        if (!in || !out) return false;
+
+        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+        if (!cctx) return false;
+        ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, clamp_level(level, 1, 19));
+
+        const size_t inChunk  = ZSTD_CStreamInSize();
+        const size_t outChunk = ZSTD_CStreamOutSize();
+        std::vector<char> inBuf(inChunk), outBuf(outChunk);
+
+        for (;;) {
+            in.read(&inBuf[0], inBuf.size());
+            size_t rd = static_cast<size_t>(in.gcount());
+            ZSTD_inBuffer zin  = { &inBuf[0], rd, 0 };
+            const int last = in.eof() ? 1 : 0;
+
+            while (zin.pos < zin.size || last) {
+                ZSTD_outBuffer zout = { &outBuf[0], outBuf.size(), 0 };
+                size_t ret = last
+                    ? ZSTD_compressStream2(cctx, &zout, &zin, ZSTD_e_end)
+                    : ZSTD_compressStream2(cctx, &zout, &zin, ZSTD_e_continue);
+                if (ZSTD_isError(ret)) { ZSTD_freeCCtx(cctx); return false; }
+                out.write(&outBuf[0], static_cast<std::streamsize>(zout.pos));
+                if (!out) { ZSTD_freeCCtx(cctx); return false; }
+                if (last && ret == 0 && zin.pos == zin.size) {
+                    ZSTD_freeCCtx(cctx);
+                    out.flush();
+                    return out.good();
+                }
+                if (!last && zin.pos == zin.size) break;
+            }
+        }
+#       else
+        (void)src; (void)dst_tmp; (void)level; return false;
+#       endif
+    }
+
+    inline bool compress_file_external(const std::string& src,
+                                       const std::string& cmd_tpl,
+                                       int level) {
+        if (cmd_tpl.empty()) return false;
+        std::string cmd = cmd_tpl;
+        size_t pos = cmd.find("{file}");
+        if (pos != std::string::npos) cmd.replace(pos, 6, "\"" + src + "\"");
+        pos = cmd.find("{level}");
+        if (pos != std::string::npos) cmd.replace(pos, 7, std::to_string(level));
+        return std::system(cmd.c_str()) == 0;
+    }
+
+    inline bool compress_file(CompressType type,
+                              const std::string& src,
+                              int level,
+                              const std::string& external_cmd) {
+        if (type == CompressType::NONE) return true;
+        if (type == CompressType::EXTERNAL_CMD) {
+            return compress_file_external(src, external_cmd, level);
+        }
+        std::string dst = src + (type == CompressType::GZIP ? ".gz" : ".zst");
+        std::string tmp = dst + ".tmp";
+        bool ok = false;
+        if (type == CompressType::GZIP) ok = compress_file_gzip(src, tmp, level);
+        else ok = compress_file_zstd(src, tmp, level);
+        if (!ok) { std::remove(tmp.c_str()); return false; }
+        if (std::rename(tmp.c_str(), dst.c_str()) != 0) { std::remove(tmp.c_str()); return false; }
+        std::remove(src.c_str());
+        return true;
+    }
+
+    inline CompressionWorker::CompressionWorker(CompressType type,
+                                                int level,
+                                                std::string external_cmd)
+        : m_type(type), m_level(level), m_external_cmd(std::move(external_cmd)) {
+        if (m_type != CompressType::NONE) {
+            m_thread = std::thread(&CompressionWorker::run, this);
+        }
+    }
+
+    inline CompressionWorker::~CompressionWorker() {
+        {
+            std::lock_guard<std::mutex> lk(m_mx);
+            m_stop = true;
+            m_cv.notify_all();
+        }
+        if (m_thread.joinable()) m_thread.join();
+    }
+
+    inline void CompressionWorker::enqueue(std::string path) {
+        std::lock_guard<std::mutex> lk(m_mx);
+        m_q.push(std::move(path));
+        m_cv.notify_one();
+    }
+
+    inline void CompressionWorker::wait() {
+        std::unique_lock<std::mutex> lk(m_mx);
+        m_cv_idle.wait(lk, [this]{ return m_q.empty() && !m_busy; });
+    }
+
+    inline void CompressionWorker::run() {
+        for (;;) {
+            std::string src;
+            {
+                std::unique_lock<std::mutex> lk(m_mx);
+                m_cv.wait(lk, [this]{ return m_stop || !m_q.empty(); });
+                if (m_stop && m_q.empty()) break;
+                src = std::move(m_q.front());
+                m_q.pop();
+                m_busy = true;
+            }
+
+            compress_file(m_type, src, m_level, m_external_cmd);
+
+            {
+                std::lock_guard<std::mutex> lk(m_mx);
+                m_busy = false;
+                if (m_q.empty()) m_cv_idle.notify_all();
+            }
+        }
+    }
+
+}} // namespace logit::detail
+
+#endif // _LOGIT_COMPRESSION_WORKER_HPP_INCLUDED
+

include/logit_cpp/logit/enums.hpp

@@ -51,6 +51,23 @@ namespace logit {
         TimeSinceLastLog       ///< The time elapsed since the last log in seconds.
     };
 
+    /// \enum CompressType
+    /// \brief Supported compression algorithms for rotated files.
+    enum class CompressType {
+        NONE,        ///< Do not compress rotated files.
+        GZIP,        ///< Compress using gzip (zlib).
+        ZSTD,        ///< Compress using zstd.
+        EXTERNAL_CMD ///< Use an external command for compression.
+    };
+
+    /// \enum RotationNaming
+    /// \brief Naming policy for rotated log files.
+    enum class RotationNaming {
+        Sequence,    ///< Append a numeric sequence: YYYY-MM-DD.001.log
+        Timestamp,   ///< Append HHMMSS timestamp: YYYY-MM-DD_HHMMSS.log
+        TimestampMs  ///< Append HHMMSSmmm timestamp: YYYY-MM-DD_HHMMSSmmm.log
+    };
+
     /// \brief Convert LogLevel to a C-style string representation.
     /// \param level The log level.
     /// \param mode The output mode (0 for full name, 1 for abbreviation).