diff --git a/.config/typos.toml b/.config/typos.toml index 6aef279d64a..2d4d98016d2 100644 --- a/.config/typos.toml +++ b/.config/typos.toml @@ -14,6 +14,7 @@ exat = "exat" optin = "optin" smove = "smove" Parth = "Parth" # seems like the spellchecker does not like it is similar to "Path" +Collet = "Collet" # LZ4 author Yann Collet nd = "nd" [default] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 350bfaf5748..639a17f762e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -197,6 +197,39 @@ jobs: - name: test-tls-builtin run: ./runtest --verbose --single unit/tls --dump-logs --tls + test-replication-compression: + # Re-runs the replication suite with repl-compression=lz4 applied + # globally via --config. Test files opt in by carrying the top-level "repl-compression" + # tag (see tests/support/server.tcl), so this job selects them with + # --tags repl-compression instead of a hardcoded filename list that would + # silently drift if a file is renamed. Exercises the streaming-compression + # transport across the replication surface (full sync, dual-channel, buffer + # management, AOF-sync, etc.) to catch regressions that only surface under + # compressed replication. + # Not tagged for this job: integration/replication-psync (start_server lives + # inside a proc, top-level tags cannot apply) and the replication-buffer / + # dual-channel buffer-memory tests (assert exact byte volumes that compression changes). + runs-on: ubuntu-latest + steps: + - name: Install libbacktrace + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: ianlancetaylor/libbacktrace + ref: b9e40069c0b47a722286b94eb5231f7f05c08713 + path: libbacktrace + - run: cd libbacktrace && ./configure && make && sudo make install + - name: Checkout Valkey + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: make + run: make -j4 SERVER_CFLAGS='-Werror' USE_LIBBACKTRACE=yes + - name: install test dependencies + run: sudo apt-get install -y tcl8.6 tclx + - name: replication tests with streaming compression enabled + run: | + ./runtest --verbose --dump-logs \ + --tags "repl-compression -slow" \ + --config repl-compression lz4 + build-debian-old: runs-on: ubuntu-latest container: debian:bullseye diff --git a/REUSE.toml b/REUSE.toml index 9d9e1e38b96..10c453742b4 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -90,6 +90,12 @@ precedence = "aggregate" SPDX-FileCopyrightText = ["2010-2016, Redis Ltd.", "2010-2013, Pieter Noordhuis ", "2024-present, Valkey contributors"] SPDX-License-Identifier = "BSD-2-Clause" +[[annotations]] +path = "deps/lz4/**" +precedence = "aggregate" +SPDX-FileCopyrightText = "2011-2023, Yann Collet" +SPDX-License-Identifier = "BSD-2-Clause" + [[annotations]] path = "deps/hdr_histogram/**" precedence = "aggregate" diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake index 46a7ea81d60..6dab7c0c6b7 100644 --- a/cmake/Modules/SourceFiles.cmake +++ b/cmake/Modules/SourceFiles.cmake @@ -121,7 +121,11 @@ set(VALKEY_SERVER_SRCS ${CMAKE_SOURCE_DIR}/src/vset.c ${CMAKE_SOURCE_DIR}/src/fifo.c ${CMAKE_SOURCE_DIR}/src/mutexqueue.c - ${CMAKE_SOURCE_DIR}/src/queues.c) + ${CMAKE_SOURCE_DIR}/src/queues.c + ${CMAKE_SOURCE_DIR}/src/compression.c + ${CMAKE_SOURCE_DIR}/src/compression_lz4.c + ${CMAKE_SOURCE_DIR}/src/compression_stream.c + ${CMAKE_SOURCE_DIR}/src/compression_repl.c) # valkey-cli diff --git a/cmake/Modules/ValkeySetup.cmake b/cmake/Modules/ValkeySetup.cmake index 14fe910d741..ef05d1d6ee6 100644 --- a/cmake/Modules/ValkeySetup.cmake +++ b/cmake/Modules/ValkeySetup.cmake @@ -282,6 +282,7 @@ include_directories("${CMAKE_SOURCE_DIR}/src/modules/lua") include_directories("${CMAKE_SOURCE_DIR}/deps/linenoise") include_directories("${CMAKE_SOURCE_DIR}/deps/hdr_histogram") include_directories("${CMAKE_SOURCE_DIR}/deps/fpconv") +include_directories("${CMAKE_SOURCE_DIR}/deps/lz4") add_subdirectory("${CMAKE_SOURCE_DIR}/deps") diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index b385d9df7e7..84ba919e1bc 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -33,6 +33,7 @@ add_subdirectory(linenoise) add_subdirectory(fpconv) add_subdirectory(hdr_histogram) add_subdirectory(fast_float) +add_subdirectory(lz4) # Clear any cached variables passed to libvalkey from the cache unset(BUILD_SHARED_LIBS CACHE) diff --git a/deps/Makefile b/deps/Makefile index 2b8af56041a..6bddbafcbe5 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -42,6 +42,7 @@ distclean: -(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true -(cd hdr_histogram && $(MAKE) clean) > /dev/null || true -(cd fpconv && $(MAKE) clean) > /dev/null || true + -(cd lz4 && $(MAKE) clean) > /dev/null || true -(rm -f .make-*) .PHONY: distclean @@ -132,3 +133,9 @@ gtest-parallel: .make-prerequisites rm -rf gtest-parallel; \ git clone --depth 1 https://github.com/google/gtest-parallel.git gtest-parallel; \ fi + +lz4: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd lz4 && $(MAKE) CFLAGS="$(CFLAGS)" + +.PHONY: lz4 diff --git a/deps/README.md b/deps/README.md index 89122e61d44..b2fb264cbe8 100644 --- a/deps/README.md +++ b/deps/README.md @@ -5,6 +5,7 @@ should be provided by the operating system. * **libvalkey** is the official C client library for Valkey. It is used by valkey-cli, valkey-benchmark and Valkey Sentinel. It is managed in a separate project and updated as needed. * **linenoise** is a readline replacement. It is developed by the same authors of Valkey but is managed as a separated project and updated as needed. * **lua** is Lua 5.1 with minor changes for security and additional libraries. +* **LZ4** is the v1.10.0 streaming compression library used for whole-file RDB compression. * **hdr_histogram** Used for per-command latency tracking histograms. * **ffc.h** is a C99 port of the fast_float library, used as a replacement for strtod to convert strings to floats efficiently. * **gtest-parallel** is a script for running googletest tests in parallel. @@ -62,6 +63,24 @@ following: 1. Remove the linenoise directory. 2. Substitute it with the new linenoise source tree. +LZ4 +--- + +LZ4 is imported from the upstream release archive. The vendored copy currently +uses version 1.10.0. `Makefile` and `CMakeLists.txt` are maintained locally for +the Valkey build, and xxHash symbols are namespaced to avoid conflicts with +modules loaded by Valkey. + +To upgrade LZ4: + +1. Download the new release archive from https://github.com/lz4/lz4/releases. +2. Replace `lz4.c`, `lz4.h`, `lz4hc.c`, `lz4hc.h`, `lz4frame.c`, + `lz4frame.h`, `xxhash.c`, `xxhash.h`, and `LICENSE` with the versions from + the release's `lib` directory. +3. Preserve the local `Makefile` and `CMakeLists.txt`, including the + `XXH_NAMESPACE` definition. +4. Update the version recorded above. + Lua --- diff --git a/deps/lz4/CMakeLists.txt b/deps/lz4/CMakeLists.txt new file mode 100644 index 00000000000..c80300d6430 --- /dev/null +++ b/deps/lz4/CMakeLists.txt @@ -0,0 +1,15 @@ +project(lz4) + +set(SRCS + "${CMAKE_CURRENT_LIST_DIR}/lz4.c" + "${CMAKE_CURRENT_LIST_DIR}/lz4.h" + "${CMAKE_CURRENT_LIST_DIR}/lz4hc.c" + "${CMAKE_CURRENT_LIST_DIR}/lz4hc.h" + "${CMAKE_CURRENT_LIST_DIR}/lz4frame.c" + "${CMAKE_CURRENT_LIST_DIR}/lz4frame.h" + "${CMAKE_CURRENT_LIST_DIR}/xxhash.c" + "${CMAKE_CURRENT_LIST_DIR}/xxhash.h") + +add_library(lz4 STATIC ${SRCS}) +target_compile_definitions(lz4 PRIVATE XXH_NAMESPACE=VALKEY_LZ4_) +set_target_properties(lz4 PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/deps/lz4/LICENSE b/deps/lz4/LICENSE new file mode 100644 index 00000000000..488491695a6 --- /dev/null +++ b/deps/lz4/LICENSE @@ -0,0 +1,24 @@ +LZ4 Library +Copyright (c) 2011-2020, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/deps/lz4/Makefile b/deps/lz4/Makefile new file mode 100644 index 00000000000..bfa81717b15 --- /dev/null +++ b/deps/lz4/Makefile @@ -0,0 +1,29 @@ +STD= +WARN= -Wall +OPT= -Os + +R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) -DXXH_NAMESPACE=VALKEY_LZ4_ +R_LDFLAGS= $(LDFLAGS) +DEBUG= -g + +R_CC=$(CC) $(R_CFLAGS) +R_LD=$(CC) $(R_LDFLAGS) + +AR= ar +ARFLAGS= rcs + +liblz4.a: lz4.o lz4hc.o lz4frame.o xxhash.o + $(AR) $(ARFLAGS) $@ $^ + +lz4.o: lz4.c lz4.h +lz4hc.o: lz4hc.c lz4hc.h lz4.h +lz4frame.o: lz4frame.c lz4frame.h lz4.h lz4hc.h xxhash.h +xxhash.o: xxhash.c xxhash.h + +.c.o: + $(R_CC) -fPIC -c $< + +.PHONY: clean +clean: + rm -f *.o + rm -f *.a diff --git a/deps/lz4/lz4.c b/deps/lz4/lz4.c new file mode 100644 index 00000000000..a2f7abee19f --- /dev/null +++ b/deps/lz4/lz4.c @@ -0,0 +1,2829 @@ +/* + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011-2023, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + +/*-************************************ +* Tuning parameters +**************************************/ +/* + * LZ4_HEAPMODE : + * Select how stateless compression functions like `LZ4_compress_default()` + * allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifndef LZ4_HEAPMODE +# define LZ4_HEAPMODE 0 +#endif + +/* + * LZ4_ACCELERATION_DEFAULT : + * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 + */ +#define LZ4_ACCELERATION_DEFAULT 1 +/* + * LZ4_ACCELERATION_MAX : + * Any "acceleration" value higher than this threshold + * get treated as LZ4_ACCELERATION_MAX instead (fix #876) + */ +#define LZ4_ACCELERATION_MAX 65537 + + +/*-************************************ +* CPU Feature Detection +**************************************/ +/* LZ4_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ +# if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define LZ4_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) || defined(_MSC_VER) +# define LZ4_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/* + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count + */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ +# undef LZ4_FORCE_SW_BITCOUNT /* avoid double def */ +# define LZ4_FORCE_SW_BITCOUNT +#endif + + + +/*-************************************ +* Dependency +**************************************/ +/* + * LZ4_SRC_INCLUDED: + * Amalgamation flag, whether lz4.c is included + */ +#ifndef LZ4_SRC_INCLUDED +# define LZ4_SRC_INCLUDED 1 +#endif + +#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS +# define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ +#endif + +#ifndef LZ4_STATIC_LINKING_ONLY +# define LZ4_STATIC_LINKING_ONLY +#endif +#include "lz4.h" +/* see also "memory routines" below */ + + +/*-************************************ +* Compiler Options +**************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */ +# include /* only present in VS2005+ */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 6237) /* disable: C6237: conditional expression is always 0 */ +# pragma warning(disable : 6239) /* disable: C6239: ( && ) always evaluates to the result of */ +# pragma warning(disable : 6240) /* disable: C6240: ( && ) always evaluates to the result of */ +# pragma warning(disable : 6326) /* disable: C6326: Potential comparison of a constant with another constant */ +#endif /* _MSC_VER */ + +#ifndef LZ4_FORCE_INLINE +# if defined (_MSC_VER) && !defined (__clang__) /* MSVC */ +# define LZ4_FORCE_INLINE static __forceinline +# else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# if defined (__GNUC__) || defined (__clang__) +# define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define LZ4_FORCE_INLINE static inline +# endif +# else +# define LZ4_FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +# endif /* _MSC_VER */ +#endif /* LZ4_FORCE_INLINE */ + +/* LZ4_FORCE_O2 and LZ4_FORCE_INLINE + * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, + * together with a simple 8-byte copy loop as a fall-back path. + * However, this optimization hurts the decompression speed by >30%, + * because the execution does not go to the optimized loop + * for typical compressible data, and all of the preamble checks + * before going to the fall-back path become useless overhead. + * This optimization happens only with the -O3 flag, and -O2 generates + * a simple 8-byte copy loop. + * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8 + * functions are annotated with __attribute__((optimize("O2"))), + * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute + * of LZ4_wildCopy8 does not affect the compression speed. + */ +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) +# define LZ4_FORCE_O2 __attribute__((optimize("O2"))) +# undef LZ4_FORCE_INLINE +# define LZ4_FORCE_INLINE static __inline __attribute__((optimize("O2"),always_inline)) +#else +# define LZ4_FORCE_O2 +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#ifndef likely +#define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely +#define unlikely(expr) expect((expr) != 0, 0) +#endif + +/* Should the alignment test prove unreliable, for some reason, + * it can be disabled by setting LZ4_ALIGN_TEST to 0 */ +#ifndef LZ4_ALIGN_TEST /* can be externally provided */ +# define LZ4_ALIGN_TEST 1 +#endif + + +/*-************************************ +* Memory routines +**************************************/ + +/*! LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION : + * Disable relatively high-level LZ4/HC functions that use dynamic memory + * allocation functions (malloc(), calloc(), free()). + * + * Note that this is a compile-time switch. And since it disables + * public/stable LZ4 v1 API functions, we don't recommend using this + * symbol to generate a library for distribution. + * + * The following public functions are removed when this symbol is defined. + * - lz4 : LZ4_createStream, LZ4_freeStream, + * LZ4_createStreamDecode, LZ4_freeStreamDecode, LZ4_create (deprecated) + * - lz4hc : LZ4_createStreamHC, LZ4_freeStreamHC, + * LZ4_createHC (deprecated), LZ4_freeHC (deprecated) + * - lz4frame, lz4file : All LZ4F_* functions + */ +#if defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +# define ALLOC(s) lz4_error_memory_allocation_is_disabled +# define ALLOC_AND_ZERO(s) lz4_error_memory_allocation_is_disabled +# define FREEMEM(p) lz4_error_memory_allocation_is_disabled +#elif defined(LZ4_USER_MEMORY_FUNCTIONS) +/* memory management functions can be customized by user project. + * Below functions must exist somewhere in the Project + * and be available at link time */ +void* LZ4_malloc(size_t s); +void* LZ4_calloc(size_t n, size_t s); +void LZ4_free(void* p); +# define ALLOC(s) LZ4_malloc(s) +# define ALLOC_AND_ZERO(s) LZ4_calloc(1,s) +# define FREEMEM(p) LZ4_free(p) +#else +# include /* malloc, calloc, free */ +# define ALLOC(s) malloc(s) +# define ALLOC_AND_ZERO(s) calloc(1,s) +# define FREEMEM(p) free(p) +#endif + +#if ! LZ4_FREESTANDING +# include /* memset, memcpy */ +#endif +#if !defined(LZ4_memset) +# define LZ4_memset(p,v,s) memset((p),(v),(s)) +#endif +#define MEM_INIT(p,v,s) LZ4_memset((p),(v),(s)) + + +/*-************************************ +* Common Constants +**************************************/ +#define MINMATCH 4 + +#define WILDCOPYLENGTH 8 +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 +static const int LZ4_minLength = (MFLIMIT+1); + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 +#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ +# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" +#endif + +#define ML_BITS 4 +#define ML_MASK ((1U<=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include + static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ " %i: ", __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + +static int LZ4_isAligned(const void* ptr, size_t alignment) +{ + return ((size_t)ptr & (alignment -1)) == 0; +} + + +/*-************************************ +* Types +**************************************/ +#include +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef uintptr_t uptrval; +#else +# if UINT_MAX != 4294967295UL +# error "LZ4 code (when not C++ or C99) assumes that sizeof(int) == 4" +# endif + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef size_t uptrval; /* generally true, except OpenVMS-64 */ +#endif + +#if defined(__x86_64__) + typedef U64 reg_t; /* 64-bits in x32 mode */ +#else + typedef size_t reg_t; /* 32-bits in x32 mode */ +#endif + +typedef enum { + notLimited = 0, + limitedOutput = 1, + fillOutput = 2 +} limitedOutput_directive; + + +/*-************************************ +* Reading and writing into memory +**************************************/ + +/** + * LZ4 relies on memcpy with a constant size being inlined. In freestanding + * environments, the compiler can't assume the implementation of memcpy() is + * standard compliant, so it can't apply its specialized memcpy() inlining + * logic. When possible, use __builtin_memcpy() to tell the compiler to analyze + * memcpy() as if it were standard compliant, so it can inline it in freestanding + * environments. This is needed when decompressing the Linux Kernel, for example. + */ +#if !defined(LZ4_memcpy) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +# else +# define LZ4_memcpy(dst, src, size) memcpy(dst, src, size) +# endif +#endif + +#if !defined(LZ4_memmove) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memmove __builtin_memmove +# else +# define LZ4_memmove memmove +# endif +#endif + +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(__GNUC__) || defined(__INTEL_COMPILER) +#define LZ4_PACK( __Declaration__ ) __Declaration__ __attribute__((__packed__)) +#elif defined(_MSC_VER) +#define LZ4_PACK( __Declaration__ ) __pragma( pack(push, 1) ) __Declaration__ __pragma( pack(pop)) +#endif + +#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) +/* lie to the compiler about data alignment; use with caution */ + +static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } +static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } +static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } + +static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } + +#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +LZ4_PACK(typedef struct { U16 u16; }) LZ4_unalign16; +LZ4_PACK(typedef struct { U32 u32; }) LZ4_unalign32; +LZ4_PACK(typedef struct { reg_t uArch; }) LZ4_unalignST; + +static U16 LZ4_read16(const void* ptr) { return ((const LZ4_unalign16*)ptr)->u16; } +static U32 LZ4_read32(const void* ptr) { return ((const LZ4_unalign32*)ptr)->u32; } +static reg_t LZ4_read_ARCH(const void* ptr) { return ((const LZ4_unalignST*)ptr)->uArch; } + +static void LZ4_write16(void* memPtr, U16 value) { ((LZ4_unalign16*)memPtr)->u16 = value; } +static void LZ4_write32(void* memPtr, U32 value) { ((LZ4_unalign32*)memPtr)->u32 = value; } + +#else /* safe and portable access using memcpy() */ + +static U16 LZ4_read16(const void* memPtr) +{ + U16 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static U32 LZ4_read32(const void* memPtr) +{ + U32 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static reg_t LZ4_read_ARCH(const void* memPtr) +{ + reg_t val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static void LZ4_write16(void* memPtr, U16 value) +{ + LZ4_memcpy(memPtr, &value, sizeof(value)); +} + +static void LZ4_write32(void* memPtr, U32 value) +{ + LZ4_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* LZ4_FORCE_MEMORY_ACCESS */ + + +static U16 LZ4_readLE16(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read16(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] | (p[1]<<8)); + } +} + +#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT +static U32 LZ4_readLE32(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read32(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U32)p[0] | (p[1]<<8) | (p[2]<<16) | (p[3]<<24); + } +} +#endif + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if (LZ4_isLittleEndian()) { + LZ4_write16(memPtr, value); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } +} + +/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ +LZ4_FORCE_INLINE +void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_memcpy(d,s,8); d+=8; s+=8; } while (d= 16. */ +LZ4_FORCE_INLINE void +LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_memcpy(d,s,16); LZ4_memcpy(d+16,s+16,16); d+=32; s+=32; } while (d= dstPtr + MINMATCH + * - there is at least 12 bytes available to write after dstEnd */ +LZ4_FORCE_INLINE void +LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) +{ + BYTE v[8]; + + assert(dstEnd >= dstPtr + MINMATCH); + + switch(offset) { + case 1: + MEM_INIT(v, *srcPtr, 8); + break; + case 2: + LZ4_memcpy(v, srcPtr, 2); + LZ4_memcpy(&v[2], srcPtr, 2); +#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */ +# pragma warning(push) +# pragma warning(disable : 6385) /* warning C6385: Reading invalid data from 'v'. */ +#endif + LZ4_memcpy(&v[4], v, 4); +#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */ +# pragma warning(pop) +#endif + break; + case 4: + LZ4_memcpy(v, srcPtr, 4); + LZ4_memcpy(&v[4], srcPtr, 4); + break; + default: + LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); + return; + } + + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + while (dstPtr < dstEnd) { + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + } +} +#endif + + +/*-************************************ +* Common functions +**************************************/ +static unsigned LZ4_NbCommonBytes (reg_t val) +{ + assert(val != 0); + if (LZ4_isLittleEndian()) { + if (sizeof(val) == 8) { +# if defined(_MSC_VER) && (_MSC_VER >= 1800) && (defined(_M_AMD64) && !defined(_M_ARM64EC)) && !defined(LZ4_FORCE_SW_BITCOUNT) +/*-************************************************************************************************* +* ARM64EC is a Microsoft-designed ARM64 ABI compatible with AMD64 applications on ARM64 Windows 11. +* The ARM64EC ABI does not support AVX/AVX2/AVX512 instructions, nor their relevant intrinsics +* including _tzcnt_u64. Therefore, we need to neuter the _tzcnt_u64 code path for ARM64EC. +****************************************************************************************************/ +# if defined(__clang__) && (__clang_major__ < 10) + /* Avoid undefined clang-cl intrinsics issue. + * See https://github.com/lz4/lz4/pull/1017 for details. */ + return (unsigned)__builtin_ia32_tzcnt_u64(val) >> 3; +# else + /* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */ + return (unsigned)_tzcnt_u64(val) >> 3; +# endif +# elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64(&r, (U64)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctzll((U64)val) >> 3; +# else + const U64 m = 0x0101010101010101ULL; + val ^= val - 1; + return (unsigned)(((U64)((val & (m - 1)) * m)) >> 56); +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward(&r, (U32)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctz((U32)val) >> 3; +# else + const U32 m = 0x01010101; + return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24; +# endif + } + } else /* Big Endian CPU */ { + if (sizeof(val)==8) { +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clzll((U64)val) >> 3; +# else +#if 1 + /* this method is probably faster, + * but adds a 128 bytes lookup table */ + static const unsigned char ctz7_tab[128] = { + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + }; + U64 const mask = 0x0101010101010101ULL; + U64 const t = (((val >> 8) - mask) | val) & mask; + return ctz7_tab[(t * 0x0080402010080402ULL) >> 57]; +#else + /* this method doesn't consume memory space like the previous one, + * but it contains several branches, + * that may end up slowing execution */ + static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. + Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. + Note that this code path is never triggered in 32-bits mode. */ + unsigned r; + if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +#endif +# endif + } else /* 32 bits */ { +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clz((U32)val) >> 3; +# else + val >>= 8; + val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | + (val + 0x00FF0000)) >> 24; + return (unsigned)val ^ 3; +# endif + } + } +} + + +#define STEPSIZE sizeof(reg_t) +LZ4_FORCE_INLINE +unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + if (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { + pIn+=STEPSIZE; pMatch+=STEPSIZE; + } else { + return LZ4_NbCommonBytes(diff); + } } + + while (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } + pIn += LZ4_NbCommonBytes(diff); + return (unsigned)(pIn - pStart); + } + + if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn compression run slower on incompressible data */ + + +/*-************************************ +* Local Structures and types +**************************************/ +typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; + +/** + * This enum distinguishes several different modes of accessing previous + * content in the stream. + * + * - noDict : There is no preceding content. + * - withPrefix64k : Table entries up to ctx->dictSize before the current blob + * blob being compressed are valid and refer to the preceding + * content (of length ctx->dictSize), which is available + * contiguously preceding in memory the content currently + * being compressed. + * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere + * else in memory, starting at ctx->dictionary with length + * ctx->dictSize. + * - usingDictCtx : Everything concerning the preceding content is + * in a separate context, pointed to by ctx->dictCtx. + * ctx->dictionary, ctx->dictSize, and table entries + * in the current context that refer to positions + * preceding the beginning of the current compression are + * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx + * ->dictSize describe the location and size of the preceding + * content, and matches are found by looking in the ctx + * ->dictCtx->hashTable. + */ +typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; +typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; + + +/*-************************************ +* Local Utils +**************************************/ +int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +int LZ4_sizeofState(void) { return sizeof(LZ4_stream_t); } + + +/*-**************************************** +* Internal Definitions, used only in Tests +*******************************************/ +#if defined (__cplusplus) +extern "C" { +#endif + +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); + +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize); +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize); +#if defined (__cplusplus) +} +#endif + +/*-****************************** +* Compression functions +********************************/ +LZ4_FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +{ + if (tableType == byU16) + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +{ + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + if (LZ4_isLittleEndian()) { + const U64 prime5bytes = 889523592379ULL; + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + } else { + const U64 prime8bytes = 11400714785074694791ULL; + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + } +} + +LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) +{ + if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + +#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT + return LZ4_hash4(LZ4_readLE32(p), tableType); +#else + return LZ4_hash4(LZ4_read32(p), tableType); +#endif +} + +LZ4_FORCE_INLINE void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ assert(0); return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } + case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } + } +} + +/* LZ4_putPosition*() : only used in byPtr mode */ +LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, + void* tableBase, tableType_t const tableType) +{ + const BYTE** const hashTable = (const BYTE**)tableBase; + assert(tableType == byPtr); (void)tableType; + hashTable[h] = p; +} + +LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType) +{ + U32 const h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType); +} + +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +LZ4_FORCE_INLINE U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { + const U32* const hashTable = (const U32*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-2))); + return hashTable[h]; + } + if (tableType == byU16) { + const U16* const hashTable = (const U16*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-1))); + return hashTable[h]; + } + assert(0); return 0; /* forbidden case */ +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + assert(tableType == byPtr); (void)tableType; + { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } +} + +LZ4_FORCE_INLINE const BYTE* +LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType) +{ + U32 const h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType); +} + +LZ4_FORCE_INLINE void +LZ4_prepareTable(LZ4_stream_t_internal* const cctx, + const int inputSize, + const tableType_t tableType) { + /* If the table hasn't been used, it's guaranteed to be zeroed out, and is + * therefore safe to use no matter what mode we're in. Otherwise, we figure + * out if it's safe to leave as is or whether it needs to be reset. + */ + if ((tableType_t)cctx->tableType != clearedTable) { + assert(inputSize >= 0); + if ((tableType_t)cctx->tableType != tableType + || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) + || ((tableType == byU32) && cctx->currentOffset > 1 GB) + || tableType == byPtr + || inputSize >= 4 KB) + { + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); + MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); + cctx->currentOffset = 0; + cctx->tableType = (U32)clearedTable; + } else { + DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); + } + } + + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, + * is faster than compressing without a gap. + * However, compressing with currentOffset == 0 is faster still, + * so we preserve that case. + */ + if (cctx->currentOffset != 0 && tableType == byU32) { + DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); + cctx->currentOffset += 64 KB; + } + + /* Finally, clear history */ + cctx->dictCtx = NULL; + cctx->dictionary = NULL; + cctx->dictSize = 0; +} + +/** LZ4_compress_generic_validated() : + * inlined, to ensure branches are decided at compilation time. + * The following conditions are presumed already validated: + * - source != NULL + * - inputSize > 0 + */ +LZ4_FORCE_INLINE int LZ4_compress_generic_validated( + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + int* inputConsumed, /* only written when outputDirective == fillOutput */ + const int maxOutputSize, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + int result; + const BYTE* ip = (const BYTE*)source; + + U32 const startIndex = cctx->currentOffset; + const BYTE* base = (const BYTE*)source - startIndex; + const BYTE* lowLimit; + + const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; + const BYTE* const dictionary = + dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; + const U32 dictSize = + dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; + const U32 dictDelta = + (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with indexes in current context */ + + int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ + const BYTE* const dictEnd = dictionary ? dictionary + dictSize : dictionary; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; + const BYTE* const matchlimit = iend - LASTLITERALS; + + /* the dictCtx currentOffset is indexed on the start of the dictionary, + * while a dictionary in the current context precedes the currentOffset */ + const BYTE* dictBase = (dictionary == NULL) ? NULL : + (dictDirective == usingDictCtx) ? + dictionary + dictSize - dictCtx->currentOffset : + dictionary + dictSize - startIndex; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 offset = 0; + U32 forwardH; + + DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", inputSize, tableType); + assert(ip != NULL); + if (tableType == byU16) assert(inputSize= 1); + + lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); + + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; + } + cctx->currentOffset += (U32)inputSize; + cctx->tableType = (U32)tableType; + + if (inputSizehashTable, byPtr); + } else { + LZ4_putIndexOnHash(startIndex, h, cctx->hashTable, tableType); + } } + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + BYTE* token; + const BYTE* filledIp; + + /* Find a match */ + if (tableType == byPtr) { + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType); + + } while ( (match+LZ4_DISTANCE_MAX < ip) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + + } else { /* byU32, byU16 */ + + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const current = (U32)(forwardIp - base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex <= current); + assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + matchIndex += dictDelta; /* make dictCtx index comparable with current context */ + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else if (dictDirective == usingExtDict) { + if (matchIndex < startIndex) { + DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); + assert(startIndex - matchIndex >= MINMATCH); + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else { /* single continuous memory segment */ + match = base + matchIndex; + } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + + DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ + assert(matchIndex < current); + if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + && (matchIndex+LZ4_DISTANCE_MAX < current)) { + continue; + } /* too far */ + assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ + + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_extMem) offset = current - matchIndex; + break; /* match found */ + } + + } while(1); + } + + /* Catch up */ + filledIp = ip; + assert(ip > anchor); /* this is always true as ip has been advanced before entering the main loop */ + if ((match > lowLimit) && unlikely(ip[-1] == match[-1])) { + do { ip--; match--; } while (((ip > anchor) & (match > lowLimit)) && (unlikely(ip[-1] == match[-1]))); + } + + /* Encode Literals */ + { unsigned const litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) { + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + if ((outputDirective == fillOutput) && + (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { + op--; + goto _last_literals; + } + if (litLength >= RUN_MASK) { + unsigned len = litLength - RUN_MASK; + *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< olimit)) { + /* the match was too close to the end, rewind and go to last literals */ + op = token; + goto _last_literals; + } + + /* Encode Offset */ + if (maybe_extMem) { /* static test */ + DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); + assert(offset <= LZ4_DISTANCE_MAX && offset > 0); + LZ4_writeLE16(op, (U16)offset); op+=2; + } else { + DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); + assert(ip-match <= LZ4_DISTANCE_MAX); + LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + } + + /* Encode MatchLength */ + { unsigned matchCode; + + if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) + && (lowLimit==dictionary) /* match within extDict */ ) { + const BYTE* limit = ip + (dictEnd-match); + assert(dictEnd > match); + if (limit > matchlimit) limit = matchlimit; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += (size_t)matchCode + MINMATCH; + if (ip==limit) { + unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); + matchCode += more; + ip += more; + } + DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += (size_t)matchCode + MINMATCH; + DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); + } + + if ((outputDirective) && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) { + if (outputDirective == fillOutput) { + /* Match description too long : reduce it */ + U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255; + ip -= matchCode - newMatchCode; + assert(newMatchCode < matchCode); + matchCode = newMatchCode; + if (unlikely(ip <= filledIp)) { + /* We have already filled up to filledIp so if ip ends up less than filledIp + * we have positions in the hash table beyond the current position. This is + * a problem if we reuse the hash table. So we have to remove these positions + * from the hash table. + */ + const BYTE* ptr; + DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip)); + for (ptr = ip; ptr <= filledIp; ++ptr) { + U32 const h = LZ4_hashPosition(ptr, tableType); + LZ4_clearHash(h, cctx->hashTable, tableType); + } + } + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) { + op+=4; + LZ4_write32(op, 0xFFFFFFFF); + matchCode -= 4*255; + } + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); + } + /* Ensure we have enough space for the last literals. */ + assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit)); + + anchor = ip; + + /* Test end of chunk */ + if (ip >= mflimitPlusOne) break; + + /* Fill table */ + { U32 const h = LZ4_hashPosition(ip-2, tableType); + if (tableType == byPtr) { + LZ4_putPositionOnHash(ip-2, h, cctx->hashTable, byPtr); + } else { + U32 const idx = (U32)((ip-2) - base); + LZ4_putIndexOnHash(idx, h, cctx->hashTable, tableType); + } } + + /* Test next position */ + if (tableType == byPtr) { + + match = LZ4_getPosition(ip, cctx->hashTable, tableType); + LZ4_putPosition(ip, cctx->hashTable, tableType); + if ( (match+LZ4_DISTANCE_MAX >= ip) + && (LZ4_read32(match) == LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + } else { /* byU32, byU16 */ + + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const current = (U32)(ip-base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex < current); + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + matchIndex += dictDelta; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else { /* single memory segment */ + match = base + matchIndex; + } + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + assert(matchIndex < current); + if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) + && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) + && (LZ4_read32(match) == LZ4_read32(ip)) ) { + token=op++; + *token=0; + if (maybe_extMem) offset = current - matchIndex; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", + (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); + goto _next_match; + } + } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + + } + +_last_literals: + /* Encode Last Literals */ + { size_t lastRun = (size_t)(iend - anchor); + if ( (outputDirective) && /* Check output buffer overflow */ + (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) { + if (outputDirective == fillOutput) { + /* adapt lastRun to fill 'dst' */ + assert(olimit >= op); + lastRun = (size_t)(olimit-op) - 1/*token*/; + lastRun -= (lastRun + 256 - RUN_MASK) / 256; /*additional length tokens*/ + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRun); + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun< 0); + DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, result); + return result; +} + +/** LZ4_compress_generic() : + * inlined, to ensure branches are decided at compilation time; + * takes care of src == (NULL, 0) + * and forward the rest to LZ4_compress_generic_validated */ +LZ4_FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal* const cctx, + const char* const src, + char* const dst, + const int srcSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ + const int dstCapacity, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, dstCapacity=%i", + srcSize, dstCapacity); + + if ((U32)srcSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported srcSize, too large (or negative) */ + if (srcSize == 0) { /* src == NULL supported if srcSize == 0 */ + if (outputDirective != notLimited && dstCapacity <= 0) return 0; /* no output, can't write anything */ + DEBUGLOG(5, "Generating an empty block"); + assert(outputDirective == notLimited || dstCapacity >= 1); + assert(dst != NULL); + dst[0] = 0; + if (outputDirective == fillOutput) { + assert (inputConsumed != NULL); + *inputConsumed = 0; + } + return 1; + } + assert(src != NULL); + + return LZ4_compress_generic_validated(cctx, src, dst, srcSize, + inputConsumed, /* only written into if outputDirective == fillOutput */ + dstCapacity, outputDirective, + tableType, dictDirective, dictIssue, acceleration); +} + + +int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; + assert(ctx != NULL); + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + +/** + * LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of + * "correctly initialized"). + */ +int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) +{ + LZ4_stream_t_internal* const ctx = &((LZ4_stream_t*)state)->internal_donotuse; + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + assert(ctx != NULL); + + if (dstCapacity >= LZ4_compressBound(srcSize)) { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + + +int LZ4_compress_fast(const char* src, char* dest, int srcSize, int dstCapacity, int acceleration) +{ + int result; +#if (LZ4_HEAPMODE) + LZ4_stream_t* const ctxPtr = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctxPtr == NULL) return 0; +#else + LZ4_stream_t ctx; + LZ4_stream_t* const ctxPtr = &ctx; +#endif + result = LZ4_compress_fast_extState(ctxPtr, src, dest, srcSize, dstCapacity, acceleration); + +#if (LZ4_HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} + + +int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast(src, dst, srcSize, dstCapacity, 1); +} + + +/* Note!: This function leaves the stream in an unclean/broken state! + * It is not safe to subsequently use the same state with a _fastReset() or + * _continue() call without resetting it. */ +static int LZ4_compress_destSize_extState_internal(LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration) +{ + void* const s = LZ4_initStream(state, sizeof (*state)); + assert(s != NULL); (void)s; + + if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, acceleration); + } else { + if (*srcSizePtr < LZ4_64Klimit) { + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, acceleration); + } else { + tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, acceleration); + } } +} + +int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration) +{ + int const r = LZ4_compress_destSize_extState_internal((LZ4_stream_t*)state, src, dst, srcSizePtr, targetDstSize, acceleration); + /* clean the state on exit */ + LZ4_initStream(state, sizeof (LZ4_stream_t)); + return r; +} + + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ +#if (LZ4_HEAPMODE) + LZ4_stream_t* const ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctx == NULL) return 0; +#else + LZ4_stream_t ctxBody; + LZ4_stream_t* const ctx = &ctxBody; +#endif + + int result = LZ4_compress_destSize_extState_internal(ctx, src, dst, srcSizePtr, targetDstSize, 1); + +#if (LZ4_HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} + + + +/*-****************************** +* Streaming functions +********************************/ + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); + LZ4_STATIC_ASSERT(sizeof(LZ4_stream_t) >= sizeof(LZ4_stream_t_internal)); + DEBUGLOG(4, "LZ4_createStream %p", lz4s); + if (lz4s == NULL) return NULL; + LZ4_initStream(lz4s, sizeof(*lz4s)); + return lz4s; +} +#endif + +static size_t LZ4_stream_t_alignment(void) +{ +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_stream_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_stream_t); +#else + return 1; /* effectively disabled */ +#endif +} + +LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) +{ + DEBUGLOG(5, "LZ4_initStream"); + if (buffer == NULL) { return NULL; } + if (size < sizeof(LZ4_stream_t)) { return NULL; } + if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) return NULL; + MEM_INIT(buffer, 0, sizeof(LZ4_stream_t_internal)); + return (LZ4_stream_t*)buffer; +} + +/* resetStream is now deprecated, + * prefer initStream() which is more general */ +void LZ4_resetStream (LZ4_stream_t* LZ4_stream) +{ + DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); +} + +void LZ4_resetStream_fast(LZ4_stream_t* ctx) { + LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +int LZ4_freeStream (LZ4_stream_t* LZ4_stream) +{ + if (!LZ4_stream) return 0; /* support free on NULL */ + DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream); + FREEMEM(LZ4_stream); + return (0); +} +#endif + + +typedef enum { _ld_fast, _ld_slow } LoadDict_mode_e; +#define HASH_UNIT sizeof(reg_t) +int LZ4_loadDict_internal(LZ4_stream_t* LZ4_dict, + const char* dictionary, int dictSize, + LoadDict_mode_e _ld) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + const tableType_t tableType = byU32; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + U32 idx32; + + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); + + /* It's necessary to reset the context, + * and not just continue it with prepareTable() + * to avoid any risk of generating overflowing matchIndex + * when compressing using this dictionary */ + LZ4_resetStream(LZ4_dict); + + /* We always increment the offset by 64 KB, since, if the dict is longer, + * we truncate it to the last 64k, and if it's shorter, we still want to + * advance by a whole window length so we can provide the guarantee that + * there are only valid offsets in the window, which allows an optimization + * in LZ4_compress_fast_continue() where it uses noDictIssue even when the + * dictionary isn't a full 64k. */ + dict->currentOffset += 64 KB; + + if (dictSize < (int)HASH_UNIT) { + return 0; + } + + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->tableType = (U32)tableType; + idx32 = dict->currentOffset - dict->dictSize; + + while (p <= dictEnd-HASH_UNIT) { + U32 const h = LZ4_hashPosition(p, tableType); + /* Note: overwriting => favors positions end of dictionary */ + LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType); + p+=3; idx32+=3; + } + + if (_ld == _ld_slow) { + /* Fill hash table with additional references, to improve compression capability */ + p = dict->dictionary; + idx32 = dict->currentOffset - dict->dictSize; + while (p <= dictEnd-HASH_UNIT) { + U32 const h = LZ4_hashPosition(p, tableType); + U32 const limit = dict->currentOffset - 64 KB; + if (LZ4_getIndexOnHash(h, dict->hashTable, tableType) <= limit) { + /* Note: not overwriting => favors positions beginning of dictionary */ + LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType); + } + p++; idx32++; + } + } + + return (int)dict->dictSize; +} + +int LZ4_loadDict(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_fast); +} + +int LZ4_loadDictSlow(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_slow); +} + +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) +{ + const LZ4_stream_t_internal* dictCtx = (dictionaryStream == NULL) ? NULL : + &(dictionaryStream->internal_donotuse); + + DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", + workingStream, dictionaryStream, + dictCtx != NULL ? dictCtx->dictSize : 0); + + if (dictCtx != NULL) { + /* If the current offset is zero, we will never look in the + * external dictionary context, since there is no value a table + * entry can take that indicate a miss. In that case, we need + * to bump the offset to something non-zero. + */ + if (workingStream->internal_donotuse.currentOffset == 0) { + workingStream->internal_donotuse.currentOffset = 64 KB; + } + + /* Don't actually attach an empty dictionary. + */ + if (dictCtx->dictSize == 0) { + dictCtx = NULL; + } + } + workingStream->internal_donotuse.dictCtx = dictCtx; +} + + +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) +{ + assert(nextSize >= 0); + if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ + /* rescale hash table */ + U32 const delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + DEBUGLOG(4, "LZ4_renormDictT"); + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; + } +} + + +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, + const char* source, char* dest, + int inputSize, int maxOutputSize, + int acceleration) +{ + const tableType_t tableType = byU32; + LZ4_stream_t_internal* const streamPtr = &LZ4_stream->internal_donotuse; + const char* dictEnd = streamPtr->dictSize ? (const char*)streamPtr->dictionary + streamPtr->dictSize : NULL; + + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", inputSize, streamPtr->dictSize); + + LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */ + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + + /* invalidate tiny dictionaries */ + if ( (streamPtr->dictSize < 4) /* tiny dictionary : not enough for a hash */ + && (dictEnd != source) /* prefix mode */ + && (inputSize > 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */ + && (streamPtr->dictCtx == NULL) /* usingDictCtx */ + ) { + DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); + /* remove dictionary existence from history, to employ faster prefix mode */ + streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)source; + dictEnd = source; + } + + /* Check overlapping input/dictionary space */ + { const char* const sourceEnd = source + inputSize; + if ((sourceEnd > (const char*)streamPtr->dictionary) && (sourceEnd < dictEnd)) { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == source) { + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); + else + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); + } + + /* external dictionary mode */ + { int result; + if (streamPtr->dictCtx) { + /* We depend here on the fact that dictCtx'es (produced by + * LZ4_loadDict) guarantee that their tables contain no references + * to offsets between dictCtx->currentOffset - 64 KB and + * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe + * to use noDictIssue even when the dict isn't a full 64 KB. + */ + if (inputSize > 4 KB) { + /* For compressing large blobs, it is faster to pay the setup + * cost to copy the dictionary's tables into the active context, + * so that the compression loop is only looking into one table. + */ + LZ4_memcpy(streamPtr, streamPtr->dictCtx, sizeof(*streamPtr)); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); + } + } else { /* small data <= 4 KB */ + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } + } + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + return result; + } +} + + +/* Hidden debug function, to force-test external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) +{ + LZ4_stream_t_internal* const streamPtr = &LZ4_dict->internal_donotuse; + int result; + + LZ4_renormDictT(streamPtr, srcSize); + + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + } + + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)srcSize; + + return result; +} + + +/*! LZ4_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : no need to call LZ4_loadDict() afterwards, dictionary is immediately usable, + * one can therefore call LZ4_compress_fast_continue() right after. + * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ +int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + + DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer); + + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } + + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) { + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + assert(dict->dictionary); + LZ4_memmove(safeBuffer, previousDictEnd - dictSize, (size_t)dictSize); + } + + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; + + return dictSize; +} + + + +/*-******************************* + * Decompression functions + ********************************/ + +typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; + +#undef MIN +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + + +/* variant for decompress_unsafe() + * does not know end of input + * presumes input is well formed + * note : will consume at least one byte */ +static size_t read_long_length_no_check(const BYTE** pp) +{ + size_t b, l = 0; + do { b = **pp; (*pp)++; l += b; } while (b==255); + DEBUGLOG(6, "read_long_length_no_check: +length=%zu using %zu input bytes", l, l/255 + 1) + return l; +} + +/* core decoder variant for LZ4_decompress_fast*() + * for legacy support only : these entry points are deprecated. + * - Presumes input is correctly formed (no defense vs malformed inputs) + * - Does not know input size (presume input buffer is "large enough") + * - Decompress a full block (only) + * @return : nb of bytes read from input. + * Note : this variant is not optimized for speed, just for maintenance. + * the goal is to remove support of decompress_fast*() variants by v2.0 +**/ +LZ4_FORCE_INLINE int +LZ4_decompress_unsafe_generic( + const BYTE* const istart, + BYTE* const ostart, + int decompressedSize, + + size_t prefixSize, + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note: =0 if dictStart==NULL */ + ) +{ + const BYTE* ip = istart; + BYTE* op = (BYTE*)ostart; + BYTE* const oend = ostart + decompressedSize; + const BYTE* const prefixStart = ostart - prefixSize; + + DEBUGLOG(5, "LZ4_decompress_unsafe_generic"); + if (dictStart == NULL) assert(dictSize == 0); + + while (1) { + /* start new sequence */ + unsigned token = *ip++; + + /* literals */ + { size_t ll = token >> ML_BITS; + if (ll==15) { + /* long literal length */ + ll += read_long_length_no_check(&ip); + } + if ((size_t)(oend-op) < ll) return -1; /* output buffer overflow */ + LZ4_memmove(op, ip, ll); /* support in-place decompression */ + op += ll; + ip += ll; + if ((size_t)(oend-op) < MFLIMIT) { + if (op==oend) break; /* end of block */ + DEBUGLOG(5, "invalid: literals end at distance %zi from end of block", oend-op); + /* incorrect end of block : + * last match must start at least MFLIMIT==12 bytes before end of output block */ + return -1; + } } + + /* match */ + { size_t ml = token & 15; + size_t const offset = LZ4_readLE16(ip); + ip+=2; + + if (ml==15) { + /* long literal length */ + ml += read_long_length_no_check(&ip); + } + ml += MINMATCH; + + if ((size_t)(oend-op) < ml) return -1; /* output buffer overflow */ + + { const BYTE* match = op - offset; + + /* out of range */ + if (offset > (size_t)(op - prefixStart) + dictSize) { + DEBUGLOG(6, "offset out of range"); + return -1; + } + + /* check special case : extDict */ + if (offset > (size_t)(op - prefixStart)) { + /* extDict scenario */ + const BYTE* const dictEnd = dictStart + dictSize; + const BYTE* extMatch = dictEnd - (offset - (size_t)(op-prefixStart)); + size_t const extml = (size_t)(dictEnd - extMatch); + if (extml > ml) { + /* match entirely within extDict */ + LZ4_memmove(op, extMatch, ml); + op += ml; + ml = 0; + } else { + /* match split between extDict & prefix */ + LZ4_memmove(op, extMatch, extml); + op += extml; + ml -= extml; + } + match = prefixStart; + } + + /* match copy - slow variant, supporting overlap copy */ + { size_t u; + for (u=0; u= ipmax before start of loop. Returns initial_error if so. + * @error (output) - error code. Must be set to 0 before call. +**/ +typedef size_t Rvl_t; +static const Rvl_t rvl_error = (Rvl_t)(-1); +LZ4_FORCE_INLINE Rvl_t +read_variable_length(const BYTE** ip, const BYTE* ilimit, + int initial_check) +{ + Rvl_t s, length = 0; + assert(ip != NULL); + assert(*ip != NULL); + assert(ilimit != NULL); + if (initial_check && unlikely((*ip) >= ilimit)) { /* read limit reached */ + return rvl_error; + } + s = **ip; + (*ip)++; + length += s; + if (unlikely((*ip) > ilimit)) { /* read limit reached */ + return rvl_error; + } + /* accumulator overflow detection (32-bit mode only) */ + if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { + return rvl_error; + } + if (likely(s != 255)) return length; + do { + s = **ip; + (*ip)++; + length += s; + if (unlikely((*ip) > ilimit)) { /* read limit reached */ + return rvl_error; + } + /* accumulator overflow detection (32-bit mode only) */ + if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { + return rvl_error; + } + } while (s == 255); + + return length; +} + +/*! LZ4_decompress_generic() : + * This generic decompression function covers all use cases. + * It shall be instantiated several times, using different sets of directives. + * Note that it is important for performance that this function really get inlined, + * in order to remove useless branches during compilation optimization. + */ +LZ4_FORCE_INLINE int +LZ4_decompress_generic( + const char* const src, + char* const dst, + int srcSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ + + earlyEnd_directive partialDecoding, /* full, partial */ + dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + if ((src == NULL) || (outputSize < 0)) { return -1; } + + { const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; + + const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; + + const int checkOffset = (dictSize < (int)(64 KB)); + + + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - 14 /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - 14 /*maxLL*/ - 18 /*maxML*/; + + const BYTE* match; + size_t offset; + unsigned token; + size_t length; + + + DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); + + /* Special cases */ + assert(lowPrefix <= op); + if (unlikely(outputSize==0)) { + /* Empty output buffer */ + if (partialDecoding) return 0; + return ((srcSize==1) && (*ip==0)) ? 0 : -1; + } + if (unlikely(srcSize==0)) { return -1; } + + /* LZ4_FAST_DEC_LOOP: + * designed for modern OoO performance cpus, + * where copying reliably 32-bytes is preferable to an unpredictable branch. + * note : fast loop may show a regression for some client arm chips. */ +#if LZ4_FAST_DEC_LOOP + if ((oend - op) < FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(6, "move to safe decode loop"); + goto safe_decode; + } + + /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE */ + DEBUGLOG(6, "using fast decode loop"); + while (1) { + /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ + assert(oend - op >= FASTLOOP_SAFE_DISTANCE); + assert(ip < iend); + token = *ip++; + length = token >> ML_BITS; /* literal length */ + DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + /* decode literal length */ + if (length == RUN_MASK) { + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { + DEBUGLOG(6, "error reading long literal length"); + goto _output_error; + } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + + /* copy literals */ + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ((op+length>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } + LZ4_wildCopy32(op, ip, op+length); + ip += length; op += length; + } else if (ip <= iend-(16 + 1/*max lit + offset + nextToken*/)) { + /* We don't need to check oend, since we check it once for each loop below */ + DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); + /* Literals can only be <= 14, but hope compilers optimize better when copy by a register size */ + LZ4_memcpy(op, ip, 16); + ip += length; op += length; + } else { + goto safe_literal_copy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + DEBUGLOG(6, "blockPos%6u: offset = %u", (unsigned)(op-(BYTE*)dst), (unsigned)offset); + match = op - offset; + assert(match <= op); /* overflow check */ + + /* get matchlength */ + length = token & ML_MASK; + DEBUGLOG(7, " match length token = %u (len==%u)", (unsigned)length, (unsigned)length+MINMATCH); + + if (length == ML_MASK) { + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { + DEBUGLOG(5, "error reading long match length"); + goto _output_error; + } + length += addl; + length += MINMATCH; + DEBUGLOG(7, " long match length == %u", (unsigned)length); + if (unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + } else { + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(7, "moving to safe_match_copy (ml==%u)", (unsigned)length); + goto safe_match_copy; + } + + /* Fastpath check: skip LZ4_wildCopy32 when true */ + if ((dict == withPrefix64k) || (match >= lowPrefix)) { + if (offset >= 8) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + + LZ4_memcpy(op, match, 8); + LZ4_memcpy(op+8, match+8, 8); + LZ4_memcpy(op+16, match+16, 2); + op += length; + continue; + } } } + + if ( checkOffset && (unlikely(match + dictSize < lowPrefix)) ) { + DEBUGLOG(5, "Error : pos=%zi, offset=%zi => outside buffers", op-lowPrefix, op-match); + goto _output_error; + } + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) { + DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd"); + length = MIN(length, (size_t)(oend-op)); + } else { + DEBUGLOG(6, "end-of-block condition violated") + goto _output_error; + } } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) { *op++ = *copyFrom++; } + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + + assert((op <= oend) && (oend-op >= 32)); + if (unlikely(offset<16)) { + LZ4_memcpy_using_offset(op, match, cpy, offset); + } else { + LZ4_wildCopy32(op, match, cpy); + } + + op = cpy; /* wildcopy correction */ + } + safe_decode: +#endif + + /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + DEBUGLOG(6, "using safe decode loop"); + while (1) { + assert(ip < iend); + token = *ip++; + length = token >> ML_BITS; /* literal length */ + DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (length != RUN_MASK) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((ip < shortiend) & (op <= shortoend)) ) { + /* Copy the literals */ + LZ4_memcpy(op, ip, 16); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + DEBUGLOG(7, "blockPos%6u: matchLength token = %u (len=%u)", (unsigned)(op-(BYTE*)dst), (unsigned)length, (unsigned)length + 4); + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + LZ4_memcpy(op + 0, match + 0, 8); + LZ4_memcpy(op + 8, match + 8, 8); + LZ4_memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; + } + + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; + } + + /* decode literal length */ + if (length == RUN_MASK) { + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + } + +#if LZ4_FAST_DEC_LOOP + safe_literal_copy: +#endif + /* copy literals */ + cpy = op+length; + + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) { + /* We've either hit the input parsing restriction or the output parsing restriction. + * In the normal scenario, decoding a full block, it must be the last sequence, + * otherwise it's an error (invalid input or dimensions). + * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow. + */ + if (partialDecoding) { + /* Since we are partial decoding we may be in this block because of the output parsing + * restriction, which is not valid since the output buffer is allowed to be undersized. + */ + DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end") + DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length); + DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op)); + DEBUGLOG(7, "partialDecoding: remaining space in srcBuffer : %i", (int)(iend - ip)); + /* Finishing in the middle of a literals segment, + * due to lack of input. + */ + if (ip+length > iend) { + length = (size_t)(iend-ip); + cpy = op + length; + } + /* Finishing in the middle of a literals segment, + * due to lack of output space. + */ + if (cpy > oend) { + cpy = oend; + assert(op<=oend); + length = (size_t)(oend-op); + } + } else { + /* We must be on the last sequence (or invalid) because of the parsing limitations + * so check that we exactly consume the input and don't overrun the output buffer. + */ + if ((ip+length != iend) || (cpy > oend)) { + DEBUGLOG(5, "should have been last run of literals") + DEBUGLOG(5, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend); + DEBUGLOG(5, "or cpy(%p) > (oend-MFLIMIT)(%p)", cpy, oend-MFLIMIT); + DEBUGLOG(5, "after writing %u bytes / %i bytes available", (unsigned)(op-(BYTE*)dst), outputSize); + goto _output_error; + } + } + LZ4_memmove(op, ip, length); /* supports overlapping memory regions, for in-place decompression scenarios */ + ip += length; + op += length; + /* Necessarily EOF when !partialDecoding. + * When partialDecoding, it is EOF if we've either + * filled the output buffer or + * can't proceed with reading an offset for following match. + */ + if (!partialDecoding || (cpy == oend) || (ip >= (iend-2))) { + break; + } + } else { + LZ4_wildCopy8(op, ip, cpy); /* can overwrite up to 8 bytes beyond cpy */ + ip += length; op = cpy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + + /* get matchlength */ + length = token & ML_MASK; + DEBUGLOG(7, "blockPos%6u: matchLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + _copy_match: + if (length == ML_MASK) { + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + +#if LZ4_FAST_DEC_LOOP + safe_match_copy: +#endif + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + assert(match >= lowPrefix); + + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may end anywhere within the block */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, mlen); + } + op = copyEnd; + if (op == oend) { break; } + continue; + } + + if (unlikely(offset<8)) { + LZ4_write32(op, 0); /* silence msan warning when offset==0 */ + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += inc32table[offset]; + LZ4_memcpy(op+4, match, 4); + match -= dec64table[offset]; + } else { + LZ4_memcpy(op, match, 8); + match += 8; + } + op += 8; + + if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy8(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, 8); + if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } + } + op = cpy; /* wildcopy correction */ + } + + /* end of decoding */ + DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + + /* Overflow error detected */ + _output_error: + return (int) (-(((const char*)ip)-src))-1; + } +} + + +/*===== Instantiate the API decoding functions. =====*/ + +LZ4_FORCE_O2 +int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, + decode_full_block, noDict, + (BYTE*)dest, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, + partial_decode, + noDict, (BYTE*)dst, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_fast(const char* source, char* dest, int originalSize) +{ + DEBUGLOG(5, "LZ4_decompress_fast"); + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, NULL, 0); +} + +/*===== Instantiate a few more decoding cases, used more than once. =====*/ + +LZ4_FORCE_O2 /* Exported, an obsolete API function. */ +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withPrefix64k(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +/* Another obsolete API function, paired with the previous one. */ +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 64 KB, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withSmallPrefix(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, + size_t prefixSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize) +{ + DEBUGLOG(5, "LZ4_decompress_safe_forceExtDict"); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, (const BYTE*)dictStart, dictSize); +} + +/* The "double dictionary" mode, for use with e.g. ring buffers: the first part + * of the dictionary is passed as prefix, and the second via dictStart + dictSize. + * These routines are used only once, in LZ4_decompress_*_continue(). + */ +LZ4_FORCE_INLINE +int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} + +/*===== streaming decompression functions =====*/ + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_streamDecode_t* LZ4_createStreamDecode(void) +{ + LZ4_STATIC_ASSERT(sizeof(LZ4_streamDecode_t) >= sizeof(LZ4_streamDecode_t_internal)); + return (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); +} + +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) +{ + if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ + FREEMEM(LZ4_stream); + return 0; +} +#endif + +/*! LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * @return : 1 if OK, 0 if error + */ +int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + lz4sd->prefixSize = (size_t)dictSize; + if (dictSize) { + assert(dictionary != NULL); + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + } else { + lz4sd->prefixEnd = (const BYTE*) dictionary; + } + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; +} + +/*! LZ4_decoderRingBufferSize() : + * when setting a ring buffer for streaming decompression (optional scenario), + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * Note : in a ring buffer scenario, + * blocks are presumed decompressed next to each other. + * When not enough space remains for next block (remainingSize < maxBlockSize), + * decoding resumes from beginning of ring buffer. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +int LZ4_decoderRingBufferSize(int maxBlockSize) +{ + if (maxBlockSize < 0) return 0; + if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; + if (maxBlockSize < 16) maxBlockSize = 16; + return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ4_setStreamDecode() +*/ +LZ4_FORCE_O2 +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixSize == 0) { + /* The first call, no dictionary yet. */ + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + /* They're rolling the current segment. */ + if (lz4sd->prefixSize >= 64 KB - 1) + result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + else if (lz4sd->extDictSize == 0) + result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize); + else + result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)result; + lz4sd->prefixEnd += result; + } else { + /* The buffer wraps around, or they're switching to another buffer. */ + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + +LZ4_FORCE_O2 int +LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* source, char* dest, int originalSize) +{ + LZ4_streamDecode_t_internal* const lz4sd = + (assert(LZ4_streamDecode!=NULL), &LZ4_streamDecode->internal_donotuse); + int result; + + DEBUGLOG(5, "LZ4_decompress_fast_continue (toDecodeSize=%i)", originalSize); + assert(originalSize >= 0); + + if (lz4sd->prefixSize == 0) { + DEBUGLOG(5, "first invocation : no prefix nor extDict"); + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_fast(source, dest, originalSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + DEBUGLOG(5, "continue using existing prefix"); + result = LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + lz4sd->prefixSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)originalSize; + lz4sd->prefixEnd += originalSize; + } else { + DEBUGLOG(5, "prefix becomes extDict"); + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_fast_extDict(source, dest, originalSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); +} + +int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_safe_partial(source, dest, compressedSize, targetOutputSize, dstCapacity); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_partial_withPrefix64k(source, dest, compressedSize, targetOutputSize, dstCapacity); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_withSmallPrefix(source, dest, compressedSize, targetOutputSize, dstCapacity, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_forceExtDict(source, dest, compressedSize, targetOutputSize, dstCapacity, dictStart, (size_t)dictSize); +} + +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + if (dictSize==0 || dictStart+dictSize == dest) + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + (size_t)dictSize, NULL, 0); + assert(dictSize >= 0); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); +} + + +/*=************************************************* +* Obsolete Functions +***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_default(source, dest, inputSize, maxOutputSize); +} +int LZ4_compress(const char* src, char* dest, int srcSize) +{ + return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); +} +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); +} +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); +} +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1); +} +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); +} + +/* +These decompression functions are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ4_uncompress is totally equivalent to LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe +*/ +int LZ4_uncompress (const char* source, char* dest, int outputSize) +{ + return LZ4_decompress_fast(source, dest, outputSize); +} +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) +{ + return LZ4_decompress_safe(source, dest, isize, maxOutputSize); +} + +/* Obsolete Streaming functions */ + +int LZ4_sizeofStreamState(void) { return sizeof(LZ4_stream_t); } + +int LZ4_resetStreamState(void* state, char* inputBuffer) +{ + (void)inputBuffer; + LZ4_resetStream((LZ4_stream_t*)state); + return 0; +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +void* LZ4_create (char* inputBuffer) +{ + (void)inputBuffer; + return LZ4_createStream(); +} +#endif + +char* LZ4_slideInputBuffer (void* state) +{ + /* avoid const char * -> char * conversion warning */ + return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary; +} + +#endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/deps/lz4/lz4.h b/deps/lz4/lz4.h new file mode 100644 index 00000000000..80e3e5ca04d --- /dev/null +++ b/deps/lz4/lz4.h @@ -0,0 +1,884 @@ +/* + * LZ4 - Fast LZ compression algorithm + * Header File + * Copyright (C) 2011-2023, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef LZ4_H_2983827168210 +#define LZ4_H_2983827168210 + +/* --- Dependency --- */ +#include /* size_t */ + + +/** + Introduction + + LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, + scalable with multi-cores CPU. It features an extremely fast decoder, with speed in + multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. + + The LZ4 compression library provides in-memory compression and decompression functions. + It gives full buffer control to user. + Compression can be done in: + - a single step (described as Simple Functions) + - a single step, reusing a context (described in Advanced Functions) + - unbounded multiple steps (described as Streaming compression) + + lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). + Decompressing such a compressed block requires additional metadata. + Exact metadata depends on exact decompression function. + For the typical case of LZ4_decompress_safe(), + metadata includes block's compressed size, and maximum bound of decompressed size. + Each application is free to encode and pass such metadata in whichever way it wants. + + lz4.h only handle blocks, it can not generate Frames. + + Blocks are different from Frames (doc/lz4_Frame_format.md). + Frames bundle both blocks and metadata in a specified manner. + Embedding metadata is required for compressed data to be self-contained and portable. + Frame format is delivered through a companion API, declared in lz4frame.h. + The `lz4` CLI can only manage frames. +*/ + +/*^*************************************************************** +* Export parameters +*****************************************************************/ +/* +* LZ4_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +* LZ4LIB_VISIBILITY : +* Control library symbols visibility. +*/ +#ifndef LZ4LIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4LIB_VISIBILITY +# endif +#endif +#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) +# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY +#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) +# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define LZ4LIB_API LZ4LIB_VISIBILITY +#endif + +/*! LZ4_FREESTANDING : + * When this macro is set to 1, it enables "freestanding mode" that is + * suitable for typical freestanding environment which doesn't support + * standard C library. + * + * - LZ4_FREESTANDING is a compile-time switch. + * - It requires the following macros to be defined: + * LZ4_memcpy, LZ4_memmove, LZ4_memset. + * - It only enables LZ4/HC functions which don't use heap. + * All LZ4F_* functions are not supported. + * - See tests/freestanding.c to check its basic setup. + */ +#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) +# define LZ4_HEAPMODE 0 +# define LZ4HC_HEAPMODE 0 +# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1 +# if !defined(LZ4_memcpy) +# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'." +# endif +# if !defined(LZ4_memset) +# error "LZ4_FREESTANDING requires macro 'LZ4_memset'." +# endif +# if !defined(LZ4_memmove) +# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'." +# endif +#elif ! defined(LZ4_FREESTANDING) +# define LZ4_FREESTANDING 0 +#endif + + +/*------ Version ------*/ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 10 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ + +#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) + +#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE +#define LZ4_QUOTE(str) #str +#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) +#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */ + +LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version; requires v1.3.0+ */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version; requires v1.7.5+ */ + + +/*-************************************ +* Tuning memory usage +**************************************/ +/*! + * LZ4_MEMORY_USAGE : + * Can be selected at compile time, by setting LZ4_MEMORY_USAGE. + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB) + * Increasing memory usage improves compression ratio, generally at the cost of speed. + * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality. + * Default value is 14, for 16KB, which nicely fits into most L1 caches. + */ +#ifndef LZ4_MEMORY_USAGE +# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT +#endif + +/* These are absolute limits, they should not be changed by users */ +#define LZ4_MEMORY_USAGE_MIN 10 +#define LZ4_MEMORY_USAGE_DEFAULT 14 +#define LZ4_MEMORY_USAGE_MAX 20 + +#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN) +# error "LZ4_MEMORY_USAGE is too small !" +#endif + +#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX) +# error "LZ4_MEMORY_USAGE is too large !" +#endif + +/*-************************************ +* Simple Functions +**************************************/ +/*! LZ4_compress_default() : + * Compresses 'srcSize' bytes from buffer 'src' + * into already allocated 'dst' buffer of size 'dstCapacity'. + * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). + * It also runs faster, so it's a recommended setting. + * If the function cannot compress 'src' into a more limited 'dst' budget, + * compression stops *immediately*, and the function result is zero. + * In which case, 'dst' content is undefined (invalid). + * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + * dstCapacity : size of buffer 'dst' (which must be already allocated) + * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails + * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + */ +LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); + +/*! LZ4_decompress_safe() : + * @compressedSize : is the exact complete size of the compressed block. + * @dstCapacity : is the size of destination buffer (which must be already allocated), + * presumed an upper bound of decompressed size. + * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * Note 1 : This function is protected against malicious data packets : + * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, + * even if the compressed block is maliciously modified to order the decoder to do these actions. + * In such case, the decoder stops immediately, and considers the compressed block malformed. + * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. + * The implementation is free to send / store / derive this information in whichever way is most beneficial. + * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. + */ +LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); + + +/*-************************************ +* Advanced Functions +**************************************/ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) + +/*! LZ4_compressBound() : + Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is incorrect (too large or negative) +*/ +LZ4LIB_API int LZ4_compressBound(int inputSize); + +/*! LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows selection of "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c). + Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c). +*/ +LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + + +/*! LZ4_compress_fast_extState() : + * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. + * Use LZ4_sizeofState() to know how much memory must be allocated, + * and allocate it on 8-bytes boundaries (using `malloc()` typically). + * Then, provide this buffer as `void* state` to compression function. + */ +LZ4LIB_API int LZ4_sizeofState(void); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_compress_destSize() : + * Reverse the logic : compresses as much data as possible from 'src' buffer + * into already allocated buffer 'dst', of size >= 'dstCapacity'. + * This function either compresses the entire 'src' content into 'dst' if it's large enough, + * or fill 'dst' buffer completely with as much data as possible from 'src'. + * note: acceleration parameter is fixed to "default". + * + * *srcSizePtr : in+out parameter. Initially contains size of input. + * Will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + * New value is necessarily <= input value. + * @return : Nb bytes written into 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails. + * + * Note : from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+): + * the produced compressed content could, in specific circumstances, + * require to be decompressed into a destination buffer larger + * by at least 1 byte than the content to decompress. + * If an application uses `LZ4_compress_destSize()`, + * it's highly recommended to update liblz4 to v1.9.2 or better. + * If this can't be done or ensured, + * the receiving decompression function should provide + * a dstCapacity which is > decompressedSize, by at least 1 byte. + * See https://github.com/lz4/lz4/issues/859 for details + */ +LZ4LIB_API int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize); + +/*! LZ4_decompress_safe_partial() : + * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', + * into destination buffer 'dst' of size 'dstCapacity'. + * Up to 'targetOutputSize' bytes will be decoded. + * The function stops decoding on reaching this objective. + * This can be useful to boost performance + * whenever only the beginning of a block is required. + * + * @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize) + * If source stream is detected malformed, function returns a negative result. + * + * Note 1 : @return can be < targetOutputSize, if compressed block contains less data. + * + * Note 2 : targetOutputSize must be <= dstCapacity + * + * Note 3 : this function effectively stops decoding on reaching targetOutputSize, + * so dstCapacity is kind of redundant. + * This is because in older versions of this function, + * decoding operation would still write complete sequences. + * Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize, + * it could write more bytes, though only up to dstCapacity. + * Some "margin" used to be required for this operation to work properly. + * Thankfully, this is no longer necessary. + * The function nonetheless keeps the same signature, in an effort to preserve API compatibility. + * + * Note 4 : If srcSize is the exact size of the block, + * then targetOutputSize can be any value, + * including larger than the block's decompressed size. + * The function will, at most, generate block's decompressed size. + * + * Note 5 : If srcSize is _larger_ than block's compressed size, + * then targetOutputSize **MUST** be <= block's decompressed size. + * Otherwise, *silent corruption will occur*. + */ +LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); + + +/*-********************************************* +* Streaming Compression Functions +***********************************************/ +typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ + +/*! + Note about RC_INVOKED + + - RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio). + https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros + + - Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars) + and reports warning "RC4011: identifier truncated". + + - To eliminate the warning, we surround long preprocessor symbol with + "#if !defined(RC_INVOKED) ... #endif" block that means + "skip this block when rc.exe is trying to read it". +*/ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); +LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif + +/*! LZ4_resetStream_fast() : v1.9.0+ + * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks + * (e.g., LZ4_compress_fast_continue()). + * + * An LZ4_stream_t must be initialized once before usage. + * This is automatically done when created by LZ4_createStream(). + * However, should the LZ4_stream_t be simply declared on stack (for example), + * it's necessary to initialize it first, using LZ4_initStream(). + * + * After init, start any new stream with LZ4_resetStream_fast(). + * A same LZ4_stream_t can be re-used multiple times consecutively + * and compress multiple streams, + * provided that it starts each new stream with LZ4_resetStream_fast(). + * + * LZ4_resetStream_fast() is much faster than LZ4_initStream(), + * but is not compatible with memory regions containing garbage data. + * + * Note: it's only useful to call LZ4_resetStream_fast() + * in the context of streaming compression. + * The *extState* functions perform their own resets. + * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive. + */ +LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); + +/*! LZ4_loadDict() : + * Use this function to reference a static dictionary into LZ4_stream_t. + * The dictionary must remain available during compression. + * LZ4_loadDict() triggers a reset, so any previous data will be forgotten. + * The same dictionary will have to be loaded on decompression side for successful decoding. + * Dictionary are useful for better compression of small data (KB range). + * While LZ4 itself accepts any input as dictionary, dictionary efficiency is also a topic. + * When in doubt, employ the Zstandard's Dictionary Builder. + * Loading a size of 0 is allowed, and is the same as reset. + * @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded) + */ +LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_loadDictSlow() : v1.10.0+ + * Same as LZ4_loadDict(), + * but uses a bit more cpu to reference the dictionary content more thoroughly. + * This is expected to slightly improve compression ratio. + * The extra-cpu cost is likely worth it if the dictionary is re-used across multiple sessions. + * @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded) + */ +LZ4LIB_API int LZ4_loadDictSlow(LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_attach_dictionary() : stable since v1.10.0 + * + * This allows efficient re-use of a static dictionary multiple times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a + * working LZ4_stream_t, this function introduces a no-copy setup mechanism, + * in which the working stream references @dictionaryStream in-place. + * + * Several assumptions are made about the state of @dictionaryStream. + * Currently, only states which have been prepared by LZ4_loadDict() or + * LZ4_loadDictSlow() should be expected to work. + * + * Alternatively, the provided @dictionaryStream may be NULL, + * in which case any existing dictionary stream is unset. + * + * If a dictionary is provided, it replaces any pre-existing stream history. + * The dictionary contents are the only history that can be referenced and + * logically immediately precede the data compressed in the first subsequent + * compression call. + * + * The dictionary will only remain attached to the working stream through the + * first compression call, at the end of which it is cleared. + * @dictionaryStream stream (and source buffer) must remain in-place / accessible / unchanged + * through the completion of the compression session. + * + * Note: there is no equivalent LZ4_attach_*() method on the decompression side + * because there is no initialization cost, hence no need to share the cost across multiple sessions. + * To decompress LZ4 blocks using dictionary, attached or not, + * just employ the regular LZ4_setStreamDecode() for streaming, + * or the stateless LZ4_decompress_safe_usingDict() for one-shot decompression. + */ +LZ4LIB_API void +LZ4_attach_dictionary(LZ4_stream_t* workingStream, + const LZ4_stream_t* dictionaryStream); + +/*! LZ4_compress_fast_continue() : + * Compress 'src' content using data from previously compressed blocks, for better compression ratio. + * 'dst' buffer must be already allocated. + * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * + * @return : size of compressed block + * or 0 if there is an error (typically, cannot fit into 'dst'). + * + * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. + * Each block has precise boundaries. + * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. + * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. + * + * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! + * + * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. + * Make sure that buffers are separated, by at least one byte. + * This construction ensures that each block only depends on previous block. + * + * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * + * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. + */ +LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_saveDict() : + * If last 64KB data cannot be guaranteed to remain available at its current memory location, + * save it into a safer place (char* safeBuffer). + * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), + * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. + * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. + */ +LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize); + + +/*-********************************************** +* Streaming Decompression Functions +* Bufferless synchronous API +************************************************/ +typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ + +/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : + * creation / destruction of streaming decompression tracking context. + * A tracking context can be re-used multiple times. + */ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); +LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif + +/*! LZ4_setStreamDecode() : + * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. + * Use this function to start decompression of a new stream of blocks. + * A dictionary can optionally be set. Use NULL or size 0 for a reset order. + * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. + * @return : 1 if OK, 0 if error + */ +LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); + +/*! LZ4_decoderRingBufferSize() : v1.8.2+ + * Note : in a ring buffer scenario (optional), + * blocks are presumed decompressed next to each other + * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), + * at which stage it resumes from beginning of ring buffer. + * When setting such a ring buffer for streaming decompression, + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ + +/*! LZ4_decompress_safe_continue() : + * This decoding function allows decompression of consecutive blocks in "streaming" mode. + * The difference with the usual independent blocks is that + * new blocks are allowed to find references into former blocks. + * A block is an unsplittable entity, and must be presented entirely to the decompression function. + * LZ4_decompress_safe_continue() only accepts one block at a time. + * It's modeled after `LZ4_decompress_safe()` and behaves similarly. + * + * @LZ4_streamDecode : decompression state, tracking the position in memory of past data + * @compressedSize : exact complete size of one compressed block. + * @dstCapacity : size of destination buffer (which must be already allocated), + * must be an upper bound of decompressed size. + * @return : number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * + * The last 64KB of previously decoded data *must* remain available and unmodified + * at the memory position where they were previously decoded. + * If less than 64KB of data has been decoded, all the data must be present. + * + * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : + * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). + * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. + * In which case, encoding and decoding buffers do not need to be synchronized. + * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. + * - Synchronized mode : + * Decompression buffer size is _exactly_ the same as compression buffer size, + * and follows exactly same update rule (block boundaries at same positions), + * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), + * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). + * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including small ones ( < 64 KB). + * + * Whenever these conditions are not possible, + * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, + * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. +*/ +LZ4LIB_API int +LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* src, char* dst, + int srcSize, int dstCapacity); + + +/*! LZ4_decompress_safe_usingDict() : + * Works the same as + * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_safe_continue() + * However, it's stateless: it doesn't need any LZ4_streamDecode_t state. + * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. + */ +LZ4LIB_API int +LZ4_decompress_safe_usingDict(const char* src, char* dst, + int srcSize, int dstCapacity, + const char* dictStart, int dictSize); + +/*! LZ4_decompress_safe_partial_usingDict() : + * Behaves the same as LZ4_decompress_safe_partial() + * with the added ability to specify a memory segment for past data. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. + */ +LZ4LIB_API int +LZ4_decompress_safe_partial_usingDict(const char* src, char* dst, + int compressedSize, + int targetOutputSize, int maxOutputSize, + const char* dictStart, int dictSize); + +#endif /* LZ4_H_2983827168210 */ + + +/*^************************************* + * !!!!!! STATIC LINKING ONLY !!!!!! + ***************************************/ + +/*-**************************************************************************** + * Experimental section + * + * Symbols declared in this section must be considered unstable. Their + * signatures or semantics may change, or they may be removed altogether in the + * future. They are therefore only safe to depend on when the caller is + * statically linked against the library. + * + * To protect against unsafe usage, not only are the declarations guarded, + * the definitions are hidden by default + * when building LZ4 as a shared/dynamic library. + * + * In order to access these declarations, + * define LZ4_STATIC_LINKING_ONLY in your application + * before including LZ4's headers. + * + * In order to make their implementations accessible dynamically, you must + * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. + ******************************************************************************/ + +#ifdef LZ4_STATIC_LINKING_ONLY + +#ifndef LZ4_STATIC_3504398509 +#define LZ4_STATIC_3504398509 + +#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS +# define LZ4LIB_STATIC_API LZ4LIB_API +#else +# define LZ4LIB_STATIC_API +#endif + + +/*! LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. + * It is only safe to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). + * From a high level, the difference is that + * this function initializes the provided state with a call to something like LZ4_resetStream_fast() + * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). + */ +LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_compress_destSize_extState() : introduced in v1.10.0 + * Same as LZ4_compress_destSize(), but using an externally allocated state. + * Also: exposes @acceleration + */ +int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration); + +/*! In-place compression and decompression + * + * It's possible to have input and output sharing the same buffer, + * for highly constrained memory environments. + * In both cases, it requires input to lay at the end of the buffer, + * and decompression to start at beginning of the buffer. + * Buffer size must feature some margin, hence be larger than final size. + * + * |<------------------------buffer--------------------------------->| + * |<-----------compressed data--------->| + * |<-----------decompressed size------------------>| + * |<----margin---->| + * + * This technique is more useful for decompression, + * since decompressed size is typically larger, + * and margin is short. + * + * In-place decompression will work inside any buffer + * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + * This presumes that decompressedSize > compressedSize. + * Otherwise, it means compression actually expanded data, + * and it would be more efficient to store such data with a flag indicating it's not compressed. + * This can happen when data is not compressible (already compressed, or encrypted). + * + * For in-place compression, margin is larger, as it must be able to cope with both + * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + * and data expansion, which can happen when input is not compressible. + * As a consequence, buffer size requirements are much higher, + * and memory savings offered by in-place compression are more limited. + * + * There are ways to limit this cost for compression : + * - Reduce history size, by modifying LZ4_DISTANCE_MAX. + * Note that it is a compile-time constant, so all compressions will apply this limit. + * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + * so it's a reasonable trick when inputs are known to be small. + * - Require the compressor to deliver a "maximum compressed size". + * This is the `dstCapacity` parameter in `LZ4_compress*()`. + * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + * in which case, the return code will be 0 (zero). + * The caller must be ready for these cases to happen, + * and typically design a backup scheme to send data uncompressed. + * The combination of both techniques can significantly reduce + * the amount of margin required for in-place compression. + * + * In-place compression can work in any buffer + * which size is >= (maxCompressedSize) + * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + * so it's possible to reduce memory requirements by playing with them. + */ + +#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) +#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ + +#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ +# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ +#endif + +#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ +#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ + +#endif /* LZ4_STATIC_3504398509 */ +#endif /* LZ4_STATIC_LINKING_ONLY */ + + + +#ifndef LZ4_H_98237428734687 +#define LZ4_H_98237428734687 + +/*-************************************************************ + * Private Definitions + ************************************************************** + * Do not use these definitions directly. + * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Accessing members will expose user code to API and/or ABI break in future versions of the library. + **************************************************************/ +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef int8_t LZ4_i8; + typedef uint8_t LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; +#else + typedef signed char LZ4_i8; + typedef unsigned char LZ4_byte; + typedef unsigned short LZ4_u16; + typedef unsigned int LZ4_u32; +#endif + +/*! LZ4_stream_t : + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_stream_t object. +**/ + +typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; +struct LZ4_stream_t_internal { + LZ4_u32 hashTable[LZ4_HASH_SIZE_U32]; + const LZ4_byte* dictionary; + const LZ4_stream_t_internal* dictCtx; + LZ4_u32 currentOffset; + LZ4_u32 tableType; + LZ4_u32 dictSize; + /* Implicit padding to ensure structure is aligned */ +}; + +#define LZ4_STREAM_MINSIZE ((1UL << (LZ4_MEMORY_USAGE)) + 32) /* static size, for inter-version compatibility */ +union LZ4_stream_u { + char minStateSize[LZ4_STREAM_MINSIZE]; + LZ4_stream_t_internal internal_donotuse; +}; /* previously typedef'd to LZ4_stream_t */ + + +/*! LZ4_initStream() : v1.9.0+ + * An LZ4_stream_t structure must be initialized at least once. + * This is automatically done when invoking LZ4_createStream(), + * but it's not when the structure is simply declared on stack (for example). + * + * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t. + * It can also initialize any arbitrary buffer of sufficient size, + * and will @return a pointer of proper type upon initialization. + * + * Note : initialization fails if size and alignment conditions are not respected. + * In which case, the function will @return NULL. + * Note2: An LZ4_stream_t structure guarantees correct alignment and size. + * Note3: Before v1.9.0, use LZ4_resetStream() instead +**/ +LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* stateBuffer, size_t size); + + +/*! LZ4_streamDecode_t : + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_streamDecode_t object. +**/ +typedef struct { + const LZ4_byte* externalDict; + const LZ4_byte* prefixEnd; + size_t extDictSize; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#define LZ4_STREAMDECODE_MINSIZE 32 +union LZ4_streamDecode_u { + char minStateSize[LZ4_STREAMDECODE_MINSIZE]; + LZ4_streamDecode_t_internal internal_donotuse; +} ; /* previously typedef'd to LZ4_streamDecode_t */ + + + +/*-************************************ +* Obsolete Functions +**************************************/ + +/*! Deprecation warnings + * + * Deprecated functions make the compiler generate a warning when invoked. + * This is meant to invite users to update their source code. + * Should deprecation warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc + * or _CRT_SECURE_NO_WARNINGS in Visual. + * + * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS + * before including the header file. + */ +#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS +# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define LZ4_DEPRECATED(message) [[deprecated(message)]] +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45)) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# else +# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler") +# define LZ4_DEPRECATED(message) /* disabled */ +# endif +#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ + +/*! Obsolete compression functions (since v1.7.3) */ +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/*! Obsolete decompression functions (since v1.8.0) */ +LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); +LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); + +/* Obsolete streaming functions (since v1.7.0) + * degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, they don't + * actually retain any history between compression calls. The compression ratio + * achieved will therefore be no better than compressing each chunk + * independently. + */ +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); + +/*! Obsolete streaming decoding functions (since v1.7.0) */ +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + +/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) : + * These functions used to be faster than LZ4_decompress_safe(), + * but this is no longer the case. They are now slower. + * This is because LZ4_decompress_fast() doesn't know the input size, + * and therefore must progress more cautiously into the input buffer to not read beyond the end of block. + * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. + * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. + * + * The last remaining LZ4_decompress_fast() specificity is that + * it can decompress a block without knowing its compressed size. + * Such functionality can be achieved in a more secure manner + * by employing LZ4_decompress_safe_partial(). + * + * Parameters: + * originalSize : is the uncompressed size to regenerate. + * `dst` must be already allocated, its size must be >= 'originalSize' bytes. + * @return : number of bytes read from source buffer (== compressed size). + * The function expects to finish at block's end exactly. + * If the source stream is detected malformed, the function stops decoding and returns a negative result. + * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer. + * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds. + * Also, since match offsets are not validated, match reads from 'src' may underflow too. + * These issues never happen if input (compressed) data is correct. + * But they may happen if input data is invalid (error or intentional tampering). + * As a consequence, use these functions in trusted environments with trusted data **only**. + */ +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial() instead") +LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider migrating towards LZ4_decompress_safe_continue() instead. " + "Note that the contract will change (requires block's compressed size, instead of decompressed size)") +LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial_usingDict() instead") +LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); + +/*! LZ4_resetStream() : + * An LZ4_stream_t structure must be initialized at least once. + * This is done with LZ4_initStream(), or LZ4_resetStream(). + * Consider switching to LZ4_initStream(), + * invoking LZ4_resetStream() will trigger deprecation warnings in the future. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); + + +#endif /* LZ4_H_98237428734687 */ + + +#if defined (__cplusplus) +} +#endif diff --git a/deps/lz4/lz4frame.c b/deps/lz4/lz4frame.c new file mode 100644 index 00000000000..f89c0557995 --- /dev/null +++ b/deps/lz4/lz4frame.c @@ -0,0 +1,2136 @@ +/* + * LZ4 auto-framing library + * Copyright (C) 2011-2016, Yann Collet. + * + * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at : + * - LZ4 homepage : http://www.lz4.org + * - LZ4 source repository : https://github.com/lz4/lz4 + */ + +/* LZ4F is a stand-alone API to create LZ4-compressed Frames + * in full conformance with specification v1.6.1 . + * This library rely upon memory management capabilities (malloc, free) + * provided either by , + * or redirected towards another library of user's choice + * (see Memory Routines below). + */ + + +/*-************************************ +* Compiler Options +**************************************/ +#include +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/*-************************************ +* Tuning parameters +**************************************/ +/* + * LZ4F_HEAPMODE : + * Control how LZ4F_compressFrame allocates the Compression State, + * either on stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifndef LZ4F_HEAPMODE +# define LZ4F_HEAPMODE 0 +#endif + + +/*-************************************ +* Library declarations +**************************************/ +#define LZ4F_STATIC_LINKING_ONLY +#include "lz4frame.h" +#define LZ4_STATIC_LINKING_ONLY +#include "lz4.h" +#define LZ4_HC_STATIC_LINKING_ONLY +#include "lz4hc.h" +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + + +/*-************************************ +* Memory routines +**************************************/ +/* + * User may redirect invocations of + * malloc(), calloc() and free() + * towards another library or solution of their choice + * by modifying below section. +**/ + +#include /* memset, memcpy, memmove */ +#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ +# define MEM_INIT(p,v,s) memset((p),(v),(s)) +#endif + +#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ +# include /* malloc, calloc, free */ +# define ALLOC(s) malloc(s) +# define ALLOC_AND_ZERO(s) calloc(1,(s)) +# define FREEMEM(p) free(p) +#endif + +static void* LZ4F_calloc(size_t s, LZ4F_CustomMem cmem) +{ + /* custom calloc defined : use it */ + if (cmem.customCalloc != NULL) { + return cmem.customCalloc(cmem.opaqueState, s); + } + /* nothing defined : use default 's calloc() */ + if (cmem.customAlloc == NULL) { + return ALLOC_AND_ZERO(s); + } + /* only custom alloc defined : use it, and combine it with memset() */ + { void* const p = cmem.customAlloc(cmem.opaqueState, s); + if (p != NULL) MEM_INIT(p, 0, s); + return p; +} } + +static void* LZ4F_malloc(size_t s, LZ4F_CustomMem cmem) +{ + /* custom malloc defined : use it */ + if (cmem.customAlloc != NULL) { + return cmem.customAlloc(cmem.opaqueState, s); + } + /* nothing defined : use default 's malloc() */ + return ALLOC(s); +} + +static void LZ4F_free(void* p, LZ4F_CustomMem cmem) +{ + if (p == NULL) return; + if (cmem.customFree != NULL) { + /* custom allocation defined : use it */ + cmem.customFree(cmem.opaqueState, p); + return; + } + /* nothing defined : use default 's free() */ + FREEMEM(p); +} + + +/*-************************************ +* Debug +**************************************/ +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) && !defined(DEBUGLOG) +# include +static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ " (%i): ", __LINE__ ); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + +/*-************************************ +* Basic Types +**************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +/* unoptimized version; solves endianness & alignment issues */ +static U32 LZ4F_readLE32 (const void* src) +{ + const BYTE* const srcPtr = (const BYTE*)src; + U32 value32 = srcPtr[0]; + value32 |= ((U32)srcPtr[1])<< 8; + value32 |= ((U32)srcPtr[2])<<16; + value32 |= ((U32)srcPtr[3])<<24; + return value32; +} + +static void LZ4F_writeLE32 (void* dst, U32 value32) +{ + BYTE* const dstPtr = (BYTE*)dst; + dstPtr[0] = (BYTE)value32; + dstPtr[1] = (BYTE)(value32 >> 8); + dstPtr[2] = (BYTE)(value32 >> 16); + dstPtr[3] = (BYTE)(value32 >> 24); +} + +static U64 LZ4F_readLE64 (const void* src) +{ + const BYTE* const srcPtr = (const BYTE*)src; + U64 value64 = srcPtr[0]; + value64 |= ((U64)srcPtr[1]<<8); + value64 |= ((U64)srcPtr[2]<<16); + value64 |= ((U64)srcPtr[3]<<24); + value64 |= ((U64)srcPtr[4]<<32); + value64 |= ((U64)srcPtr[5]<<40); + value64 |= ((U64)srcPtr[6]<<48); + value64 |= ((U64)srcPtr[7]<<56); + return value64; +} + +static void LZ4F_writeLE64 (void* dst, U64 value64) +{ + BYTE* const dstPtr = (BYTE*)dst; + dstPtr[0] = (BYTE)value64; + dstPtr[1] = (BYTE)(value64 >> 8); + dstPtr[2] = (BYTE)(value64 >> 16); + dstPtr[3] = (BYTE)(value64 >> 24); + dstPtr[4] = (BYTE)(value64 >> 32); + dstPtr[5] = (BYTE)(value64 >> 40); + dstPtr[6] = (BYTE)(value64 >> 48); + dstPtr[7] = (BYTE)(value64 >> 56); +} + + +/*-************************************ +* Constants +**************************************/ +#ifndef LZ4_SRC_INCLUDED /* avoid double definition */ +# define KB *(1<<10) +# define MB *(1<<20) +# define GB *(1<<30) +#endif + +#define _1BIT 0x01 +#define _2BITS 0x03 +#define _3BITS 0x07 +#define _4BITS 0x0F +#define _8BITS 0xFF + +#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U +#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB + +static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */ +static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ +static const size_t BHSize = LZ4F_BLOCK_HEADER_SIZE; /* block header : size, and compress flag */ +static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checksum (optional) */ + + +/*-************************************ +* Structures and local types +**************************************/ + +typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_BlockCompressMode_e; +typedef enum { ctxNone, ctxFast, ctxHC } LZ4F_CtxType_e; + +typedef struct LZ4F_cctx_s +{ + LZ4F_CustomMem cmem; + LZ4F_preferences_t prefs; + U32 version; + U32 cStage; /* 0 : compression uninitialized ; 1 : initialized, can compress */ + const LZ4F_CDict* cdict; + size_t maxBlockSize; + size_t maxBufferSize; + BYTE* tmpBuff; /* internal buffer, for streaming */ + BYTE* tmpIn; /* starting position of data compress within internal buffer (>= tmpBuff) */ + size_t tmpInSize; /* amount of data to compress after tmpIn */ + U64 totalInSize; + XXH32_state_t xxh; + void* lz4CtxPtr; + U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ + U16 lz4CtxType; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ + LZ4F_BlockCompressMode_e blockCompressMode; +} LZ4F_cctx_t; + + +/*-************************************ +* Error management +**************************************/ +#define LZ4F_GENERATE_STRING(STRING) #STRING, +static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; + + +unsigned LZ4F_isError(LZ4F_errorCode_t code) +{ + return (code > (LZ4F_errorCode_t)(-LZ4F_ERROR_maxCode)); +} + +const char* LZ4F_getErrorName(LZ4F_errorCode_t code) +{ + static const char* codeError = "Unspecified error code"; + if (LZ4F_isError(code)) return LZ4F_errorStrings[-(int)(code)]; + return codeError; +} + +LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult) +{ + if (!LZ4F_isError(functionResult)) return LZ4F_OK_NoError; + return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult); +} + +static LZ4F_errorCode_t LZ4F_returnErrorCode(LZ4F_errorCodes code) +{ + /* A compilation error here means sizeof(ptrdiff_t) is not large enough */ + LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); + return (LZ4F_errorCode_t)-(ptrdiff_t)code; +} + +#define RETURN_ERROR(e) return LZ4F_returnErrorCode(LZ4F_ERROR_ ## e) + +#define RETURN_ERROR_IF(c,e) do { \ + if (c) { \ + DEBUGLOG(3, "Error: " #c); \ + RETURN_ERROR(e); \ + } \ + } while (0) + +#define FORWARD_IF_ERROR(r) do { if (LZ4F_isError(r)) return (r); } while (0) + +unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } + +int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; } + +size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID) +{ + static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; + + if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB) + RETURN_ERROR(maxBlockSize_invalid); + { int const blockSizeIdx = (int)blockSizeID - (int)LZ4F_max64KB; + return blockSizes[blockSizeIdx]; +} } + +/*-************************************ +* Private functions +**************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +static BYTE LZ4F_headerChecksum (const void* header, size_t length) +{ + U32 const xxh = XXH32(header, length, 0); + return (BYTE)(xxh >> 8); +} + + +/*-************************************ +* Simple-pass compression functions +**************************************/ +static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, + const size_t srcSize) +{ + LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB; + size_t maxBlockSize = 64 KB; + while (requestedBSID > proposedBSID) { + if (srcSize <= maxBlockSize) + return proposedBSID; + proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1); + maxBlockSize <<= 2; + } + return requestedBSID; +} + +/*! LZ4F_compressBound_internal() : + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario. + * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + */ +static size_t LZ4F_compressBound_internal(size_t srcSize, + const LZ4F_preferences_t* preferencesPtr, + size_t alreadyBuffered) +{ + LZ4F_preferences_t prefsNull = LZ4F_INIT_PREFERENCES; + prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ + prefsNull.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; /* worst case */ + { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + U32 const flush = prefsPtr->autoFlush | (srcSize==0); + LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID; + size_t const blockSize = LZ4F_getBlockSize(blockID); + size_t const maxBuffered = blockSize - 1; + size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); + size_t const maxSrcSize = srcSize + bufferedSize; + unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); + size_t const partialBlockSize = maxSrcSize & (blockSize-1); + size_t const lastBlockSize = flush ? partialBlockSize : 0; + unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); + + size_t const blockCRCSize = BFSize * prefsPtr->frameInfo.blockChecksumFlag; + size_t const frameEnd = BHSize + (prefsPtr->frameInfo.contentChecksumFlag*BFSize); + + return ((BHSize + blockCRCSize) * nbBlocks) + + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd; + } +} + +size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_preferences_t prefs; + size_t const headerSize = maxFHSize; /* max header size, including optional fields */ + + if (preferencesPtr!=NULL) prefs = *preferencesPtr; + else MEM_INIT(&prefs, 0, sizeof(prefs)); + prefs.autoFlush = 1; + + return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);; +} + + +/*! LZ4F_compressFrame_usingCDict() : + * Compress srcBuffer using a dictionary, in a single step. + * cdict can be NULL, in which case, no dictionary is used. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, + * however, it's the only way to provide a dictID, so it's not recommended. + * @return : number of bytes written into dstBuffer, + * or an error code if it fails (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_preferences_t prefs; + LZ4F_compressOptions_t options; + BYTE* const dstStart = (BYTE*) dstBuffer; + BYTE* dstPtr = dstStart; + BYTE* const dstEnd = dstStart + dstCapacity; + + DEBUGLOG(4, "LZ4F_compressFrame_usingCDict (srcSize=%u)", (unsigned)srcSize); + if (preferencesPtr!=NULL) + prefs = *preferencesPtr; + else + MEM_INIT(&prefs, 0, sizeof(prefs)); + if (prefs.frameInfo.contentSize != 0) + prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ + + prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); + prefs.autoFlush = 1; + if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) + prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */ + + MEM_INIT(&options, 0, sizeof(options)); + options.stableSrc = 1; + + RETURN_ERROR_IF(dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs), dstMaxSize_tooSmall); + + { size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */ + FORWARD_IF_ERROR(headerSize); + dstPtr += headerSize; /* header size */ } + + assert(dstEnd >= dstPtr); + { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options); + FORWARD_IF_ERROR(cSize); + dstPtr += cSize; } + + assert(dstEnd >= dstPtr); + { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */ + FORWARD_IF_ERROR(tailSize); + dstPtr += tailSize; } + + assert(dstEnd >= dstStart); + return (size_t)(dstPtr - dstStart); +} + + +/*! LZ4F_compressFrame() : + * Compress an entire srcBuffer into a valid LZ4 frame, in a single step. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr) +{ + size_t result; +#if (LZ4F_HEAPMODE) + LZ4F_cctx_t* cctxPtr; + result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION); + FORWARD_IF_ERROR(result); +#else + LZ4F_cctx_t cctx; + LZ4_stream_t lz4ctx; + LZ4F_cctx_t* const cctxPtr = &cctx; + + MEM_INIT(&cctx, 0, sizeof(cctx)); + cctx.version = LZ4F_VERSION; + cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */ + if ( preferencesPtr == NULL + || preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN ) { + LZ4_initStream(&lz4ctx, sizeof(lz4ctx)); + cctxPtr->lz4CtxPtr = &lz4ctx; + cctxPtr->lz4CtxAlloc = 1; + cctxPtr->lz4CtxType = ctxFast; + } +#endif + DEBUGLOG(4, "LZ4F_compressFrame"); + + result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity, + srcBuffer, srcSize, + NULL, preferencesPtr); + +#if (LZ4F_HEAPMODE) + LZ4F_freeCompressionContext(cctxPtr); +#else + if ( preferencesPtr != NULL + && preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN ) { + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); + } +#endif + return result; +} + + +/*-*************************************************** +* Dictionary compression +*****************************************************/ + +struct LZ4F_CDict_s { + LZ4F_CustomMem cmem; + void* dictContent; + LZ4_stream_t* fastCtx; + LZ4_streamHC_t* HCCtx; +}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */ + +LZ4F_CDict* +LZ4F_createCDict_advanced(LZ4F_CustomMem cmem, const void* dictBuffer, size_t dictSize) +{ + const char* dictStart = (const char*)dictBuffer; + LZ4F_CDict* const cdict = (LZ4F_CDict*)LZ4F_malloc(sizeof(*cdict), cmem); + DEBUGLOG(4, "LZ4F_createCDict_advanced"); + if (!cdict) return NULL; + cdict->cmem = cmem; + if (dictSize > 64 KB) { + dictStart += dictSize - 64 KB; + dictSize = 64 KB; + } + cdict->dictContent = LZ4F_malloc(dictSize, cmem); + /* note: using @cmem to allocate => can't use default create */ + cdict->fastCtx = (LZ4_stream_t*)LZ4F_malloc(sizeof(LZ4_stream_t), cmem); + cdict->HCCtx = (LZ4_streamHC_t*)LZ4F_malloc(sizeof(LZ4_streamHC_t), cmem); + if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) { + LZ4F_freeCDict(cdict); + return NULL; + } + memcpy(cdict->dictContent, dictStart, dictSize); + LZ4_initStream(cdict->fastCtx, sizeof(LZ4_stream_t)); + LZ4_loadDictSlow(cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize); + LZ4_initStreamHC(cdict->HCCtx, sizeof(LZ4_streamHC_t)); + /* note: we don't know at this point which compression level is going to be used + * as a consequence, HCCtx is created for the more common HC mode */ + LZ4_setCompressionLevel(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT); + LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize); + return cdict; +} + +/*! LZ4F_createCDict() : + * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict + * @return : digested dictionary for compression, or NULL if failed */ +LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) +{ + DEBUGLOG(4, "LZ4F_createCDict"); + return LZ4F_createCDict_advanced(LZ4F_defaultCMem, dictBuffer, dictSize); +} + +void LZ4F_freeCDict(LZ4F_CDict* cdict) +{ + if (cdict==NULL) return; /* support free on NULL */ + LZ4F_free(cdict->dictContent, cdict->cmem); + LZ4F_free(cdict->fastCtx, cdict->cmem); + LZ4F_free(cdict->HCCtx, cdict->cmem); + LZ4F_free(cdict, cdict->cmem); +} + + +/*-********************************* +* Advanced compression functions +***********************************/ + +LZ4F_cctx* +LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version) +{ + LZ4F_cctx* const cctxPtr = + (LZ4F_cctx*)LZ4F_calloc(sizeof(LZ4F_cctx), customMem); + if (cctxPtr==NULL) return NULL; + + cctxPtr->cmem = customMem; + cctxPtr->version = version; + cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */ + + return cctxPtr; +} + +/*! LZ4F_createCompressionContext() : + * The first thing to do is to create a compressionContext object, which will be used in all compression operations. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. + * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries. + * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. + * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. + * Object can release its memory using LZ4F_freeCompressionContext(); +**/ +LZ4F_errorCode_t +LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version) +{ + assert(LZ4F_compressionContextPtr != NULL); /* considered a violation of narrow contract */ + /* in case it nonetheless happen in production */ + RETURN_ERROR_IF(LZ4F_compressionContextPtr == NULL, parameter_null); + + *LZ4F_compressionContextPtr = LZ4F_createCompressionContext_advanced(LZ4F_defaultCMem, version); + RETURN_ERROR_IF(*LZ4F_compressionContextPtr==NULL, allocation_failed); + return LZ4F_OK_NoError; +} + +LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr) +{ + if (cctxPtr != NULL) { /* support free on NULL */ + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */ + LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem); + LZ4F_free(cctxPtr, cctxPtr->cmem); + } + return LZ4F_OK_NoError; +} + + +/** + * This function prepares the internal LZ4(HC) stream for a new compression, + * resetting the context and attaching the dictionary, if there is one. + * + * It needs to be called at the beginning of each independent compression + * stream (i.e., at the beginning of a frame in blockLinked mode, or at the + * beginning of each block in blockIndependent mode). + */ +static void LZ4F_initStream(void* ctx, + const LZ4F_CDict* cdict, + int level, + LZ4F_blockMode_t blockMode) { + if (level < LZ4HC_CLEVEL_MIN) { + if (cdict || blockMode == LZ4F_blockLinked) { + /* In these cases, we will call LZ4_compress_fast_continue(), + * which needs an already reset context. Otherwise, we'll call a + * one-shot API. The non-continued APIs internally perform their own + * resets at the beginning of their calls, where they know what + * tableType they need the context to be in. So in that case this + * would be misguided / wasted work. */ + LZ4_resetStream_fast((LZ4_stream_t*)ctx); + if (cdict) + LZ4_attach_dictionary((LZ4_stream_t*)ctx, cdict->fastCtx); + } + /* In these cases, we'll call a one-shot API. + * The non-continued APIs internally perform their own resets + * at the beginning of their calls, where they know + * which tableType they need the context to be in. + * Therefore, a reset here would be wasted work. */ + } else { + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level); + if (cdict) + LZ4_attach_HC_dictionary((LZ4_streamHC_t*)ctx, cdict->HCCtx); + } +} + +static int ctxTypeID_to_size(int ctxTypeID) { + switch(ctxTypeID) { + case 1: + return LZ4_sizeofState(); + case 2: + return LZ4_sizeofStateHC(); + default: + return 0; + } +} + +/* LZ4F_compressBegin_internal() + * Note: only accepts @cdict _or_ @dictBuffer as non NULL. + */ +size_t LZ4F_compressBegin_internal(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* dictBuffer, size_t dictSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_preferences_t const prefNull = LZ4F_INIT_PREFERENCES; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + + RETURN_ERROR_IF(dstCapacity < maxFHSize, dstMaxSize_tooSmall); + if (preferencesPtr == NULL) preferencesPtr = &prefNull; + cctx->prefs = *preferencesPtr; + + /* cctx Management */ + { U16 const ctxTypeID = (cctx->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; + int requiredSize = ctxTypeID_to_size(ctxTypeID); + int allocatedSize = ctxTypeID_to_size(cctx->lz4CtxAlloc); + if (allocatedSize < requiredSize) { + /* not enough space allocated */ + LZ4F_free(cctx->lz4CtxPtr, cctx->cmem); + if (cctx->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { + /* must take ownership of memory allocation, + * in order to respect custom allocator contract */ + cctx->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_stream_t), cctx->cmem); + if (cctx->lz4CtxPtr) + LZ4_initStream(cctx->lz4CtxPtr, sizeof(LZ4_stream_t)); + } else { + cctx->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_streamHC_t), cctx->cmem); + if (cctx->lz4CtxPtr) + LZ4_initStreamHC(cctx->lz4CtxPtr, sizeof(LZ4_streamHC_t)); + } + RETURN_ERROR_IF(cctx->lz4CtxPtr == NULL, allocation_failed); + cctx->lz4CtxAlloc = ctxTypeID; + cctx->lz4CtxType = ctxTypeID; + } else if (cctx->lz4CtxType != ctxTypeID) { + /* otherwise, a sufficient buffer is already allocated, + * but we need to reset it to the correct context type */ + if (cctx->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { + LZ4_initStream((LZ4_stream_t*)cctx->lz4CtxPtr, sizeof(LZ4_stream_t)); + } else { + LZ4_initStreamHC((LZ4_streamHC_t*)cctx->lz4CtxPtr, sizeof(LZ4_streamHC_t)); + LZ4_setCompressionLevel((LZ4_streamHC_t*)cctx->lz4CtxPtr, cctx->prefs.compressionLevel); + } + cctx->lz4CtxType = ctxTypeID; + } } + + /* Buffer Management */ + if (cctx->prefs.frameInfo.blockSizeID == 0) + cctx->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + cctx->maxBlockSize = LZ4F_getBlockSize(cctx->prefs.frameInfo.blockSizeID); + + { size_t const requiredBuffSize = preferencesPtr->autoFlush ? + ((cctx->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 64 KB : 0) : /* only needs past data up to window size */ + cctx->maxBlockSize + ((cctx->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 128 KB : 0); + + if (cctx->maxBufferSize < requiredBuffSize) { + cctx->maxBufferSize = 0; + LZ4F_free(cctx->tmpBuff, cctx->cmem); + cctx->tmpBuff = (BYTE*)LZ4F_malloc(requiredBuffSize, cctx->cmem); + RETURN_ERROR_IF(cctx->tmpBuff == NULL, allocation_failed); + cctx->maxBufferSize = requiredBuffSize; + } } + cctx->tmpIn = cctx->tmpBuff; + cctx->tmpInSize = 0; + (void)XXH32_reset(&(cctx->xxh), 0); + + /* context init */ + cctx->cdict = cdict; + if (cctx->prefs.frameInfo.blockMode == LZ4F_blockLinked) { + /* frame init only for blockLinked : blockIndependent will be init at each block */ + LZ4F_initStream(cctx->lz4CtxPtr, cdict, cctx->prefs.compressionLevel, LZ4F_blockLinked); + } + if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) { + LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctx->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); + } + if (dictBuffer) { + assert(cdict == NULL); + RETURN_ERROR_IF(dictSize > INT_MAX, parameter_invalid); + if (cctx->lz4CtxType == ctxFast) { + /* lz4 fast*/ + LZ4_loadDict((LZ4_stream_t*)cctx->lz4CtxPtr, (const char*)dictBuffer, (int)dictSize); + } else { + /* lz4hc */ + assert(cctx->lz4CtxType == ctxHC); + LZ4_loadDictHC((LZ4_streamHC_t*)cctx->lz4CtxPtr, (const char*)dictBuffer, (int)dictSize); + } + } + + /* Stage 2 : Write Frame Header */ + + /* Magic Number */ + LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); + dstPtr += 4; + { BYTE* const headerStart = dstPtr; + + /* FLG Byte */ + *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ + + ((cctx->prefs.frameInfo.blockMode & _1BIT ) << 5) + + ((cctx->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4) + + ((unsigned)(cctx->prefs.frameInfo.contentSize > 0) << 3) + + ((cctx->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) + + (cctx->prefs.frameInfo.dictID > 0) ); + /* BD Byte */ + *dstPtr++ = (BYTE)((cctx->prefs.frameInfo.blockSizeID & _3BITS) << 4); + /* Optional Frame content size field */ + if (cctx->prefs.frameInfo.contentSize) { + LZ4F_writeLE64(dstPtr, cctx->prefs.frameInfo.contentSize); + dstPtr += 8; + cctx->totalInSize = 0; + } + /* Optional dictionary ID field */ + if (cctx->prefs.frameInfo.dictID) { + LZ4F_writeLE32(dstPtr, cctx->prefs.frameInfo.dictID); + dstPtr += 4; + } + /* Header CRC Byte */ + *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart)); + dstPtr++; + } + + cctx->cStage = 1; /* header written, now request input data block */ + return (size_t)(dstPtr - dstStart); +} + +size_t LZ4F_compressBegin(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressBegin_internal(cctx, dstBuffer, dstCapacity, + NULL, 0, + NULL, preferencesPtr); +} + +/* LZ4F_compressBegin_usingDictOnce: + * Hidden implementation, + * employed for multi-threaded compression + * when frame defines linked blocks */ +size_t LZ4F_compressBegin_usingDictOnce(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* dict, size_t dictSize, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressBegin_internal(cctx, dstBuffer, dstCapacity, + dict, dictSize, + NULL, preferencesPtr); +} + +size_t LZ4F_compressBegin_usingDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* dict, size_t dictSize, + const LZ4F_preferences_t* preferencesPtr) +{ + /* note : incorrect implementation : + * this will only use the dictionary once, + * instead of once *per* block when frames defines independent blocks */ + return LZ4F_compressBegin_usingDictOnce(cctx, dstBuffer, dstCapacity, + dict, dictSize, + preferencesPtr); +} + +size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressBegin_internal(cctx, dstBuffer, dstCapacity, + NULL, 0, + cdict, preferencesPtr); +} + + +/* LZ4F_compressBound() : + * @return minimum capacity of dstBuffer for a given srcSize to handle worst case scenario. + * LZ4F_preferences_t structure is optional : if NULL, preferences will be set to cover worst case scenario. + * This function cannot fail. + */ +size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +{ + if (preferencesPtr && preferencesPtr->autoFlush) { + return LZ4F_compressBound_internal(srcSize, preferencesPtr, 0); + } + return LZ4F_compressBound_internal(srcSize, preferencesPtr, (size_t)-1); +} + + +typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict); + + +/*! LZ4F_makeBlock(): + * compress a single block, add header and optional checksum. + * assumption : dst buffer capacity is >= BHSize + srcSize + crcSize + */ +static size_t LZ4F_makeBlock(void* dst, + const void* src, size_t srcSize, + compressFunc_t compress, void* lz4ctx, int level, + const LZ4F_CDict* cdict, + LZ4F_blockChecksum_t crcFlag) +{ + BYTE* const cSizePtr = (BYTE*)dst; + U32 cSize; + assert(compress != NULL); + cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize), + (int)(srcSize), (int)(srcSize-1), + level, cdict); + + if (cSize == 0 || cSize >= srcSize) { + cSize = (U32)srcSize; + LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); + memcpy(cSizePtr+BHSize, src, srcSize); + } else { + LZ4F_writeLE32(cSizePtr, cSize); + } + if (crcFlag) { + U32 const crc32 = XXH32(cSizePtr+BHSize, cSize, 0); /* checksum of compressed data */ + LZ4F_writeLE32(cSizePtr+BHSize+cSize, crc32); + } + return BHSize + cSize + ((U32)crcFlag)*BFSize; +} + + +static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + int const acceleration = (level < 0) ? -level + 1 : 1; + DEBUGLOG(5, "LZ4F_compressBlock (srcSize=%i)", srcSize); + LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent); + if (cdict) { + return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); + } else { + return LZ4_compress_fast_extState_fastReset(ctx, src, dst, srcSize, dstCapacity, acceleration); + } +} + +static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + int const acceleration = (level < 0) ? -level + 1 : 1; + (void)cdict; /* init once at beginning of frame */ + DEBUGLOG(5, "LZ4F_compressBlock_continue (srcSize=%i)", srcSize); + return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); +} + +static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent); + if (cdict) { + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); + } + return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level); +} + +static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + (void)level; (void)cdict; /* init once at beginning of frame */ + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); +} + +static int LZ4F_doNotCompressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + (void)ctx; (void)src; (void)dst; (void)srcSize; (void)dstCapacity; (void)level; (void)cdict; + return 0; +} + +static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level, LZ4F_BlockCompressMode_e compressMode) +{ + if (compressMode == LZ4B_UNCOMPRESSED) + return LZ4F_doNotCompressBlock; + if (level < LZ4HC_CLEVEL_MIN) { + if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock; + return LZ4F_compressBlock_continue; + } + if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC; + return LZ4F_compressBlockHC_continue; +} + +/* Save history (up to 64KB) into @tmpBuff */ +static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) +{ + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) + return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); + return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); +} + +typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; + +static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } }; + + + /*! LZ4F_compressUpdateImpl() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * If the block compression does not match the compression of the previous block, the old data is flushed + * and operations continue with the new compression mode. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on. + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr, + LZ4F_BlockCompressMode_e blockCompression) + { + size_t const blockSize = cctxPtr->maxBlockSize; + const BYTE* srcPtr = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcPtr + srcSize; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + LZ4F_lastBlockStatus lastBlockCompressed = notDone; + compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, blockCompression); + size_t bytesWritten; + DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize); + + RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); /* state must be initialized and waiting for next block */ + if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) + RETURN_ERROR(dstMaxSize_tooSmall); + + if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize) + RETURN_ERROR(dstMaxSize_tooSmall); + + /* flush currently written block, to continue with new block compression */ + if (cctxPtr->blockCompressMode != blockCompression) { + bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr); + dstPtr += bytesWritten; + cctxPtr->blockCompressMode = blockCompression; + } + + if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull; + + /* complete tmp buffer */ + if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ + size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize; + assert(blockSize > cctxPtr->tmpInSize); + if (sizeToCopy > srcSize) { + /* add src to tmpIn buffer */ + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); + srcPtr = srcEnd; + cctxPtr->tmpInSize += srcSize; + /* still needs some CRC */ + } else { + /* complete tmpIn block and then compress it */ + lastBlockCompressed = fromTmpBuffer; + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); + srcPtr += sizeToCopy; + + dstPtr += LZ4F_makeBlock(dstPtr, + cctxPtr->tmpIn, blockSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); + if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; + cctxPtr->tmpInSize = 0; + } } + + while ((size_t)(srcEnd - srcPtr) >= blockSize) { + /* compress full blocks */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ4F_makeBlock(dstPtr, + srcPtr, blockSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); + srcPtr += blockSize; + } + + if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { + /* autoFlush : remaining input (< blockSize) is compressed */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ4F_makeBlock(dstPtr, + srcPtr, (size_t)(srcEnd - srcPtr), + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); + srcPtr = srcEnd; + } + + /* preserve dictionary within @tmpBuff whenever necessary */ + if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { + /* linked blocks are only supported in compressed mode, see LZ4F_uncompressedUpdate */ + assert(blockCompression == LZ4B_COMPRESSED); + if (compressOptionsPtr->stableSrc) { + cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */ + } else { + int const realDictSize = LZ4F_localSaveDict(cctxPtr); + assert(0 <= realDictSize && realDictSize <= 64 KB); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + } + + /* keep tmpIn within limits */ + if (!(cctxPtr->prefs.autoFlush) /* no autoflush : there may be some data left within internal buffer */ + && (cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) ) /* not enough room to store next block */ + { + /* only preserve 64KB within internal buffer. Ensures there is enough room for next block. + * note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */ + int const realDictSize = LZ4F_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)); + } + + /* some input data left, necessarily < blockSize */ + if (srcPtr < srcEnd) { + /* fill tmp buffer */ + size_t const sizeToCopy = (size_t)(srcEnd - srcPtr); + memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); + cctxPtr->tmpInSize = sizeToCopy; + } + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) + (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); + + cctxPtr->totalInSize += srcSize; + return (size_t)(dstPtr - dstStart); +} + +/*! LZ4F_compressUpdate() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) +{ + return LZ4F_compressUpdateImpl(cctxPtr, + dstBuffer, dstCapacity, + srcBuffer, srcSize, + compressOptionsPtr, LZ4B_COMPRESSED); +} + +/*! LZ4F_uncompressedUpdate() : + * Same as LZ4F_compressUpdate(), but requests blocks to be sent uncompressed. + * This symbol is only supported when LZ4F_blockIndependent is used + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) +{ + return LZ4F_compressUpdateImpl(cctxPtr, + dstBuffer, dstCapacity, + srcBuffer, srcSize, + compressOptionsPtr, LZ4B_UNCOMPRESSED); +} + + +/*! LZ4F_flush() : + * When compressed data must be sent immediately, without waiting for a block to be filled, + * invoke LZ4_flush(), which will immediately compress any remaining data stored within LZ4F_cctx. + * The result of the function is the number of bytes written into dstBuffer. + * It can be zero, this means there was no data left within LZ4F_cctx. + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + * LZ4F_compressOptions_t* is optional. NULL is a valid argument. + */ +size_t LZ4F_flush(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* compressOptionsPtr) +{ + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + compressFunc_t compress; + + if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ + RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); + RETURN_ERROR_IF(dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize), dstMaxSize_tooSmall); + (void)compressOptionsPtr; /* not useful (yet) */ + + /* select compression function */ + compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, cctxPtr->blockCompressMode); + + /* compress tmp buffer */ + dstPtr += LZ4F_makeBlock(dstPtr, + cctxPtr->tmpIn, cctxPtr->tmpInSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); + assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity)); + + if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) + cctxPtr->tmpIn += cctxPtr->tmpInSize; + cctxPtr->tmpInSize = 0; + + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */ + int const realDictSize = LZ4F_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + + return (size_t)(dstPtr - dstStart); +} + + +/*! LZ4F_compressEnd() : + * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). + * It will flush whatever data remained within compressionContext (like LZ4_flush()) + * but also properly finalize the frame, with an endMark and an (optional) checksum. + * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * @return: the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) + * or an error code if it fails (can be tested using LZ4F_isError()) + * The context can then be used again to compress a new frame, starting with LZ4F_compressBegin(). + */ +size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* compressOptionsPtr) +{ + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + + size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr); + DEBUGLOG(5,"LZ4F_compressEnd: dstCapacity=%u", (unsigned)dstCapacity); + FORWARD_IF_ERROR(flushSize); + dstPtr += flushSize; + + assert(flushSize <= dstCapacity); + dstCapacity -= flushSize; + + RETURN_ERROR_IF(dstCapacity < 4, dstMaxSize_tooSmall); + LZ4F_writeLE32(dstPtr, 0); + dstPtr += 4; /* endMark */ + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) { + U32 const xxh = XXH32_digest(&(cctxPtr->xxh)); + RETURN_ERROR_IF(dstCapacity < 8, dstMaxSize_tooSmall); + DEBUGLOG(5,"Writing 32-bit content checksum (0x%0X)", xxh); + LZ4F_writeLE32(dstPtr, xxh); + dstPtr+=4; /* content Checksum */ + } + + cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ + + if (cctxPtr->prefs.frameInfo.contentSize) { + if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) + RETURN_ERROR(frameSize_wrong); + } + + return (size_t)(dstPtr - dstStart); +} + + +/*-*************************************************** +* Frame Decompression +*****************************************************/ + +typedef enum { + dstage_getFrameHeader=0, dstage_storeFrameHeader, + dstage_init, + dstage_getBlockHeader, dstage_storeBlockHeader, + dstage_copyDirect, dstage_getBlockChecksum, + dstage_getCBlock, dstage_storeCBlock, + dstage_flushOut, + dstage_getSuffix, dstage_storeSuffix, + dstage_getSFrameSize, dstage_storeSFrameSize, + dstage_skipSkippable +} dStage_t; + +struct LZ4F_dctx_s { + LZ4F_CustomMem cmem; + LZ4F_frameInfo_t frameInfo; + U32 version; + dStage_t dStage; + U64 frameRemainingSize; + size_t maxBlockSize; + size_t maxBufferSize; + BYTE* tmpIn; + size_t tmpInSize; + size_t tmpInTarget; + BYTE* tmpOutBuffer; + const BYTE* dict; + size_t dictSize; + BYTE* tmpOut; + size_t tmpOutSize; + size_t tmpOutStart; + XXH32_state_t xxh; + XXH32_state_t blockChecksum; + int skipChecksum; + BYTE header[LZ4F_HEADER_SIZE_MAX]; +}; /* typedef'd to LZ4F_dctx in lz4frame.h */ + + +LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version) +{ + LZ4F_dctx* const dctx = (LZ4F_dctx*)LZ4F_calloc(sizeof(LZ4F_dctx), customMem); + if (dctx == NULL) return NULL; + + dctx->cmem = customMem; + dctx->version = version; + return dctx; +} + +/*! LZ4F_createDecompressionContext() : + * Create a decompressionContext object, which will track all decompression operations. + * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object. + * Object can later be released using LZ4F_freeDecompressionContext(). + * @return : if != 0, there was an error during context creation. + */ +LZ4F_errorCode_t +LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) +{ + assert(LZ4F_decompressionContextPtr != NULL); /* violation of narrow contract */ + RETURN_ERROR_IF(LZ4F_decompressionContextPtr == NULL, parameter_null); /* in case it nonetheless happen in production */ + + *LZ4F_decompressionContextPtr = LZ4F_createDecompressionContext_advanced(LZ4F_defaultCMem, versionNumber); + if (*LZ4F_decompressionContextPtr == NULL) { /* failed allocation */ + RETURN_ERROR(allocation_failed); + } + return LZ4F_OK_NoError; +} + +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx) +{ + LZ4F_errorCode_t result = LZ4F_OK_NoError; + if (dctx != NULL) { /* can accept NULL input, like free() */ + result = (LZ4F_errorCode_t)dctx->dStage; + LZ4F_free(dctx->tmpIn, dctx->cmem); + LZ4F_free(dctx->tmpOutBuffer, dctx->cmem); + LZ4F_free(dctx, dctx->cmem); + } + return result; +} + + +/*==--- Streaming Decompression operations ---==*/ +void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) +{ + DEBUGLOG(5, "LZ4F_resetDecompressionContext"); + dctx->dStage = dstage_getFrameHeader; + dctx->dict = NULL; + dctx->dictSize = 0; + dctx->skipChecksum = 0; + dctx->frameRemainingSize = 0; +} + + +/*! LZ4F_decodeHeader() : + * input : `src` points at the **beginning of the frame** + * output : set internal values of dctx, such as + * dctx->frameInfo and dctx->dStage. + * Also allocates internal buffers. + * @return : nb Bytes read from src (necessarily <= srcSize) + * or an error code (testable with LZ4F_isError()) + */ +static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize) +{ + unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID; + size_t frameHeaderSize; + const BYTE* srcPtr = (const BYTE*)src; + + DEBUGLOG(5, "LZ4F_decodeHeader"); + /* need to decode header to get frameInfo */ + RETURN_ERROR_IF(srcSize < minFHSize, frameHeader_incomplete); /* minimal frame header size */ + MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo)); + + /* special case : skippable frames */ + if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) { + dctx->frameInfo.frameType = LZ4F_skippableFrame; + if (src == (void*)(dctx->header)) { + dctx->tmpInSize = srcSize; + dctx->tmpInTarget = 8; + dctx->dStage = dstage_storeSFrameSize; + return srcSize; + } else { + dctx->dStage = dstage_getSFrameSize; + return 4; + } } + + /* control magic number */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) { + DEBUGLOG(4, "frame header error : unknown magic number"); + RETURN_ERROR(frameType_unknown); + } +#endif + dctx->frameInfo.frameType = LZ4F_frame; + + /* Flags */ + { U32 const FLG = srcPtr[4]; + U32 const version = (FLG>>6) & _2BITS; + blockChecksumFlag = (FLG>>4) & _1BIT; + blockMode = (FLG>>5) & _1BIT; + contentSizeFlag = (FLG>>3) & _1BIT; + contentChecksumFlag = (FLG>>2) & _1BIT; + dictIDFlag = FLG & _1BIT; + /* validate */ + if (((FLG>>1)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */ + if (version != 1) RETURN_ERROR(headerVersion_wrong); /* Version Number, only supported value */ + } + DEBUGLOG(6, "contentSizeFlag: %u", contentSizeFlag); + + /* Frame Header Size */ + frameHeaderSize = minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0); + + if (srcSize < frameHeaderSize) { + /* not enough input to fully decode frame header */ + if (srcPtr != dctx->header) + memcpy(dctx->header, srcPtr, srcSize); + dctx->tmpInSize = srcSize; + dctx->tmpInTarget = frameHeaderSize; + dctx->dStage = dstage_storeFrameHeader; + return srcSize; + } + + { U32 const BD = srcPtr[5]; + blockSizeID = (BD>>4) & _3BITS; + /* validate */ + if (((BD>>7)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */ + if (blockSizeID < 4) RETURN_ERROR(maxBlockSize_invalid); /* 4-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bits */ + } + + /* check header */ + assert(frameHeaderSize > 5); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); + RETURN_ERROR_IF(HC != srcPtr[frameHeaderSize-1], headerChecksum_invalid); + } +#endif + + /* save */ + dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; + dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag; + dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag; + dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; + dctx->maxBlockSize = LZ4F_getBlockSize((LZ4F_blockSizeID_t)blockSizeID); + if (contentSizeFlag) { + dctx->frameRemainingSize = dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); + } + if (dictIDFlag) + dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5); + + dctx->dStage = dstage_init; + + return frameHeaderSize; +} + + +/*! LZ4F_headerSize() : + * @return : size of frame header + * or an error code, which can be tested using LZ4F_isError() + */ +size_t LZ4F_headerSize(const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(src == NULL, srcPtr_wrong); + + /* minimal srcSize to determine header size */ + if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH) + RETURN_ERROR(frameHeader_incomplete); + + /* special case : skippable frames */ + if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) + return 8; + + /* control magic number */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) + RETURN_ERROR(frameType_unknown); +#endif + + /* Frame Header Size */ + { BYTE const FLG = ((const BYTE*)src)[4]; + U32 const contentSizeFlag = (FLG>>3) & _1BIT; + U32 const dictIDFlag = FLG & _1BIT; + return minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0); + } +} + +/*! LZ4F_getFrameInfo() : + * This function extracts frame parameters (max blockSize, frame checksum, etc.). + * Usage is optional. Objective is to provide relevant information for allocation purposes. + * This function works in 2 situations : + * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process. + * Amount of input data provided must be large enough to successfully decode the frame header. + * A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum. + * - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx. + * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + * Decompression must resume from (srcBuffer + *srcSizePtr). + * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, + * or an error code which can be tested using LZ4F_isError() + * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped. + * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. + */ +LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr) +{ + LZ4F_STATIC_ASSERT(dstage_getFrameHeader < dstage_storeFrameHeader); + if (dctx->dStage > dstage_storeFrameHeader) { + /* frameInfo already decoded */ + size_t o=0, i=0; + *srcSizePtr = 0; + *frameInfoPtr = dctx->frameInfo; + /* returns : recommended nb of bytes for LZ4F_decompress() */ + return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL); + } else { + if (dctx->dStage == dstage_storeFrameHeader) { + /* frame decoding already started, in the middle of header => automatic fail */ + *srcSizePtr = 0; + RETURN_ERROR(frameDecoding_alreadyStarted); + } else { + size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr); + if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; } + if (*srcSizePtr < hSize) { + *srcSizePtr=0; + RETURN_ERROR(frameHeader_incomplete); + } + + { size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize); + if (LZ4F_isError(decodeResult)) { + *srcSizePtr = 0; + } else { + *srcSizePtr = decodeResult; + decodeResult = BHSize; /* block header size */ + } + *frameInfoPtr = dctx->frameInfo; + return decodeResult; + } } } +} + + +/* LZ4F_updateDict() : + * only used for LZ4F_blockLinked mode + * Condition : @dstPtr != NULL + */ +static void LZ4F_updateDict(LZ4F_dctx* dctx, + const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart, + unsigned withinTmp) +{ + assert(dstPtr != NULL); + if (dctx->dictSize==0) dctx->dict = (const BYTE*)dstPtr; /* will lead to prefix mode */ + assert(dctx->dict != NULL); + + if (dctx->dict + dctx->dictSize == dstPtr) { /* prefix mode, everything within dstBuffer */ + dctx->dictSize += dstSize; + return; + } + + assert(dstPtr >= dstBufferStart); + if ((size_t)(dstPtr - dstBufferStart) + dstSize >= 64 KB) { /* history in dstBuffer becomes large enough to become dictionary */ + dctx->dict = (const BYTE*)dstBufferStart; + dctx->dictSize = (size_t)(dstPtr - dstBufferStart) + dstSize; + return; + } + + assert(dstSize < 64 KB); /* if dstSize >= 64 KB, dictionary would be set into dstBuffer directly */ + + /* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOutBuffer */ + assert(dctx->tmpOutBuffer != NULL); + + if (withinTmp && (dctx->dict == dctx->tmpOutBuffer)) { /* continue history within tmpOutBuffer */ + /* withinTmp expectation : content of [dstPtr,dstSize] is same as [dict+dictSize,dstSize], so we just extend it */ + assert(dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart); + dctx->dictSize += dstSize; + return; + } + + if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ + size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer); + size_t copySize = 64 KB - dctx->tmpOutSize; + const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; + if (dctx->tmpOutSize > 64 KB) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + + memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize; + return; + } + + if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */ + if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */ + size_t const preserveSize = 64 KB - dstSize; + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); + dctx->dictSize = preserveSize; + } + memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize); + dctx->dictSize += dstSize; + return; + } + + /* join dict & dest into tmp */ + { size_t preserveSize = 64 KB - dstSize; + if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize; + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); + memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize); + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dstSize; + } +} + + +/*! LZ4F_decompress() : + * Call this function repetitively to regenerate compressed data in srcBuffer. + * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer + * into dstBuffer of capacity *dstSizePtr. + * + * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + * + * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * If number of bytes read is < number of bytes provided, then decompression operation is not complete. + * Remaining data will have to be presented again in a subsequent invocation. + * + * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling. + * Note that this is just a hint, and it's always possible to any srcSize value. + * When a frame is fully decoded, @return will be 0. + * If decompression failed, @return is an error code which can be tested using LZ4F_isError(). + */ +size_t LZ4F_decompress(LZ4F_dctx* dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* decompressOptionsPtr) +{ + LZ4F_decompressOptions_t optionsNull; + const BYTE* const srcStart = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcStart + *srcSizePtr; + const BYTE* srcPtr = srcStart; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* const dstEnd = dstStart ? dstStart + *dstSizePtr : NULL; + BYTE* dstPtr = dstStart; + const BYTE* selectedIn = NULL; + unsigned doAnotherStage = 1; + size_t nextSrcSizeHint = 1; + + + DEBUGLOG(5, "LZ4F_decompress: src[%p](%u) => dst[%p](%u)", + srcBuffer, (unsigned)*srcSizePtr, dstBuffer, (unsigned)*dstSizePtr); + if (dstBuffer == NULL) assert(*dstSizePtr == 0); + MEM_INIT(&optionsNull, 0, sizeof(optionsNull)); + if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; + *srcSizePtr = 0; + *dstSizePtr = 0; + assert(dctx != NULL); + dctx->skipChecksum |= (decompressOptionsPtr->skipChecksums != 0); /* once set, disable for the remainder of the frame */ + + /* behaves as a state machine */ + + while (doAnotherStage) { + + switch(dctx->dStage) + { + + case dstage_getFrameHeader: + DEBUGLOG(6, "dstage_getFrameHeader"); + if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */ + size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */ + FORWARD_IF_ERROR(hSize); + srcPtr += hSize; + break; + } + dctx->tmpInSize = 0; + if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */ + dctx->tmpInTarget = minFHSize; /* minimum size to decode header */ + dctx->dStage = dstage_storeFrameHeader; + /* fall-through */ + + case dstage_storeFrameHeader: + DEBUGLOG(6, "dstage_storeFrameHeader"); + { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr)); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + } + if (dctx->tmpInSize < dctx->tmpInTarget) { + nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ + doAnotherStage = 0; /* not enough src data, ask for some more */ + break; + } + FORWARD_IF_ERROR( LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget) ); /* will update dStage appropriately */ + break; + + case dstage_init: + DEBUGLOG(6, "dstage_init"); + if (dctx->frameInfo.contentChecksumFlag) (void)XXH32_reset(&(dctx->xxh), 0); + /* internal buffers allocation */ + { size_t const bufferNeeded = dctx->maxBlockSize + + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0); + if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ + dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ + LZ4F_free(dctx->tmpIn, dctx->cmem); + dctx->tmpIn = (BYTE*)LZ4F_malloc(dctx->maxBlockSize + BFSize /* block checksum */, dctx->cmem); + RETURN_ERROR_IF(dctx->tmpIn == NULL, allocation_failed); + LZ4F_free(dctx->tmpOutBuffer, dctx->cmem); + dctx->tmpOutBuffer= (BYTE*)LZ4F_malloc(bufferNeeded, dctx->cmem); + RETURN_ERROR_IF(dctx->tmpOutBuffer== NULL, allocation_failed); + dctx->maxBufferSize = bufferNeeded; + } } + dctx->tmpInSize = 0; + dctx->tmpInTarget = 0; + dctx->tmpOut = dctx->tmpOutBuffer; + dctx->tmpOutStart = 0; + dctx->tmpOutSize = 0; + + dctx->dStage = dstage_getBlockHeader; + /* fall-through */ + + case dstage_getBlockHeader: + if ((size_t)(srcEnd - srcPtr) >= BHSize) { + selectedIn = srcPtr; + srcPtr += BHSize; + } else { + /* not enough input to read cBlockSize field */ + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeBlockHeader; + } + + if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */ + case dstage_storeBlockHeader: + { size_t const remainingInput = (size_t)(srcEnd - srcPtr); + size_t const wantedData = BHSize - dctx->tmpInSize; + size_t const sizeToCopy = MIN(wantedData, remainingInput); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctx->tmpInSize += sizeToCopy; + + if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */ + nextSrcSizeHint = BHSize - dctx->tmpInSize; + doAnotherStage = 0; + break; + } + selectedIn = dctx->tmpIn; + } /* if (dctx->dStage == dstage_storeBlockHeader) */ + + /* decode block header */ + { U32 const blockHeader = LZ4F_readLE32(selectedIn); + size_t const nextCBlockSize = blockHeader & 0x7FFFFFFFU; + size_t const crcSize = dctx->frameInfo.blockChecksumFlag * BFSize; + if (blockHeader==0) { /* frameEnd signal, no more block */ + DEBUGLOG(5, "end of frame"); + dctx->dStage = dstage_getSuffix; + break; + } + if (nextCBlockSize > dctx->maxBlockSize) { + RETURN_ERROR(maxBlockSize_invalid); + } + if (blockHeader & LZ4F_BLOCKUNCOMPRESSED_FLAG) { + /* next block is uncompressed */ + dctx->tmpInTarget = nextCBlockSize; + DEBUGLOG(5, "next block is uncompressed (size %u)", (U32)nextCBlockSize); + if (dctx->frameInfo.blockChecksumFlag) { + (void)XXH32_reset(&dctx->blockChecksum, 0); + } + dctx->dStage = dstage_copyDirect; + break; + } + /* next block is a compressed block */ + dctx->tmpInTarget = nextCBlockSize + crcSize; + dctx->dStage = dstage_getCBlock; + if (dstPtr==dstEnd || srcPtr==srcEnd) { + nextSrcSizeHint = BHSize + nextCBlockSize + crcSize; + doAnotherStage = 0; + } + break; + } + + case dstage_copyDirect: /* uncompressed block */ + DEBUGLOG(6, "dstage_copyDirect"); + { size_t sizeToCopy; + if (dstPtr == NULL) { + sizeToCopy = 0; + } else { + size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr)); + sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); + memcpy(dstPtr, srcPtr, sizeToCopy); + if (!dctx->skipChecksum) { + if (dctx->frameInfo.blockChecksumFlag) { + (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + } + if (dctx->frameInfo.contentChecksumFlag) + (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); + } + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= sizeToCopy; + + /* history management (linked blocks only)*/ + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) { + LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0); + } + srcPtr += sizeToCopy; + dstPtr += sizeToCopy; + } + if (sizeToCopy == dctx->tmpInTarget) { /* all done */ + if (dctx->frameInfo.blockChecksumFlag) { + dctx->tmpInSize = 0; + dctx->dStage = dstage_getBlockChecksum; + } else + dctx->dStage = dstage_getBlockHeader; /* new block */ + break; + } + dctx->tmpInTarget -= sizeToCopy; /* need to copy more */ + } + nextSrcSizeHint = dctx->tmpInTarget + + +(dctx->frameInfo.blockChecksumFlag ? BFSize : 0) + + BHSize /* next header size */; + doAnotherStage = 0; + break; + + /* check block checksum for recently transferred uncompressed block */ + case dstage_getBlockChecksum: + DEBUGLOG(6, "dstage_getBlockChecksum"); + { const void* crcSrc; + if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) { + crcSrc = srcPtr; + srcPtr += 4; + } else { + size_t const stillToCopy = 4 - dctx->tmpInSize; + size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr)); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctx->tmpInSize < 4) { /* all input consumed */ + doAnotherStage = 0; + break; + } + crcSrc = dctx->header; + } + if (!dctx->skipChecksum) { + U32 const readCRC = LZ4F_readLE32(crcSrc); + U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + DEBUGLOG(6, "compare block checksum"); + if (readCRC != calcCRC) { + DEBUGLOG(4, "incorrect block checksum: %08X != %08X", + readCRC, calcCRC); + RETURN_ERROR(blockChecksum_invalid); + } +#else + (void)readCRC; + (void)calcCRC; +#endif + } } + dctx->dStage = dstage_getBlockHeader; /* new block */ + break; + + case dstage_getCBlock: + DEBUGLOG(6, "dstage_getCBlock"); + if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) { + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeCBlock; + break; + } + /* input large enough to read full block directly */ + selectedIn = srcPtr; + srcPtr += dctx->tmpInTarget; + + if (0) /* always jump over next block */ + case dstage_storeCBlock: + { size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize; + size_t const inputLeft = (size_t)(srcEnd-srcPtr); + size_t const sizeToCopy = MIN(wantedData, inputLeft); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */ + nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + + (dctx->frameInfo.blockChecksumFlag ? BFSize : 0) + + BHSize /* next header size */; + doAnotherStage = 0; + break; + } + selectedIn = dctx->tmpIn; + } + + /* At this stage, input is large enough to decode a block */ + + /* First, decode and control block checksum if it exists */ + if (dctx->frameInfo.blockChecksumFlag) { + assert(dctx->tmpInTarget >= 4); + dctx->tmpInTarget -= 4; + assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */ + { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget); + U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + RETURN_ERROR_IF(readBlockCrc != calcBlockCrc, blockChecksum_invalid); +#else + (void)readBlockCrc; + (void)calcBlockCrc; +#endif + } } + + /* decode directly into destination buffer if there is enough room */ + if ( ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) + /* unless the dictionary is stored in tmpOut: + * in which case it's faster to decode within tmpOut + * to benefit from prefix speedup */ + && !(dctx->dict!= NULL && (const BYTE*)dctx->dict + dctx->dictSize == dctx->tmpOut) ) + { + const char* dict = (const char*)dctx->dict; + size_t dictSize = dctx->dictSize; + int decodedSize; + assert(dstPtr != NULL); + if (dict && dictSize > 1 GB) { + /* overflow control : dctx->dictSize is an int, avoid truncation / sign issues */ + dict += dictSize - 64 KB; + dictSize = 64 KB; + } + decodedSize = LZ4_decompress_safe_usingDict( + (const char*)selectedIn, (char*)dstPtr, + (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, + dict, (int)dictSize); + RETURN_ERROR_IF(decodedSize < 0, decompressionFailed); + if ((dctx->frameInfo.contentChecksumFlag) && (!dctx->skipChecksum)) + XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize); + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= (size_t)decodedSize; + + /* dictionary management */ + if (dctx->frameInfo.blockMode==LZ4F_blockLinked) { + LZ4F_updateDict(dctx, dstPtr, (size_t)decodedSize, dstStart, 0); + } + + dstPtr += decodedSize; + dctx->dStage = dstage_getBlockHeader; /* end of block, let's get another one */ + break; + } + + /* not enough place into dst : decode into tmpOut */ + + /* manage dictionary */ + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) { + if (dctx->dict == dctx->tmpOutBuffer) { + /* truncate dictionary to 64 KB if too big */ + if (dctx->dictSize > 128 KB) { + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB); + dctx->dictSize = 64 KB; + } + dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize; + } else { /* dict not within tmpOut */ + size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB); + dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace; + } } + + /* Decode block into tmpOut */ + { const char* dict = (const char*)dctx->dict; + size_t dictSize = dctx->dictSize; + int decodedSize; + if (dict && dictSize > 1 GB) { + /* the dictSize param is an int, avoid truncation / sign issues */ + dict += dictSize - 64 KB; + dictSize = 64 KB; + } + decodedSize = LZ4_decompress_safe_usingDict( + (const char*)selectedIn, (char*)dctx->tmpOut, + (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, + dict, (int)dictSize); + RETURN_ERROR_IF(decodedSize < 0, decompressionFailed); + if (dctx->frameInfo.contentChecksumFlag && !dctx->skipChecksum) + XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize); + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= (size_t)decodedSize; + dctx->tmpOutSize = (size_t)decodedSize; + dctx->tmpOutStart = 0; + dctx->dStage = dstage_flushOut; + } + /* fall-through */ + + case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ + DEBUGLOG(6, "dstage_flushOut"); + if (dstPtr != NULL) { + size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr)); + memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy); + + /* dictionary management */ + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) + LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1 /*withinTmp*/); + + dctx->tmpOutStart += sizeToCopy; + dstPtr += sizeToCopy; + } + if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */ + dctx->dStage = dstage_getBlockHeader; /* get next block */ + break; + } + /* could not flush everything : stop there, just request a block header */ + doAnotherStage = 0; + nextSrcSizeHint = BHSize; + break; + + case dstage_getSuffix: + RETURN_ERROR_IF(dctx->frameRemainingSize, frameSize_wrong); /* incorrect frame size decoded */ + if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */ + nextSrcSizeHint = 0; + LZ4F_resetDecompressionContext(dctx); + doAnotherStage = 0; + break; + } + if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */ + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeSuffix; + } else { + selectedIn = srcPtr; + srcPtr += 4; + } + + if (dctx->dStage == dstage_storeSuffix) /* can be skipped */ + case dstage_storeSuffix: + { size_t const remainingInput = (size_t)(srcEnd - srcPtr); + size_t const wantedData = 4 - dctx->tmpInSize; + size_t const sizeToCopy = MIN(wantedData, remainingInput); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctx->tmpInSize += sizeToCopy; + if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */ + nextSrcSizeHint = 4 - dctx->tmpInSize; + doAnotherStage=0; + break; + } + selectedIn = dctx->tmpIn; + } /* if (dctx->dStage == dstage_storeSuffix) */ + + /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */ + if (!dctx->skipChecksum) { + U32 const readCRC = LZ4F_readLE32(selectedIn); + U32 const resultCRC = XXH32_digest(&(dctx->xxh)); + DEBUGLOG(4, "frame checksum: stored 0x%0X vs 0x%0X processed", readCRC, resultCRC); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + RETURN_ERROR_IF(readCRC != resultCRC, contentChecksum_invalid); +#else + (void)readCRC; + (void)resultCRC; +#endif + } + nextSrcSizeHint = 0; + LZ4F_resetDecompressionContext(dctx); + doAnotherStage = 0; + break; + + case dstage_getSFrameSize: + if ((srcEnd - srcPtr) >= 4) { + selectedIn = srcPtr; + srcPtr += 4; + } else { + /* not enough input to read cBlockSize field */ + dctx->tmpInSize = 4; + dctx->tmpInTarget = 8; + dctx->dStage = dstage_storeSFrameSize; + } + + if (dctx->dStage == dstage_storeSFrameSize) + case dstage_storeSFrameSize: + { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, + (size_t)(srcEnd - srcPtr) ); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctx->tmpInSize += sizeToCopy; + if (dctx->tmpInSize < dctx->tmpInTarget) { + /* not enough input to get full sBlockSize; wait for more */ + nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize; + doAnotherStage = 0; + break; + } + selectedIn = dctx->header + 4; + } /* if (dctx->dStage == dstage_storeSFrameSize) */ + + /* case dstage_decodeSFrameSize: */ /* no direct entry */ + { size_t const SFrameSize = LZ4F_readLE32(selectedIn); + dctx->frameInfo.contentSize = SFrameSize; + dctx->tmpInTarget = SFrameSize; + dctx->dStage = dstage_skipSkippable; + break; + } + + case dstage_skipSkippable: + { size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr)); + srcPtr += skipSize; + dctx->tmpInTarget -= skipSize; + doAnotherStage = 0; + nextSrcSizeHint = dctx->tmpInTarget; + if (nextSrcSizeHint) break; /* still more to skip */ + /* frame fully skipped : prepare context for a new frame */ + LZ4F_resetDecompressionContext(dctx); + break; + } + } /* switch (dctx->dStage) */ + } /* while (doAnotherStage) */ + + /* preserve history within tmpOut whenever necessary */ + LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2); + if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */ + && (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */ + && (dctx->dict != NULL) /* dictionary exists */ + && (!decompressOptionsPtr->stableDst) /* cannot rely on dst data to remain there for next call */ + && ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */ + { + if (dctx->dStage == dstage_flushOut) { + size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer); + size_t copySize = 64 KB - dctx->tmpOutSize; + const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; + if (dctx->tmpOutSize > 64 KB) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + assert(dctx->tmpOutBuffer != NULL); + + memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dctx->tmpOutStart; + } else { + const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize; + size_t const newDictSize = MIN(dctx->dictSize, 64 KB); + + memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); + + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = newDictSize; + dctx->tmpOut = dctx->tmpOutBuffer + newDictSize; + } + } + + *srcSizePtr = (size_t)(srcPtr - srcStart); + *dstSizePtr = (size_t)(dstPtr - dstStart); + return nextSrcSizeHint; +} + +/*! LZ4F_decompress_usingDict() : + * Same as LZ4F_decompress(), using a predefined dictionary. + * Dictionary is used "in place", without any preprocessing. + * It must remain accessible throughout the entire frame decoding. + */ +size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr) +{ + if (dctx->dStage <= dstage_init) { + dctx->dict = (const BYTE*)dict; + dctx->dictSize = dictSize; + } + return LZ4F_decompress(dctx, dstBuffer, dstSizePtr, + srcBuffer, srcSizePtr, + decompressOptionsPtr); +} diff --git a/deps/lz4/lz4frame.h b/deps/lz4/lz4frame.h new file mode 100644 index 00000000000..b8ae3227670 --- /dev/null +++ b/deps/lz4/lz4frame.h @@ -0,0 +1,751 @@ +/* + LZ4F - LZ4-Frame library + Header File + Copyright (C) 2011-2020, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/* LZ4F is a stand-alone API able to create and decode LZ4 frames + * conformant with specification v1.6.1 in doc/lz4_Frame_format.md . + * Generated frames are compatible with `lz4` CLI. + * + * LZ4F also offers streaming capabilities. + * + * lz4.h is not required when using lz4frame.h, + * except to extract common constants such as LZ4_VERSION_NUMBER. + * */ + +#ifndef LZ4F_H_09782039843 +#define LZ4F_H_09782039843 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* --- Dependency --- */ +#include /* size_t */ + + +/** + * Introduction + * + * lz4frame.h implements LZ4 frame specification: see doc/lz4_Frame_format.md . + * LZ4 Frames are compatible with `lz4` CLI, + * and designed to be interoperable with any system. +**/ + +/*-*************************************************************** + * Compiler specifics + *****************************************************************/ +/* LZ4_DLL_EXPORT : + * Enable exporting of functions when building a Windows DLL + * LZ4FLIB_VISIBILITY : + * Control library symbols visibility. + */ +#ifndef LZ4FLIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4FLIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4FLIB_VISIBILITY +# endif +#endif +#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) +# define LZ4FLIB_API __declspec(dllexport) LZ4FLIB_VISIBILITY +#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) +# define LZ4FLIB_API __declspec(dllimport) LZ4FLIB_VISIBILITY +#else +# define LZ4FLIB_API LZ4FLIB_VISIBILITY +#endif + +#ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS +# define LZ4F_DEPRECATE(x) x +#else +# if defined(_MSC_VER) +# define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */ +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6)) +# define LZ4F_DEPRECATE(x) x __attribute__((deprecated)) +# else +# define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */ +# endif +#endif + + +/*-************************************ + * Error management + **************************************/ +typedef size_t LZ4F_errorCode_t; + +LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells when a function result is an error code */ +LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; for debugging */ + + +/*-************************************ + * Frame compression types + ************************************* */ +/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */ +#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS +# define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x +#else +# define LZ4F_OBSOLETE_ENUM(x) +#endif + +/* The larger the block size, the (slightly) better the compression ratio, + * though there are diminishing returns. + * Larger blocks also increase memory usage on both compression and decompression sides. + */ +typedef enum { + LZ4F_default=0, + LZ4F_max64KB=4, + LZ4F_max256KB=5, + LZ4F_max1MB=6, + LZ4F_max4MB=7 + LZ4F_OBSOLETE_ENUM(max64KB) + LZ4F_OBSOLETE_ENUM(max256KB) + LZ4F_OBSOLETE_ENUM(max1MB) + LZ4F_OBSOLETE_ENUM(max4MB) +} LZ4F_blockSizeID_t; + +/* Linked blocks sharply reduce inefficiencies when using small blocks, + * they compress better. + * However, some LZ4 decoders are only compatible with independent blocks */ +typedef enum { + LZ4F_blockLinked=0, + LZ4F_blockIndependent + LZ4F_OBSOLETE_ENUM(blockLinked) + LZ4F_OBSOLETE_ENUM(blockIndependent) +} LZ4F_blockMode_t; + +typedef enum { + LZ4F_noContentChecksum=0, + LZ4F_contentChecksumEnabled + LZ4F_OBSOLETE_ENUM(noContentChecksum) + LZ4F_OBSOLETE_ENUM(contentChecksumEnabled) +} LZ4F_contentChecksum_t; + +typedef enum { + LZ4F_noBlockChecksum=0, + LZ4F_blockChecksumEnabled +} LZ4F_blockChecksum_t; + +typedef enum { + LZ4F_frame=0, + LZ4F_skippableFrame + LZ4F_OBSOLETE_ENUM(skippableFrame) +} LZ4F_frameType_t; + +#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS +typedef LZ4F_blockSizeID_t blockSizeID_t; +typedef LZ4F_blockMode_t blockMode_t; +typedef LZ4F_frameType_t frameType_t; +typedef LZ4F_contentChecksum_t contentChecksum_t; +#endif + +/*! LZ4F_frameInfo_t : + * makes it possible to set or read frame parameters. + * Structure must be first init to 0, using memset() or LZ4F_INIT_FRAMEINFO, + * setting all parameters to default. + * It's then possible to update selectively some parameters */ +typedef struct { + LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB; 0 == default (LZ4F_max64KB) */ + LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default (LZ4F_blockLinked) */ + LZ4F_contentChecksum_t contentChecksumFlag; /* 1: add a 32-bit checksum of frame's decompressed data; 0 == default (disabled) */ + LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */ + unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */ + unsigned dictID; /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */ + LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0 == default (disabled) */ +} LZ4F_frameInfo_t; + +#define LZ4F_INIT_FRAMEINFO { LZ4F_max64KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0ULL, 0U, LZ4F_noBlockChecksum } /* v1.8.3+ */ + +/*! LZ4F_preferences_t : + * makes it possible to supply advanced compression instructions to streaming interface. + * Structure must be first init to 0, using memset() or LZ4F_INIT_PREFERENCES, + * setting all parameters to default. + * All reserved fields must be set to zero. */ +typedef struct { + LZ4F_frameInfo_t frameInfo; + int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */ + unsigned autoFlush; /* 1: always flush; reduces usage of internal buffers */ + unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4HC_CLEVEL_OPT_MIN) */ /* v1.8.2+ */ + unsigned reserved[3]; /* must be zero for forward compatibility */ +} LZ4F_preferences_t; + +#define LZ4F_INIT_PREFERENCES { LZ4F_INIT_FRAMEINFO, 0, 0u, 0u, { 0u, 0u, 0u } } /* v1.8.3+ */ + + +/*-********************************* +* Simple compression function +***********************************/ + +/*! LZ4F_compressFrame() : + * Compress srcBuffer content into an LZ4-compressed frame. + * It's a one shot operation, all input content is consumed, and all output is generated. + * + * Note : it's a stateless operation (no LZ4F_cctx state needed). + * In order to reduce load on the allocator, LZ4F_compressFrame(), by default, + * uses the stack to allocate space for the compression state and some table. + * If this usage of the stack is too much for your application, + * consider compiling `lz4frame.c` with compile-time macro LZ4F_HEAPMODE set to 1 instead. + * All state allocations will use the Heap. + * It also means each invocation of LZ4F_compressFrame() will trigger several internal alloc/free invocations. + * + * @dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * @preferencesPtr is optional : one can provide NULL, in which case all preferences are set to default. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) + */ +LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr); + +/*! LZ4F_compressFrameBound() : + * Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences. + * `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences. + * Note : this result is only usable with LZ4F_compressFrame(). + * It may also be relevant to LZ4F_compressUpdate() _only if_ no flush() operation is ever performed. + */ +LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); + + +/*! LZ4F_compressionLevel_max() : + * @return maximum allowed compression level (currently: 12) + */ +LZ4FLIB_API int LZ4F_compressionLevel_max(void); /* v1.8.0+ */ + + +/*-*********************************** +* Advanced compression functions +*************************************/ +typedef struct LZ4F_cctx_s LZ4F_cctx; /* incomplete type */ +typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with older APIs, prefer using LZ4F_cctx */ + +typedef struct { + unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */ + unsigned reserved[3]; +} LZ4F_compressOptions_t; + +/*--- Resource Management ---*/ + +#define LZ4F_VERSION 100 /* This number can be used to check for an incompatible API breaking change */ +LZ4FLIB_API unsigned LZ4F_getVersion(void); + +/*! LZ4F_createCompressionContext() : + * The first thing to do is to create a compressionContext object, + * which will keep track of operation state during streaming compression. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version, + * and a pointer to LZ4F_cctx*, to write the resulting pointer into. + * @version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. + * The function provides a pointer to a fully allocated LZ4F_cctx object. + * @cctxPtr MUST be != NULL. + * If @return != zero, context creation failed. + * A created compression context can be employed multiple times for consecutive streaming operations. + * Once all streaming compression jobs are completed, + * the state object can be released using LZ4F_freeCompressionContext(). + * Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored. + * Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing). +**/ +LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); +LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); + + +/*---- Compression ----*/ + +#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected parameters */ +#define LZ4F_HEADER_SIZE_MAX 19 + +/* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */ +#define LZ4F_BLOCK_HEADER_SIZE 4 + +/* Size in bytes of a block checksum footer in little-endian format. */ +#define LZ4F_BLOCK_CHECKSUM_SIZE 4 + +/* Size in bytes of the content checksum. */ +#define LZ4F_CONTENT_CHECKSUM_SIZE 4 + +/*! LZ4F_compressBegin() : + * will write the frame header into dstBuffer. + * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * `prefsPtr` is optional : NULL can be provided to set all preferences to default. + * @return : number of bytes written into dstBuffer for the header + * or an error code (which can be tested using LZ4F_isError()) + */ +LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* prefsPtr); + +/*! LZ4F_compressBound() : + * Provides minimum dstCapacity required to guarantee success of + * LZ4F_compressUpdate(), given a srcSize and preferences, for a worst case scenario. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() instead. + * Note that the result is only valid for a single invocation of LZ4F_compressUpdate(). + * When invoking LZ4F_compressUpdate() multiple times, + * if the output buffer is gradually filled up instead of emptied and re-used from its start, + * one must check if there is enough remaining capacity before each invocation, using LZ4F_compressBound(). + * @return is always the same for a srcSize and prefsPtr. + * prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario. + * tech details : + * @return if automatic flushing is not enabled, includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. + * It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd(). + * @return doesn't include frame header, as it was already generated by LZ4F_compressBegin(). + */ +LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); + +/*! LZ4F_compressUpdate() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations. + * This value is provided by LZ4F_compressBound(). + * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). + * After an error, the state is left in a UB state, and must be re-initialized or freed. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. + * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). + * or an error code if it fails (which can be tested using LZ4F_isError()) + */ +LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* cOptPtr); + +/*! LZ4F_flush() : + * When data must be generated and sent immediately, without waiting for a block to be completely filled, + * it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. + * `dstCapacity` must be large enough to ensure the operation will be successful. + * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx) + * or an error code if it fails (which can be tested using LZ4F_isError()) + * Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). + */ +LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* cOptPtr); + +/*! LZ4F_compressEnd() : + * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). + * It will flush whatever data remained within `cctx` (like LZ4_flush()) + * and properly finalize the frame, with an endMark and a checksum. + * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. + * @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark), + * or an error code if it fails (which can be tested using LZ4F_isError()) + * Note : LZ4F_compressEnd() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). + * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. + */ +LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* cOptPtr); + + +/*-********************************* +* Decompression functions +***********************************/ +typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */ +typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */ + +typedef struct { + unsigned stableDst; /* pledges that last 64KB decompressed data is present right before @dstBuffer pointer. + * This optimization skips internal storage operations. + * Once set, this pledge must remain valid up to the end of current frame. */ + unsigned skipChecksums; /* disable checksum calculation and verification, even when one is present in frame, to save CPU time. + * Setting this option to 1 once disables all checksums for the rest of the frame. */ + unsigned reserved1; /* must be set to zero for forward compatibility */ + unsigned reserved0; /* idem */ +} LZ4F_decompressOptions_t; + + +/* Resource management */ + +/*! LZ4F_createDecompressionContext() : + * Create an LZ4F_dctx object, to track all decompression operations. + * @version provided MUST be LZ4F_VERSION. + * @dctxPtr MUST be valid. + * The function fills @dctxPtr with the value of a pointer to an allocated and initialized LZ4F_dctx object. + * The @return is an errorCode, which can be tested using LZ4F_isError(). + * dctx memory can be released using LZ4F_freeDecompressionContext(); + * Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released. + * That is, it should be == 0 if decompression has been completed fully and correctly. + */ +LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version); +LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); + + +/*-*********************************** +* Streaming decompression functions +*************************************/ + +#define LZ4F_MAGICNUMBER 0x184D2204U +#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U +#define LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH 5 + +/*! LZ4F_headerSize() : v1.9.0+ + * Provide the header size of a frame starting at `src`. + * `srcSize` must be >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH, + * which is enough to decode the header length. + * @return : size of frame header + * or an error code, which can be tested using LZ4F_isError() + * note : Frame header size is variable, but is guaranteed to be + * >= LZ4F_HEADER_SIZE_MIN bytes, and <= LZ4F_HEADER_SIZE_MAX bytes. + */ +LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize); + +/*! LZ4F_getFrameInfo() : + * This function extracts frame parameters (max blockSize, dictID, etc.). + * Its usage is optional: user can also invoke LZ4F_decompress() directly. + * + * Extracted information will fill an existing LZ4F_frameInfo_t structure. + * This can be useful for allocation and dictionary identification purposes. + * + * LZ4F_getFrameInfo() can work in the following situations : + * + * 1) At the beginning of a new frame, before any invocation of LZ4F_decompress(). + * It will decode header from `srcBuffer`, + * consuming the header and starting the decoding process. + * + * Input size must be large enough to contain the full frame header. + * Frame header size can be known beforehand by LZ4F_headerSize(). + * Frame header size is variable, but is guaranteed to be >= LZ4F_HEADER_SIZE_MIN bytes, + * and not more than <= LZ4F_HEADER_SIZE_MAX bytes. + * Hence, blindly providing LZ4F_HEADER_SIZE_MAX bytes or more will always work. + * It's allowed to provide more input data than the header size, + * LZ4F_getFrameInfo() will only consume the header. + * + * If input size is not large enough, + * aka if it's smaller than header size, + * function will fail and return an error code. + * + * 2) After decoding has been started, + * it's possible to invoke LZ4F_getFrameInfo() anytime + * to extract already decoded frame parameters stored within dctx. + * + * Note that, if decoding has barely started, + * and not yet read enough information to decode the header, + * LZ4F_getFrameInfo() will fail. + * + * The number of bytes consumed from srcBuffer will be updated in *srcSizePtr (necessarily <= original value). + * LZ4F_getFrameInfo() only consumes bytes when decoding has not yet started, + * and when decoding the header has been successful. + * Decompression must then resume from (srcBuffer + *srcSizePtr). + * + * @return : a hint about how many srcSize bytes LZ4F_decompress() expects for next call, + * or an error code which can be tested using LZ4F_isError(). + * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. + * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. + */ +LZ4FLIB_API size_t +LZ4F_getFrameInfo(LZ4F_dctx* dctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr); + +/*! LZ4F_decompress() : + * Call this function repetitively to regenerate data compressed in `srcBuffer`. + * + * The function requires a valid dctx state. + * It will read up to *srcSizePtr bytes from srcBuffer, + * and decompress data into dstBuffer, of capacity *dstSizePtr. + * + * The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value). + * The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value). + * + * The function does not necessarily read all input bytes, so always check value in *srcSizePtr. + * Unconsumed source data must be presented again in subsequent invocations. + * + * `dstBuffer` can freely change between each consecutive function invocation. + * `dstBuffer` content will be overwritten. + * + * Note: if `LZ4F_getFrameInfo()` is called before `LZ4F_decompress()`, srcBuffer must be updated to reflect + * the number of bytes consumed after reading the frame header. Failure to update srcBuffer before calling + * `LZ4F_decompress()` will cause decompression failure or, even worse, successful but incorrect decompression. + * See the `LZ4F_getFrameInfo()` docs for details. + * + * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides some small speed benefit, because it skips intermediate buffers. + * This is just a hint though, it's always possible to provide any srcSize. + * + * When a frame is fully decoded, @return will be 0 (no more data expected). + * When provided with more bytes than necessary to decode a frame, + * LZ4F_decompress() will stop reading exactly at end of current frame, and @return 0. + * + * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). + * After a decompression error, the `dctx` context is not resumable. + * Use LZ4F_resetDecompressionContext() to return to clean state. + * + * After a frame is fully decoded, dctx can be used again to decompress another frame. + */ +LZ4FLIB_API size_t +LZ4F_decompress(LZ4F_dctx* dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* dOptPtr); + + +/*! LZ4F_resetDecompressionContext() : added in v1.8.0 + * In case of an error, the context is left in "undefined" state. + * In which case, it's necessary to reset it, before re-using it. + * This method can also be used to abruptly stop any unfinished decompression, + * and start a new one using same context resources. */ +LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */ + + +/********************************** + * Dictionary compression API + *********************************/ + +/* A Dictionary is useful for the compression of small messages (KB range). + * It dramatically improves compression efficiency. + * + * LZ4 can ingest any input as dictionary, though only the last 64 KB are useful. + * Better results are generally achieved by using Zstandard's Dictionary Builder + * to generate a high-quality dictionary from a set of samples. + * + * The same dictionary will have to be used on the decompression side + * for decoding to be successful. + * To help identify the correct dictionary at decoding stage, + * the frame header allows optional embedding of a dictID field. + */ + +/*! LZ4F_compressBegin_usingDict() : stable since v1.10 + * Inits dictionary compression streaming, and writes the frame header into dstBuffer. + * @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * @prefsPtr is optional : one may provide NULL as argument, + * however, it's the only way to provide dictID in the frame header. + * @dictBuffer must outlive the compression session. + * @return : number of bytes written into dstBuffer for the header, + * or an error code (which can be tested using LZ4F_isError()) + * NOTE: The LZ4Frame spec allows each independent block to be compressed with the dictionary, + * but this entry supports a more limited scenario, where only the first block uses the dictionary. + * This is still useful for small data, which only need one block anyway. + * For larger inputs, one may be more interested in LZ4F_compressFrame_usingCDict() below. + */ +LZ4FLIB_API size_t +LZ4F_compressBegin_usingDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* dictBuffer, size_t dictSize, + const LZ4F_preferences_t* prefsPtr); + +/*! LZ4F_decompress_usingDict() : stable since v1.10 + * Same as LZ4F_decompress(), using a predefined dictionary. + * Dictionary is used "in place", without any preprocessing. +** It must remain accessible throughout the entire frame decoding. */ +LZ4FLIB_API size_t +LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr); + +/***************************************** + * Bulk processing dictionary compression + *****************************************/ + +/* Loading a dictionary has a cost, since it involves construction of tables. + * The Bulk processing dictionary API makes it possible to share this cost + * over an arbitrary number of compression jobs, even concurrently, + * markedly improving compression latency for these cases. + * + * Note that there is no corresponding bulk API for the decompression side, + * because dictionary does not carry any initialization cost for decompression. + * Use the regular LZ4F_decompress_usingDict() there. + */ +typedef struct LZ4F_CDict_s LZ4F_CDict; + +/*! LZ4_createCDict() : stable since v1.10 + * When compressing multiple messages / blocks using the same dictionary, it's recommended to initialize it just once. + * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after LZ4_CDict creation, since its content is copied within CDict. */ +LZ4FLIB_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); +LZ4FLIB_API void LZ4F_freeCDict(LZ4F_CDict* CDict); + +/*! LZ4_compressFrame_usingCDict() : stable since v1.10 + * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary. + * @cctx must point to a context created by LZ4F_createCompressionContext(). + * If @cdict==NULL, compress without a dictionary. + * @dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * If this condition is not respected, function will fail (@return an errorCode). + * The LZ4F_preferences_t structure is optional : one may provide NULL as argument, + * but it's not recommended, as it's the only way to provide @dictID in the frame header. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) + * Note: for larger inputs generating multiple independent blocks, + * this entry point uses the dictionary for each block. */ +LZ4FLIB_API size_t +LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr); + +/*! LZ4F_compressBegin_usingCDict() : stable since v1.10 + * Inits streaming dictionary compression, and writes the frame header into dstBuffer. + * @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * @prefsPtr is optional : one may provide NULL as argument, + * note however that it's the only way to insert a @dictID in the frame header. + * @cdict must outlive the compression session. + * @return : number of bytes written into dstBuffer for the header, + * or an error code, which can be tested using LZ4F_isError(). */ +LZ4FLIB_API size_t +LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* prefsPtr); + + +#if defined (__cplusplus) +} +#endif + +#endif /* LZ4F_H_09782039843 */ + +#if defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) +#define LZ4F_H_STATIC_09782039843 + +/* Note : + * The below declarations are not stable and may change in the future. + * They are therefore only safe to depend on + * when the caller is statically linked against the library. + * To access their declarations, define LZ4F_STATIC_LINKING_ONLY. + * + * By default, these symbols aren't published into shared/dynamic libraries. + * You can override this behavior and force them to be published + * by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. + * Use at your own risk. + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS +# define LZ4FLIB_STATIC_API LZ4FLIB_API +#else +# define LZ4FLIB_STATIC_API +#endif + + +/* --- Error List --- */ +#define LZ4F_LIST_ERRORS(ITEM) \ + ITEM(OK_NoError) \ + ITEM(ERROR_GENERIC) \ + ITEM(ERROR_maxBlockSize_invalid) \ + ITEM(ERROR_blockMode_invalid) \ + ITEM(ERROR_parameter_invalid) \ + ITEM(ERROR_compressionLevel_invalid) \ + ITEM(ERROR_headerVersion_wrong) \ + ITEM(ERROR_blockChecksum_invalid) \ + ITEM(ERROR_reservedFlag_set) \ + ITEM(ERROR_allocation_failed) \ + ITEM(ERROR_srcSize_tooLarge) \ + ITEM(ERROR_dstMaxSize_tooSmall) \ + ITEM(ERROR_frameHeader_incomplete) \ + ITEM(ERROR_frameType_unknown) \ + ITEM(ERROR_frameSize_wrong) \ + ITEM(ERROR_srcPtr_wrong) \ + ITEM(ERROR_decompressionFailed) \ + ITEM(ERROR_headerChecksum_invalid) \ + ITEM(ERROR_contentChecksum_invalid) \ + ITEM(ERROR_frameDecoding_alreadyStarted) \ + ITEM(ERROR_compressionState_uninitialized) \ + ITEM(ERROR_parameter_null) \ + ITEM(ERROR_io_write) \ + ITEM(ERROR_io_read) \ + ITEM(ERROR_maxCode) + +#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, + +/* enum list is exposed, to handle specific errors */ +typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) + _LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes; + +LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); + +/********************************** + * Advanced compression operations + *********************************/ + +/*! LZ4F_getBlockSize() : + * @return, in scalar format (size_t), + * the maximum block size associated with @blockSizeID, + * or an error code (can be tested using LZ4F_isError()) if @blockSizeID is invalid. +**/ +LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID); + +/*! LZ4F_uncompressedUpdate() : + * LZ4F_uncompressedUpdate() can be called repetitively to add data stored as uncompressed blocks. + * Important rule: dstCapacity MUST be large enough to store the entire source buffer as + * no compression is done for this operation + * If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode). + * After an error, the state is left in a UB state, and must be re-initialized or freed. + * If previously a compressed block was written, buffered data is flushed first, + * before appending uncompressed data is continued. + * This operation is only supported when LZ4F_blockIndependent is used. + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. + * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). + * or an error code if it fails (which can be tested using LZ4F_isError()) + */ +LZ4FLIB_STATIC_API size_t +LZ4F_uncompressedUpdate(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* cOptPtr); + +/********************************** + * Custom memory allocation + *********************************/ + +/*! Custom memory allocation : v1.9.4+ + * These prototypes make it possible to pass custom allocation/free functions. + * LZ4F_customMem is provided at state creation time, using LZ4F_create*_advanced() listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*LZ4F_AllocFunction) (void* opaqueState, size_t size); +typedef void* (*LZ4F_CallocFunction) (void* opaqueState, size_t size); +typedef void (*LZ4F_FreeFunction) (void* opaqueState, void* address); +typedef struct { + LZ4F_AllocFunction customAlloc; + LZ4F_CallocFunction customCalloc; /* optional; when not defined, uses customAlloc + memset */ + LZ4F_FreeFunction customFree; + void* opaqueState; +} LZ4F_CustomMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); +LZ4FLIB_STATIC_API LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); +LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict_advanced(LZ4F_CustomMem customMem, const void* dictBuffer, size_t dictSize); + + +#if defined (__cplusplus) +} +#endif + +#endif /* defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) */ diff --git a/deps/lz4/lz4hc.c b/deps/lz4/lz4hc.c new file mode 100644 index 00000000000..4d8c36a6978 --- /dev/null +++ b/deps/lz4/lz4hc.c @@ -0,0 +1,2192 @@ +/* + LZ4 HC - High Compression Mode of LZ4 + Copyright (C) 2011-2020, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ + + +/* ************************************* +* Tuning Parameter +***************************************/ + +/*! HEAPMODE : + * Select how stateless HC compression functions like `LZ4_compress_HC()` + * allocate memory for their workspace: + * in stack (0:fastest), or in heap (1:default, requires malloc()). + * Since workspace is rather large, heap mode is recommended. +**/ +#ifndef LZ4HC_HEAPMODE +# define LZ4HC_HEAPMODE 1 +#endif + + +/*=== Dependency ===*/ +#define LZ4_HC_STATIC_LINKING_ONLY +#include "lz4hc.h" +#include + + +/*=== Shared lz4.c code ===*/ +#ifndef LZ4_SRC_INCLUDED +# if defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-function" +# endif +# if defined (__clang__) +# pragma clang diagnostic ignored "-Wunused-function" +# endif +# define LZ4_COMMONDEFS_ONLY +# include "lz4.c" /* LZ4_count, constants, mem */ +#endif + + +/*=== Enums ===*/ +typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; + + +/*=== Constants ===*/ +#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) +#define LZ4_OPT_NUM (1<<12) + + +/*=== Macros ===*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) + + +/*=== Levels definition ===*/ +typedef enum { lz4mid, lz4hc, lz4opt } lz4hc_strat_e; +typedef struct { + lz4hc_strat_e strat; + int nbSearches; + U32 targetLength; +} cParams_t; +static const cParams_t k_clTable[LZ4HC_CLEVEL_MAX+1] = { + { lz4mid, 2, 16 }, /* 0, unused */ + { lz4mid, 2, 16 }, /* 1, unused */ + { lz4mid, 2, 16 }, /* 2 */ + { lz4hc, 4, 16 }, /* 3 */ + { lz4hc, 8, 16 }, /* 4 */ + { lz4hc, 16, 16 }, /* 5 */ + { lz4hc, 32, 16 }, /* 6 */ + { lz4hc, 64, 16 }, /* 7 */ + { lz4hc, 128, 16 }, /* 8 */ + { lz4hc, 256, 16 }, /* 9 */ + { lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/ + { lz4opt, 512,128 }, /*11 */ + { lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ +}; + +static cParams_t LZ4HC_getCLevelParams(int cLevel) +{ + /* note : clevel convention is a bit different from lz4frame, + * possibly something worth revisiting for consistency */ + if (cLevel < 1) + cLevel = LZ4HC_CLEVEL_DEFAULT; + cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel); + return k_clTable[cLevel]; +} + + +/*=== Hashing ===*/ +#define LZ4HC_HASHSIZE 4 +#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) +static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } + +#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) +/* lie to the compiler about data alignment; use with caution */ +static U64 LZ4_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) +/* __pack instructions are safer, but compiler specific */ +LZ4_PACK(typedef struct { U64 u64; }) LZ4_unalign64; +static U64 LZ4_read64(const void* ptr) { return ((const LZ4_unalign64*)ptr)->u64; } + +#else /* safe and portable access using memcpy() */ +static U64 LZ4_read64(const void* memPtr) +{ + U64 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +#endif /* LZ4_FORCE_MEMORY_ACCESS */ + +#define LZ4MID_HASHSIZE 8 +#define LZ4MID_HASHLOG (LZ4HC_HASH_LOG-1) +#define LZ4MID_HASHTABLESIZE (1 << LZ4MID_HASHLOG) + +static U32 LZ4MID_hash4(U32 v) { return (v * 2654435761U) >> (32-LZ4MID_HASHLOG); } +static U32 LZ4MID_hash4Ptr(const void* ptr) { return LZ4MID_hash4(LZ4_read32(ptr)); } +/* note: hash7 hashes the lower 56-bits. + * It presumes input was read using little endian.*/ +static U32 LZ4MID_hash7(U64 v) { return (U32)(((v << (64-56)) * 58295818150454627ULL) >> (64-LZ4MID_HASHLOG)) ; } +static U64 LZ4_readLE64(const void* memPtr); +static U32 LZ4MID_hash8Ptr(const void* ptr) { return LZ4MID_hash7(LZ4_readLE64(ptr)); } + +static U64 LZ4_readLE64(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read64(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + /* note: relies on the compiler to simplify this expression */ + return (U64)p[0] | ((U64)p[1]<<8) | ((U64)p[2]<<16) | ((U64)p[3]<<24) + | ((U64)p[4]<<32) | ((U64)p[5]<<40) | ((U64)p[6]<<48) | ((U64)p[7]<<56); + } +} + + +/*=== Count match length ===*/ +LZ4_FORCE_INLINE +unsigned LZ4HC_NbCommonBytes32(U32 val) +{ + assert(val != 0); + if (LZ4_isLittleEndian()) { +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)((31 - r) >> 3); +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clz(val) >> 3; +# else + val >>= 8; + val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | + (val + 0x00FF0000)) >> 24; + return (unsigned)val ^ 3; +# endif + } else { +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)(r >> 3); +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctz(val) >> 3; +# else + const U32 m = 0x01010101; + return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24; +# endif + } +} + +/** LZ4HC_countBack() : + * @return : negative value, nb of common bytes before ip/match */ +LZ4_FORCE_INLINE +int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, + const BYTE* const iMin, const BYTE* const mMin) +{ + int back = 0; + int const min = (int)MAX(iMin - ip, mMin - match); + assert(min <= 0); + assert(ip >= iMin); assert((size_t)(ip-iMin) < (1U<<31)); + assert(match >= mMin); assert((size_t)(match - mMin) < (1U<<31)); + + while ((back - min) > 3) { + U32 const v = LZ4_read32(ip + back - 4) ^ LZ4_read32(match + back - 4); + if (v) { + return (back - (int)LZ4HC_NbCommonBytes32(v)); + } else back -= 4; /* 4-byte step */ + } + /* check remainder if any */ + while ( (back > min) + && (ip[back-1] == match[back-1]) ) + back--; + return back; +} + +/*=== Chain table updates ===*/ +#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ +/* Make fields passed to, and updated by LZ4HC_encodeSequence explicit */ +#define UPDATABLE(ip, op, anchor) &ip, &op, &anchor + + +/************************************** +* Init +**************************************/ +static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) +{ + MEM_INIT(hc4->hashTable, 0, sizeof(hc4->hashTable)); + MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); +} + +static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start) +{ + size_t const bufferSize = (size_t)(hc4->end - hc4->prefixStart); + size_t newStartingOffset = bufferSize + hc4->dictLimit; + DEBUGLOG(5, "LZ4HC_init_internal"); + assert(newStartingOffset >= bufferSize); /* check overflow */ + if (newStartingOffset > 1 GB) { + LZ4HC_clearTables(hc4); + newStartingOffset = 0; + } + newStartingOffset += 64 KB; + hc4->nextToUpdate = (U32)newStartingOffset; + hc4->prefixStart = start; + hc4->end = start; + hc4->dictStart = start; + hc4->dictLimit = (U32)newStartingOffset; + hc4->lowLimit = (U32)newStartingOffset; +} + + +/************************************** +* Encode +**************************************/ +/* LZ4HC_encodeSequence() : + * @return : 0 if ok, + * 1 if buffer issue detected */ +LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( + const BYTE** _ip, + BYTE** _op, + const BYTE** _anchor, + int matchLength, + int offset, + limitedOutput_directive limit, + BYTE* oend) +{ +#define ip (*_ip) +#define op (*_op) +#define anchor (*_anchor) + + size_t length; + BYTE* const token = op++; + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6) + static const BYTE* start = NULL; + static U32 totalCost = 0; + U32 const pos = (start==NULL) ? 0 : (U32)(anchor - start); + U32 const ll = (U32)(ip - anchor); + U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0; + U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0; + U32 const cost = 1 + llAdd + ll + 2 + mlAdd; + if (start==NULL) start = anchor; /* only works for single segment */ + /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */ + DEBUGLOG(6, "pos:%7u -- literals:%4u, match:%4i, offset:%5i, cost:%4u + %5u", + pos, + (U32)(ip - anchor), matchLength, offset, + cost, totalCost); + totalCost += cost; +#endif + + /* Encode Literal length */ + length = (size_t)(ip - anchor); + LZ4_STATIC_ASSERT(notLimited == 0); + /* Check output limit */ + if (limit && ((op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) { + DEBUGLOG(6, "Not enough room to write %i literals (%i bytes remaining)", + (int)length, (int)(oend - op)); + return 1; + } + if (length >= RUN_MASK) { + size_t len = length - RUN_MASK; + *token = (RUN_MASK << ML_BITS); + for(; len >= 255 ; len -= 255) *op++ = 255; + *op++ = (BYTE)len; + } else { + *token = (BYTE)(length << ML_BITS); + } + + /* Copy Literals */ + LZ4_wildCopy8(op, anchor, op + length); + op += length; + + /* Encode Offset */ + assert(offset <= LZ4_DISTANCE_MAX ); + assert(offset > 0); + LZ4_writeLE16(op, (U16)(offset)); op += 2; + + /* Encode MatchLength */ + assert(matchLength >= MINMATCH); + length = (size_t)matchLength - MINMATCH; + if (limit && (op + (length / 255) + (1 + LASTLITERALS) > oend)) { + DEBUGLOG(6, "Not enough room to write match length"); + return 1; /* Check output limit */ + } + if (length >= ML_MASK) { + *token += ML_MASK; + length -= ML_MASK; + for(; length >= 510 ; length -= 510) { *op++ = 255; *op++ = 255; } + if (length >= 255) { length -= 255; *op++ = 255; } + *op++ = (BYTE)length; + } else { + *token += (BYTE)(length); + } + + /* Prepare next loop */ + ip += matchLength; + anchor = ip; + + return 0; + +#undef ip +#undef op +#undef anchor +} + + +typedef struct { + int off; + int len; + int back; /* negative value */ +} LZ4HC_match_t; + +LZ4HC_match_t LZ4HC_searchExtDict(const BYTE* ip, U32 ipIndex, + const BYTE* const iLowLimit, const BYTE* const iHighLimit, + const LZ4HC_CCtx_internal* dictCtx, U32 gDictEndIndex, + int currentBestML, int nbAttempts) +{ + size_t const lDictEndIndex = (size_t)(dictCtx->end - dictCtx->prefixStart) + dictCtx->dictLimit; + U32 lDictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; + U32 matchIndex = lDictMatchIndex + gDictEndIndex - (U32)lDictEndIndex; + int offset = 0, sBack = 0; + assert(lDictEndIndex <= 1 GB); + if (lDictMatchIndex>0) + DEBUGLOG(7, "lDictEndIndex = %zu, lDictMatchIndex = %u", lDictEndIndex, lDictMatchIndex); + while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) { + const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + lDictMatchIndex; + + if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { + int mlt; + int back = 0; + const BYTE* vLimit = ip + (lDictEndIndex - lDictMatchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + back = (ip > iLowLimit) ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->prefixStart) : 0; + mlt -= back; + if (mlt > currentBestML) { + currentBestML = mlt; + offset = (int)(ipIndex - matchIndex); + sBack = back; + DEBUGLOG(7, "found match of length %i within extDictCtx", currentBestML); + } } + + { U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, lDictMatchIndex); + lDictMatchIndex -= nextOffset; + matchIndex -= nextOffset; + } } + + { LZ4HC_match_t md; + md.len = currentBestML; + md.off = offset; + md.back = sBack; + return md; + } +} + +typedef LZ4HC_match_t (*LZ4MID_searchIntoDict_f)(const BYTE* ip, U32 ipIndex, + const BYTE* const iHighLimit, + const LZ4HC_CCtx_internal* dictCtx, U32 gDictEndIndex); + +static LZ4HC_match_t LZ4MID_searchHCDict(const BYTE* ip, U32 ipIndex, + const BYTE* const iHighLimit, + const LZ4HC_CCtx_internal* dictCtx, U32 gDictEndIndex) +{ + return LZ4HC_searchExtDict(ip,ipIndex, + ip, iHighLimit, + dictCtx, gDictEndIndex, + MINMATCH-1, 2); +} + +static LZ4HC_match_t LZ4MID_searchExtDict(const BYTE* ip, U32 ipIndex, + const BYTE* const iHighLimit, + const LZ4HC_CCtx_internal* dictCtx, U32 gDictEndIndex) +{ + size_t const lDictEndIndex = (size_t)(dictCtx->end - dictCtx->prefixStart) + dictCtx->dictLimit; + const U32* const hash4Table = dictCtx->hashTable; + const U32* const hash8Table = hash4Table + LZ4MID_HASHTABLESIZE; + DEBUGLOG(7, "LZ4MID_searchExtDict (ipIdx=%u)", ipIndex); + + /* search long match first */ + { U32 l8DictMatchIndex = hash8Table[LZ4MID_hash8Ptr(ip)]; + U32 m8Index = l8DictMatchIndex + gDictEndIndex - (U32)lDictEndIndex; + assert(lDictEndIndex <= 1 GB); + if (ipIndex - m8Index <= LZ4_DISTANCE_MAX) { + const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + l8DictMatchIndex; + const size_t safeLen = MIN(lDictEndIndex - l8DictMatchIndex, (size_t)(iHighLimit - ip)); + int mlt = (int)LZ4_count(ip, matchPtr, ip + safeLen); + if (mlt >= MINMATCH) { + LZ4HC_match_t md; + DEBUGLOG(7, "Found long ExtDict match of len=%u", mlt); + md.len = mlt; + md.off = (int)(ipIndex - m8Index); + md.back = 0; + return md; + } + } + } + + /* search for short match second */ + { U32 l4DictMatchIndex = hash4Table[LZ4MID_hash4Ptr(ip)]; + U32 m4Index = l4DictMatchIndex + gDictEndIndex - (U32)lDictEndIndex; + if (ipIndex - m4Index <= LZ4_DISTANCE_MAX) { + const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + l4DictMatchIndex; + const size_t safeLen = MIN(lDictEndIndex - l4DictMatchIndex, (size_t)(iHighLimit - ip)); + int mlt = (int)LZ4_count(ip, matchPtr, ip + safeLen); + if (mlt >= MINMATCH) { + LZ4HC_match_t md; + DEBUGLOG(7, "Found short ExtDict match of len=%u", mlt); + md.len = mlt; + md.off = (int)(ipIndex - m4Index); + md.back = 0; + return md; + } + } + } + + /* nothing found */ + { LZ4HC_match_t const md = {0, 0, 0 }; + return md; + } +} + +/************************************** +* Mid Compression (level 2) +**************************************/ + +LZ4_FORCE_INLINE void +LZ4MID_addPosition(U32* hTable, U32 hValue, U32 index) +{ + hTable[hValue] = index; +} + +#define ADDPOS8(_p, _idx) LZ4MID_addPosition(hash8Table, LZ4MID_hash8Ptr(_p), _idx) +#define ADDPOS4(_p, _idx) LZ4MID_addPosition(hash4Table, LZ4MID_hash4Ptr(_p), _idx) + +/* Fill hash tables with references into dictionary. + * The resulting table is only exploitable by LZ4MID (level 2) */ +static void +LZ4MID_fillHTable (LZ4HC_CCtx_internal* cctx, const void* dict, size_t size) +{ + U32* const hash4Table = cctx->hashTable; + U32* const hash8Table = hash4Table + LZ4MID_HASHTABLESIZE; + const BYTE* const prefixPtr = (const BYTE*)dict; + U32 const prefixIdx = cctx->dictLimit; + U32 const target = prefixIdx + (U32)size - LZ4MID_HASHSIZE; + U32 idx = cctx->nextToUpdate; + assert(dict == cctx->prefixStart); + DEBUGLOG(4, "LZ4MID_fillHTable (size:%zu)", size); + if (size <= LZ4MID_HASHSIZE) + return; + + for (; idx < target; idx += 3) { + ADDPOS4(prefixPtr+idx-prefixIdx, idx); + ADDPOS8(prefixPtr+idx+1-prefixIdx, idx+1); + } + + idx = (size > 32 KB + LZ4MID_HASHSIZE) ? target - 32 KB : cctx->nextToUpdate; + for (; idx < target; idx += 1) { + ADDPOS8(prefixPtr+idx-prefixIdx, idx); + } + + cctx->nextToUpdate = target; +} + +static LZ4MID_searchIntoDict_f select_searchDict_function(const LZ4HC_CCtx_internal* dictCtx) +{ + if (dictCtx == NULL) return NULL; + if (LZ4HC_getCLevelParams(dictCtx->compressionLevel).strat == lz4mid) + return LZ4MID_searchExtDict; + return LZ4MID_searchHCDict; +} + +static int LZ4MID_compress ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* srcSizePtr, + int const maxOutputSize, + const limitedOutput_directive limit, + const dictCtx_directive dict + ) +{ + U32* const hash4Table = ctx->hashTable; + U32* const hash8Table = hash4Table + LZ4MID_HASHTABLESIZE; + const BYTE* ip = (const BYTE*)src; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + const BYTE* const ilimit = (iend - LZ4MID_HASHSIZE); + BYTE* op = (BYTE*)dst; + BYTE* oend = op + maxOutputSize; + + const BYTE* const prefixPtr = ctx->prefixStart; + const U32 prefixIdx = ctx->dictLimit; + const U32 ilimitIdx = (U32)(ilimit - prefixPtr) + prefixIdx; + const BYTE* const dictStart = ctx->dictStart; + const U32 dictIdx = ctx->lowLimit; + const U32 gDictEndIndex = ctx->lowLimit; + const LZ4MID_searchIntoDict_f searchIntoDict = (dict == usingDictCtxHc) ? select_searchDict_function(ctx->dictCtx) : NULL; + unsigned matchLength; + unsigned matchDistance; + + /* input sanitization */ + DEBUGLOG(5, "LZ4MID_compress (%i bytes)", *srcSizePtr); + if (dict == usingDictCtxHc) DEBUGLOG(5, "usingDictCtxHc"); + assert(*srcSizePtr >= 0); + if (*srcSizePtr) assert(src != NULL); + if (maxOutputSize) assert(dst != NULL); + if (*srcSizePtr < 0) return 0; /* invalid */ + if (maxOutputSize < 0) return 0; /* invalid */ + if (*srcSizePtr > LZ4_MAX_INPUT_SIZE) { + /* forbidden: no input is allowed to be that large */ + return 0; + } + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (*srcSizePtr < LZ4_minLength) + goto _lz4mid_last_literals; /* Input too small, no compression (all literals) */ + + /* main loop */ + while (ip <= mflimit) { + const U32 ipIndex = (U32)(ip - prefixPtr) + prefixIdx; + /* search long match */ + { U32 const h8 = LZ4MID_hash8Ptr(ip); + U32 const pos8 = hash8Table[h8]; + assert(h8 < LZ4MID_HASHTABLESIZE); + assert(pos8 < ipIndex); + LZ4MID_addPosition(hash8Table, h8, ipIndex); + if (ipIndex - pos8 <= LZ4_DISTANCE_MAX) { + /* match candidate found */ + if (pos8 >= prefixIdx) { + const BYTE* const matchPtr = prefixPtr + pos8 - prefixIdx; + assert(matchPtr < ip); + matchLength = LZ4_count(ip, matchPtr, matchlimit); + if (matchLength >= MINMATCH) { + DEBUGLOG(7, "found long match at pos %u (len=%u)", pos8, matchLength); + matchDistance = ipIndex - pos8; + goto _lz4mid_encode_sequence; + } + } else { + if (pos8 >= dictIdx) { + /* extDict match candidate */ + const BYTE* const matchPtr = dictStart + (pos8 - dictIdx); + const size_t safeLen = MIN(prefixIdx - pos8, (size_t)(matchlimit - ip)); + matchLength = LZ4_count(ip, matchPtr, ip + safeLen); + if (matchLength >= MINMATCH) { + DEBUGLOG(7, "found long match at ExtDict pos %u (len=%u)", pos8, matchLength); + matchDistance = ipIndex - pos8; + goto _lz4mid_encode_sequence; + } + } + } + } } + /* search short match */ + { U32 const h4 = LZ4MID_hash4Ptr(ip); + U32 const pos4 = hash4Table[h4]; + assert(h4 < LZ4MID_HASHTABLESIZE); + assert(pos4 < ipIndex); + LZ4MID_addPosition(hash4Table, h4, ipIndex); + if (ipIndex - pos4 <= LZ4_DISTANCE_MAX) { + /* match candidate found */ + if (pos4 >= prefixIdx) { + /* only search within prefix */ + const BYTE* const matchPtr = prefixPtr + (pos4 - prefixIdx); + assert(matchPtr < ip); + assert(matchPtr >= prefixPtr); + matchLength = LZ4_count(ip, matchPtr, matchlimit); + if (matchLength >= MINMATCH) { + /* short match found, let's just check ip+1 for longer */ + U32 const h8 = LZ4MID_hash8Ptr(ip+1); + U32 const pos8 = hash8Table[h8]; + U32 const m2Distance = ipIndex + 1 - pos8; + matchDistance = ipIndex - pos4; + if ( m2Distance <= LZ4_DISTANCE_MAX + && pos8 >= prefixIdx /* only search within prefix */ + && likely(ip < mflimit) + ) { + const BYTE* const m2Ptr = prefixPtr + (pos8 - prefixIdx); + unsigned ml2 = LZ4_count(ip+1, m2Ptr, matchlimit); + if (ml2 > matchLength) { + LZ4MID_addPosition(hash8Table, h8, ipIndex+1); + ip++; + matchLength = ml2; + matchDistance = m2Distance; + } } + goto _lz4mid_encode_sequence; + } + } else { + if (pos4 >= dictIdx) { + /* extDict match candidate */ + const BYTE* const matchPtr = dictStart + (pos4 - dictIdx); + const size_t safeLen = MIN(prefixIdx - pos4, (size_t)(matchlimit - ip)); + matchLength = LZ4_count(ip, matchPtr, ip + safeLen); + if (matchLength >= MINMATCH) { + DEBUGLOG(7, "found match at ExtDict pos %u (len=%u)", pos4, matchLength); + matchDistance = ipIndex - pos4; + goto _lz4mid_encode_sequence; + } + } + } + } } + /* no match found in prefix */ + if ( (dict == usingDictCtxHc) + && (ipIndex - gDictEndIndex < LZ4_DISTANCE_MAX - 8) ) { + /* search a match into external dictionary */ + LZ4HC_match_t dMatch = searchIntoDict(ip, ipIndex, + matchlimit, + ctx->dictCtx, gDictEndIndex); + if (dMatch.len >= MINMATCH) { + DEBUGLOG(7, "found Dictionary match (offset=%i)", dMatch.off); + assert(dMatch.back == 0); + matchLength = (unsigned)dMatch.len; + matchDistance = (unsigned)dMatch.off; + goto _lz4mid_encode_sequence; + } + } + /* no match found */ + ip += 1 + ((ip-anchor) >> 9); /* skip faster over incompressible data */ + continue; + +_lz4mid_encode_sequence: + /* catch back */ + while (((ip > anchor) & ((U32)(ip-prefixPtr) > matchDistance)) && (unlikely(ip[-1] == ip[-(int)matchDistance-1]))) { + ip--; matchLength++; + }; + + /* fill table with beginning of match */ + ADDPOS8(ip+1, ipIndex+1); + ADDPOS8(ip+2, ipIndex+2); + ADDPOS4(ip+1, ipIndex+1); + + /* encode */ + { BYTE* const saved_op = op; + /* LZ4HC_encodeSequence always updates @op; on success, it updates @ip and @anchor */ + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + (int)matchLength, (int)matchDistance, + limit, oend) ) { + op = saved_op; /* restore @op value before failed LZ4HC_encodeSequence */ + goto _lz4mid_dest_overflow; + } + } + + /* fill table with end of match */ + { U32 endMatchIdx = (U32)(ip-prefixPtr) + prefixIdx; + U32 pos_m2 = endMatchIdx - 2; + if (pos_m2 < ilimitIdx) { + if (likely(ip - prefixPtr > 5)) { + ADDPOS8(ip-5, endMatchIdx - 5); + } + ADDPOS8(ip-3, endMatchIdx - 3); + ADDPOS8(ip-2, endMatchIdx - 2); + ADDPOS4(ip-2, endMatchIdx - 2); + ADDPOS4(ip-1, endMatchIdx - 1); + } + } + } + +_lz4mid_last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + llAdd + lastRunSize; + if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) return 0; /* not enough space in @dst */ + /* adapt lastRunSize to fill 'dest' */ + lastRunSize = (size_t)(oend - op) - 1 /*token*/; + llAdd = (lastRunSize + 256 - RUN_MASK) / 256; + lastRunSize -= llAdd; + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); + ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) + *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + assert(lastRunSize <= (size_t)(oend - op)); + LZ4_memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + DEBUGLOG(5, "compressed %i bytes into %i bytes", *srcSizePtr, (int)((char*)op - dst)); + assert(ip >= (const BYTE*)src); + assert(ip <= iend); + *srcSizePtr = (int)(ip - (const BYTE*)src); + assert((char*)op >= dst); + assert(op <= oend); + assert((char*)op - dst < INT_MAX); + return (int)((char*)op - dst); + +_lz4mid_dest_overflow: + if (limit == fillOutput) { + /* Assumption : @ip, @anchor, @optr and @matchLength must be set correctly */ + size_t const ll = (size_t)(ip - anchor); + size_t const ll_addbytes = (ll + 240) / 255; + size_t const ll_totalCost = 1 + ll_addbytes + ll; + BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ + DEBUGLOG(6, "Last sequence is overflowing : %u literals, %u remaining space", + (unsigned)ll, (unsigned)(oend-op)); + if (op + ll_totalCost <= maxLitPos) { + /* ll validated; now adjust match length */ + size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); + size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); + assert(maxMlSize < INT_MAX); + if ((size_t)matchLength > maxMlSize) matchLength= (unsigned)maxMlSize; + if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + matchLength >= MFLIMIT) { + DEBUGLOG(6, "Let's encode a last sequence (ll=%u, ml=%u)", (unsigned)ll, matchLength); + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + (int)matchLength, (int)matchDistance, + notLimited, oend); + } } + DEBUGLOG(6, "Let's finish with a run of literals (%u bytes left)", (unsigned)(oend-op)); + goto _lz4mid_last_literals; + } + /* compression failed */ + return 0; +} + + +/************************************** +* HC Compression - Search +**************************************/ + +/* Update chains up to ip (excluded) */ +LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) +{ + U16* const chainTable = hc4->chainTable; + U32* const hashTable = hc4->hashTable; + const BYTE* const prefixPtr = hc4->prefixStart; + U32 const prefixIdx = hc4->dictLimit; + U32 const target = (U32)(ip - prefixPtr) + prefixIdx; + U32 idx = hc4->nextToUpdate; + assert(ip >= prefixPtr); + assert(target >= prefixIdx); + + while (idx < target) { + U32 const h = LZ4HC_hashPtr(prefixPtr+idx-prefixIdx); + size_t delta = idx - hashTable[h]; + if (delta>LZ4_DISTANCE_MAX) delta = LZ4_DISTANCE_MAX; + DELTANEXTU16(chainTable, idx) = (U16)delta; + hashTable[h] = idx; + idx++; + } + + hc4->nextToUpdate = target; +} + +#if defined(_MSC_VER) +# define LZ4HC_rotl32(x,r) _rotl(x,r) +#else +# define LZ4HC_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif + + +static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern) +{ + size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3; + if (bitsToRotate == 0) return pattern; + return LZ4HC_rotl32(pattern, (int)bitsToRotate); +} + +/* LZ4HC_countPattern() : + * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */ +static unsigned +LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32) +{ + const BYTE* const iStart = ip; + reg_t const pattern = (sizeof(pattern)==8) ? + (reg_t)pattern32 + (((reg_t)pattern32) << (sizeof(pattern)*4)) : pattern32; + + while (likely(ip < iEnd-(sizeof(pattern)-1))) { + reg_t const diff = LZ4_read_ARCH(ip) ^ pattern; + if (!diff) { ip+=sizeof(pattern); continue; } + ip += LZ4_NbCommonBytes(diff); + return (unsigned)(ip - iStart); + } + + if (LZ4_isLittleEndian()) { + reg_t patternByte = pattern; + while ((ip>= 8; + } + } else { /* big endian */ + U32 bitOffset = (sizeof(pattern)*8) - 8; + while (ip < iEnd) { + BYTE const byte = (BYTE)(pattern >> bitOffset); + if (*ip != byte) break; + ip ++; bitOffset -= 8; + } } + + return (unsigned)(ip - iStart); +} + +/* LZ4HC_reverseCountPattern() : + * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) + * read using natural platform endianness */ +static unsigned +LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) +{ + const BYTE* const iStart = ip; + + while (likely(ip >= iLow+4)) { + if (LZ4_read32(ip-4) != pattern) break; + ip -= 4; + } + { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianness */ + while (likely(ip>iLow)) { + if (ip[-1] != *bytePtr) break; + ip--; bytePtr--; + } } + return (unsigned)(iStart - ip); +} + +/* LZ4HC_protectDictEnd() : + * Checks if the match is in the last 3 bytes of the dictionary, so reading the + * 4 byte MINMATCH would overflow. + * @returns true if the match index is okay. + */ +static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex) +{ + return ((U32)((dictLimit - 1) - matchIndex) >= 3); +} + +typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; +typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; + + +LZ4_FORCE_INLINE LZ4HC_match_t +LZ4HC_InsertAndGetWiderMatch ( + LZ4HC_CCtx_internal* const hc4, + const BYTE* const ip, + const BYTE* const iLowLimit, const BYTE* const iHighLimit, + int longest, + const int maxNbAttempts, + const int patternAnalysis, const int chainSwap, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ + U16* const chainTable = hc4->chainTable; + U32* const hashTable = hc4->hashTable; + const LZ4HC_CCtx_internal* const dictCtx = hc4->dictCtx; + const BYTE* const prefixPtr = hc4->prefixStart; + const U32 prefixIdx = hc4->dictLimit; + const U32 ipIndex = (U32)(ip - prefixPtr) + prefixIdx; + const int withinStartDistance = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex); + const U32 lowestMatchIndex = (withinStartDistance) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; + const BYTE* const dictStart = hc4->dictStart; + const U32 dictIdx = hc4->lowLimit; + const BYTE* const dictEnd = dictStart + prefixIdx - dictIdx; + int const lookBackLength = (int)(ip-iLowLimit); + int nbAttempts = maxNbAttempts; + U32 matchChainPos = 0; + U32 const pattern = LZ4_read32(ip); + U32 matchIndex; + repeat_state_e repeat = rep_untested; + size_t srcPatternLength = 0; + int offset = 0, sBack = 0; + + DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch"); + /* First Match */ + LZ4HC_Insert(hc4, ip); /* insert all prior positions up to ip (excluded) */ + matchIndex = hashTable[LZ4HC_hashPtr(ip)]; + DEBUGLOG(7, "First candidate match for pos %u found at index %u / %u (lowestMatchIndex)", + ipIndex, matchIndex, lowestMatchIndex); + + while ((matchIndex>=lowestMatchIndex) && (nbAttempts>0)) { + int matchLength=0; + nbAttempts--; + assert(matchIndex < ipIndex); + if (favorDecSpeed && (ipIndex - matchIndex < 8)) { + /* do nothing: + * favorDecSpeed intentionally skips matches with offset < 8 */ + } else if (matchIndex >= prefixIdx) { /* within current Prefix */ + const BYTE* const matchPtr = prefixPtr + (matchIndex - prefixIdx); + assert(matchPtr < ip); + assert(longest >= 1); + if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) { + if (LZ4_read32(matchPtr) == pattern) { + int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, prefixPtr) : 0; + matchLength = MINMATCH + (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); + matchLength -= back; + if (matchLength > longest) { + longest = matchLength; + offset = (int)(ipIndex - matchIndex); + sBack = back; + DEBUGLOG(7, "Found match of len=%i within prefix, offset=%i, back=%i", longest, offset, -back); + } } } + } else { /* lowestMatchIndex <= matchIndex < dictLimit : within Ext Dict */ + const BYTE* const matchPtr = dictStart + (matchIndex - dictIdx); + assert(matchIndex >= dictIdx); + if ( likely(matchIndex <= prefixIdx - 4) + && (LZ4_read32(matchPtr) == pattern) ) { + int back = 0; + const BYTE* vLimit = ip + (prefixIdx - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + matchLength = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) + matchLength += LZ4_count(ip+matchLength, prefixPtr, iHighLimit); + back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0; + matchLength -= back; + if (matchLength > longest) { + longest = matchLength; + offset = (int)(ipIndex - matchIndex); + sBack = back; + DEBUGLOG(7, "Found match of len=%i within dict, offset=%i, back=%i", longest, offset, -back); + } } } + + if (chainSwap && matchLength==longest) { /* better match => select a better chain */ + assert(lookBackLength==0); /* search forward only */ + if (matchIndex + (U32)longest <= ipIndex) { + int const kTrigger = 4; + U32 distanceToNextMatch = 1; + int const end = longest - MINMATCH + 1; + int step = 1; + int accel = 1 << kTrigger; + int pos; + for (pos = 0; pos < end; pos += step) { + U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos); + step = (accel++ >> kTrigger); + if (candidateDist > distanceToNextMatch) { + distanceToNextMatch = candidateDist; + matchChainPos = (U32)pos; + accel = 1 << kTrigger; + } } + if (distanceToNextMatch > 1) { + if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ + matchIndex -= distanceToNextMatch; + continue; + } } } + + { U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex); + if (patternAnalysis && distNextMatch==1 && matchChainPos==0) { + U32 const matchCandidateIdx = matchIndex-1; + /* may be a repeated pattern */ + if (repeat == rep_untested) { + if ( ((pattern & 0xFFFF) == (pattern >> 16)) + & ((pattern & 0xFF) == (pattern >> 24)) ) { + DEBUGLOG(7, "Repeat pattern detected, char %02X", pattern >> 24); + repeat = rep_confirmed; + srcPatternLength = LZ4HC_countPattern(ip+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); + } else { + repeat = rep_not; + } } + if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex) + && LZ4HC_protectDictEnd(prefixIdx, matchCandidateIdx) ) { + const int extDict = matchCandidateIdx < prefixIdx; + const BYTE* const matchPtr = extDict ? dictStart + (matchCandidateIdx - dictIdx) : prefixPtr + (matchCandidateIdx - prefixIdx); + if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ + const BYTE* const iLimit = extDict ? dictEnd : iHighLimit; + size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern); + if (extDict && matchPtr + forwardPatternLength == iLimit) { + U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern); + forwardPatternLength += LZ4HC_countPattern(prefixPtr, iHighLimit, rotatedPattern); + } + { const BYTE* const lowestMatchPtr = extDict ? dictStart : prefixPtr; + size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); + size_t currentSegmentLength; + if (!extDict + && matchPtr - backLength == prefixPtr + && dictIdx < prefixIdx) { + U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern); + backLength += LZ4HC_reverseCountPattern(dictEnd, dictStart, rotatedPattern); + } + /* Limit backLength not go further than lowestMatchIndex */ + backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex); + assert(matchCandidateIdx - backLength >= lowestMatchIndex); + currentSegmentLength = backLength + forwardPatternLength; + /* Adjust to end of pattern if the source pattern fits, otherwise the beginning of the pattern */ + if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ + && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ + U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ + if (LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) + matchIndex = newMatchIndex; + else { + /* Can only happen if started in the prefix */ + assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict); + matchIndex = prefixIdx; + } + } else { + U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ + if (!LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) { + assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict); + matchIndex = prefixIdx; + } else { + matchIndex = newMatchIndex; + if (lookBackLength==0) { /* no back possible */ + size_t const maxML = MIN(currentSegmentLength, srcPatternLength); + if ((size_t)longest < maxML) { + assert(prefixPtr - prefixIdx + matchIndex != ip); + if ((size_t)(ip - prefixPtr) + prefixIdx - matchIndex > LZ4_DISTANCE_MAX) break; + assert(maxML < 2 GB); + longest = (int)maxML; + offset = (int)(ipIndex - matchIndex); + assert(sBack == 0); + DEBUGLOG(7, "Found repeat pattern match of len=%i, offset=%i", longest, offset); + } + { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); + if (distToNextPattern > matchIndex) break; /* avoid overflow */ + matchIndex -= distToNextPattern; + } } } } } + continue; + } } + } } /* PA optimization */ + + /* follow current chain */ + matchIndex -= DELTANEXTU16(chainTable, matchIndex + matchChainPos); + + } /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */ + + if ( dict == usingDictCtxHc + && nbAttempts > 0 + && withinStartDistance) { + size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->prefixStart) + dictCtx->dictLimit; + U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; + assert(dictEndOffset <= 1 GB); + matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; + if (dictMatchIndex>0) DEBUGLOG(7, "dictEndOffset = %zu, dictMatchIndex = %u => relative matchIndex = %i", dictEndOffset, dictMatchIndex, (int)dictMatchIndex - (int)dictEndOffset); + while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) { + const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + dictMatchIndex; + + if (LZ4_read32(matchPtr) == pattern) { + int mlt; + int back = 0; + const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->prefixStart) : 0; + mlt -= back; + if (mlt > longest) { + longest = mlt; + offset = (int)(ipIndex - matchIndex); + sBack = back; + DEBUGLOG(7, "found match of length %i within extDictCtx", longest); + } } + + { U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex); + dictMatchIndex -= nextOffset; + matchIndex -= nextOffset; + } } } + + { LZ4HC_match_t md; + assert(longest >= 0); + md.len = longest; + md.off = offset; + md.back = sBack; + return md; + } +} + +LZ4_FORCE_INLINE LZ4HC_match_t +LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, + const int maxNbAttempts, + const int patternAnalysis, + const dictCtx_directive dict) +{ + DEBUGLOG(7, "LZ4HC_InsertAndFindBestMatch"); + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio); +} + + +LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( + LZ4HC_CCtx_internal* const ctx, + const char* const source, + char* const dest, + int* srcSizePtr, + int const maxOutputSize, + int maxNbAttempts, + const limitedOutput_directive limit, + const dictCtx_directive dict + ) +{ + const int inputSize = *srcSizePtr; + const int patternAnalysis = (maxNbAttempts > 128); /* levels 9+ */ + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + + BYTE* optr = (BYTE*) dest; + BYTE* op = (BYTE*) dest; + BYTE* oend = op + maxOutputSize; + + const BYTE* start0; + const BYTE* start2 = NULL; + const BYTE* start3 = NULL; + LZ4HC_match_t m0, m1, m2, m3; + const LZ4HC_match_t nomatch = {0, 0, 0}; + + /* init */ + DEBUGLOG(5, "LZ4HC_compress_hashChain (dict?=>%i)", dict); + *srcSizePtr = 0; + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + + /* Main Loop */ + while (ip <= mflimit) { + m1 = LZ4HC_InsertAndFindBestMatch(ctx, ip, matchlimit, maxNbAttempts, patternAnalysis, dict); + if (m1.len encode ML1 immediately */ + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + m1.len, m1.off, + limit, oend) ) + goto _dest_overflow; + continue; + } + + if (start0 < ip) { /* first match was skipped at least once */ + if (start2 < ip + m0.len) { /* squeezing ML1 between ML0(original ML1) and ML2 */ + ip = start0; m1 = m0; /* restore initial Match1 */ + } } + + /* Here, start0==ip */ + if ((start2 - ip) < 3) { /* First Match too small : removed */ + ip = start2; + m1 = m2; + goto _Search2; + } + +_Search3: + if ((start2 - ip) < OPTIMAL_ML) { + int correction; + int new_ml = m1.len; + if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; + if (ip+new_ml > start2 + m2.len - MINMATCH) + new_ml = (int)(start2 - ip) + m2.len - MINMATCH; + correction = new_ml - (int)(start2 - ip); + if (correction > 0) { + start2 += correction; + m2.len -= correction; + } + } + + if (start2 + m2.len <= mflimit) { + start3 = start2 + m2.len - 3; + m3 = LZ4HC_InsertAndGetWiderMatch(ctx, + start3, start2, matchlimit, m2.len, + maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio); + start3 += m3.back; + } else { + m3 = nomatch; /* do not search further */ + } + + if (m3.len <= m2.len) { /* No better match => encode ML1 and ML2 */ + /* ip & ref are known; Now for ml */ + if (start2 < ip+m1.len) m1.len = (int)(start2 - ip); + /* Now, encode 2 sequences */ + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + m1.len, m1.off, + limit, oend) ) + goto _dest_overflow; + ip = start2; + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + m2.len, m2.off, + limit, oend) ) { + m1 = m2; + goto _dest_overflow; + } + continue; + } + + if (start3 < ip+m1.len+3) { /* Not enough space for match 2 : remove it */ + if (start3 >= (ip+m1.len)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ + if (start2 < ip+m1.len) { + int correction = (int)(ip+m1.len - start2); + start2 += correction; + m2.len -= correction; + if (m2.len < MINMATCH) { + start2 = start3; + m2 = m3; + } + } + + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + m1.len, m1.off, + limit, oend) ) + goto _dest_overflow; + ip = start3; + m1 = m3; + + start0 = start2; + m0 = m2; + goto _Search2; + } + + start2 = start3; + m2 = m3; + goto _Search3; + } + + /* + * OK, now we have 3 ascending matches; + * let's write the first one ML1. + * ip & ref are known; Now decide ml. + */ + if (start2 < ip+m1.len) { + if ((start2 - ip) < OPTIMAL_ML) { + int correction; + if (m1.len > OPTIMAL_ML) m1.len = OPTIMAL_ML; + if (ip + m1.len > start2 + m2.len - MINMATCH) + m1.len = (int)(start2 - ip) + m2.len - MINMATCH; + correction = m1.len - (int)(start2 - ip); + if (correction > 0) { + start2 += correction; + m2.len -= correction; + } + } else { + m1.len = (int)(start2 - ip); + } + } + optr = op; + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + m1.len, m1.off, + limit, oend) ) + goto _dest_overflow; + + /* ML2 becomes ML1 */ + ip = start2; m1 = m2; + + /* ML3 becomes ML2 */ + start2 = start3; m2 = m3; + + /* let's find a new ML3 */ + goto _Search3; + } + +_last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + llAdd + lastRunSize; + if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) return 0; + /* adapt lastRunSize to fill 'dest' */ + lastRunSize = (size_t)(oend - op) - 1 /*token*/; + llAdd = (lastRunSize + 256 - RUN_MASK) / 256; + lastRunSize -= llAdd; + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); + ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + LZ4_memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); + return (int) (((char*)op)-dest); + +_dest_overflow: + if (limit == fillOutput) { + /* Assumption : @ip, @anchor, @optr and @m1 must be set correctly */ + size_t const ll = (size_t)(ip - anchor); + size_t const ll_addbytes = (ll + 240) / 255; + size_t const ll_totalCost = 1 + ll_addbytes + ll; + BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ + DEBUGLOG(6, "Last sequence overflowing"); + op = optr; /* restore correct out pointer */ + if (op + ll_totalCost <= maxLitPos) { + /* ll validated; now adjust match length */ + size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); + size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); + assert(maxMlSize < INT_MAX); assert(m1.len >= 0); + if ((size_t)m1.len > maxMlSize) m1.len = (int)maxMlSize; + if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + m1.len >= MFLIMIT) { + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, m1.off, notLimited, oend); + } } + goto _last_literals; + } + /* compression failed */ + return 0; +} + + +static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx, + const char* const source, char* dst, + int* srcSizePtr, int dstCapacity, + int const nbSearches, size_t sufficient_len, + const limitedOutput_directive limit, int const fullUpdate, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed); + +LZ4_FORCE_INLINE int +LZ4HC_compress_generic_internal ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + const limitedOutput_directive limit, + const dictCtx_directive dict + ) +{ + DEBUGLOG(5, "LZ4HC_compress_generic_internal(src=%p, srcSize=%d)", + src, *srcSizePtr); + + if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ + + ctx->end += *srcSizePtr; + { cParams_t const cParam = LZ4HC_getCLevelParams(cLevel); + HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio; + int result; + + if (cParam.strat == lz4mid) { + result = LZ4MID_compress(ctx, + src, dst, srcSizePtr, dstCapacity, + limit, dict); + } else if (cParam.strat == lz4hc) { + result = LZ4HC_compress_hashChain(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, limit, dict); + } else { + assert(cParam.strat == lz4opt); + result = LZ4HC_compress_optimal(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, cParam.targetLength, limit, + cLevel >= LZ4HC_CLEVEL_MAX, /* ultra mode */ + dict, favor); + } + if (result <= 0) ctx->dirty = 1; + return result; + } +} + +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock); + +static int +LZ4HC_compress_generic_noDictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + assert(ctx->dictCtx == NULL); + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx); +} + +static int isStateCompatible(const LZ4HC_CCtx_internal* ctx1, const LZ4HC_CCtx_internal* ctx2) +{ + int const isMid1 = LZ4HC_getCLevelParams(ctx1->compressionLevel).strat == lz4mid; + int const isMid2 = LZ4HC_getCLevelParams(ctx2->compressionLevel).strat == lz4mid; + return !(isMid1 ^ isMid2); +} + +static int +LZ4HC_compress_generic_dictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + const size_t position = (size_t)(ctx->end - ctx->prefixStart) + (ctx->dictLimit - ctx->lowLimit); + assert(ctx->dictCtx != NULL); + if (position >= 64 KB) { + ctx->dictCtx = NULL; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else if (position == 0 && *srcSizePtr > 4 KB && isStateCompatible(ctx, ctx->dictCtx)) { + LZ4_memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); + LZ4HC_setExternalDict(ctx, (const BYTE *)src); + ctx->compressionLevel = (short)cLevel; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtxHc); + } +} + +static int +LZ4HC_compress_generic ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + if (ctx->dictCtx == NULL) { + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + return LZ4HC_compress_generic_dictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } +} + + +int LZ4_sizeofStateHC(void) { return (int)sizeof(LZ4_streamHC_t); } + +static size_t LZ4_streamHC_t_alignment(void) +{ +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_streamHC_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_streamHC_t); +#else + return 1; /* effectively disabled */ +#endif +} + +/* state is presumed correctly initialized, + * in which case its size and alignment have already been validate */ +int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; + if (!LZ4_isAligned(state, LZ4_streamHC_t_alignment())) return 0; + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel); + LZ4HC_init_internal (ctx, (const BYTE*)src); + if (dstCapacity < LZ4_compressBound(srcSize)) + return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput); + else + return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, notLimited); +} + +int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); + if (ctx==NULL) return 0; /* init failure */ + return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel); +} + +int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + int cSize; +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); + if (statePtr==NULL) return 0; +#else + LZ4_streamHC_t state; + LZ4_streamHC_t* const statePtr = &state; +#endif + DEBUGLOG(5, "LZ4_compress_HC") + cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + FREEMEM(statePtr); +#endif + return cSize; +} + +/* state is presumed sized correctly (>= sizeof(LZ4_streamHC_t)) */ +int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) +{ + LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); + if (ctx==NULL) return 0; /* init failure */ + LZ4HC_init_internal(&ctx->internal_donotuse, (const BYTE*) source); + LZ4_setCompressionLevel(ctx, cLevel); + return LZ4HC_compress_generic(&ctx->internal_donotuse, source, dest, sourceSizePtr, targetDestSize, cLevel, fillOutput); +} + + + +/************************************** +* Streaming Functions +**************************************/ +/* allocation */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_streamHC_t* LZ4_createStreamHC(void) +{ + LZ4_streamHC_t* const state = + (LZ4_streamHC_t*)ALLOC_AND_ZERO(sizeof(LZ4_streamHC_t)); + if (state == NULL) return NULL; + LZ4_setCompressionLevel(state, LZ4HC_CLEVEL_DEFAULT); + return state; +} + +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) +{ + DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr); + if (!LZ4_streamHCPtr) return 0; /* support free on NULL */ + FREEMEM(LZ4_streamHCPtr); + return 0; +} +#endif + + +LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size) +{ + LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer; + DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", buffer, (unsigned)size); + /* check conditions */ + if (buffer == NULL) return NULL; + if (size < sizeof(LZ4_streamHC_t)) return NULL; + if (!LZ4_isAligned(buffer, LZ4_streamHC_t_alignment())) return NULL; + /* init */ + { LZ4HC_CCtx_internal* const hcstate = &(LZ4_streamHCPtr->internal_donotuse); + MEM_INIT(hcstate, 0, sizeof(*hcstate)); } + LZ4_setCompressionLevel(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT); + return LZ4_streamHCPtr; +} + +/* just a stub */ +void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); +} + +void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + LZ4HC_CCtx_internal* const s = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(5, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); + if (s->dirty) { + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + } else { + assert(s->end >= s->prefixStart); + s->dictLimit += (U32)(s->end - s->prefixStart); + s->prefixStart = NULL; + s->end = NULL; + s->dictCtx = NULL; + } + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); +} + +void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + DEBUGLOG(5, "LZ4_setCompressionLevel(%p, %d)", LZ4_streamHCPtr, compressionLevel); + if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; + if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; + LZ4_streamHCPtr->internal_donotuse.compressionLevel = (short)compressionLevel; +} + +void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor) +{ + LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor!=0); +} + +/* LZ4_loadDictHC() : + * LZ4_streamHCPtr is presumed properly initialized */ +int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, + const char* dictionary, int dictSize) +{ + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + cParams_t cp; + DEBUGLOG(4, "LZ4_loadDictHC(ctx:%p, dict:%p, dictSize:%d, clevel=%d)", LZ4_streamHCPtr, dictionary, dictSize, ctxPtr->compressionLevel); + assert(dictSize >= 0); + assert(LZ4_streamHCPtr != NULL); + if (dictSize > 64 KB) { + dictionary += (size_t)dictSize - 64 KB; + dictSize = 64 KB; + } + /* need a full initialization, there are bad side-effects when using resetFast() */ + { int const cLevel = ctxPtr->compressionLevel; + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + LZ4_setCompressionLevel(LZ4_streamHCPtr, cLevel); + cp = LZ4HC_getCLevelParams(cLevel); + } + LZ4HC_init_internal (ctxPtr, (const BYTE*)dictionary); + ctxPtr->end = (const BYTE*)dictionary + dictSize; + if (cp.strat == lz4mid) { + LZ4MID_fillHTable (ctxPtr, dictionary, (size_t)dictSize); + } else { + if (dictSize >= LZ4HC_HASHSIZE) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); + } + return dictSize; +} + +void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream) { + working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL; +} + +/* compression */ + +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) +{ + DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock); + if ( (ctxPtr->end >= ctxPtr->prefixStart + 4) + && (LZ4HC_getCLevelParams(ctxPtr->compressionLevel).strat != lz4mid) ) { + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + } + + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictStart = ctxPtr->prefixStart; + ctxPtr->dictLimit += (U32)(ctxPtr->end - ctxPtr->prefixStart); + ctxPtr->prefixStart = newBlock; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ + + /* cannot reference an extDict and a dictCtx at the same time */ + ctxPtr->dictCtx = NULL; +} + +static int +LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int dstCapacity, + limitedOutput_directive limit) +{ + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(5, "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d, limit=%d)", + LZ4_streamHCPtr, src, *srcSizePtr, limit); + assert(ctxPtr != NULL); + /* auto-init if forgotten */ + if (ctxPtr->prefixStart == NULL) + LZ4HC_init_internal (ctxPtr, (const BYTE*) src); + + /* Check overflow */ + if ((size_t)(ctxPtr->end - ctxPtr->prefixStart) + ctxPtr->dictLimit > 2 GB) { + size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->prefixStart); + if (dictSize > 64 KB) dictSize = 64 KB; + LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); + } + + /* Check if blocks follow each other */ + if ((const BYTE*)src != ctxPtr->end) + LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src); + + /* Check overlapping input/dictionary space */ + { const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr; + const BYTE* const dictBegin = ctxPtr->dictStart; + const BYTE* const dictEnd = ctxPtr->dictStart + (ctxPtr->dictLimit - ctxPtr->lowLimit); + if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) { + if (sourceEnd > dictEnd) sourceEnd = dictEnd; + ctxPtr->lowLimit += (U32)(sourceEnd - ctxPtr->dictStart); + ctxPtr->dictStart += (U32)(sourceEnd - ctxPtr->dictStart); + /* invalidate dictionary is it's too small */ + if (ctxPtr->dictLimit - ctxPtr->lowLimit < LZ4HC_HASHSIZE) { + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictStart = ctxPtr->prefixStart; + } } } + + return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit); +} + +int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity) +{ + DEBUGLOG(5, "LZ4_compress_HC_continue"); + if (dstCapacity < LZ4_compressBound(srcSize)) + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput); + else + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, notLimited); +} + +int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize) +{ + return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput); +} + + +/* LZ4_saveDictHC : + * save history content + * into a user-provided buffer + * which is then used to continue compression + */ +int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) +{ + LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; + int const prefixSize = (int)(streamPtr->end - streamPtr->prefixStart); + DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); + assert(prefixSize >= 0); + if (dictSize > 64 KB) dictSize = 64 KB; + if (dictSize < 4) dictSize = 0; + if (dictSize > prefixSize) dictSize = prefixSize; + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) + LZ4_memmove(safeBuffer, streamPtr->end - dictSize, (size_t)dictSize); + { U32 const endIndex = (U32)(streamPtr->end - streamPtr->prefixStart) + streamPtr->dictLimit; + streamPtr->end = (safeBuffer == NULL) ? NULL : (const BYTE*)safeBuffer + dictSize; + streamPtr->prefixStart = (const BYTE*)safeBuffer; + streamPtr->dictLimit = endIndex - (U32)dictSize; + streamPtr->lowLimit = endIndex - (U32)dictSize; + streamPtr->dictStart = streamPtr->prefixStart; + if (streamPtr->nextToUpdate < streamPtr->dictLimit) + streamPtr->nextToUpdate = streamPtr->dictLimit; + } + return dictSize; +} + + +/* ================================================ + * LZ4 Optimal parser (levels [LZ4HC_CLEVEL_OPT_MIN - LZ4HC_CLEVEL_MAX]) + * ===============================================*/ +typedef struct { + int price; + int off; + int mlen; + int litlen; +} LZ4HC_optimal_t; + +/* price in bytes */ +LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) +{ + int price = litlen; + assert(litlen >= 0); + if (litlen >= (int)RUN_MASK) + price += 1 + ((litlen-(int)RUN_MASK) / 255); + return price; +} + +/* requires mlen >= MINMATCH */ +LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) +{ + int price = 1 + 2 ; /* token + 16-bit offset */ + assert(litlen >= 0); + assert(mlen >= MINMATCH); + + price += LZ4HC_literalsPrice(litlen); + + if (mlen >= (int)(ML_MASK+MINMATCH)) + price += 1 + ((mlen-(int)(ML_MASK+MINMATCH)) / 255); + + return price; +} + +LZ4_FORCE_INLINE LZ4HC_match_t +LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, + const BYTE* ip, const BYTE* const iHighLimit, + int minLen, int nbSearches, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ + LZ4HC_match_t const match0 = { 0 , 0, 0 }; + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + ** so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + LZ4HC_match_t md = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed); + assert(md.back == 0); + if (md.len <= minLen) return match0; + if (favorDecSpeed) { + if ((md.len>18) & (md.len<=36)) md.len=18; /* favor dec.speed (shortcut) */ + } + return md; +} + + +static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, + const char* const source, + char* dst, + int* srcSizePtr, + int dstCapacity, + int const nbSearches, + size_t sufficient_len, + const limitedOutput_directive limit, + int const fullUpdate, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ + int retval = 0; +#define TRAILING_LITERALS 3 +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + LZ4HC_optimal_t* const opt = (LZ4HC_optimal_t*)ALLOC(sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS)); +#else + LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */ +#endif + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + BYTE* op = (BYTE*) dst; + BYTE* opSaved = (BYTE*) dst; + BYTE* oend = op + dstCapacity; + int ovml = MINMATCH; /* overflow - last sequence */ + int ovoff = 0; + + /* init */ +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + if (opt == NULL) goto _return_label; +#endif + DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity); + *srcSizePtr = 0; + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; + + /* Main Loop */ + while (ip <= mflimit) { + int const llen = (int)(ip - anchor); + int best_mlen, best_off; + int cur, last_match_pos = 0; + + LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); + if (firstMatch.len==0) { ip++; continue; } + + if ((size_t)firstMatch.len > sufficient_len) { + /* good enough solution : immediate encoding */ + int const firstML = firstMatch.len; + opSaved = op; + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, firstMatch.off, limit, oend) ) { /* updates ip, op and anchor */ + ovml = firstML; + ovoff = firstMatch.off; + goto _dest_overflow; + } + continue; + } + + /* set prices for first positions (literals) */ + { int rPos; + for (rPos = 0 ; rPos < MINMATCH ; rPos++) { + int const cost = LZ4HC_literalsPrice(llen + rPos); + opt[rPos].mlen = 1; + opt[rPos].off = 0; + opt[rPos].litlen = llen + rPos; + opt[rPos].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + rPos, cost, opt[rPos].litlen); + } } + /* set prices using initial match */ + { int const matchML = firstMatch.len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ + int const offset = firstMatch.off; + int mlen; + assert(matchML < LZ4_OPT_NUM); + for (mlen = MINMATCH ; mlen <= matchML ; mlen++) { + int const cost = LZ4HC_sequencePrice(llen, mlen); + opt[mlen].mlen = mlen; + opt[mlen].off = offset; + opt[mlen].litlen = llen; + opt[mlen].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup", + mlen, cost, mlen); + } } + last_match_pos = firstMatch.len; + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + + /* check further positions */ + for (cur = 1; cur < last_match_pos; cur++) { + const BYTE* const curPtr = ip + cur; + LZ4HC_match_t newMatch; + + if (curPtr > mflimit) break; + DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u", + cur, opt[cur].price, opt[cur+1].price, cur+1); + if (fullUpdate) { + /* not useful to search here if next position has same (or lower) cost */ + if ( (opt[cur+1].price <= opt[cur].price) + /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */ + && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) ) + continue; + } else { + /* not useful to search here if next position has same (or lower) cost */ + if (opt[cur+1].price <= opt[cur].price) continue; + } + + DEBUGLOG(7, "search at rPos:%u", cur); + if (fullUpdate) + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); + else + /* only test matches of minimum length; slightly faster, but misses a few bytes */ + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict, favorDecSpeed); + if (!newMatch.len) continue; + + if ( ((size_t)newMatch.len > sufficient_len) + || (newMatch.len + cur >= LZ4_OPT_NUM) ) { + /* immediate encoding */ + best_mlen = newMatch.len; + best_off = newMatch.off; + last_match_pos = cur + 1; + goto encode; + } + + /* before match : set price with literals at beginning */ + { int const baseLitlen = opt[cur].litlen; + int litlen; + for (litlen = 1; litlen < MINMATCH; litlen++) { + int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen); + int const pos = cur + litlen; + if (price < opt[pos].price) { + opt[pos].mlen = 1; /* literal */ + opt[pos].off = 0; + opt[pos].litlen = baseLitlen+litlen; + opt[pos].price = price; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", + pos, price, opt[pos].litlen); + } } } + + /* set prices using match at position = cur */ + { int const matchML = newMatch.len; + int ml = MINMATCH; + + assert(cur + newMatch.len < LZ4_OPT_NUM); + for ( ; ml <= matchML ; ml++) { + int const pos = cur + ml; + int const offset = newMatch.off; + int price; + int ll; + DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)", + pos, last_match_pos); + if (opt[cur].mlen == 1) { + ll = opt[cur].litlen; + price = ((cur > ll) ? opt[cur - ll].price : 0) + + LZ4HC_sequencePrice(ll, ml); + } else { + ll = 0; + price = opt[cur].price + LZ4HC_sequencePrice(0, ml); + } + + assert((U32)favorDecSpeed <= 1); + if (pos > last_match_pos+TRAILING_LITERALS + || price <= opt[pos].price - (int)favorDecSpeed) { + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)", + pos, price, ml); + assert(pos < LZ4_OPT_NUM); + if ( (ml == matchML) /* last pos of last match */ + && (last_match_pos < pos) ) + last_match_pos = pos; + opt[pos].mlen = ml; + opt[pos].off = offset; + opt[pos].litlen = ll; + opt[pos].price = price; + } } } + /* complete following positions with literals */ + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + } /* for (cur = 1; cur <= last_match_pos; cur++) */ + + assert(last_match_pos < LZ4_OPT_NUM + TRAILING_LITERALS); + best_mlen = opt[last_match_pos].mlen; + best_off = opt[last_match_pos].off; + cur = last_match_pos - best_mlen; + +encode: /* cur, last_match_pos, best_mlen, best_off must be set */ + assert(cur < LZ4_OPT_NUM); + assert(last_match_pos >= 1); /* == 1 when only one candidate */ + DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos); + { int candidate_pos = cur; + int selected_matchLength = best_mlen; + int selected_offset = best_off; + while (1) { /* from end to beginning */ + int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */ + int const next_offset = opt[candidate_pos].off; + DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength); + opt[candidate_pos].mlen = selected_matchLength; + opt[candidate_pos].off = selected_offset; + selected_matchLength = next_matchLength; + selected_offset = next_offset; + if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */ + assert(next_matchLength > 0); /* can be 1, means literal */ + candidate_pos -= next_matchLength; + } } + + /* encode all recorded sequences in order */ + { int rPos = 0; /* relative position (to ip) */ + while (rPos < last_match_pos) { + int const ml = opt[rPos].mlen; + int const offset = opt[rPos].off; + if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */ + rPos += ml; + assert(ml >= MINMATCH); + assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX)); + opSaved = op; + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, offset, limit, oend) ) { /* updates ip, op and anchor */ + ovml = ml; + ovoff = offset; + goto _dest_overflow; + } } } + } /* while (ip <= mflimit) */ + +_last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + llAdd + lastRunSize; + if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) { /* Check output limit */ + retval = 0; + goto _return_label; + } + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (size_t)(oend - op) - 1 /*token*/; + llAdd = (lastRunSize + 256 - RUN_MASK) / 256; + lastRunSize -= llAdd; + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); + ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + LZ4_memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); + retval = (int) ((char*)op-dst); + goto _return_label; + +_dest_overflow: +if (limit == fillOutput) { + /* Assumption : ip, anchor, ovml and ovref must be set correctly */ + size_t const ll = (size_t)(ip - anchor); + size_t const ll_addbytes = (ll + 240) / 255; + size_t const ll_totalCost = 1 + ll_addbytes + ll; + BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ + DEBUGLOG(6, "Last sequence overflowing (only %i bytes remaining)", (int)(oend-1-opSaved)); + op = opSaved; /* restore correct out pointer */ + if (op + ll_totalCost <= maxLitPos) { + /* ll validated; now adjust match length */ + size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); + size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); + assert(maxMlSize < INT_MAX); assert(ovml >= 0); + if ((size_t)ovml > maxMlSize) ovml = (int)maxMlSize; + if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + ovml >= MFLIMIT) { + DEBUGLOG(6, "Space to end : %i + ml (%i)", (int)((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1), ovml); + DEBUGLOG(6, "Before : ip = %p, anchor = %p", ip, anchor); + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ovml, ovoff, notLimited, oend); + DEBUGLOG(6, "After : ip = %p, anchor = %p", ip, anchor); + } } + goto _last_literals; +} +_return_label: +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + if (opt) FREEMEM(opt); +#endif + return retval; +} + + +/*************************************************** +* Deprecated Functions +***************************************************/ + +/* These functions currently generate deprecation warnings */ + +/* Wrappers for deprecated compression functions */ +int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); } +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); } + + +/* Deprecated streaming functions */ +int LZ4_sizeofStreamStateHC(void) { return sizeof(LZ4_streamHC_t); } + +/* state is presumed correctly sized, aka >= sizeof(LZ4_streamHC_t) + * @return : 0 on success, !=0 if error */ +int LZ4_resetStreamStateHC(void* state, char* inputBuffer) +{ + LZ4_streamHC_t* const hc4 = LZ4_initStreamHC(state, sizeof(*hc4)); + if (hc4 == NULL) return 1; /* init failed */ + LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + return 0; +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +void* LZ4_createHC (const char* inputBuffer) +{ + LZ4_streamHC_t* const hc4 = LZ4_createStreamHC(); + if (hc4 == NULL) return NULL; /* not enough memory */ + LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + return hc4; +} + +int LZ4_freeHC (void* LZ4HC_Data) +{ + if (!LZ4HC_Data) return 0; /* support free on NULL */ + FREEMEM(LZ4HC_Data); + return 0; +} +#endif + +int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel) +{ + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, notLimited); +} + +int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel) +{ + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, dstCapacity, cLevel, limitedOutput); +} + +char* LZ4_slideInputBufferHC(void* LZ4HC_Data) +{ + LZ4HC_CCtx_internal* const s = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; + const BYTE* const bufferStart = s->prefixStart - s->dictLimit + s->lowLimit; + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)LZ4HC_Data, s->compressionLevel); + /* ugly conversion trick, required to evade (const char*) -> (char*) cast-qual warning :( */ + return (char*)(uptrval)bufferStart; +} diff --git a/deps/lz4/lz4hc.h b/deps/lz4/lz4hc.h new file mode 100644 index 00000000000..992bc8cdd77 --- /dev/null +++ b/deps/lz4/lz4hc.h @@ -0,0 +1,414 @@ +/* + LZ4 HC - High Compression Mode of LZ4 + Header File + Copyright (C) 2011-2020, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +#ifndef LZ4_HC_H_19834876238432 +#define LZ4_HC_H_19834876238432 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* --- Dependency --- */ +/* note : lz4hc requires lz4.h/lz4.c for compilation */ +#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ + + +/* --- Useful constants --- */ +#define LZ4HC_CLEVEL_MIN 2 +#define LZ4HC_CLEVEL_DEFAULT 9 +#define LZ4HC_CLEVEL_OPT_MIN 10 +#define LZ4HC_CLEVEL_MAX 12 + + +/*-************************************ + * Block Compression + **************************************/ +/*! LZ4_compress_HC() : + * Compress data from `src` into `dst`, using the powerful but slower "HC" algorithm. + * `dst` must be already allocated. + * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") + * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") + * `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work. + * Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX. + * @return : the number of bytes written into 'dst' + * or 0 if compression fails. + */ +LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); + + +/* Note : + * Decompression functions are provided within "lz4.h" (BSD license) + */ + + +/*! LZ4_compress_HC_extStateHC() : + * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. + * `state` size is provided by LZ4_sizeofStateHC(). + * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly). + */ +LZ4LIB_API int LZ4_sizeofStateHC(void); +LZ4LIB_API int LZ4_compress_HC_extStateHC(void* stateHC, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); + + +/*! LZ4_compress_HC_destSize() : v1.9.0+ + * Will compress as much data as possible from `src` + * to fit into `targetDstSize` budget. + * Result is provided in 2 parts : + * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) + * or 0 if compression fails. + * `srcSizePtr` : on success, *srcSizePtr is updated to indicate how much bytes were read from `src` + */ +LZ4LIB_API int LZ4_compress_HC_destSize(void* stateHC, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize, + int compressionLevel); + + +/*-************************************ + * Streaming Compression + * Bufferless synchronous API + **************************************/ + typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */ + +/*! LZ4_createStreamHC() and LZ4_freeStreamHC() : + * These functions create and release memory for LZ4 HC streaming state. + * Newly created states are automatically initialized. + * A same state can be used multiple times consecutively, + * starting with LZ4_resetStreamHC_fast() to start a new stream of blocks. + */ +LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); +LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); + +/* + These functions compress data in successive blocks of any size, + using previous blocks as dictionary, to improve compression ratio. + One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. + There is an exception for ring buffers, which can be smaller than 64 KB. + Ring-buffer scenario is automatically detected and handled within LZ4_compress_HC_continue(). + + Before starting compression, state must be allocated and properly initialized. + LZ4_createStreamHC() does both, though compression level is set to LZ4HC_CLEVEL_DEFAULT. + + Selecting the compression level can be done with LZ4_resetStreamHC_fast() (starts a new stream) + or LZ4_setCompressionLevel() (anytime, between blocks in the same stream) (experimental). + LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once, + which is automatically the case when state is created using LZ4_createStreamHC(). + + After reset, a first "fictional block" can be designated as initial dictionary, + using LZ4_loadDictHC() (Optional). + Note: In order for LZ4_loadDictHC() to create the correct data structure, + it is essential to set the compression level _before_ loading the dictionary. + + Invoke LZ4_compress_HC_continue() to compress each successive block. + The number of blocks is unlimited. + Previous input blocks, including initial dictionary when present, + must remain accessible and unmodified during compression. + + It's allowed to update compression level anytime between blocks, + using LZ4_setCompressionLevel() (experimental). + + @dst buffer should be sized to handle worst case scenarios + (see LZ4_compressBound(), it ensures compression success). + In case of failure, the API does not guarantee recovery, + so the state _must_ be reset. + To ensure compression success + whenever @dst buffer size cannot be made >= LZ4_compressBound(), + consider using LZ4_compress_HC_continue_destSize(). + + Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks, + it's possible to copy the last blocks into a more stable memory space, using LZ4_saveDictHC(). + Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer' (<= 64 KB) + + After completing a streaming compression, + it's possible to start a new stream of blocks, using the same LZ4_streamHC_t state, + just by resetting it, using LZ4_resetStreamHC_fast(). +*/ + +LZ4LIB_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t* streamHCPtr, int compressionLevel); /* v1.9.0+ */ +LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); + +LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, + const char* src, char* dst, + int srcSize, int maxDstSize); + +/*! LZ4_compress_HC_continue_destSize() : v1.9.0+ + * Similar to LZ4_compress_HC_continue(), + * but will read as much data as possible from `src` + * to fit into `targetDstSize` budget. + * Result is provided into 2 parts : + * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) + * or 0 if compression fails. + * `srcSizePtr` : on success, *srcSizePtr will be updated to indicate how much bytes were read from `src`. + * Note that this function may not consume the entire input. + */ +LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize); + +LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); + + +/*! LZ4_attach_HC_dictionary() : stable since v1.10.0 + * This API allows for the efficient re-use of a static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a + * working LZ4_streamHC_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDictHC() should + * be expected to work. + * + * Alternatively, the provided dictionary stream pointer may be NULL, in which + * case any existing dictionary stream is unset. + * + * A dictionary should only be attached to a stream without any history (i.e., + * a stream that has just been reset). + * + * The dictionary will remain attached to the working stream only for the + * current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the + * dictionary context association from the working stream. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the lifetime of the stream session. + */ +LZ4LIB_API void +LZ4_attach_HC_dictionary(LZ4_streamHC_t* working_stream, + const LZ4_streamHC_t* dictionary_stream); + + +/*^********************************************** + * !!!!!! STATIC LINKING ONLY !!!!!! + ***********************************************/ + +/*-****************************************************************** + * PRIVATE DEFINITIONS : + * Do not use these definitions directly. + * They are merely exposed to allow static allocation of `LZ4_streamHC_t`. + * Declare an `LZ4_streamHC_t` directly, rather than any type below. + * Even then, only do so in the context of static linking, as definitions may change between versions. + ********************************************************************/ + +#define LZ4HC_DICTIONARY_LOGSIZE 16 +#define LZ4HC_MAXD (1<= LZ4HC_CLEVEL_OPT_MIN. + */ +LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed( + LZ4_streamHC_t* LZ4_streamHCPtr, int favor); + +/*! LZ4_resetStreamHC_fast() : v1.9.0+ + * When an LZ4_streamHC_t is known to be in a internally coherent state, + * it can often be prepared for a new compression with almost no work, only + * sometimes falling back to the full, expensive reset that is always required + * when the stream is in an indeterminate state (i.e., the reset performed by + * LZ4_resetStreamHC()). + * + * LZ4_streamHCs are guaranteed to be in a valid state when: + * - returned from LZ4_createStreamHC() + * - reset by LZ4_resetStreamHC() + * - memset(stream, 0, sizeof(LZ4_streamHC_t)) + * - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast() + * - the stream was in a valid state and was then used in any compression call + * that returned success + * - the stream was in an indeterminate state and was used in a compression + * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that + * returned success + * + * Note: + * A stream that was last used in a compression call that returned an error + * may be passed to this function. However, it will be fully reset, which will + * clear any existing history and settings from the context. + */ +LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast( + LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); + +/*! LZ4_compress_HC_extStateHC_fastReset() : + * A variant of LZ4_compress_HC_extStateHC(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStreamHC_fast() for a definition of + * "correctly initialized"). From a high level, the difference is that this + * function initializes the provided state with a call to + * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a + * call to LZ4_resetStreamHC(). + */ +LZ4LIB_STATIC_API int LZ4_compress_HC_extStateHC_fastReset ( + void* state, + const char* src, char* dst, + int srcSize, int dstCapacity, + int compressionLevel); + +#if defined (__cplusplus) +} +#endif + +#endif /* LZ4_HC_SLO_098092834 */ +#endif /* LZ4_HC_STATIC_LINKING_ONLY */ diff --git a/deps/lz4/xxhash.c b/deps/lz4/xxhash.c new file mode 100644 index 00000000000..82c31ec29cb --- /dev/null +++ b/deps/lz4/xxhash.c @@ -0,0 +1,1032 @@ +/* +* xxHash - Fast Hash algorithm +* Copyright (C) 2012-2016, Yann Collet +* +* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following disclaimer +* in the documentation and/or other materials provided with the +* distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +* You can contact the author at : +* - xxHash homepage: http://www.xxhash.com +* - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + + +/* ************************************* +* Tuning parameters +***************************************/ +/*!XXH_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. + * It can generate buggy code on targets which do not support unaligned memory accesses. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://stackoverflow.com/a/32095106/646947 for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \ + || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \ + || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define XXH_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \ + || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \ + || defined(__ARM_ARCH_7S__) || defined(__aarch64__) )) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/*!XXH_ACCEPT_NULL_INPUT_POINTER : + * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault. + * When this macro is enabled, xxHash actively checks input for null pointer. + * It it is, result for null input pointers is the same as a null-length input. + */ +#ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ +# define XXH_ACCEPT_NULL_INPUT_POINTER 0 +#endif + +/*!XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independence be of no importance for your application, you may set the #define below to 1, + * to improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ +#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ +# define XXH_FORCE_NATIVE_FORMAT 0 +#endif + +/*!XXH_FORCE_ALIGN_CHECK : + * This is a minor performance trick, only useful with lots of very small keys. + * It means : check for aligned/unaligned input. + * The check costs one initial branch per hash; + * set it to 0 when the input is guaranteed to be aligned, + * or when alignment doesn't matter for performance. + */ +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_FORCE_ALIGN_CHECK 0 +# elif defined(__aarch64__) +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/*! Modify the local functions below should you wish to use some other memory routines +* for malloc(), free() */ +#include +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +/*! and for memcpy() */ +#include +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + +#include /* assert */ + +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#if defined (_MSC_VER) && !defined (__clang__) /* MSVC */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# define FORCE_INLINE static __forceinline +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# if defined (__GNUC__) || defined (__clang__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************* +* Basic Types +***************************************/ +#ifndef MEM_MODULE +# if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; +# else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; +# endif +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U32 u32; } __attribute__((packed)) unalign; +static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ +static U32 XXH_read32(const void* memPtr) +{ + U32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* ************************************* +* Architecture Macros +***************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianness; + +/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ +#ifndef XXH_CPU_LITTLE_ENDIAN +static int XXH_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +#endif + + +/* *************************** +* Memory reads +*****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianness endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianness endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +static U32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} + + +/* ************************************* +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_sa = 1/(int)(!!(c)) }; } /* use after variable declarations */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +static const U32 PRIME32_1 = 2654435761U; +static const U32 PRIME32_2 = 2246822519U; +static const U32 PRIME32_3 = 3266489917U; +static const U32 PRIME32_4 = 668265263U; +static const U32 PRIME32_5 = 374761393U; + +static U32 XXH32_round(U32 seed, U32 input) +{ + seed += input * PRIME32_2; + seed = XXH_rotl32(seed, 13); + seed *= PRIME32_1; + return seed; +} + +/* mix all bits */ +static U32 XXH32_avalanche(U32 h32) +{ + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +static U32 +XXH32_finalize(U32 h32, const void* ptr, size_t len, + XXH_endianness endian, XXH_alignment align) + +{ + const BYTE* p = (const BYTE*)ptr; + +#define PROCESS1 \ + h32 += (*p++) * PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; + +#define PROCESS4 \ + h32 += XXH_get32bits(p) * PRIME32_3; \ + p+=4; \ + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + + switch(len&15) /* or switch(bEnd - p) */ + { + case 12: PROCESS4; + /* fallthrough */ + case 8: PROCESS4; + /* fallthrough */ + case 4: PROCESS4; + return XXH32_avalanche(h32); + + case 13: PROCESS4; + /* fallthrough */ + case 9: PROCESS4; + /* fallthrough */ + case 5: PROCESS4; + PROCESS1; + return XXH32_avalanche(h32); + + case 14: PROCESS4; + /* fallthrough */ + case 10: PROCESS4; + /* fallthrough */ + case 6: PROCESS4; + PROCESS1; + PROCESS1; + return XXH32_avalanche(h32); + + case 15: PROCESS4; + /* fallthrough */ + case 11: PROCESS4; + /* fallthrough */ + case 7: PROCESS4; + /* fallthrough */ + case 3: PROCESS1; + /* fallthrough */ + case 2: PROCESS1; + /* fallthrough */ + case 1: PROCESS1; + /* fallthrough */ + case 0: return XXH32_avalanche(h32); + } + assert(0); + return h32; /* reaching this point is deemed impossible */ +} + + +FORCE_INLINE U32 +XXH32_endian_align(const void* input, size_t len, U32 seed, + XXH_endianness endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; + +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) { + const BYTE* const limit = bEnd - 15; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; + v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; + v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; + v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; + } while (p < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + PRIME32_5; + } + + h32 += (U32)len; + + return XXH32_finalize(h32, p, len&15, endian, align); +} + + +XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, input, len); + return XXH32_digest(&state); +#else + XXH_endianness endian_detected = (XXH_endianness)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + + +/*====== Hash streaming ======*/ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.v1 = seed + PRIME32_1 + PRIME32_2; + state.v2 = seed + PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME32_1; + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); + return XXH_OK; +} + + +FORCE_INLINE XXH_errorcode +XXH32_update_endian(XXH32_state_t* state, const void* input, size_t len, XXH_endianness endian) +{ + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; +#endif + + { const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + + state->total_len_32 += (unsigned)len; + state->large_len |= (len>=16) | (state->total_len_32>=16); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (unsigned)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianness endian_detected = (XXH_endianness)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + +FORCE_INLINE U32 +XXH32_digest_endian (const XXH32_state_t* state, XXH_endianness endian) +{ + U32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v1, 1) + + XXH_rotl32(state->v2, 7) + + XXH_rotl32(state->v3, 12) + + XXH_rotl32(state->v4, 18); + } else { + h32 = state->v3 /* == seed */ + PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, state->mem32, state->memsize, endian, XXH_aligned); +} + + +XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianness endian_detected = (XXH_endianness)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + +/*====== Canonical representation ======*/ + +/*! Default XXH result types are basic unsigned 32 and 64 bits. +* The canonical representation follows human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file or buffer, remaining comparable across different systems. +*/ + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ + +/*====== Memory access ======*/ + +#ifndef MEM_MODULE +# define MEM_MODULE +# if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t U64; +# else + /* if compiler doesn't support unsigned long long, replace by another 64-bit type */ + typedef unsigned long long U64; +# endif +#endif + + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64; +static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; } + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ + +static U64 XXH_read64(const void* memPtr) +{ + U64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianness endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianness endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + +static U64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} + + +/*====== xxh64 ======*/ + +static const U64 PRIME64_1 = 11400714785074694791ULL; +static const U64 PRIME64_2 = 14029467366897019727ULL; +static const U64 PRIME64_3 = 1609587929392839161ULL; +static const U64 PRIME64_4 = 9650029242287828579ULL; +static const U64 PRIME64_5 = 2870177450012600261ULL; + +static U64 XXH64_round(U64 acc, U64 input) +{ + acc += input * PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= PRIME64_1; + return acc; +} + +static U64 XXH64_mergeRound(U64 acc, U64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; +} + +static U64 XXH64_avalanche(U64 h64) +{ + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +static U64 +XXH64_finalize(U64 h64, const void* ptr, size_t len, + XXH_endianness endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)ptr; + +#define PROCESS1_64 \ + h64 ^= (*p++) * PRIME64_5; \ + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + +#define PROCESS4_64 \ + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \ + p+=4; \ + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + +#define PROCESS8_64 { \ + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \ + p+=8; \ + h64 ^= k1; \ + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \ +} + + switch(len&31) { + case 24: PROCESS8_64; + /* fallthrough */ + case 16: PROCESS8_64; + /* fallthrough */ + case 8: PROCESS8_64; + return XXH64_avalanche(h64); + + case 28: PROCESS8_64; + /* fallthrough */ + case 20: PROCESS8_64; + /* fallthrough */ + case 12: PROCESS8_64; + /* fallthrough */ + case 4: PROCESS4_64; + return XXH64_avalanche(h64); + + case 25: PROCESS8_64; + /* fallthrough */ + case 17: PROCESS8_64; + /* fallthrough */ + case 9: PROCESS8_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 29: PROCESS8_64; + /* fallthrough */ + case 21: PROCESS8_64; + /* fallthrough */ + case 13: PROCESS8_64; + /* fallthrough */ + case 5: PROCESS4_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 26: PROCESS8_64; + /* fallthrough */ + case 18: PROCESS8_64; + /* fallthrough */ + case 10: PROCESS8_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 30: PROCESS8_64; + /* fallthrough */ + case 22: PROCESS8_64; + /* fallthrough */ + case 14: PROCESS8_64; + /* fallthrough */ + case 6: PROCESS4_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 27: PROCESS8_64; + /* fallthrough */ + case 19: PROCESS8_64; + /* fallthrough */ + case 11: PROCESS8_64; + PROCESS1_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 31: PROCESS8_64; + /* fallthrough */ + case 23: PROCESS8_64; + /* fallthrough */ + case 15: PROCESS8_64; + /* fallthrough */ + case 7: PROCESS4_64; + /* fallthrough */ + case 3: PROCESS1_64; + /* fallthrough */ + case 2: PROCESS1_64; + /* fallthrough */ + case 1: PROCESS1_64; + /* fallthrough */ + case 0: return XXH64_avalanche(h64); + } + + /* impossible to reach */ + assert(0); + return 0; /* unreachable, but some compilers complain without it */ +} + +FORCE_INLINE U64 +XXH64_endian_align(const void* input, size_t len, U64 seed, + XXH_endianness endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U64 h64; + +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; + v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; + v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; + v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; + } while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + return XXH64_finalize(h64, p, len, endian, align); +} + + +XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, input, len); + return XXH64_digest(&state); +#else + XXH_endianness endian_detected = (XXH_endianness)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +/*====== Hash Streaming ======*/ + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) +{ + XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.v1 = seed + PRIME64_1 + PRIME64_2; + state.v2 = seed + PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME64_1; + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); + return XXH_OK; +} + +FORCE_INLINE XXH_errorcode +XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianness endian) +{ + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; +#endif + + { const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianness endian_detected = (XXH_endianness)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianness endian) +{ + U64 h64; + + if (state->total_len >= 32) { + U64 const v1 = state->v1; + U64 const v2 = state->v2; + U64 const v3 = state->v3; + U64 const v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { + h64 = state->v3 /*seed*/ + PRIME64_5; + } + + h64 += (U64) state->total_len; + + return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, endian, XXH_aligned); +} + +XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianness endian_detected = (XXH_endianness)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + +/*====== Canonical representation ======*/ + +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#endif /* XXH_NO_LONG_LONG */ diff --git a/deps/lz4/xxhash.h b/deps/lz4/xxhash.h new file mode 100644 index 00000000000..d6bad943358 --- /dev/null +++ b/deps/lz4/xxhash.h @@ -0,0 +1,328 @@ +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************** +* Definitions +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/* **************************** + * API modifier + ******************************/ +/** XXH_INLINE_ALL (and XXH_PRIVATE_API) + * This is useful to include xxhash functions in `static` mode + * in order to inline them, and remove their symbol from the public list. + * Inlining can offer dramatic performance improvement on small keys. + * Methodology : + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * `xxhash.c` is automatically included. + * It's not useful to compile and link it as a separate module. + */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +# endif +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif +#else +# define XXH_PUBLIC_API /* do nothing */ +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/*! XXH_NAMESPACE, aka Namespace Emulation : + * + * If you want to include _and expose_ xxHash functions from within your own library, + * but also want to avoid symbol collisions with other libraries which may also include xxHash, + * + * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library + * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). + * + * Note that no change is required within the calling program as long as it includes `xxhash.h` : + * regular symbol name will be automatically translated by this header. + */ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 6 +#define XXH_VERSION_RELEASE 5 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +typedef unsigned int XXH32_hash_t; + +/*! XXH32() : + Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); + +/*====== Streaming ======*/ +typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +/* + * Streaming functions generate the xxHash of an input provided in multiple segments. + * Note that, for small input, they are slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * XXH state must first be allocated, using XXH*_createState() . + * + * Start a new hash by initializing state with a seed, using XXH*_reset(). + * + * Then, feed the hash state by calling XXH*_update() as many times as necessary. + * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using XXH*_digest(). + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a digest, + * and generate some new hashes later on, by calling again XXH*_digest(). + * + * When done, free XXH state space if it was allocated dynamically. + */ + +/*====== Canonical representation ======*/ + +typedef struct { unsigned char digest[4]; } XXH32_canonical_t; +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + +/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. + * The canonical representation uses human-readable write convention, aka big-endian (large digits first). + * These functions allow transformation of hash result into and from its canonical format. + * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. + */ + + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +typedef unsigned long long XXH64_hash_t; + +/*! XXH64() : + Calculate the 64-bit hash of sequence of length "len" stored at memory address "input". + "seed" can be used to alter the result predictably. + This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark). +*/ +XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); + +/*====== Streaming ======*/ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/*====== Canonical representation ======*/ +typedef struct { unsigned char digest[8]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); +#endif /* XXH_NO_LONG_LONG */ + + + +#ifdef XXH_STATIC_LINKING_ONLY + +/* ================================================================================================ + This section contains declarations which are not guaranteed to remain stable. + They may change in future versions, becoming incompatible with a different version of the library. + These declarations should only be used with static linking. + Never use them in association with dynamic linking ! +=================================================================================================== */ + +/* These definitions are only present to allow + * static allocation of XXH state, on stack or in a struct for example. + * Never **ever** use members directly. */ + +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + +struct XXH32_state_s { + uint32_t total_len_32; + uint32_t large_len; + uint32_t v1; + uint32_t v2; + uint32_t v3; + uint32_t v4; + uint32_t mem32[4]; + uint32_t memsize; + uint32_t reserved; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +struct XXH64_state_s { + uint64_t total_len; + uint64_t v1; + uint64_t v2; + uint64_t v3; + uint64_t v4; + uint64_t mem64[4]; + uint32_t memsize; + uint32_t reserved[2]; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ + +# else + +struct XXH32_state_s { + unsigned total_len_32; + unsigned large_len; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; + unsigned memsize; + unsigned reserved; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +# ifndef XXH_NO_LONG_LONG /* remove 64-bit support */ +struct XXH64_state_s { + unsigned long long total_len; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; + unsigned memsize; + unsigned reserved[2]; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ +# endif + +# endif + + +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ +#endif + +#endif /* XXH_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* XXHASH_H_5627135585666179 */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dcf36903da3..71452b7f20f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,6 +11,7 @@ get_valkey_server_linker_option(VALKEY_SERVER_LDFLAGS) list(APPEND SERVER_LIBS "fpconv") list(APPEND SERVER_LIBS "hdr_histogram") list(APPEND SERVER_LIBS "ffc") +list(APPEND SERVER_LIBS "lz4") valkey_build_and_install_bin(valkey-server "${VALKEY_SERVER_SRCS}" "${VALKEY_SERVER_LDFLAGS}" "${SERVER_LIBS}" "redis-server") add_dependencies(valkey-server generate_commands_def) diff --git a/src/Makefile b/src/Makefile index 2930c527094..8d49f2f2853 100644 --- a/src/Makefile +++ b/src/Makefile @@ -31,7 +31,7 @@ endif ifneq ($(OPTIMIZATION),-O0) OPTIMIZATION+=-fno-omit-frame-pointer endif -DEPENDENCY_TARGETS=libvalkey linenoise hdr_histogram fpconv +DEPENDENCY_TARGETS=libvalkey linenoise hdr_histogram fpconv lz4 NODEPS:=clean distclean # Default settings @@ -256,7 +256,7 @@ ifdef OPENSSL_PREFIX endif # Include paths to dependencies -FINAL_CFLAGS+= -I../deps/libvalkey/include -I../deps/linenoise -I../deps/hdr_histogram -I../deps/fpconv -I../deps/fast_float +FINAL_CFLAGS+= -I../deps/libvalkey/include -I../deps/linenoise -I../deps/hdr_histogram -I../deps/fpconv -I../deps/fast_float -I../deps/lz4 # Lua scripting engine module ifeq ($(BUILD_LUA),no) @@ -488,6 +488,10 @@ ENGINE_SERVER_OBJ = \ cluster_slot_stats.o \ commandlog.o \ commands.o \ + compression.o \ + compression_lz4.o \ + compression_stream.o \ + compression_repl.o \ config.o \ connection.o \ crc16.o \ @@ -709,7 +713,7 @@ endif # valkey-server $(SERVER_NAME): $(ENGINE_SERVER_OBJ) $(LUA_MODULE) - $(SERVER_LD) -o $@ $(ENGINE_SERVER_OBJ) ../deps/libvalkey/lib/libvalkey.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS) $(LUA_LDFLAGS) + $(SERVER_LD) -o $@ $(ENGINE_SERVER_OBJ) ../deps/libvalkey/lib/libvalkey.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a ../deps/lz4/liblz4.a $(FINAL_LIBS) $(LUA_LDFLAGS) # Valkey static library, used to compile against for unit testing $(ENGINE_LIB_NAME): $(ENGINE_SERVER_OBJ) diff --git a/src/compression.c b/src/compression.c new file mode 100644 index 00000000000..be315343ab0 --- /dev/null +++ b/src/compression.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "compression.h" +#include "compression_lz4.h" +#include "server.h" +#include "serverassert.h" +#include + +const char *compressionAlgoName(compressionAlgo algo) { + switch (algo) { + case ALGO_NONE: + return "none"; + case ALGO_LZF: + return "lzf"; + case ALGO_LZ4: + return "lz4"; + default: + return "unknown"; + } +} + +/* ===== Compressor ===== */ + +int streamCompressorInit(streamCompressor *compressor, + compressionAlgo algo, + int level, + bool codec_checksum) { + memset(compressor, 0, sizeof(*compressor)); + compressor->algo = algo; + compressor->level = level; + compressor->codec_checksum = codec_checksum; + compressor->content_checksum = codec_checksum; + + switch (algo) { + case ALGO_LZ4: + return compressionLz4CompressorInit(compressor); + default: + return C_ERR; + } +} + +void streamCompressorSetContentChecksum(streamCompressor *compressor, bool enabled) { + /* The flag is written into the frame header, so it cannot change once the + * stream has started. */ + assert(!compressor->stream_started); + compressor->content_checksum = enabled; +} + +size_t streamCompressorOutputBound(const streamCompressor *compressor, size_t input_len) { + switch (compressor->algo) { + case ALGO_LZ4: + return compressionLz4OutputBound(input_len); + default: + panic("Unsupported stream compression algorithm: %d", compressor->algo); + } +} + +ssize_t streamCompressorFeed(streamCompressor *compressor, + uint8_t *output, + size_t output_capacity, + const uint8_t *input, + size_t input_len, + compressFlushMode flush_mode) { + switch (compressor->algo) { + case ALGO_LZ4: + return compressionLz4CompressFeed(compressor, output, output_capacity, input, input_len, flush_mode); + default: + panic("Unsupported stream compression algorithm: %d", compressor->algo); + } +} + +void streamCompressorFree(streamCompressor *compressor) { + switch (compressor->algo) { + case ALGO_LZ4: + compressionLz4CompressorFree(compressor); + break; + default: + break; + } +} + +/* ===== Decompressor ===== */ + +int streamDecompressorInit(streamDecompressor *decompressor, + compressionAlgo algo, + bool skip_codec_checksum_validation) { + memset(decompressor, 0, sizeof(*decompressor)); + decompressor->algo = algo; + decompressor->skip_codec_checksum_validation = skip_codec_checksum_validation; + + switch (algo) { + case ALGO_LZ4: + return compressionLz4DecompressorInit(decompressor); + default: + return C_ERR; + } +} + +ssize_t streamDecompressorFeed(streamDecompressor *decompressor, + uint8_t *output, + size_t output_capacity, + const uint8_t *input, + size_t input_len, + size_t *input_consumed) { + *input_consumed = 0; + if (decompressor->frame_done) return 0; + + switch (decompressor->algo) { + case ALGO_LZ4: + return compressionLz4DecompressFeed(decompressor, output, output_capacity, + input, input_len, input_consumed); + default: + panic("Unsupported stream decompression algorithm: %d", decompressor->algo); + } +} + +void streamDecompressorFree(streamDecompressor *decompressor) { + switch (decompressor->algo) { + case ALGO_NONE: + break; + case ALGO_LZ4: + compressionLz4DecompressorFree(decompressor); + break; + default: + panic("Unsupported stream decompression algorithm: %d", decompressor->algo); + } +} diff --git a/src/compression.h b/src/compression.h new file mode 100644 index 00000000000..c526690eadd --- /dev/null +++ b/src/compression.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef COMPRESSION_H +#define COMPRESSION_H + +#include "fmacros.h" +#include +#include +#include +#include + +typedef enum { + ALGO_NONE = 0, + ALGO_LZF = 1, /* Per-string LZF inside the RDB payload (legacy). */ + ALGO_LZ4 = 2, +} compressionAlgo; + +typedef enum { + COMPRESS_FLUSH_CONTINUE = 0, /* Buffer internally. */ + COMPRESS_FLUSH_SYNC = 1, /* Drain buffered bytes, keep frame open. */ + COMPRESS_FLUSH_END = 2, /* Finalize frame. */ +} compressFlushMode; + +/* Returns a static algorithm name for logs and config output. */ +const char *compressionAlgoName(compressionAlgo algo); + +/* ===== Compressor ===== */ + +typedef struct { + compressionAlgo algo; + int level; /* 0 selects the codec default. */ + void *ctx; + bool stream_started; + bool codec_checksum; + bool content_checksum; /* Whole-frame checksum; defaults to codec_checksum. Off for + * streams that never end their frame (it would be computed + * on every byte but never emitted or validated). */ +} streamCompressor; + +/* Compressor lifecycle. Codec dispatch used by streamWriter; the writer owns + * sticky error state while these functions manage only codec state. */ +int streamCompressorInit(streamCompressor *compressor, compressionAlgo algo, int level, bool codec_checksum); +/* Override the frame content checksum. Must be called before the stream starts. */ +void streamCompressorSetContentChecksum(streamCompressor *compressor, bool enabled); +size_t streamCompressorOutputBound(const streamCompressor *compressor, size_t input_len); +/* Feeds raw input into the compressor and writes compressed bytes to output. + * Called repeatedly to build a complete frame: COMPRESS_FLUSH_CONTINUE keeps + * buffering, COMPRESS_FLUSH_SYNC drains buffered bytes but leaves the frame + * open, and COMPRESS_FLUSH_END closes it. output must be at least + * streamCompressorOutputBound(compressor, input_len) bytes. Returns bytes + * written, or -1 on error. */ +ssize_t streamCompressorFeed(streamCompressor *compressor, + uint8_t *output, + size_t output_capacity, + const uint8_t *input, + size_t input_len, + compressFlushMode flush_mode); +void streamCompressorFree(streamCompressor *compressor); + +/* ===== Decompressor ===== */ + +typedef struct { + compressionAlgo algo; + bool frame_done; + bool skip_codec_checksum_validation; + void *ctx; + size_t input_hint; /* Preferred compressed bytes for next feed, 0 if unknown. */ +} streamDecompressor; + +/* Decompressor lifecycle. Codec dispatch used by streamReader; the reader owns + * buffering and sticky error state. */ +int streamDecompressorInit(streamDecompressor *decompressor, + compressionAlgo algo, + bool skip_codec_checksum_validation); +ssize_t streamDecompressorFeed(streamDecompressor *decompressor, + uint8_t *output, + size_t output_capacity, + const uint8_t *input, + size_t input_len, + size_t *input_consumed); +void streamDecompressorFree(streamDecompressor *decompressor); + +#endif /* COMPRESSION_H */ diff --git a/src/compression_lz4.c b/src/compression_lz4.c new file mode 100644 index 00000000000..d4e344185a0 --- /dev/null +++ b/src/compression_lz4.c @@ -0,0 +1,166 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "compression_lz4.h" +#include "server.h" +#include "serverassert.h" +#include "zmalloc.h" +#include + +#define LZ4F_STATIC_LINKING_ONLY +#include + +static void *lz4Zmalloc(void *opaque, size_t size) { + (void)opaque; + return zmalloc(size); +} + +static void *lz4Zcalloc(void *opaque, size_t size) { + (void)opaque; + return zcalloc(size); +} + +static void lz4Zfree(void *opaque, void *address) { + (void)opaque; + zfree(address); +} + +static const LZ4F_CustomMem lz4f_mem = { + .customAlloc = lz4Zmalloc, + .customCalloc = lz4Zcalloc, + .customFree = lz4Zfree, + .opaqueState = NULL, +}; + +/* Shared bound-calc preferences. The actual compress level and checksum mode + * are overridden per stream before LZ4F_compressBegin. */ +static const LZ4F_preferences_t lz4f_prefs = { + .frameInfo = { + .blockChecksumFlag = LZ4F_blockChecksumEnabled, + .contentChecksumFlag = LZ4F_contentChecksumEnabled, + .blockSizeID = LZ4F_max64KB, + .blockMode = LZ4F_blockLinked, + }, + .compressionLevel = 0, +}; + +/* ===== Compressor ===== */ + +int compressionLz4CompressorInit(streamCompressor *compressor) { + compressor->ctx = LZ4F_createCompressionContext_advanced(lz4f_mem, LZ4F_VERSION); + return compressor->ctx != NULL ? C_OK : C_ERR; +} + +size_t compressionLz4OutputBound(size_t input_len) { + return LZ4F_compressBound(input_len, &lz4f_prefs) + LZ4F_HEADER_SIZE_MAX + LZ4F_compressBound(0, &lz4f_prefs); +} + +ssize_t compressionLz4CompressFeed(streamCompressor *compressor, + uint8_t *output, + size_t output_capacity, + const uint8_t *input, + size_t input_len, + compressFlushMode flush_mode) { + assert(compressor->ctx != NULL); + + LZ4F_cctx *cctx = (LZ4F_cctx *)compressor->ctx; + size_t offset = 0; + + if (!compressor->stream_started) { + LZ4F_preferences_t prefs = lz4f_prefs; + prefs.compressionLevel = compressor->level; + prefs.frameInfo.blockChecksumFlag = compressor->codec_checksum + ? LZ4F_blockChecksumEnabled + : LZ4F_noBlockChecksum; + prefs.frameInfo.contentChecksumFlag = compressor->content_checksum + ? LZ4F_contentChecksumEnabled + : LZ4F_noContentChecksum; + size_t r = LZ4F_compressBegin(cctx, output, output_capacity, &prefs); + if (LZ4F_isError(r)) return -1; + offset = r; + compressor->stream_started = true; + } + + if (input_len > 0) { + if (offset >= output_capacity) return -1; + size_t r = LZ4F_compressUpdate(cctx, output + offset, output_capacity - offset, input, input_len, NULL); + if (LZ4F_isError(r)) return -1; + offset += r; + } + + switch (flush_mode) { + case COMPRESS_FLUSH_CONTINUE: + break; + case COMPRESS_FLUSH_SYNC: { + /* Replication batch path caller: drain buffered codec bytes, keep frame open. */ + if (offset >= output_capacity) return -1; + size_t r = LZ4F_flush(cctx, output + offset, output_capacity - offset, NULL); + if (LZ4F_isError(r)) return -1; + offset += r; + break; + } + case COMPRESS_FLUSH_END: { + if (offset >= output_capacity) return -1; + size_t r = LZ4F_compressEnd(cctx, output + offset, output_capacity - offset, NULL); + if (LZ4F_isError(r)) return -1; + offset += r; + compressor->stream_started = false; + break; + } + default: + panic("Invalid compression flush mode: %d", flush_mode); + } + + if (offset > (size_t)SSIZE_MAX) return -1; + return (ssize_t)offset; +} + +void compressionLz4CompressorFree(streamCompressor *compressor) { + if (compressor->ctx) { + LZ4F_freeCompressionContext((LZ4F_cctx *)compressor->ctx); + compressor->ctx = NULL; + } +} + +/* ===== Decompressor ===== */ + +int compressionLz4DecompressorInit(streamDecompressor *decompressor) { + decompressor->ctx = LZ4F_createDecompressionContext_advanced(lz4f_mem, LZ4F_VERSION); + if (decompressor->ctx == NULL) return C_ERR; + decompressor->input_hint = LZ4F_HEADER_SIZE_MIN; + return C_OK; +} + +ssize_t compressionLz4DecompressFeed(streamDecompressor *decompressor, + uint8_t *output, + size_t output_capacity, + const uint8_t *input, + size_t input_len, + size_t *input_consumed) { + assert(decompressor->ctx != NULL); + *input_consumed = 0; + if (decompressor->frame_done) return 0; + + LZ4F_dctx *dctx = (LZ4F_dctx *)decompressor->ctx; + size_t dst_size = output_capacity; + size_t src_size = input_len; + LZ4F_decompressOptions_t options = { + .skipChecksums = decompressor->skip_codec_checksum_validation, + }; + size_t ret = LZ4F_decompress(dctx, output, &dst_size, input, &src_size, &options); + if (LZ4F_isError(ret)) return -1; + *input_consumed = src_size; + decompressor->input_hint = ret; + if (ret == 0) decompressor->frame_done = true; + return (ssize_t)dst_size; +} + +void compressionLz4DecompressorFree(streamDecompressor *decompressor) { + if (decompressor->ctx) { + LZ4F_freeDecompressionContext((LZ4F_dctx *)decompressor->ctx); + decompressor->ctx = NULL; + } +} diff --git a/src/compression_lz4.h b/src/compression_lz4.h new file mode 100644 index 00000000000..cf6518a5932 --- /dev/null +++ b/src/compression_lz4.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef COMPRESSION_LZ4_H +#define COMPRESSION_LZ4_H + +#include "compression.h" + +/* Compressor lifecycle. Init returns C_OK/C_ERR. OutputBound includes enough + * space for frame start and any requested flush mode. Feed returns -1 on codec + * failure; input may be NULL when input_len is zero. */ +int compressionLz4CompressorInit(streamCompressor *compressor); +size_t compressionLz4OutputBound(size_t input_len); +ssize_t compressionLz4CompressFeed(streamCompressor *compressor, + uint8_t *output, + size_t output_capacity, + const uint8_t *input, + size_t input_len, + compressFlushMode flush_mode); +void compressionLz4CompressorFree(streamCompressor *compressor); + +/* Decompressor lifecycle. Init returns C_OK/C_ERR. Feed returns produced bytes + * or -1 and reports consumed compressed bytes through input_consumed. A zero + * return does not by itself mean frame end; callers inspect + * decompressor->frame_done. */ +int compressionLz4DecompressorInit(streamDecompressor *decompressor); +ssize_t compressionLz4DecompressFeed(streamDecompressor *decompressor, + uint8_t *output, + size_t output_capacity, + const uint8_t *input, + size_t input_len, + size_t *input_consumed); +void compressionLz4DecompressorFree(streamDecompressor *decompressor); + +#endif /* COMPRESSION_LZ4_H */ diff --git a/src/compression_repl.c b/src/compression_repl.c new file mode 100644 index 00000000000..74034ac5a94 --- /dev/null +++ b/src/compression_repl.c @@ -0,0 +1,216 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "compression_repl.h" +#include "server.h" +#include "zmalloc.h" + +/* Release the decode scratch buffer once it grows past this so a replica does + * not retain peak allocation. ~16 x PROTO_REPLY_CHUNK_BYTES, kept local so the + * threshold reads clearly at its use site. */ +#define REPL_COMPRESSION_RETAIN_LIMIT (256 * 1024) + +/* Retain the encode staging buffer up to a full batch's LZ4 worst-case output. */ +#define REPL_COMPRESSION_ENCODE_RETAIN_LIMIT \ + (REPL_COMPRESSION_BATCH_LIMIT + (REPL_COMPRESSION_BATCH_LIMIT / 255) + 1024) + +/* Decoded-output room offered to the codec per feed iteration. */ +#define REPL_DECODE_CHUNK (16 * 1024) + +/* ===== Primary-side per-replica compressor ===== */ + +replCompressor *replCompressorCreate(compressionAlgo algo) { + replCompressor *rc = zcalloc(sizeof(*rc)); + + rc->out_buf = sdsempty(); + /* Block checksums ON (codec_checksum=true). */ + if (streamWriterInit(&rc->writer, algo, true, NULL, NULL) != C_OK) { + sdsfree(rc->out_buf); + zfree(rc); + return NULL; + } + streamWriterSetStreamKind(&rc->writer, VCS_STREAM_REPL); + /* Repl frames never end: a content checksum would never be emitted or + * validated. Block checksums stay. */ + streamCompressorSetContentChecksum(&rc->writer.compressor, false); + streamWriterSetSink(&rc->writer, &rc->out_buf); + return rc; +} + +void replCompressorDestroy(replCompressor *rc) { + if (!rc) return; + streamWriterFree(&rc->writer); + sdsfree(rc->out_buf); + zfree(rc); +} + +int replCompressorWrite(replCompressor *rc, const void *buf, size_t len) { + return streamWriterWrite(&rc->writer, buf, len); +} + +int replCompressorFlush(replCompressor *rc) { + return streamWriterFlush(&rc->writer); +} + +void replCompressorResetBatch(replCompressor *rc) { + /* Decide retention from the payload length, not sdsalloc: SDS grows capacity + * greedily, so a normal batch would otherwise be reclaimed and reallocated + * every cycle. A batch is capped at REPL_COMPRESSION_BATCH_LIMIT, so its + * output stays within the bound; only an oversized payload is reclaimed. */ + size_t used = sdslen(rc->out_buf); + sdsclear(rc->out_buf); + rc->out_buf_pos = 0; + rc->raw_bytes = 0; + if (used > REPL_COMPRESSION_ENCODE_RETAIN_LIMIT) { + sdsfree(rc->out_buf); + rc->out_buf = sdsempty(); + } +} + +size_t replCompressorMemUsage(const replCompressor *rc) { + if (!rc) return 0; + /* Codec context memory is small and fixed; only the staging SDS is measured. */ + size_t total = sizeof(*rc); + if (rc->out_buf) total += sdsalloc(rc->out_buf); + return total; +} + +compressionAlgo replCompressorAlgo(const replCompressor *rc) { + return rc ? rc->writer.compressor.algo : ALGO_NONE; +} + +/* ===== Replica-side decompressor ===== */ + +replDecompressor *replDecompressorCreate(void) { + replDecompressor *rd = zcalloc(sizeof(*rd)); + rd->decode_buf = sdsempty(); + return rd; +} + +void replDecompressorDestroy(replDecompressor *rd) { + if (!rd) return; + if (rd->mode == REPL_DECODE_MODE_COMPRESSED) streamDecompressorFree(&rd->decompressor); + sdsfree(rd->decode_buf); + zfree(rd); +} + +/* Append raw bytes to the decode buffer (passthrough), enforcing output_max. */ +static replDecodeResult replDecodeEmit(replDecompressor *rd, const uint8_t *in, size_t len, size_t output_max) { + if (len == 0) return REPL_DECODE_OK; + if (sdslen(rd->decode_buf) + len > output_max) return REPL_DECODE_OVERFLOW; + rd->decode_buf = sdscatlen(rd->decode_buf, in, len); + return REPL_DECODE_OK; +} + +/* Drain compressed bytes [in, in+len) through the codec into rd->decode_buf, + * bounded by output_max (decompression-bomb guard). */ +static replDecodeResult replDecodeFeed(replDecompressor *rd, const uint8_t *in, size_t len, size_t output_max) { + size_t off = 0; + size_t room = 0; + ssize_t produced = 0; + do { + size_t used = sdslen(rd->decode_buf); + /* Budget exhausted with more output possibly pending: the stream + * expands past the bomb-guard cap. Checked up front so the codec is + * never handed more room than the remaining budget allows. */ + if (used >= output_max) return REPL_DECODE_OVERFLOW; + room = REPL_DECODE_CHUNK; + if (room > output_max - used) room = output_max - used; + rd->decode_buf = sdsMakeRoomFor(rd->decode_buf, room); + size_t consumed = 0; + produced = streamDecompressorFeed(&rd->decompressor, + (uint8_t *)rd->decode_buf + used, + room, + in + off, len - off, &consumed); + if (produced < 0 || consumed > len - off) return REPL_DECODE_ERR; + if (produced > 0) sdsIncrLen(rd->decode_buf, (size_t)produced); + off += consumed; + /* A long-lived replication stream must never reach a compressed frame + * end. If it does, the stream is corrupt or the primary sent an + * unexpected terminator: the caller should disconnect. */ + if (rd->decompressor.frame_done) return REPL_DECODE_FRAME_DONE; + /* The codec always makes progress given input and output room; no + * progress with input still pending is a stuck state. Fail rather + * than let the caller drop the unconsumed tail. Gated on pending + * input: empty-input drain iterations legitimately produce 0. */ + if (off < len && consumed == 0 && produced == 0) return REPL_DECODE_ERR; + /* Keep draining with empty input while the codec may hold buffered + * output, which is only the case when it filled the entire room. */ + } while (off < len || (size_t)produced == room); + return REPL_DECODE_OK; +} + +replDecodeResult replDecompressorDecode(replDecompressor *rd, + const void *src, + size_t len, + size_t output_max, + size_t *out_len) { + static const uint8_t vcs_magic[VCS_MAGIC_SIZE] = {VCS_MAGIC_0, VCS_MAGIC_1, VCS_MAGIC_2}; + + if (out_len) *out_len = 0; + sdsclear(rd->decode_buf); + + const uint8_t *in = src; + size_t off = 0; + + /* Probe phase: classify the stream from its leading bytes. The magic may + * arrive split across feeds, so bytes accumulate until the prefix matches + * or rules out the VCS magic. */ + if (rd->mode == REPL_DECODE_MODE_PROBE) { + while (rd->envelope_len < VCS_MAGIC_SIZE && off < len) { + if (in[off] != vcs_magic[rd->envelope_len]) { + rd->mode = REPL_DECODE_MODE_PASSTHROUGH; /* Not a VCS stream. */ + break; + } + rd->envelope[rd->envelope_len++] = in[off++]; + } + + if (rd->mode == REPL_DECODE_MODE_PROBE) { + /* Magic matches so far; gather the rest of the envelope. */ + while (rd->envelope_len < VCS_ENVELOPE_SIZE && off < len) rd->envelope[rd->envelope_len++] = in[off++]; + if (rd->envelope_len < VCS_ENVELOPE_SIZE) return REPL_DECODE_OK; /* Need more header. */ + + compressionAlgo algo = ALGO_NONE; + if (streamParseVcsEnvelope(rd->envelope, VCS_ENVELOPE_SIZE, VCS_STREAM_REPL, &algo) != C_OK) + return REPL_DECODE_ERR; + if (streamDecompressorInit(&rd->decompressor, algo, false) != C_OK) return REPL_DECODE_ERR; + rd->mode = REPL_DECODE_MODE_COMPRESSED; + } + } + + if (rd->mode == REPL_DECODE_MODE_PASSTHROUGH) { + /* Replay any buffered magic-prefix bytes once, then forward the rest. */ + replDecodeResult r = replDecodeEmit(rd, rd->envelope, rd->envelope_len, output_max); + rd->envelope_len = 0; + if (r != REPL_DECODE_OK) return r; + r = replDecodeEmit(rd, in + off, len - off, output_max); + if (r != REPL_DECODE_OK) return r; + } else if (off < len) { + replDecodeResult r = replDecodeFeed(rd, in + off, len - off, output_max); + if (r != REPL_DECODE_OK) return r; + } + + /* Shrink the scratch buffer if it grew large and is now mostly empty. */ + if (sdsalloc(rd->decode_buf) > REPL_COMPRESSION_RETAIN_LIMIT && + sdslen(rd->decode_buf) < sdsalloc(rd->decode_buf) / 4) { + rd->decode_buf = sdsRemoveFreeSpace(rd->decode_buf, 0); + } + + if (out_len) *out_len = sdslen(rd->decode_buf); + return REPL_DECODE_OK; +} + +sds replDecompressorBuf(replDecompressor *rd) { + return rd ? rd->decode_buf : NULL; +} + +bool replDecompressorIsPassthrough(const replDecompressor *rd) { + return rd && rd->mode == REPL_DECODE_MODE_PASSTHROUGH; +} + +bool replDecompressorIsCompressed(const replDecompressor *rd) { + return rd && rd->mode == REPL_DECODE_MODE_COMPRESSED; +} diff --git a/src/compression_repl.h b/src/compression_repl.h new file mode 100644 index 00000000000..a7d7c5d1a8f --- /dev/null +++ b/src/compression_repl.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef COMPRESSION_REPL_H +#define COMPRESSION_REPL_H + +/* Replication compression adapter on top of compression_stream. The rio-based + * streamWriter/streamReader path serves blocking RDB save/load; replication + * reads a non-blocking socket in the event loop where a pull-mode reader + * cannot work, so this adapter is push-mode. The incoming stream may be + * compressed (VCS envelope) or plaintext (primary compresses only if its own + * config is enabled) and is classified from its leading bytes. */ + +#include "compression.h" +#include "compression_stream.h" +#include "sds.h" + +/* Max raw replication-backlog bytes compressed per write dispatch cycle; + * bounds per-batch latency and the staging buffer. */ +#define REPL_COMPRESSION_BATCH_LIMIT (1024 * 1024) + +/* ===== Primary-side per-replica compressor ===== */ + +typedef struct replCompressor { + streamWriter writer; /* VCS/LZ4 frame encoder (VCS_STREAM_REPL). */ + sds out_buf; /* Compressed bytes staged for the socket. */ + size_t out_buf_pos; /* Next unsent byte offset within out_buf. */ + size_t raw_bytes; /* Raw backlog bytes represented by out_buf. */ +} replCompressor; + +/* Compressor API. Write and Flush return C_OK on success and C_ERR on error; + * the encoder compresses directly into out_buf. Create returns NULL on error; + * Destroy is NULL-safe. */ +replCompressor *replCompressorCreate(compressionAlgo algo); +void replCompressorDestroy(replCompressor *rc); +int replCompressorWrite(replCompressor *rc, const void *buf, size_t len); +int replCompressorFlush(replCompressor *rc); +/* Clears the staging buffer for a new batch; reclaims only oversized payloads. */ +void replCompressorResetBatch(replCompressor *rc); +/* Approximate heap usage for client-output-buffer accounting. */ +size_t replCompressorMemUsage(const replCompressor *rc); +/* Algorithm the compressor was initialized with; ALGO_NONE if rc is NULL. */ +compressionAlgo replCompressorAlgo(const replCompressor *rc); + +/* ===== Replica-side decompressor ===== */ + +typedef enum { + REPL_DECODE_OK = 0, /* Decoded (possibly 0) bytes; need more input next tick. */ + REPL_DECODE_ERR = -1, /* IO/feed/decoder error: disconnect. */ + REPL_DECODE_FRAME_DONE = -2, /* Frame ended on a live link: protocol corruption. */ + REPL_DECODE_OVERFLOW = -3, /* Decoded output exceeded the bomb-guard cap. */ +} replDecodeResult; + +typedef enum { + REPL_DECODE_MODE_PROBE = 0, /* Still classifying the stream. */ + REPL_DECODE_MODE_COMPRESSED, /* VCS envelope seen; decoder initialized. */ + REPL_DECODE_MODE_PASSTHROUGH /* Non-VCS stream; bytes forwarded as-is. */ +} replDecodeMode; + +/* Decoder for the single primary link. Accumulates the leading bytes (they may + * span several reads), classifies the stream, then either feeds the codec or + * forwards plaintext untouched. */ +typedef struct replDecompressor { + streamDecompressor decompressor; /* Valid once mode == COMPRESSED. */ + replDecodeMode mode; + uint8_t envelope[VCS_ENVELOPE_SIZE]; /* Leading bytes gathered during PROBE. */ + size_t envelope_len; + sds decode_buf; /* Most recent decoded bytes. */ +} replDecompressor; + +/* Decoder API. Create returns NULL on error; Destroy is NULL-safe. Decode + * feeds len transport bytes and drains decoded output into decode_buf + * (cleared first). On REPL_DECODE_OK, *out_len is the decoded byte count + * (0 means a partial frame was buffered; resume next tick). output_max caps + * decoded output as a decompression-bomb guard. */ +replDecompressor *replDecompressorCreate(void); +void replDecompressorDestroy(replDecompressor *rd); +replDecodeResult replDecompressorDecode(replDecompressor *rd, + const void *src, + size_t len, + size_t output_max, + size_t *out_len); +/* Decode scratch buffer (valid bytes = sdslen) after a decode. */ +sds replDecompressorBuf(replDecompressor *rd); +/* True once the probe classified the stream as plaintext (non-VCS). */ +bool replDecompressorIsPassthrough(const replDecompressor *rd); +/* True once the probe classified the stream as compressed (VCS envelope). */ +bool replDecompressorIsCompressed(const replDecompressor *rd); + +#endif /* COMPRESSION_REPL_H */ diff --git a/src/compression_stream.c b/src/compression_stream.c new file mode 100644 index 00000000000..38065ec2596 --- /dev/null +++ b/src/compression_stream.c @@ -0,0 +1,494 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "compression_stream.h" +#include "server.h" +#include "serverassert.h" +#include "zmalloc.h" +#include +#include + +/* ===== VCS envelope ===== */ + +static const uint8_t VCS_MAGIC[VCS_MAGIC_SIZE] = { + VCS_MAGIC_0, + VCS_MAGIC_1, + VCS_MAGIC_2, +}; + +/* True when the first len bytes of buf match the VCS magic. When len is below + * VCS_MAGIC_SIZE this only compares that prefix. */ +static bool vcsHasMagicPrefix(const uint8_t *buf, size_t len) { + size_t n = len < VCS_MAGIC_SIZE ? len : VCS_MAGIC_SIZE; + return memcmp(buf, VCS_MAGIC, n) == 0; +} + +/* Fill the 7-byte VCS envelope. Returns C_ERR if algo has no wire codec id. */ +static int buildVcsEnvelope(uint8_t *out, compressionAlgo algo, uint8_t stream_kind) { + uint8_t codec; + switch (algo) { + case ALGO_LZ4: + codec = VCS_CODEC_LZ4; + break; + default: + return C_ERR; + } + + uint8_t envelope[VCS_ENVELOPE_SIZE] = { + VCS_MAGIC_0, + VCS_MAGIC_1, + VCS_MAGIC_2, + [VCS_OFFSET_VERSION] = VCS_VERSION, + [VCS_OFFSET_CODEC] = codec, + [VCS_OFFSET_RESERVED] = 0, + [VCS_OFFSET_STREAM_KIND] = stream_kind, + }; + memcpy(out, envelope, VCS_ENVELOPE_SIZE); + return C_OK; +} + +static int writeVcsEnvelope(streamWriterWriteFn write_cb, + void *ctx, + compressionAlgo algo, + uint8_t stream_kind) { + uint8_t envelope[VCS_ENVELOPE_SIZE]; + if (buildVcsEnvelope(envelope, algo, stream_kind) == C_ERR) return C_ERR; + return write_cb(ctx, envelope, VCS_ENVELOPE_SIZE); +} + +/* Reject a nonzero reserved byte so a future envelope extension fails loudly + * rather than being silently misinterpreted. */ +static int readVcsEnvelope(const uint8_t *buf, uint8_t expected_stream_kind, compressionAlgo *algo) { + if (buf[VCS_OFFSET_VERSION] != VCS_VERSION) return C_ERR; + + switch (buf[VCS_OFFSET_CODEC]) { + case VCS_CODEC_LZ4: + *algo = ALGO_LZ4; + break; + default: + return C_ERR; + } + if (buf[VCS_OFFSET_RESERVED] != 0) return C_ERR; + if (buf[VCS_OFFSET_STREAM_KIND] != expected_stream_kind) return C_ERR; + return C_OK; +} + +int streamParseVcsEnvelope(const uint8_t *buf, size_t len, uint8_t expected_stream_kind, compressionAlgo *algo) { + if (len < VCS_ENVELOPE_SIZE || !vcsHasMagicPrefix(buf, VCS_MAGIC_SIZE)) return C_ERR; + return readVcsEnvelope(buf, expected_stream_kind, algo); +} + +/* ===== Streaming Writer ===== */ + +#define STREAM_WRITER_INPUT_CHUNK_SIZE (1024 * 1024) + +int streamWriterInit(streamWriter *writer, compressionAlgo algo, bool codec_checksum, streamWriterWriteFn write_cb, void *write_ctx) { + memset(writer, 0, sizeof(*writer)); + writer->write_cb = write_cb; + writer->write_ctx = write_ctx; + writer->stream_kind = VCS_STREAM_RDB; + + if (streamCompressorInit(&writer->compressor, algo, 0, codec_checksum) == C_ERR) { + writer->state = STREAM_WRITER_STATE_ERROR; + return C_ERR; + } + return C_OK; +} + +void streamWriterSetSink(streamWriter *writer, sds *sink) { + /* Sink must be installed before the frame starts, so all output lands in + * one destination. */ + assert(writer->state == STREAM_WRITER_STATE_INITIAL); + writer->sink = sink; +} + +void streamWriterSetStreamKind(streamWriter *writer, uint8_t stream_kind) { + /* The kind is written into the envelope, so it can only change before the + * frame starts. */ + assert(writer->state == STREAM_WRITER_STATE_INITIAL); + writer->stream_kind = stream_kind; +} + +/* Envelope is emitted lazily so a writer that's created but never written + * doesn't leave a stub envelope on the sink. */ +static int streamWriterEnsureEnvelope(streamWriter *writer) { + if (writer->state == STREAM_WRITER_STATE_ACTIVE) return C_OK; + if (writer->state != STREAM_WRITER_STATE_INITIAL) return C_ERR; + assert(writer->sink != NULL || writer->write_cb != NULL); + if (writer->sink) { + uint8_t envelope[VCS_ENVELOPE_SIZE]; + if (buildVcsEnvelope(envelope, writer->compressor.algo, writer->stream_kind) == C_ERR) { + writer->state = STREAM_WRITER_STATE_ERROR; + return C_ERR; + } + *writer->sink = sdscatlen(*writer->sink, (char *)envelope, VCS_ENVELOPE_SIZE); + } else if (writeVcsEnvelope(writer->write_cb, writer->write_ctx, writer->compressor.algo, + writer->stream_kind) == C_ERR) { + writer->state = STREAM_WRITER_STATE_ERROR; + return C_ERR; + } + writer->state = STREAM_WRITER_STATE_ACTIVE; + return C_OK; +} + +/* Sink path: compress directly into the caller's sds tail. */ +static int streamWriterFeedToSink(streamWriter *writer, + const uint8_t *input, + size_t input_len, + compressFlushMode flush_mode) { + const size_t bound = streamCompressorOutputBound(&writer->compressor, input_len); + if (bound == 0) { + writer->state = STREAM_WRITER_STATE_ERROR; + return C_ERR; + } + *writer->sink = sdsMakeRoomFor(*writer->sink, bound); + const ssize_t compressed = streamCompressorFeed(&writer->compressor, + (uint8_t *)(*writer->sink) + sdslen(*writer->sink), + sdsavail(*writer->sink), input, input_len, flush_mode); + if (compressed < 0) { + writer->state = STREAM_WRITER_STATE_ERROR; + return C_ERR; + } + sdsIncrLen(*writer->sink, (size_t)compressed); + return C_OK; +} + +static int streamWriterFeedAndWrite(streamWriter *writer, + const uint8_t *input, + size_t input_len, + compressFlushMode flush_mode) { + if (writer->sink) return streamWriterFeedToSink(writer, input, input_len, flush_mode); + + const size_t needed = streamCompressorOutputBound(&writer->compressor, input_len); + if (needed > writer->out_buf_size) { + writer->out_buf = zrealloc(writer->out_buf, needed); + writer->out_buf_size = needed; + } + + const ssize_t compressed = streamCompressorFeed(&writer->compressor, writer->out_buf, + writer->out_buf_size, + input, input_len, flush_mode); + if (compressed < 0) { + writer->state = STREAM_WRITER_STATE_ERROR; + return C_ERR; + } + if (compressed > 0 && writer->write_cb(writer->write_ctx, writer->out_buf, (size_t)compressed) == C_ERR) { + writer->state = STREAM_WRITER_STATE_ERROR; + return C_ERR; + } + return C_OK; +} + +int streamWriterWrite(streamWriter *writer, const void *buf, size_t len) { + /* Writes after finish are a caller bug; silently dropping them would + * corrupt the consumer's view of the stream. */ + if (writer->state == STREAM_WRITER_STATE_FINISHED || writer->state == STREAM_WRITER_STATE_ERROR) return C_ERR; + if (len == 0) return C_OK; + + const uint8_t *src = (const uint8_t *)buf; + size_t remaining = len; + if (streamWriterEnsureEnvelope(writer) == C_ERR) return C_ERR; + while (remaining > 0) { + size_t chunk_len = remaining < STREAM_WRITER_INPUT_CHUNK_SIZE + ? remaining + : STREAM_WRITER_INPUT_CHUNK_SIZE; + if (streamWriterFeedAndWrite(writer, src, chunk_len, COMPRESS_FLUSH_CONTINUE) == C_ERR) return C_ERR; + src += chunk_len; + remaining -= chunk_len; + } + return C_OK; +} + +int streamWriterFlush(streamWriter *writer) { + if (writer->state == STREAM_WRITER_STATE_ERROR) return C_ERR; + /* Flush after finish is a no-op: frame is already closed. */ + if (writer->state == STREAM_WRITER_STATE_FINISHED) return C_OK; + /* Nothing emitted yet: no envelope, no buffered bytes to drain. */ + if (writer->state == STREAM_WRITER_STATE_INITIAL) return C_OK; + return streamWriterFeedAndWrite(writer, NULL, 0, COMPRESS_FLUSH_SYNC); +} + +int streamWriterFinish(streamWriter *writer) { + if (writer->state == STREAM_WRITER_STATE_ERROR) return C_ERR; + if (writer->state == STREAM_WRITER_STATE_FINISHED) return C_OK; + + /* Even an empty stream produces a valid envelope + empty frame so the + * loader sees a well-formed file. */ + if (streamWriterEnsureEnvelope(writer) == C_ERR) return C_ERR; + if (streamWriterFeedAndWrite(writer, NULL, 0, COMPRESS_FLUSH_END) == C_ERR) return C_ERR; + writer->state = STREAM_WRITER_STATE_FINISHED; + return C_OK; +} + +void streamWriterFree(streamWriter *writer) { + streamCompressorFree(&writer->compressor); + zfree(writer->out_buf); + writer->out_buf = NULL; + writer->out_buf_size = 0; +} + +/* ===== Streaming Reader ===== */ + +static void streamReaderSetError(streamReader *reader, streamReaderErrorKind error_kind) { + if (reader->error_kind == STREAM_READER_ERROR_NONE) reader->error_kind = error_kind; +} + +int streamReaderInit(streamReader *reader, const streamReaderConfig *cfg, streamReaderReadFn read_cb, void *read_ctx, compressionAlgo *detected_algo) { + memset(reader, 0, sizeof(*reader)); + reader->read_cb = read_cb; + reader->read_ctx = read_ctx; + reader->buffer_size = cfg->buffer_size < STREAM_READER_BUFFER_SIZE_MIN + ? STREAM_READER_BUFFER_SIZE_MIN + : cfg->buffer_size; + compressionAlgo algo = ALGO_NONE; + while (true) { + size_t need = reader->probe.header_len < VCS_MAGIC_SIZE + ? VCS_MAGIC_SIZE - reader->probe.header_len + : VCS_ENVELOPE_SIZE - reader->probe.header_len; + ssize_t got = reader->read_cb(reader->read_ctx, + reader->probe.header + reader->probe.header_len, + need); + + if (got < 0 || (size_t)got > need) { + streamReaderSetError(reader, STREAM_READER_ERROR_IO); + return C_ERR; + } + reader->probe.header_len += (size_t)got; + + if (reader->probe.header_len >= VCS_MAGIC_SIZE && + !vcsHasMagicPrefix(reader->probe.header, VCS_MAGIC_SIZE)) { + if (!cfg->allow_passthrough) { + streamReaderSetError(reader, STREAM_READER_ERROR_INCOMPATIBLE); + return C_ERR; + } + reader->state = STREAM_READER_STATE_PASSTHROUGH; + break; + } + + if (reader->probe.header_len == VCS_ENVELOPE_SIZE) { + if (readVcsEnvelope(reader->probe.header, VCS_STREAM_RDB, &algo) == C_ERR) { + streamReaderSetError(reader, STREAM_READER_ERROR_INCOMPATIBLE); + return C_ERR; + } + if (streamDecompressorInit(&reader->decompressor, algo, + cfg->skip_codec_checksum_validation) == C_ERR) { + streamReaderSetError(reader, STREAM_READER_ERROR_IO); + return C_ERR; + } + reader->compressed_buf_size = STREAM_READER_COMPRESSED_BUFFER_SIZE; + reader->compressed_buf = zmalloc(reader->compressed_buf_size); + reader->decompressed_buf = zmalloc(reader->buffer_size); + reader->state = STREAM_READER_STATE_COMPRESSED; + break; + } + + if (got > 0) continue; + + /* EOF mid-magic looks like a truncated VCS, not passthrough. */ + if ((reader->probe.header_len > 0 && + vcsHasMagicPrefix(reader->probe.header, reader->probe.header_len)) || + !cfg->allow_passthrough) { + streamReaderSetError(reader, STREAM_READER_ERROR_INCOMPATIBLE); + return C_ERR; + } + reader->state = STREAM_READER_STATE_PASSTHROUGH; + break; + } + + if (detected_algo) *detected_algo = algo; + return C_OK; +} + +/* Initialization must consume enough bytes to distinguish a VCS envelope from + * a plain stream. If the source is plain, those probe bytes are part of the + * caller's payload and cannot be discarded. Replay them first, then continue + * reading directly from the wrapped source so passthrough is byte-for-byte + * transparent to the logical parser. */ +static ssize_t streamReaderReadPassthrough(streamReader *reader, uint8_t *dst, size_t len) { + size_t total = 0; + size_t prefix_avail = reader->probe.header_len - reader->probe_replay_pos; + if (prefix_avail > 0) { + size_t from_prefix = prefix_avail < len ? prefix_avail : len; + memcpy(dst, reader->probe.header + reader->probe_replay_pos, from_prefix); + reader->probe_replay_pos += from_prefix; + dst += from_prefix; + len -= from_prefix; + total += from_prefix; + } + if (len == 0) return (ssize_t)total; + + ssize_t got = reader->read_cb(reader->read_ctx, dst, len); + if (got < 0 || (size_t)got > len) { + streamReaderSetError(reader, STREAM_READER_ERROR_IO); + return total > 0 ? (ssize_t)total : -1; + } + return (ssize_t)(total + (size_t)got); +} + +/* Fill the decoded-output buffer by consuming buffered input or reading more + * from the source. Errors are sticky, but decoded bytes produced before an + * error remain available to the caller. */ +static int streamReaderFillDecompressedBuf(streamReader *reader) { + size_t written = 0; + int result = C_OK; + + reader->decompressed_buf_pos = 0; + reader->decompressed_buf_len = 0; + + while (written < reader->buffer_size && !reader->decompressor.frame_done) { + if (reader->compressed_buf_len > 0) { + size_t consumed = 0; + size_t feed_len = reader->compressed_buf_len; + size_t input_hint = reader->decompressor.input_hint; + if (input_hint > 0 && feed_len > input_hint) feed_len = input_hint; + + size_t output_capacity = reader->buffer_size - written; + ssize_t produced = streamDecompressorFeed( + &reader->decompressor, + reader->decompressed_buf + written, output_capacity, + reader->compressed_buf + reader->compressed_buf_pos, + feed_len, &consumed); + if (produced < 0) { + streamReaderSetError(reader, STREAM_READER_ERROR_CORRUPT); + result = C_ERR; + break; + } + if (consumed > feed_len || (size_t)produced > output_capacity) { + streamReaderSetError(reader, STREAM_READER_ERROR_INTERNAL); + result = C_ERR; + break; + } + + written += (size_t)produced; + reader->compressed_buf_pos += consumed; + reader->compressed_buf_len -= consumed; + if (consumed > 0 || produced > 0) continue; + } + + if (reader->compressed_buf_len == 0) reader->compressed_buf_pos = 0; + + /* Move any unconsumed suffix to the start of the input buffer before + * asking the source for more bytes. */ + if (reader->compressed_buf_pos > 0) { + memmove(reader->compressed_buf, reader->compressed_buf + reader->compressed_buf_pos, + reader->compressed_buf_len); + reader->compressed_buf_pos = 0; + } + + size_t read_size = reader->compressed_buf_size - reader->compressed_buf_len; + /* A full input buffer with no codec progress is invalid. */ + if (read_size == 0) { + streamReaderSetError(reader, STREAM_READER_ERROR_CORRUPT); + result = C_ERR; + break; + } + + /* Limit the source read to the codec's preferred input size so it does + * not read past the current frame into trailing data. */ + size_t input_hint = reader->decompressor.input_hint; + if (input_hint > 0 && read_size > input_hint) read_size = input_hint; + if (read_size > (size_t)SSIZE_MAX) read_size = (size_t)SSIZE_MAX; + + /* Refill the compressed-input buffer, then retry decoding. */ + ssize_t got = reader->read_cb( + reader->read_ctx, + reader->compressed_buf + reader->compressed_buf_pos + reader->compressed_buf_len, + read_size); + if (got < 0 || (size_t)got > read_size) { + streamReaderSetError(reader, STREAM_READER_ERROR_IO); + result = C_ERR; + break; + } + if (got == 0) break; + reader->compressed_buf_len += (size_t)got; + } + + reader->decompressed_buf_len = written; + return result; +} + +ssize_t streamReaderRead(streamReader *reader, void *buf, size_t len) { + if (reader->error_kind != STREAM_READER_ERROR_NONE) return -1; + if (reader->state == STREAM_READER_STATE_FINISHED) return 0; + if (len == 0) return 0; + if (len > (size_t)SSIZE_MAX) return -1; + + if (reader->state == STREAM_READER_STATE_PASSTHROUGH) { + return streamReaderReadPassthrough(reader, (uint8_t *)buf, len); + } + + uint8_t *dst = (uint8_t *)buf; + size_t remaining = len; + size_t total = 0; + while (remaining > 0) { + int fill_result = C_OK; + size_t available = reader->decompressed_buf_len - reader->decompressed_buf_pos; + if (available == 0) { + fill_result = streamReaderFillDecompressedBuf(reader); + available = reader->decompressed_buf_len; + if (available == 0 && fill_result == C_ERR) return total > 0 ? (ssize_t)total : -1; + if (available == 0 && !reader->decompressor.frame_done) { + streamReaderSetError(reader, STREAM_READER_ERROR_CORRUPT); + return total > 0 ? (ssize_t)total : -1; + } + if (available == 0) break; + } + + size_t to_copy = available < remaining ? available : remaining; + memcpy(dst, reader->decompressed_buf + reader->decompressed_buf_pos, to_copy); + reader->decompressed_buf_pos += to_copy; + dst += to_copy; + remaining -= to_copy; + total += to_copy; + if (fill_result == C_ERR) break; + } + return (ssize_t)total; +} + +int streamReaderFinish(streamReader *reader) { + uint8_t buf[4096]; + + if (reader->error_kind != STREAM_READER_ERROR_NONE) return C_ERR; + if (reader->state == STREAM_READER_STATE_FINISHED) return C_OK; + if (reader->state == STREAM_READER_STATE_PASSTHROUGH) { + reader->state = STREAM_READER_STATE_FINISHED; + return C_OK; + } + if (reader->decompressed_buf_len > reader->decompressed_buf_pos) { + streamReaderSetError(reader, STREAM_READER_ERROR_CORRUPT); + return C_ERR; + } + + while (!reader->decompressor.frame_done) { + ssize_t nread = streamReaderRead(reader, buf, sizeof(buf)); + if (nread < 0) return C_ERR; + if (nread > 0) { + streamReaderSetError(reader, STREAM_READER_ERROR_CORRUPT); + return C_ERR; + } + } + + if (reader->compressed_buf_len > 0) { + streamReaderSetError(reader, STREAM_READER_ERROR_CORRUPT); + return C_ERR; + } + + reader->state = STREAM_READER_STATE_FINISHED; + return C_OK; +} + +void streamReaderFree(streamReader *reader) { + streamDecompressorFree(&reader->decompressor); + zfree(reader->compressed_buf); + zfree(reader->decompressed_buf); + reader->compressed_buf = NULL; + reader->compressed_buf_size = 0; + reader->compressed_buf_pos = 0; + reader->compressed_buf_len = 0; + reader->decompressed_buf = NULL; + reader->decompressed_buf_len = 0; + reader->decompressed_buf_pos = 0; +} diff --git a/src/compression_stream.h b/src/compression_stream.h new file mode 100644 index 00000000000..2cc65357d33 --- /dev/null +++ b/src/compression_stream.h @@ -0,0 +1,161 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef COMPRESSION_STREAM_H +#define COMPRESSION_STREAM_H + +#include "compression.h" +#include "sds.h" + +/* VCS envelope: + * [0..2] magic "VCS" + * [3] version (currently VCS_VERSION) + * [4] codec id + * [5] reserved (must be zero) + * [6] stream kind + * + * All fields are single-byte. Future multi-byte fields must use + * network byte order. Codec ids are stable wire values independent of + * compressionAlgo. Readers reject unknown versions, codecs, stream kinds, + * and nonzero reserved fields. */ +#define VCS_MAGIC_0 0x56 /* 'V' */ +#define VCS_MAGIC_1 0x43 /* 'C' */ +#define VCS_MAGIC_2 0x53 /* 'S' */ +#define VCS_MAGIC_SIZE 3 +#define VCS_ENVELOPE_SIZE 7 +#define VCS_VERSION 1 +/* Byte offsets of each envelope field. */ +#define VCS_OFFSET_VERSION 3 +#define VCS_OFFSET_CODEC 4 +#define VCS_OFFSET_RESERVED 5 +#define VCS_OFFSET_STREAM_KIND 6 + +/* Stable wire codec identifier. */ +#define VCS_CODEC_LZ4 0x01 + +/* Identifies an RDB payload in the envelope. */ +#define VCS_STREAM_RDB 0x01 +/* Identifies a replication stream payload in the envelope. */ +#define VCS_STREAM_REPL 0x02 + +typedef int (*streamWriterWriteFn)(void *ctx, const uint8_t *data, size_t len); +/* Returns >0 bytes read, 0 on EOF, -1 on error. Partial reads allowed. */ +typedef ssize_t (*streamReaderReadFn)(void *ctx, void *buf, size_t len); + +/* ===== Writer ===== */ + +typedef enum { + STREAM_WRITER_STATE_INITIAL = 0, + STREAM_WRITER_STATE_ACTIVE, + STREAM_WRITER_STATE_FINISHED, + STREAM_WRITER_STATE_ERROR, +} streamWriterState; + +typedef struct streamWriter { + streamCompressor compressor; + uint8_t *out_buf; + size_t out_buf_size; + streamWriterWriteFn write_cb; + void *write_ctx; + sds *sink; /* When set, compress directly into *sink instead of out_buf + write_cb. */ + uint8_t stream_kind; /* Envelope stream kind (default VCS_STREAM_RDB). */ + streamWriterState state; +} streamWriter; + +/* Writer API. Init uses the codec's default compression level and configures + * its integrity checks according to codec_checksum. Init, Write, Flush, + * Finish, and write_cb use C_OK/C_ERR. The writer pushes compressed bytes to + * write_cb. Errors are sticky: later operations fail without emitting bytes. */ +int streamWriterInit(streamWriter *writer, compressionAlgo algo, bool codec_checksum, streamWriterWriteFn write_cb, void *write_ctx); +/* Redirect compressed output (envelope + frames) straight into *sink, bypassing + * the internal scratch buffer and write callback. */ +void streamWriterSetSink(streamWriter *writer, sds *sink); +/* Select the envelope stream kind (default VCS_STREAM_RDB). Must be called + * before the frame starts. */ +void streamWriterSetStreamKind(streamWriter *writer, uint8_t stream_kind); +int streamWriterWrite(streamWriter *writer, const void *buf, size_t len); +/* Emits codec-buffered bytes while leaving the frame open. */ +int streamWriterFlush(streamWriter *writer); +/* Finalizes the frame. Repeated calls after successful completion are safe. */ +int streamWriterFinish(streamWriter *writer); +/* Releases resources without implicitly finalizing the frame. */ +void streamWriterFree(streamWriter *writer); + +/* Parse a complete VCS envelope, requiring the given stream kind. Returns C_OK + * and sets *algo on success, C_ERR on any mismatch. */ +int streamParseVcsEnvelope(const uint8_t *buf, size_t len, uint8_t expected_stream_kind, compressionAlgo *algo); + +/* ===== Reader ===== */ + +/* Default decompressed-output buffer size. Tiny caller values are clamped up + * so the decoder can always make forward progress without growing internal + * state. The compressed-input buffer only needs to hold one LZ4 block. */ +#define STREAM_READER_BUFFER_SIZE_DEFAULT (1024 * 1024) +#define STREAM_READER_BUFFER_SIZE_MIN (128 * 1024) +#define STREAM_READER_COMPRESSED_BUFFER_SIZE (128 * 1024) + +/* When allow_passthrough is set, non-VCS input is forwarded as raw bytes; + * otherwise it is rejected. */ +typedef struct { + bool allow_passthrough; + bool skip_codec_checksum_validation; + size_t buffer_size; +} streamReaderConfig; + +typedef enum { + STREAM_READER_ERROR_NONE = 0, + STREAM_READER_ERROR_IO = 1, + STREAM_READER_ERROR_INCOMPATIBLE = 2, + STREAM_READER_ERROR_CORRUPT = 3, + STREAM_READER_ERROR_INTERNAL = 4, +} streamReaderErrorKind; + +typedef enum { + STREAM_READER_STATE_PASSTHROUGH = 0, + STREAM_READER_STATE_COMPRESSED, + STREAM_READER_STATE_FINISHED, +} streamReaderState; + +typedef struct streamReader { + streamReaderReadFn read_cb; + void *read_ctx; + struct { + uint8_t header[VCS_ENVELOPE_SIZE]; + size_t header_len; + } probe; + size_t probe_replay_pos; /* Passthrough bytes left to replay from probe. */ + size_t buffer_size; + streamReaderErrorKind error_kind; + streamReaderState state; + + streamDecompressor decompressor; + + uint8_t *compressed_buf; + size_t compressed_buf_size; + size_t compressed_buf_pos; + size_t compressed_buf_len; + + uint8_t *decompressed_buf; + size_t decompressed_buf_pos; + size_t decompressed_buf_len; +} streamReader; + +/* Reader API. Initialization returns C_OK/C_ERR, probes the source, and may + * call read_cb. On success, detected_algo receives ALGO_NONE for passthrough + * input or the detected codec; the output pointer is optional. On failure, + * error_kind classifies the error and the reader remains safe to free. */ +int streamReaderInit(streamReader *reader, const streamReaderConfig *cfg, streamReaderReadFn read_cb, void *read_ctx, compressionAlgo *detected_algo); +/* Returns up to len bytes, 0 on EOF, or -1 on error. An error after partial + * output is reported on the next call. */ +ssize_t streamReaderRead(streamReader *reader, void *buf, size_t len); +/* Completes and validates a compressed frame after the logical parser has + * consumed its payload. It stops at the frame boundary without requiring + * physical EOF, matching the plain RDB loader's treatment of trailing bytes. + * Returns C_OK/C_ERR. */ +int streamReaderFinish(streamReader *reader); +void streamReaderFree(streamReader *reader); + +#endif /* COMPRESSION_STREAM_H */ diff --git a/src/config.c b/src/config.c index e1f0704fdb5..1cc3093166d 100644 --- a/src/config.c +++ b/src/config.c @@ -176,6 +176,16 @@ configEnum rdb_version_check_enum[] = {{"strict", RDB_VERSION_CHECK_STRICT}, {"relaxed", RDB_VERSION_CHECK_RELAXED}, {NULL, 0}}; +configEnum rdb_compression_enum[] = {{"no", RDB_COMPRESSION_NO}, + {"yes", RDB_COMPRESSION_YES}, + {"lzf", RDB_COMPRESSION_LZF}, + {"lz4", RDB_COMPRESSION_LZ4}, + {NULL, 0}}; + +configEnum repl_compression_enum[] = {{"no", REPL_COMPRESSION_NO}, + {"lz4", REPL_COMPRESSION_LZ4}, + {NULL, 0}}; + /* Output buffer limits presets. */ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {0, 0, 0}, /* normal */ @@ -861,6 +871,10 @@ static dict *matchPatternsToConfigs(robj **patterns, int pattern_count) { * CONFIG SET implementation *----------------------------------------------------------------------------*/ +/* Set by the repl-compression apply callback. Reconnects are irreversible, so + * reconciliation is deferred until the whole CONFIG SET commits. */ +static int repl_compression_reconcile_pending = 0; + void configSetCommand(client *c) { const char *errstr = NULL; const char *invalid_arg_name = NULL; @@ -978,6 +992,10 @@ void configSetCommand(client *c) { } } + /* Reset deferred side-effect state before applies run; apply callbacks set + * it, and it is consumed only on a successful commit below. */ + repl_compression_reconcile_pending = 0; + /* Apply all configs after being set */ for (i = 0; i < config_count && apply_fns[i] != NULL; i++) { if (!apply_fns[i](&errstr)) { @@ -1000,6 +1018,11 @@ void configSetCommand(client *c) { ValkeyModuleConfigChangeV1 cc = {.num_changes = config_count, .config_names = config_names}; moduleFireServerEvent(VALKEYMODULE_EVENT_CONFIG, VALKEYMODULE_SUBEVENT_CONFIG_CHANGE, &cc); addReply(c, shared.ok); + /* CONFIG SET committed: now safe to run deferred irreversible side effects. */ + if (repl_compression_reconcile_pending) { + repl_compression_reconcile_pending = 0; + reconcileReplicaCompression(); + } goto end; err: @@ -2653,6 +2676,14 @@ static int updateJemallocBgThread(const char **err) { return 1; } +static int updateReplCompression(const char **err) { + UNUSED(err); + /* Record intent only; configSetCommand reconciles after the command commits, + * so a rolled-back CONFIG SET disconnects nothing. */ + repl_compression_reconcile_pending = 1; + return 1; +} + static int updateReplBacklogSize(const char **err) { UNUSED(err); resizeReplicationBacklog(); @@ -3348,7 +3379,6 @@ standardConfig static_configs[] = { createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0, NULL, NULL), createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, 0, NULL, NULL), createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, 1, NULL, NULL), - createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, 1, NULL, NULL), createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, server.rdb_del_sync_files, 0, NULL, NULL), createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1, NULL, NULL), createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1, NULL, NULL), @@ -3455,6 +3485,8 @@ standardConfig static_configs[] = { createEnumConfig("log-format", NULL, MODIFIABLE_CONFIG, log_format_enum, server.log_format, LOG_FORMAT_LEGACY, NULL, NULL), createEnumConfig("log-timestamp-format", NULL, MODIFIABLE_CONFIG, log_timestamp_format_enum, server.log_timestamp_format, LOG_TIMESTAMP_LEGACY, NULL, NULL), createEnumConfig("rdb-version-check", NULL, MODIFIABLE_CONFIG, rdb_version_check_enum, server.rdb_version_check, RDB_VERSION_CHECK_STRICT, NULL, NULL), + createEnumConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, rdb_compression_enum, server.rdb_compression, RDB_COMPRESSION_YES, NULL, NULL), + createEnumConfig("repl-compression", NULL, MODIFIABLE_CONFIG, repl_compression_enum, server.repl_compression, REPL_COMPRESSION_NO, NULL, updateReplCompression), /* Integer configs */ createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.config_databases, 16, INTEGER_CONFIG, NULL, NULL), diff --git a/src/io_threads.c b/src/io_threads.c index 1930600ff16..e35330cec6e 100644 --- a/src/io_threads.c +++ b/src/io_threads.c @@ -518,6 +518,9 @@ int trySendReadToIOThreads(client *c) { if (c->io_write_state == CLIENT_PENDING_IO) return C_OK; /* For simplicity, don't offload replica clients reads as read traffic from replica is negligible */ if (getClientType(c) == CLIENT_TYPE_REPLICA) return C_ERR; + /* A live replication decoder must run on the main thread; the IO-thread + * read path does not decode. Destroyed once the probe resolves to plaintext. */ + if (c->flag.primary && server.repl_decompressor) return C_ERR; /* With Lua debug client we may call connWrite directly in the main thread */ if (c->flag.lua_debug) return C_ERR; /* For simplicity let the main-thread handle the blocked clients */ diff --git a/src/networking.c b/src/networking.c index 47b5533adbd..e0b12a7e512 100644 --- a/src/networking.c +++ b/src/networking.c @@ -37,6 +37,8 @@ #include "fpconv_dtoa.h" #include "fmtargs.h" #include "io_threads.h" +#include "compression_repl.h" +#include "monotonic.h" #include "module.h" #include "connection.h" #include "zmalloc.h" @@ -1803,6 +1805,14 @@ int clientHasPendingReplies(client *c) { /* Replicas use global shared replication buffer instead of * private output buffer. */ serverAssert(c->bufpos == 0 && listLength(c->reply) == 0); + + /* Unsent compressed data counts as pending. Skip while CLIENT_PENDING_IO: + * the IO thread owns out_buf; postWriteToReplica re-checks when the job completes. */ + if (c->repl_data->repl_compressor && c->io_write_state != CLIENT_PENDING_IO && + c->repl_data->repl_compressor->out_buf_pos < sdslen(c->repl_data->repl_compressor->out_buf)) { + return 1; + } + if (c->repl_data->ref_repl_buf_node == NULL) return 0; /* If the last replication buffer block content is totally sent, @@ -2155,6 +2165,8 @@ int freeClient(client *c) { return 0; } + replDestroyCompression(c); + /* For connected clients, call the disconnection event of modules hooks. */ if (c->conn) { moduleFireServerEvent(VALKEYMODULE_EVENT_CLIENT_CHANGE, VALKEYMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED, c); @@ -2451,25 +2463,20 @@ client *lookupClientByID(uint64_t id) { return c; } -static void postWriteToReplica(client *c) { - if (c->nwritten <= 0) return; - - server.stat_net_repl_output_bytes += c->nwritten; - - /* Locate the last node which has leftover data and - * decrement reference counts of all nodes in front of it. - * Set c->ref_repl_buf_node to point to the last node and - * c->ref_block_pos to the offset within that node */ +/* Advance the replica's replication-backlog cursor (ref_repl_buf_node / + * ref_block_pos) past consumed raw bytes, releasing the reference on each + * fully-sent block. Shared by the compressed and plaintext post-write paths. */ +static void advanceReplicaRefBlock(client *c, size_t consumed) { listNode *curr = c->repl_data->ref_repl_buf_node; listNode *next = NULL; - size_t nwritten = c->nwritten + c->repl_data->ref_block_pos; + size_t remaining = consumed + c->repl_data->ref_block_pos; replBufBlock *o = listNodeValue(curr); - while (nwritten >= o->used) { + while (remaining >= o->used) { next = listNextNode(curr); if (!next) break; /* End of list */ - nwritten -= o->used; + remaining -= o->used; o->refcount--; curr = next; @@ -2477,14 +2484,171 @@ static void postWriteToReplica(client *c) { o->refcount++; } - serverAssert(nwritten <= o->used); + serverAssert(remaining <= o->used); c->repl_data->ref_repl_buf_node = curr; - c->repl_data->ref_block_pos = nwritten; + c->repl_data->ref_block_pos = remaining; +} + +static void postWriteToReplica(client *c) { + /* Check compression error first; IO thread may have flagged this. */ + if (atomic_load_explicit(&c->repl_data->compression_error, memory_order_acquire)) { + serverLog(LL_WARNING, + "Compression error on replica %s (algo=%s, raw_bytes=%zu), disconnecting", + replicationGetReplicaName(c), + compressionAlgoName(replCompressorAlgo(c->repl_data->repl_compressor)), + c->repl_data->repl_compressor ? c->repl_data->repl_compressor->raw_bytes : 0); + freeClientAsync(c); + return; + } + + if (c->nwritten <= 0) return; + + server.stat_net_repl_output_bytes += c->nwritten; + + if (c->repl_data->repl_compressor) { + replCompressor *compressor = c->repl_data->repl_compressor; + /* The cursor advances by the batch's raw bytes only once out_buf is + * fully sent; a partial send keeps it pinned so the next cycle sends + * the remainder before compressing more. */ + if (compressor->out_buf_pos == sdslen(compressor->out_buf)) { + size_t raw_bytes = compressor->raw_bytes; + + advanceReplicaRefBlock(c, raw_bytes); + + c->repl_data->repl_uncompressed_bytes_total += raw_bytes; + c->repl_data->repl_compressed_bytes_total += sdslen(compressor->out_buf); + + replCompressorResetBatch(compressor); + + incrementalTrimReplicationBacklog(REPL_BACKLOG_TRIM_BLOCKS_PER_CALL); + } + return; + } + + advanceReplicaRefBlock(c, c->nwritten); incrementalTrimReplicationBacklog(REPL_BACKLOG_TRIM_BLOCKS_PER_CALL); } +/* Compressed write path for replicas on either the IO thread or the main thread. */ +static void writeToReplicaCompressed(client *c) { + replCompressor *compressor = c->repl_data->repl_compressor; + serverAssert(compressor != NULL); + + /* Finish sending the previous batch's leftover first; compressed bytes + * must reach the socket in order. */ + if (compressor->out_buf_pos < sdslen(compressor->out_buf)) { + size_t avail = sdslen(compressor->out_buf) - compressor->out_buf_pos; + c->nwritten = connWrite(c->conn, + compressor->out_buf + compressor->out_buf_pos, + avail); + if (c->nwritten <= 0) { + c->write_flags |= WRITE_FLAGS_WRITE_ERROR; + return; + } + compressor->out_buf_pos += c->nwritten; + /* Skip trim here; postWriteToReplica trims only after the batch fully + * drains. Worst-case trim delay is one batch (REPL_COMPRESSION_BATCH_LIMIT). */ + return; + } + + /* postWriteToReplica resets the batch once it fully drains, so a fresh + * batch always starts from an empty buffer. */ + serverAssert(sdslen(compressor->out_buf) == 0 && compressor->out_buf_pos == 0 && + compressor->raw_bytes == 0); + + listNode *last_node; + size_t bufpos; + if (inMainThread()) { + last_node = listLast(server.repl_buffer_blocks); + if (!last_node) return; + bufpos = ((replBufBlock *)listNodeValue(last_node))->used; + } else { + last_node = c->io_last_reply_block; + serverAssert(last_node != NULL); + bufpos = c->io_last_bufpos; + } + listNode *first_node = c->repl_data->ref_repl_buf_node; + + /* Compress new replication-backlog bytes, capped at + * REPL_COMPRESSION_BATCH_LIMIT raw bytes per cycle to bound per-batch + * latency and keep out_buf size predictable. */ + monotime compress_start = getMonotonicUs(); + size_t total_raw = 0; + for (listNode *cur = first_node; cur != NULL; cur = listNextNode(cur)) { + replBufBlock *block = listNodeValue(cur); + size_t start = (cur == first_node) ? c->repl_data->ref_block_pos : 0; + size_t end = (cur == last_node) ? bufpos : block->used; + + if (end <= start) { + serverAssert(end >= start); + if (cur == last_node) break; + continue; + } + + size_t len = end - start; + /* Cap this write at the remaining batch budget; the cursor resumes mid-block next cycle. */ + size_t remaining = REPL_COMPRESSION_BATCH_LIMIT - total_raw; + if (len > remaining) len = remaining; + int rc = replCompressorWrite(compressor, block->buf + start, len); + if (rc == C_ERR) { + atomic_store_explicit(&c->repl_data->compression_error, 1, memory_order_release); + atomic_fetch_add_explicit(&server.repl_compression_errors, 1, memory_order_relaxed); + c->write_flags |= WRITE_FLAGS_WRITE_ERROR; + return; + } + total_raw += len; + if (total_raw >= REPL_COMPRESSION_BATCH_LIMIT) break; + if (cur == last_node) break; + } + + if (total_raw == 0) return; + + /* Flush so the batch's compressed bytes land in out_buf. */ + if (replCompressorFlush(compressor) != C_OK) { + atomic_store_explicit(&c->repl_data->compression_error, 1, memory_order_release); + atomic_fetch_add_explicit(&server.repl_compression_errors, 1, memory_order_relaxed); + c->write_flags |= WRITE_FLAGS_WRITE_ERROR; + return; + } + + /* Best-effort: a minor race with INFO reads is acceptable for timing metrics. */ + atomic_fetch_add_explicit(&c->repl_data->repl_compression_time_usec, + (long long)(getMonotonicUs() - compress_start), memory_order_relaxed); + + compressor->raw_bytes = total_raw; + + /* Send out_buf. The backlog cursor advances only after a full send + * (postWriteToReplica), so a partial send keeps it pinned to the start of + * the batch. */ + size_t avail = sdslen(compressor->out_buf); + if (avail == 0) { + if (total_raw > 0) { + /* Compressor produced no output despite non-zero input: treat as error + * to prevent infinite re-compression of the same data. */ + atomic_store_explicit(&c->repl_data->compression_error, 1, memory_order_release); + atomic_fetch_add_explicit(&server.repl_compression_errors, 1, memory_order_relaxed); + c->write_flags |= WRITE_FLAGS_WRITE_ERROR; + } + return; + } + + c->nwritten = connWrite(c->conn, compressor->out_buf, avail); + if (c->nwritten <= 0) { + c->write_flags |= WRITE_FLAGS_WRITE_ERROR; + return; + } + compressor->out_buf_pos = c->nwritten; +} + static void writeToReplica(client *c) { + /* Compressed replicas use the framed write path; the decision lives here so + * callers do not branch on the per-replica compressor. */ + if (c->repl_data->repl_compressor != NULL) { + writeToReplicaCompressed(c); + return; + } + listNode *last_node; size_t bufpos; @@ -3463,6 +3627,12 @@ void freeSharedQueryBuf(void) { thread_shared_qb = NULL; } +/* Detach the client from the thread-shared query buffer when appending addlen + * bytes would reallocate it, so the shared pointer stays valid for later reads. */ +void clientUnshareQuerybufIfNeeded(client *c, size_t addlen) { + if (c->querybuf == thread_shared_qb && sdsavail(c->querybuf) < addlen) initSharedQueryBuf(); +} + /* This function is used when we want to re-enter the event loop but there * is the risk that the client we are dealing with will be freed in some * way. This happens for instance in: @@ -4316,7 +4486,12 @@ static bool readToQueryBuf(client *c) { * parseMultibulkBuffer() can avoid copying buffers to create the * robj representing the argument. */ - if (c->reqtype == PROTO_REQ_MULTIBULK && c->multibulklen && c->bulklen != -1 && c->bulklen >= PROTO_MBULK_BIG_ARG) { + /* bulklen measures the decoded stream; a compressed primary link carries + * compressed bytes on the wire, so reads stay at PROTO_IOBUF_LEN and the + * decode path grows the buffer as decoded bytes arrive. Also bounds a + * single decode call under the decoder's output cap. */ + if (c->reqtype == PROTO_REQ_MULTIBULK && c->multibulklen && c->bulklen != -1 && c->bulklen >= PROTO_MBULK_BIG_ARG && + !(c->flag.primary && server.repl_decompressor)) { ssize_t remaining = (size_t)(c->bulklen + 2) - (qblen - c->qb_pos); big_arg = 1; @@ -4355,6 +4530,10 @@ static bool readToQueryBuf(client *c) { /* Read as much as possible from the socket to save read(2) system calls. */ readlen = sdsavail(c->querybuf); + /* A compressed primary link keeps reads at PROTO_IOBUF_LEN: spare + * querybuf capacity tracks the decoded stream, and an inflated read + * would coalesce many compressed batches into one decode call. */ + if (c->flag.primary && server.repl_decompressor && readlen > PROTO_IOBUF_LEN) readlen = PROTO_IOBUF_LEN; } if (use_thread_shared_qb) serverAssert(c->querybuf == thread_shared_qb); @@ -4381,6 +4560,8 @@ static bool readToQueryBuf(client *c) { } #define REPL_MAX_READS_PER_IO_EVENT 25 +/* Bounds decode work per event; the next event resumes where this one stopped. */ +#define REPL_DECODE_EVENT_BUDGET (8 * 1024 * 1024) void readQueryFromClient(connection *conn) { client *c = connGetPrivateData(conn); /* Check if we can send the client to be handled by the IO-thread */ @@ -4390,15 +4571,27 @@ void readQueryFromClient(connection *conn) { bool repeat = false; int iter = 0; + size_t decoded_total = 0; do { + size_t qblen_before_read = c->querybuf ? sdslen(c->querybuf) : 0; + size_t decoded = 0; bool full_read = readToQueryBuf(c); if (handleReadResult(c) == C_OK) { + if (c->flag.primary && + server.repl_decompressor && + replDecompressQueryBuf(c, qblen_before_read, &decoded) == C_ERR) { + serverLog(LL_WARNING, "Disconnecting primary due to replication stream decompression failure"); + freeClientAsync(c); + return; + } + decoded_total += decoded; if (processInputBuffer(c) == C_ERR) return; trimCommandQueue(c); } repeat = (c->flag.primary && !c->flag.close_asap && ++iter < REPL_MAX_READS_PER_IO_EVENT && + decoded_total < REPL_DECODE_EVENT_BUDGET && full_read); beforeNextClient(c); } while (repeat); @@ -6136,7 +6329,13 @@ size_t getClientOutputBufferMemoryUsage(client *c) { repl_buf_size = last->repl_offset + last->size - cur->repl_offset; repl_node_num = last->id - cur->id + 1; } - return repl_buf_size + (repl_node_size * repl_node_num); + size_t compressor_size = 0; + /* Skip while an IO thread owns the compressor; the write job may realloc + * out_buf. The shared-buffer lag term still drives COB enforcement. */ + if (c->repl_data->repl_compressor && c->io_write_state != CLIENT_PENDING_IO) { + compressor_size = replCompressorMemUsage(c->repl_data->repl_compressor); + } + return repl_buf_size + (repl_node_size * repl_node_num) + compressor_size; } size_t list_item_size = sizeof(listNode) + sizeof(clientReplyBlock); diff --git a/src/rdb.c b/src/rdb.c index f689a731a4b..b24483b2487 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -46,6 +46,8 @@ #include "module.h" #include "cluster.h" #include "cluster_migrateslots.h" +#include "compression.h" +#include "compression_stream.h" #include #include @@ -77,6 +79,23 @@ void rdbCheckSetError(const char *fmt, ...); int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadingCtx *rdb_loading_ctx); void replicationEmptyDbCallback(hashtable *ht); +/* Resolve the configured policy to an algorithm. The `yes` policy follows the + * default algorithm, while explicit algorithm names remain pinned. */ +static compressionAlgo rdbCompressionAlgorithm(rdb_compression_mode mode) { + switch (mode) { + case RDB_COMPRESSION_NO: + return ALGO_NONE; + case RDB_COMPRESSION_YES: + return ALGO_LZF; + case RDB_COMPRESSION_LZF: + return ALGO_LZF; + case RDB_COMPRESSION_LZ4: + return ALGO_LZ4; + default: + serverPanic("Unknown RDB compression mode: %d", mode); + } +} + /* Returns true if the RDB version is valid and accepted, false otherwise. This * function takes configuration into account. The parameter `is_valkey_magic` * indicates that an RDB file with the VALKEY magic string was parsed. @@ -510,13 +529,16 @@ ssize_t rdbSaveRawString(rio *rdb, unsigned char *s, size_t len) { } } - /* Try LZF compression - under 20 bytes it's unable to compress even - * aaaaaaaaaaaaaaaaaa so skip it */ - if (server.rdb_compression && len > 20) { + /* Try LZF compression. Values under 20 bytes don't compress, skip those. + * Skip per-string LZF when the rio has whole-stream compression so we + * don't compress twice; standalone rios (DUMP, AOF rewrite, diskless) + * still hit this path. rdb may be NULL when rdbSavedObjectLen() calculates + * the encoded length without writing the object. */ + if (server.rdb_compression != RDB_COMPRESSION_NO && len > 20 && !(rdb && rdb->stream_writer)) { n = rdbSaveLzfStringObject(rdb, s, len); if (n == -1) return -1; if (n > 0) return n; - /* Return value of 0 means data can't be compressed, save the old way */ + /* 0 means data can't be compressed; fall through and store verbatim. */ } /* Store verbatim */ @@ -1484,7 +1506,8 @@ int rdbSaveRio(int req, int rdbver, rio *rdb, int *error, int rdbflags, rdbSaveI long key_counter = 0; int j; - if (server.rdb_checksum) rdb->update_cksum = rioGenericUpdateChecksum; + if (server.rdb_checksum && !(rdb->flags & RIO_FLAG_SKIP_RDB_CHECKSUM)) + rdb->update_cksum = rioGenericUpdateChecksum; const char *magic_prefix = rdbUseValkeyMagic(rdbver) ? "VALKEY" : "REDIS0"; serverAssert(rdbver >= 0 && rdbver <= RDB_VERSION); snprintf(magic, sizeof(magic), "%s%03d", magic_prefix, rdbver); @@ -1510,7 +1533,7 @@ int rdbSaveRio(int req, int rdbver, rio *rdb, int *error, int rdbflags, rdbSaveI /* EOF opcode */ if (rdbSaveType(rdb, RDB_OPCODE_EOF) == -1) goto werr; - /* CRC64 checksum. It will be zero if checksum computation is disabled, the + /* RDB checksum field. It will be zero if checksum computation is disabled, the * loading code skips the check in this case. */ cksum = rdb->cksum; memrev64ifbe(&cksum); @@ -1552,12 +1575,38 @@ int rdbSaveRioWithEOFMark(int req, int rdbver, rio *rdb, int *error, rdbSaveInfo return C_ERR; } +static int rdbCompressionWrite(void *ctx, const uint8_t *data, size_t len) { + return rioWriteRaw((rio *)ctx, data, len) ? C_OK : C_ERR; +} + +static int rdbCompressionInit(rio *rdb, + streamWriter *writer, + compressionAlgo algo, + bool codec_checksum) { + if (streamWriterInit(writer, algo, codec_checksum, rdbCompressionWrite, rdb) == C_ERR) return C_ERR; + rioAttachStreamWriter(rdb, writer); + return C_OK; +} + +static void rdbCompressionFree(rio *rdb, streamWriter *writer) { + rioDetachStreamWriter(rdb); + streamWriterFree(writer); +} + static int rdbSaveInternal(int req, const char *filename, rdbSaveInfo *rsi, int rdbflags) { char cwd[MAXPATHLEN]; /* Current working dir path for error messages. */ rio rdb; int error = 0; int saved_errno; char *err_op; /* For a detailed log */ + compressionAlgo compression_algo = rdbCompressionAlgorithm(server.rdb_compression); + bool use_streaming_compression = compression_algo == ALGO_LZ4; + /* Keep replication snapshots plain until full sync negotiates compression. + * Disk-based sync snapshots can also become AOF bases, which currently do + * not record whether the reused RDB has whole-stream compression. */ + if (rdbflags & RDBFLAGS_REPLICATION) use_streaming_compression = false; + streamWriter compression_writer; + bool compression_initialized = false; FILE *fp = fopen(filename, "w"); if (!fp) { @@ -1579,12 +1628,49 @@ static int rdbSaveInternal(int req, const char *filename, rdbSaveInfo *rsi, int if (!(rdbflags & RDBFLAGS_KEEP_CACHE)) rioSetReclaimCache(&rdb, 1); } + /* The file rio remains the interface passed to RDB. When compression is + * enabled, rio sends logical RDB bytes through streamWriter, which emits + * encoded bytes to the same rio's concrete file backend: + * + * disabled: rdbSaveRio -> rdb(file) -> disk + * enabled: rdbSaveRio -> streamWriter -> rdb(file backend) -> disk + * + * rioWriteRaw lets streamWriter reach the backend without recursively + * compressing its own output. */ + if (use_streaming_compression) { + if (rdbCompressionInit(&rdb, &compression_writer, compression_algo, server.rdb_checksum) == C_ERR) { + errno = EIO; /* Compressor init failure, set errno for werr log */ + err_op = "rdbCompressionInit"; + goto werr; + } + compression_initialized = true; + } + /* Streaming-compressed RDBs use codec-frame checksums instead of the + * logical RDB CRC64 trailer. */ + if (use_streaming_compression || !server.rdb_checksum) { + rdb.flags |= RIO_FLAG_SKIP_RDB_CHECKSUM; + rdb.update_cksum = NULL; + rdb.cksum = 0; + } + if (rdbSaveRio(req, RDB_VERSION, &rdb, &error, rdbflags, rsi) == C_ERR) { errno = error; err_op = "rdbSaveRio"; goto werr; } + /* Finalize the compression frame before flushing to disk. */ + if (compression_initialized) { + if (streamWriterFinish(&compression_writer) == C_ERR) { + rdb.flags |= RIO_FLAG_WRITE_ERROR; + errno = EIO; /* Compression finalization failure */ + err_op = "streamWriterFinish"; + goto werr; + } + rdbCompressionFree(&rdb, &compression_writer); + compression_initialized = false; + } + /* Make sure data will not remain on the OS's output buffers */ if (fflush(fp)) { err_op = "fflush"; @@ -1608,6 +1694,11 @@ static int rdbSaveInternal(int req, const char *filename, rdbSaveInfo *rsi, int werr: saved_errno = errno; serverLog(LL_WARNING, "Write error while saving DB to the disk(%s): %s", err_op, strerror(errno)); + if (compression_initialized) { + /* Skip finish on error, output is being discarded (unlink below). + * Just release resources. */ + rdbCompressionFree(&rdb, &compression_writer); + } if (fp) fclose(fp); unlink(filename); errno = saved_errno; @@ -3071,12 +3162,24 @@ void stopSaving(int success) { /* Track loading progress in order to serve client's from time to time and if needed calculate rdb checksum */ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) { - if (server.rdb_checksum) rioGenericUpdateChecksum(r, buf, len); + if (server.rdb_checksum && !(r->flags & RIO_FLAG_SKIP_RDB_CHECKSUM)) + rioGenericUpdateChecksum(r, buf, len); + + /* Event scheduling uses decoded (logical) bytes so that + * processEventsWhileBlocked() fires based on actual parsing work, even + * when the stream reader is draining its internal decompressed buffer + * without advancing the transport position. */ + off_t decoded_pos = (off_t)(r->processed_bytes + len); + if (server.loading_process_events_interval_bytes && - (r->processed_bytes + len) / server.loading_process_events_interval_bytes > - r->processed_bytes / server.loading_process_events_interval_bytes) { + decoded_pos / server.loading_process_events_interval_bytes > + (off_t)r->processed_bytes / server.loading_process_events_interval_bytes) { + /* Progress reporting uses transport bytes for decompression paths so the + * loading percentage stays consistent with the file size passed to + * startLoadingFile(); plain paths report decoded bytes. */ + off_t report_pos = r->stream_reader ? rioTell(r) : decoded_pos; if (server.primary_host && server.repl_state == REPL_STATE_TRANSFER) replicationSendNewlineToPrimary(); - loadingAbsProgress(r->processed_bytes); + loadingAbsProgress(report_pos); processEventsWhileBlocked(); processModuleLoadingProgressEvent(0); } @@ -3085,6 +3188,72 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) { } } +bool rdbRioHasCorruptCompressedInput(rio *rdb) { + /* rdbLoadRio also accepts raw rios, for example AOF preamble loads. */ + if (!rdb->stream_reader) return false; + return rdb->stream_reader->error_kind == STREAM_READER_ERROR_CORRUPT; +} + +bool rdbRioHasInternalStreamReaderError(rio *rdb) { + if (!rdb->stream_reader) return false; + return rdb->stream_reader->error_kind == STREAM_READER_ERROR_INTERNAL; +} + +static ssize_t rdbStreamReadRaw(void *ctx, void *buf, size_t len) { + return rioReadRawPartial((rio *)ctx, buf, len); +} + +rdbStreamReaderInitResult rdbInitStreamReader(rio *rdb, + streamReader *reader, + bool skip_codec_checksum_validation, + compressionAlgo *algo) { + streamReaderConfig cfg = { + .allow_passthrough = true, + .skip_codec_checksum_validation = skip_codec_checksum_validation, + .buffer_size = STREAM_READER_BUFFER_SIZE_DEFAULT, + }; + compressionAlgo detected_algo = ALGO_NONE; + + if (algo) *algo = ALGO_NONE; + if (streamReaderInit(reader, &cfg, rdbStreamReadRaw, rdb, &detected_algo) == C_ERR) { + streamReaderErrorKind error_kind = reader->error_kind; + streamReaderFree(reader); + return error_kind == STREAM_READER_ERROR_INCOMPATIBLE + ? RDB_STREAM_READER_INIT_INCOMPATIBLE + : RDB_STREAM_READER_INIT_ERROR; + } + + if (detected_algo == ALGO_NONE && rioCheckType(rdb) == RIO_TYPE_FILE) { + size_t probe_len = reader->probe.header_len; + off_t rewind_len = (off_t)probe_len; + + /* File-backed RDB loads can replay the probe through the native rio + * path. This keeps plain RDBs out of the stream-reader passthrough + * path while retaining it for sources that cannot be rewound. */ + if ((size_t)rewind_len == probe_len && + probe_len <= rdb->stream_processed_bytes && + fseeko(rdb->io.file.fp, -rewind_len, SEEK_CUR) == 0) { + rdb->stream_processed_bytes -= probe_len; + } else { + rioAttachStreamReader(rdb, reader); + } + } else { + rioAttachStreamReader(rdb, reader); + } + + if (detected_algo != ALGO_NONE) { + rdb->flags |= RIO_FLAG_STREAMING_COMPRESSION | RIO_FLAG_SKIP_RDB_CHECKSUM; + if (algo) *algo = detected_algo; + } + return RDB_STREAM_READER_INIT_OK; +} + +void rdbFreeStreamReader(rio *rdb, streamReader *reader) { + rioDetachStreamReader(rdb); + rdb->flags &= ~RIO_FLAG_STREAMING_COMPRESSION; + streamReaderFree(reader); +} + /* Save the given functions_ctx to the rdb. * The err output parameter is optional and will be set with relevant error * message on failure, it is the caller responsibility to free the error @@ -3585,7 +3754,9 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin uint64_t cksum, expected = rdb->cksum; if (rioRead(rdb, &cksum, 8) == 0) goto eoferr; - if (server.rdb_checksum && !server.skip_checksum_validation) { + if (rdb->flags & RIO_FLAG_STREAMING_COMPRESSION) { + serverLog(LL_NOTICE, "Logical RDB CRC64 skipped for streaming-compressed input."); + } else if (server.rdb_checksum && !server.skip_checksum_validation) { memrev64ifbe(&cksum); if (rdb->flags & RIO_FLAG_SKIP_RDB_CHECKSUM) { serverLog(LL_NOTICE, "RDB file was saved with checksum disabled: skipped checksum for this transfer"); @@ -3616,6 +3787,16 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin * the RDB file from a socket during initial SYNC (diskless replica mode), * we'll report the error to the caller, so that we can retry. */ eoferr: + if (rdbRioHasInternalStreamReaderError(rdb)) { + serverLog(LL_WARNING, "Internal error while decoding streaming-compressed RDB input. Aborting now."); + rdbReportReadError("Internal error decoding compressed RDB stream"); + return RDB_FAILED; + } + if (rdbRioHasCorruptCompressedInput(rdb)) { + serverLog(LL_WARNING, "Corrupt streaming-compressed RDB input. Unrecoverable error, aborting now."); + rdbReportCorruptRDB("Corrupt compressed RDB stream"); + return RDB_FAILED; + } serverLog(LL_WARNING, "Short read or OOM loading DB. Unrecoverable error, aborting now."); rdbReportReadError("Unexpected EOF reading RDB file"); return RDB_FAILED; @@ -3631,7 +3812,10 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags) { FILE *fp; rio rdb; - int retval; + streamReader stream_reader; + bool stream_reader_initialized = false; + compressionAlgo streaming_algo = ALGO_NONE; + int retval = RDB_FAILED; struct stat sb; int rdb_fd; @@ -3648,8 +3832,44 @@ int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags) { startLoadingFile(sb.st_size, filename, rdbflags); rioInitWithFile(&rdb, fp); + /* Probe every on-disk RDB: + * + * plain file: rewind probe, then rdbLoadRio -> rdb(file backend) + * VCS file: rdbLoadRio -> streamReader LZ4 decode -> rdb(file backend) + * + * Non-rewindable plain sources retain the streamReader passthrough path. + * For VCS input the parser sees the header produced by the decoder. */ + bool skip_codec_checksum_validation = !server.rdb_checksum || server.skip_checksum_validation; + rdbStreamReaderInitResult init_rc = + rdbInitStreamReader(&rdb, &stream_reader, skip_codec_checksum_validation, &streaming_algo); + if (init_rc == RDB_STREAM_READER_INIT_INCOMPATIBLE) { + serverLog(LL_WARNING, + "Invalid or unsupported RDB stream envelope in %s. " + "The file may require a Valkey version with streaming RDB " + "compression support.", + filename); + retval = RDB_INCOMPATIBLE; + goto done; + } + if (init_rc == RDB_STREAM_READER_INIT_ERROR) { + serverLog(LL_WARNING, "Failed to initialize RDB stream reader for %s", filename); + goto done; + } + stream_reader_initialized = true; + + if (rdb.flags & RIO_FLAG_STREAMING_COMPRESSION) { + serverLog(LL_NOTICE, "Loading compressed RDB (algo=%s) from %s", + compressionAlgoName(streaming_algo), filename); + } + retval = rdbLoadRio(&rdb, rdbflags, rsi); + if (retval == RDB_OK && streamReaderFinish(&stream_reader) == C_ERR) { + serverLog(LL_WARNING, "Compressed RDB stream in %s did not end cleanly", filename); + retval = RDB_FAILED; + } +done: + if (stream_reader_initialized) rdbFreeStreamReader(&rdb, &stream_reader); fclose(fp); stopLoading(retval == RDB_OK); /* Reclaim the cache backed by rdb */ diff --git a/src/rdb.h b/src/rdb.h index 1c6534a5b62..7bceb4cc906 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -32,6 +32,7 @@ #include #include "rio.h" +#include "compression_stream.h" /* TBD: include only necessary headers. */ #include "server.h" @@ -221,6 +222,24 @@ int rdbSaveBinaryFloatValue(rio *rdb, float val); int rdbLoadBinaryFloatValue(rio *rdb, float *val); int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi); int rdbLoadRioWithLoadingCtxScopedRdb(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadingCtx *rdb_loading_ctx); +bool rdbRioHasCorruptCompressedInput(rio *rdb); +bool rdbRioHasInternalStreamReaderError(rio *rdb); + +typedef enum { + RDB_STREAM_READER_INIT_ERROR = -1, + RDB_STREAM_READER_INIT_OK = 0, + RDB_STREAM_READER_INIT_INCOMPATIBLE = 1, +} rdbStreamReaderInitResult; + +/* Attaches a probing stream reader that accepts both plain and VCS-wrapped + * RDB input. When non-NULL, algo receives the detected codec or ALGO_NONE for + * plain input. The caller must detach and release a successfully initialized + * reader with rdbFreeStreamReader. */ +rdbStreamReaderInitResult rdbInitStreamReader(rio *rdb, + streamReader *reader, + bool skip_codec_checksum_validation, + compressionAlgo *algo); +void rdbFreeStreamReader(rio *rdb, streamReader *reader); int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx *lib_ctx, int rdbflags, sds *err); int rdbSaveRio(int req, int rdbver, rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi); ssize_t rdbSaveFunctions(rio *rdb); diff --git a/src/replication.c b/src/replication.c index 220c542f201..73a66ee4413 100644 --- a/src/replication.c +++ b/src/replication.c @@ -41,6 +41,8 @@ #include "connection.h" #include "module.h" #include "cluster_migrateslots.h" +#include "io_threads.h" +#include "compression_repl.h" #include #include @@ -84,6 +86,204 @@ ConnectionType *connTypeOfReplication(void) { * pair. Mostly useful for logging, since we want to log a replica using its * IP address and its listening port which is more clear for the user, for * example: "Closing connection with replica 10.1.2.3:6380". */ + +/* Reconcile replica transports after a runtime repl-compression change; a live + * link cannot switch mid-stream. Disabled: drop online replicas on a compressed + * stream. Enabled: drop online replicas that are capable but still plaintext. + * Still-syncing replicas keep the frozen decision; the put-online convergence + * check reconnects them via partial resync. Async disconnect keeps the + * iteration safe. */ +void reconcileReplicaCompression(void) { + listIter li; + listNode *ln; + int disconnected = 0; + + listRewind(server.replicas, &li); + while ((ln = listNext(&li))) { + client *replica = ln->value; + if (!replica->repl_data) continue; + if (replica->repl_data->repl_state != REPLICA_STATE_ONLINE) continue; + + int has_compressor = replica->repl_data->repl_compressor != NULL; + int mismatch; + if (server.repl_compression) { + mismatch = (replica->repl_data->replica_capa & REPLICA_CAPA_COMPRESSION) && !has_compressor; + } else { + mismatch = has_compressor; + } + if (!mismatch) continue; + + serverLog(LL_NOTICE, "Disconnecting replica %s to renegotiate replication compression (now %s)", + replicationGetReplicaName(replica), server.repl_compression ? "enabled" : "disabled"); + freeClientAsync(replica); + disconnected++; + } + + if (disconnected > 0) { + serverLog(LL_NOTICE, "Disconnected %d replicas to reconcile replication compression", disconnected); + } + + /* Capability with the upstream primary was negotiated at handshake; drop the + * link so the reconnect re-advertises with the new setting. */ + if (server.primary_host) { + if (server.primary) { + serverLog(LL_NOTICE, "Disconnecting from primary to renegotiate replication compression (now %s)", + server.repl_compression ? "enabled" : "disabled"); + freeClientAsync(server.primary); + } else if (cancelReplicationHandshake(1)) { + serverLog(LL_NOTICE, "Restarting sync with primary to renegotiate replication compression (now %s)", + server.repl_compression ? "enabled" : "disabled"); + } + } +} + +static bool shouldEnableReplicaCompression(client *c) { + if (!server.repl_compression || !c || !c->repl_data) return false; + return (c->repl_data->replica_capa & REPLICA_CAPA_COMPRESSION) != 0; +} + +static int replInitCompression(client *c, compressionAlgo algo) { + if (!c || !c->repl_data) return C_ERR; + + replDestroyCompression(c); + + c->repl_data->repl_compressor = replCompressorCreate(algo); + if (!c->repl_data->repl_compressor) return C_ERR; + + atomic_store_explicit(&c->repl_data->compression_error, 0, memory_order_relaxed); + + return C_OK; +} + +/* Destroy a replica's compression state and stats. Waits for any in-flight + * IO write job first; the IO thread owns the compressor during a write. */ +void replDestroyCompression(client *c) { + if (!c || !c->repl_data) return; + + if (c->repl_data->repl_compressor) { + waitForClientIO(c); + + replCompressorDestroy(c->repl_data->repl_compressor); + c->repl_data->repl_compressor = NULL; + } + + atomic_store_explicit(&c->repl_data->compression_error, 0, memory_order_relaxed); + c->repl_data->repl_compressed_bytes_total = 0; + c->repl_data->repl_uncompressed_bytes_total = 0; + atomic_store_explicit(&c->repl_data->repl_compression_time_usec, 0, memory_order_relaxed); +} + +/* Initialize framed transport compression for a replica at PSYNC completion. + * The stream writer emits the VCS envelope lazily on its first write. */ +static int replicaInitCompressionOnPsync(client *c) { + /* Dual-channel reaches both the +CONTINUE and put-online paths; the link + * is already compressed after the first init. */ + if (c->repl_data->repl_compressor) return C_OK; + + /* Derive the codec from the configured mode, mirroring the RDB path. */ + compressionAlgo algo = server.repl_compression == REPL_COMPRESSION_LZ4 ? ALGO_LZ4 : ALGO_NONE; + + serverAssert(c->io_write_state == CLIENT_IDLE); + if (replInitCompression(c, algo) != C_OK) { + serverLog(LL_WARNING, "Failed to initialize compression for replica %s", + replicationGetReplicaName(c)); + return C_ERR; + } + + serverLog(LL_NOTICE, "Replication compression enabled for replica %s (algo=%s)", + replicationGetReplicaName(c), compressionAlgoName(algo)); + return C_OK; +} + +static void replDestroyDecompression(void) { + if (server.repl_decompressor) { + replDecompressorDestroy(server.repl_decompressor); + server.repl_decompressor = NULL; + } +} + +/* (Re)create the replica-side decompressor for a fresh stream. */ +static void replRefreshDecompression(void) { + replDestroyDecompression(); + server.repl_decompressor = replDecompressorCreate(); +} + +/* Cap on decoded output per decode call. Both feed paths (socket reads and + * dual-channel replay blocks) hand the decoder at most PROTO_IOBUF_LEN per + * call, and LZ4 expansion is bounded at ~255x, so a valid call stays under + * ~4MB. Exceeding this means corrupt or malicious framing; disconnect. */ +#define REPL_STREAM_DECODER_OUTPUT_MAX (16 * 1024 * 1024) + +/* Decompress newly-read replication stream data in the query buffer. + * Replaces the raw compressed bytes (from new_data_start onward) with + * decompressed output and adjusts the replica's read_reploff accordingly. + * Reports the decoded byte count via 'decoded' when non-NULL. */ +int replDecompressQueryBuf(client *c, size_t new_data_start, size_t *decoded) { + size_t raw_input_len, decompressed_len; + + serverAssert(server.repl_decompressor != NULL); + serverAssert(c != NULL); + serverAssert(c->querybuf != NULL); + serverAssert(new_data_start <= sdslen(c->querybuf)); + + if (decoded) *decoded = 0; + + raw_input_len = sdslen(c->querybuf) - new_data_start; + if (raw_input_len == 0) return C_OK; + + monotime decompress_start = getMonotonicUs(); + + /* Feed transport bytes and drain decoded output through the adapter. */ + replDecodeResult dr = replDecompressorDecode(server.repl_decompressor, + c->querybuf + new_data_start, raw_input_len, + REPL_STREAM_DECODER_OUTPUT_MAX, &decompressed_len); + if (dr != REPL_DECODE_OK) { + if (dr == REPL_DECODE_FRAME_DONE) + serverLog(LL_WARNING, "Primary closed compressed replication frame unexpectedly"); + server.repl_decompression_errors++; + return C_ERR; + } + + sds decode_buf = replDecompressorBuf(server.repl_decompressor); + + if (new_data_start == 0) { + sdsclear(c->querybuf); + } else { + sdsrange(c->querybuf, 0, new_data_start - 1); + } + + /* sdscatlen may reallocate c->querybuf; a client still on the thread-shared + * query buffer must take ownership of a private copy first. */ + clientUnshareQuerybufIfNeeded(c, decompressed_len); + c->querybuf = sdscatlen(c->querybuf, decode_buf, decompressed_len); + if (c->querybuf_peak < sdslen(c->querybuf)) c->querybuf_peak = sdslen(c->querybuf); + + /* Convert the transport bytes already counted in handleReadResult() into + * logical replication bytes before processInputBuffer() observes the stream. + * If the reader buffered a partial compressed frame and emitted 0 bytes, + * the logical offset stays unchanged until a later read produces output. */ + c->repl_data->read_reploff -= (long long)raw_input_len; + c->repl_data->read_reploff += (long long)decompressed_len; + + if (decoded) *decoded = decompressed_len; + + /* Stats cover only streams classified compressed; probe and passthrough + * bytes are not decompression work. */ + if (replDecompressorIsCompressed(server.repl_decompressor)) { + server.repl_decompression_time_usec += getMonotonicUs() - decompress_start; + server.repl_decompressed_bytes_total += decompressed_len; + } + + /* Plaintext stream confirmed: the decoder is pure overhead from here on. + * Drop it so later reads skip this path entirely (callers gate on + * server.repl_decompressor) and primary-link reads can use IO threads. */ + if (replDecompressorIsPassthrough(server.repl_decompressor)) { + replDecompressorDestroy(server.repl_decompressor); + server.repl_decompressor = NULL; + } + return C_OK; +} + char *replicationGetReplicaName(client *c) { static char buf[NET_HOST_PORT_STR_LEN]; char ip[NET_IP_STR_LEN]; @@ -971,6 +1171,20 @@ int primaryTryPartialResynchronization(client *c, long long psync_offset) { freeClientAsync(c); return C_OK; } + + /* Initialize compression after +CONTINUE (plaintext) and before + * addReplyReplicationBacklog so backlog data goes through the compressed path. */ + if (shouldEnableReplicaCompression(c)) { + if (replicaInitCompressionOnPsync(c) != C_OK) { + freeClientAsync(c); + return C_OK; + } + } + /* The command stream starts here: freeze the link's compression decision. + * A dual-channel replica reaches put-online later; a config flip while it + * was loading must not re-make the decision mid-stream. */ + c->repl_data->repl_compression_decided = 1; + psync_len = addReplyReplicationBacklog(c, psync_offset); serverLog( LL_NOTICE, @@ -1477,6 +1691,12 @@ void replconfCommand(client *c) { } } else if (!strcasecmp(objectGetVal(c->argv[j + 1]), REPLICA_CAPA_SKIP_RDB_CHECKSUM_STR)) c->repl_data->replica_capa |= REPLICA_CAPA_SKIP_RDB_CHECKSUM; + /* "compression": the replica can decode a compressed incremental + * replication stream. The primary compresses only when both sides + * enable repl-compression; a primary that does not understand the + * capability ignores it. */ + else if (!strcasecmp(objectGetVal(c->argv[j + 1]), REPLICA_CAPA_COMPRESSION_STR)) + c->repl_data->replica_capa |= REPLICA_CAPA_COMPRESSION; } else if (!strcasecmp(objectGetVal(c->argv[j]), "ack")) { /* REPLCONF ACK is used by replica to inform the primary the amount * of replication stream that it processed so far. It is an @@ -1625,6 +1845,27 @@ int replicaPutOnline(client *replica) { replica->repl_data->repl_state = REPLICA_STATE_ONLINE; replica->repl_data->repl_ack_time = server.unixtime; /* Prevent false timeout. */ + /* Initialize compression for full-sync replicas going online. Skip when + * the decision was already frozen at +CONTINUE (dual-channel main channel): + * that stream is live and cannot switch mid-flight. */ + if (!replica->repl_data->repl_compression_decided) { + if (shouldEnableReplicaCompression(replica)) { + if (replicaInitCompressionOnPsync(replica) != C_OK) { + freeClientAsync(replica); + return 0; + } + } + replica->repl_data->repl_compression_decided = 1; + } else if (shouldEnableReplicaCompression(replica) != (replica->repl_data->repl_compressor != NULL)) { + /* Config changed while the sync was in flight; the frozen decision kept + * the stream consistent. Reconnect so the link renegotiates with the + * current config via a partial resync. */ + serverLog(LL_NOTICE, "Reconnecting replica %s to renegotiate replication compression", + replicationGetReplicaName(replica)); + freeClientAsync(replica); + return 0; + } + refreshGoodReplicasCount(); /* Fire the replica change modules event. */ moduleFireServerEvent(VALKEYMODULE_EVENT_REPLICA_CHANGE, VALKEYMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE, NULL); @@ -2418,6 +2659,7 @@ void replicaAfterLoadPrimaryRDB(connection *conn, rdbSaveInfo *rsi, int disk_bas server.repl_down_since = 0; /* Send the initial ACK immediately to put this replica in online state. */ replicationSendAck(); + if (server.repl_provisional_compression) replRefreshDecompression(); } /* Fire the primary link modules event. */ @@ -3430,8 +3672,15 @@ int streamReplDataBufToDb(client *c) { /* Read and process repl data block */ replDataBufBlock *o = listNodeValue(cur); used = o->used; + size_t qblen_before = sdslen(c->querybuf); c->querybuf = sdscatlen(c->querybuf, o->buf, used); c->repl_data->read_reploff += used; + if (server.repl_decompressor && + replDecompressQueryBuf(c, qblen_before, NULL) == C_ERR) { + serverLog(LL_WARNING, "Dual-channel replication stream decompression failure"); + blockingOperationEnds(); + return C_ERR; + } processInputBuffer(c); server.pending_repl_data.mem -= (used + sizeof(replDataBufBlock) + sizeof(listNode)); server.pending_repl_data.len -= used; @@ -3794,6 +4043,7 @@ int dualChannelReplMainConnRecvPsyncReply(connection *conn, sds *err) { serverCommunicateSystemd("STATUS=PRIMARY <-> REPLICA sync: Partial Resynchronization accepted. Ready to " "accept connections in read-write mode.\n"); } + if (server.repl_provisional_compression) replRefreshDecompression(); dualChannelSyncHandlePsync(); return C_OK; } @@ -3922,9 +4172,10 @@ int syncWithPrimaryHandleSendHandshakeState(connection *conn) { * The primary will ignore capabilities it does not understand. */ // we can ignore primary's conditions when sending capa (is_primary_stream_verified=1) - int send_skip_rdb_checksum_capa = replicationSupportSkipRDBChecksum(conn, useDisklessLoad(), 1); - char *argv[9] = {"REPLCONF", "capa", "eof", "capa", "psync2", NULL, NULL, NULL, NULL}; - size_t lens[9] = {8, 4, 3, 4, 6, 0, 0, 0, 0}; + int use_diskless_load = useDisklessLoad(); + int send_skip_rdb_checksum_capa = replicationSupportSkipRDBChecksum(conn, use_diskless_load, 1); + char *argv[11] = {"REPLCONF", "capa", "eof", "capa", "psync2", NULL, NULL, NULL, NULL, NULL, NULL}; + size_t lens[11] = {8, 4, 3, 4, 6, 0, 0, 0, 0, 0, 0}; int argc = 5; if (send_skip_rdb_checksum_capa) { argv[argc] = "capa"; @@ -3942,6 +4193,17 @@ int syncWithPrimaryHandleSendHandshakeState(connection *conn) { lens[argc] = strlen("dual-channel"); argc++; } + /* Capture the compression decision for this handshake so the decoder is + * set up from what was advertised, not a config that may change mid-stream. */ + server.repl_provisional_compression = server.repl_compression; + if (server.repl_provisional_compression) { + argv[argc] = "capa"; + lens[argc] = strlen("capa"); + argc++; + argv[argc] = REPLICA_CAPA_COMPRESSION_STR; + lens[argc] = strlen(REPLICA_CAPA_COMPRESSION_STR); + argc++; + } err = sendCommandArgv(conn, argc, argv, lens); if (err) goto err; @@ -4319,6 +4581,7 @@ void syncWithPrimary(connection *conn) { serverCommunicateSystemd("STATUS=PRIMARY <-> REPLICA sync: Partial Resynchronization accepted. Ready to " "accept connections in read-write mode.\n"); } + if (server.repl_provisional_compression) replRefreshDecompression(); return; } @@ -4599,6 +4862,7 @@ void replicationUnsetPrimary(void) { * the replicas will be able to partially resync with us, so it will be * a very fast reconnection. */ disconnectReplicas(); + replDestroyDecompression(); server.repl_state = REPL_STATE_NONE; /* We need to make sure the new primary will start the replication stream @@ -4658,6 +4922,11 @@ void replicationHandlePrimaryDisconnection(void) { /* Any other repl_state means the state machine already moved on * (e.g. REPL_STATE_CONNECT, CONNECTING, NONE) — leave it untouched. */ + /* Tear down the compressed replication decoder so a later (possibly + * uncompressed) primary stream isn't fed into stale frame state. + * Idempotent when no decoder exists. */ + replDestroyDecompression(); + /* We lost connection with our primary, don't disconnect replicas yet, * maybe we'll be able to PSYNC with our primary later. We'll disconnect * the replicas only if we'll have to do a full resync with our primary. */ diff --git a/src/rio.c b/src/rio.c index b0142672cd2..569b3df3da3 100644 --- a/src/rio.c +++ b/src/rio.c @@ -56,6 +56,7 @@ #include "config.h" #include "server.h" #include "connhelpers.h" +#include "compression_stream.h" /* ------------------------- Buffer I/O implementation ----------------------- */ @@ -87,16 +88,17 @@ static int rioBufferFlush(rio *r) { } static const rio rioBufferIO = { - rioBufferRead, - rioBufferWrite, - rioBufferTell, - rioBufferFlush, - NULL, /* update_checksum */ - 0, /* current checksum */ - 0, /* flags */ - 0, /* bytes read or written */ - 0, /* read/write chunk size */ - {{NULL, 0}} /* union for io-specific vars */ + .read = rioBufferRead, + .write = rioBufferWrite, + .tell = rioBufferTell, + .flush = rioBufferFlush, + .read_some = NULL, + .update_cksum = NULL, + .cksum = 0, + .flags = 0, + .processed_bytes = 0, + .max_processing_chunk = 0, + .io = {{NULL, 0}}, }; void rioInitWithBuffer(rio *r, sds s) { @@ -127,7 +129,7 @@ static size_t rioFileWrite(rio *r, const void *buf, size_t len) { if (r->io.file.buffered >= r->io.file.autosync) { fflush(r->io.file.fp); - size_t processed = r->processed_bytes + nwritten; + size_t processed = r->stream_processed_bytes + nwritten; serverAssert(processed % r->io.file.autosync == 0); serverAssert(r->io.file.buffered == r->io.file.autosync); @@ -167,7 +169,14 @@ static size_t rioFileWrite(rio *r, const void *buf, size_t len) { /* Returns 1 or 0 for success/failure. */ static size_t rioFileRead(rio *r, void *buf, size_t len) { - return fread(buf, len, 1, r->io.file.fp); + return fread(buf, 1, len, r->io.file.fp) == len; +} + +/* Partial-read variant: returns bytes read (may be less than len), 0 on EOF, -1 on error. */ +static ssize_t rioFileReadSome(rio *r, void *buf, size_t len) { + size_t got = fread(buf, 1, len, r->io.file.fp); + if (got == 0 && ferror(r->io.file.fp)) return -1; + return (ssize_t)got; } /* Returns read/write position in file. */ @@ -182,16 +191,17 @@ static int rioFileFlush(rio *r) { } static const rio rioFileIO = { - rioFileRead, - rioFileWrite, - rioFileTell, - rioFileFlush, - NULL, /* update_checksum */ - 0, /* current checksum */ - 0, /* flags */ - 0, /* bytes read or written */ - 0, /* read/write chunk size */ - {{NULL, 0}} /* union for io-specific vars */ + .read = rioFileRead, + .write = rioFileWrite, + .tell = rioFileTell, + .flush = rioFileFlush, + .read_some = rioFileReadSome, + .update_cksum = NULL, + .cksum = 0, + .flags = 0, + .processed_bytes = 0, + .max_processing_chunk = 0, + .io = {{NULL, 0}}, }; void rioInitWithFile(rio *r, FILE *fp) { @@ -280,16 +290,17 @@ static int rioConnFlush(rio *r) { } static const rio rioConnIO = { - rioConnRead, - rioConnWrite, - rioConnTell, - rioConnFlush, - NULL, /* update_checksum */ - 0, /* current checksum */ - 0, /* flags */ - 0, /* bytes read or written */ - 0, /* read/write chunk size */ - {{NULL, 0}} /* union for io-specific vars */ + .read = rioConnRead, + .write = rioConnWrite, + .tell = rioConnTell, + .flush = rioConnFlush, + .read_some = NULL, + .update_cksum = NULL, + .cksum = 0, + .flags = 0, + .processed_bytes = 0, + .max_processing_chunk = 0, + .io = {{NULL, 0}}, }; /* Create an RIO that implements a buffered read from an fd @@ -396,16 +407,17 @@ static int rioFdFlush(rio *r) { } static const rio rioFdIO = { - rioFdRead, - rioFdWrite, - rioFdTell, - rioFdFlush, - NULL, /* update_checksum */ - 0, /* current checksum */ - 0, /* flags */ - 0, /* bytes read or written */ - 0, /* read/write chunk size */ - {{NULL, 0}} /* union for io-specific vars */ + .read = rioFdRead, + .write = rioFdWrite, + .tell = rioFdTell, + .flush = rioFdFlush, + .read_some = NULL, + .update_cksum = NULL, + .cksum = 0, + .flags = 0, + .processed_bytes = 0, + .max_processing_chunk = 0, + .io = {{NULL, 0}}, }; void rioInitWithFd(rio *r, int fd) { @@ -429,6 +441,81 @@ void rioGenericUpdateChecksum(rio *r, const void *buf, size_t len) { r->cksum = crc64(r->cksum, buf, len); } +void rioAttachStreamWriter(rio *r, streamWriter *writer) { + serverAssert(writer != NULL); + serverAssert(r->stream_writer == NULL); + serverAssert(r->stream_reader == NULL); + r->stream_writer = writer; +} + +void rioDetachStreamWriter(rio *r) { + r->stream_writer = NULL; +} + +void rioAttachStreamReader(rio *r, streamReader *reader) { + serverAssert(reader != NULL); + serverAssert(r->stream_reader == NULL); + serverAssert(r->stream_writer == NULL); + r->stream_reader = reader; +} + +void rioDetachStreamReader(rio *r) { + r->stream_reader = NULL; +} + +size_t rioWriteStream(rio *r, const void *buf, size_t len) { + serverAssert(r->stream_writer != NULL); + if (streamWriterWrite(r->stream_writer, buf, len) == C_OK) return 1; + r->flags |= RIO_FLAG_WRITE_ERROR; + return 0; +} + +size_t rioReadStream(rio *r, void *buf, size_t len) { + serverAssert(r->stream_reader != NULL); + + uint8_t *dst = (uint8_t *)buf; + size_t remaining = len; + while (remaining > 0) { + ssize_t nread = streamReaderRead(r->stream_reader, dst, remaining); + if (nread <= 0) { + r->flags |= RIO_FLAG_READ_ERROR; + return 0; + } + remaining -= (size_t)nread; + dst += nread; + } + return 1; +} + +/* rioRead() is for callers that know an exact logical field size. A stream + * reader instead treats its input buffer size as a capacity, so its callback + * uses this upper-bound raw read. Read up to `len` bytes from the concrete + * backend without stream dispatch or logical checksum/accounting. Returns: + * - >0 bytes read + * - 0 on EOF + * - -1 on error (sticky read error is latched on the rio) */ +ssize_t rioReadRawPartial(rio *r, void *buf, size_t len) { + if (r->flags & (RIO_FLAG_READ_ERROR | RIO_FLAG_CLOSE_ASAP)) return -1; + if (len == 0) return 0; + if (!r->read_some) { + r->flags |= RIO_FLAG_READ_ERROR; + return -1; + } + + size_t bytes_to_read = + (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; + ssize_t got = r->read_some(r, buf, bytes_to_read); + + if (got < 0) { + r->flags |= RIO_FLAG_READ_ERROR; + return -1; + } + if (got > 0) { + r->stream_processed_bytes += (size_t)got; + } + return got; +} + /* Set the file-based rio object to auto-fsync every 'bytes' file written. * By default this is set to zero that means no automatic file sync is * performed. @@ -451,20 +538,6 @@ void rioSetReclaimCache(rio *r, int enabled) { r->io.file.reclaim_cache = enabled; } -/* Check the type of rio. */ -uint8_t rioCheckType(rio *r) { - if (r->read == rioFileRead) { - return RIO_TYPE_FILE; - } else if (r->read == rioBufferRead) { - return RIO_TYPE_BUFFER; - } else if (r->read == rioConnRead) { - return RIO_TYPE_CONN; - } else { - /* r->read == rioFdRead */ - return RIO_TYPE_FD; - } -} - /* --------------------------- Higher level interface -------------------------- * * The following higher level functions use lower level rio.c functions to help @@ -604,16 +677,17 @@ static int rioConnsetFlush(rio *r) { } static const rio rioConnsetIO = { - rioConnsetRead, - rioConnsetWrite, - rioConnsetTell, - rioConnsetFlush, - NULL, /* update_checksum */ - 0, /* current checksum */ - 0, /* flags */ - 0, /* bytes read or written */ - 0, /* read/write chunk size */ - {{NULL, 0}} /* union for io-specific vars */ + .read = rioConnsetRead, + .write = rioConnsetWrite, + .tell = rioConnsetTell, + .flush = rioConnsetFlush, + .read_some = NULL, + .update_cksum = NULL, + .cksum = 0, + .flags = 0, + .processed_bytes = 0, + .max_processing_chunk = 0, + .io = {{NULL, 0}}, }; void rioInitWithConnset(rio *r, connection **conns, int numconns) { @@ -635,3 +709,17 @@ void rioFreeConnset(rio *r) { zfree(r->io.connset.state); sdsfree(r->io.connset.buf); } + +/* Check the type of rio. */ +uint8_t rioCheckType(rio *r) { + if (r->read == rioFileRead) { + return RIO_TYPE_FILE; + } else if (r->read == rioBufferRead) { + return RIO_TYPE_BUFFER; + } else if (r->read == rioConnRead) { + return RIO_TYPE_CONN; + } else { + /* r->read == rioFdRead */ + return RIO_TYPE_FD; + } +} diff --git a/src/rio.h b/src/rio.h index f7d43ef84a8..1512a27a431 100644 --- a/src/rio.h +++ b/src/rio.h @@ -33,6 +33,8 @@ #define VALKEY_RIO_H #include +#include +#include #include #include "sds.h" #include "connection.h" @@ -41,20 +43,28 @@ #define RIO_FLAG_WRITE_ERROR (1 << 1) #define RIO_FLAG_CLOSE_ASAP (1 << 2) /* Rio was closed asynchronously during the current rio operation. */ #define RIO_FLAG_SKIP_RDB_CHECKSUM (1 << 3) +#define RIO_FLAG_STREAMING_COMPRESSION (1 << 4) /* Input uses whole-stream compression. */ #define RIO_TYPE_FILE (1 << 0) #define RIO_TYPE_BUFFER (1 << 1) #define RIO_TYPE_CONN (1 << 2) #define RIO_TYPE_FD (1 << 3) +struct streamWriter; +struct streamReader; + struct _rio { - /* Backend functions. - * Since this functions do not tolerate short writes or reads the return - * value is simplified to: zero on error, non zero on complete success. */ + /* Backend functions. read and write are the exact-length interface used by + * parsers: zero means failure, nonzero means all len bytes were processed. */ size_t (*read)(struct _rio *, void *buf, size_t len); size_t (*write)(struct _rio *, const void *buf, size_t len); off_t (*tell)(struct _rio *); int (*flush)(struct _rio *); + /* Partial-read backend. Here len is a maximum, not a requirement: return + * >0 bytes read, 0 on EOF, or -1 on error. A stream decoder uses this + * because it cannot know how many encoded bytes will produce the decoded + * bytes requested by its caller. NULL when the backend cannot be read. */ + ssize_t (*read_some)(struct _rio *, void *buf, size_t len); /* The update_cksum method if not NULL is used to compute the checksum of * all the data that was read or written so far. The method should be * designed so that can be called with the current checksum, and the buf @@ -68,9 +78,25 @@ struct _rio { /* number of bytes read or written */ size_t processed_bytes; - /* maximum single read or write chunk size */ + /* Number of bytes read or written on the stream's concrete I/O path. This + * differs from processed_bytes when a stream transforms data. */ + size_t stream_processed_bytes; + + /* Maximum size of one backend operation, not a total byte limit. Zero + * means unlimited. rioRead/rioWrite split larger requests into chunks. */ size_t max_processing_chunk; + /* Optional stream transforms. The caller owns their lifetime, and rio only + * dispatches logical bytes through these opaque objects: + * + * write: rioWrite -> streamWriter -> rioWriteRaw -> backend write + * read: rioRead <- streamReader <- rioReadRawPartial <- backend read + * + * Compression policy, framing, buffers, and codec state remain in + * compression_stream. */ + struct streamWriter *stream_writer; + struct streamReader *stream_reader; + /* Backend-specific vars. */ union { /* In-memory buffer target. */ @@ -116,17 +142,41 @@ typedef struct _rio rio; * actual implementation of read / write / tell, and will update the checksum * if needed. */ -static inline size_t rioWrite(rio *r, const void *buf, size_t len) { +/* Implemented in rio.c, where the opaque stream types are visible. */ +size_t rioWriteStream(rio *r, const void *buf, size_t len); +size_t rioReadStream(rio *r, void *buf, size_t len); + +/* Write directly to the concrete backend, bypassing the stream writer and + * logical checksum/accounting. streamWriter uses this to emit encoded bytes + * without recursively invoking itself. */ +static inline size_t rioWriteRaw(rio *r, const void *buf, size_t len) { if (r->flags & RIO_FLAG_WRITE_ERROR || r->flags & RIO_FLAG_CLOSE_ASAP) return 0; while (len) { size_t bytes_to_write = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; - if (r->update_cksum) r->update_cksum(r, buf, bytes_to_write); if (r->write(r, buf, bytes_to_write) == 0) { r->flags |= RIO_FLAG_WRITE_ERROR; return 0; } - buf = (char *)buf + bytes_to_write; + buf = (const char *)buf + bytes_to_write; + len -= bytes_to_write; + r->stream_processed_bytes += bytes_to_write; + } + return 1; +} + +static inline size_t rioWrite(rio *r, const void *buf, size_t len) { + if (r->flags & RIO_FLAG_WRITE_ERROR || r->flags & RIO_FLAG_CLOSE_ASAP) return 0; + while (len) { + size_t bytes_to_write = + (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; + if (r->update_cksum) r->update_cksum(r, buf, bytes_to_write); + if (r->stream_writer) { + if (rioWriteStream(r, buf, bytes_to_write) == 0) return 0; + } else { + if (rioWriteRaw(r, buf, bytes_to_write) == 0) return 0; + } + buf = (const char *)buf + bytes_to_write; len -= bytes_to_write; r->processed_bytes += bytes_to_write; } @@ -138,9 +188,14 @@ static inline size_t rioRead(rio *r, void *buf, size_t len) { while (len) { size_t bytes_to_read = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; - if (r->read(r, buf, bytes_to_read) == 0) { - r->flags |= RIO_FLAG_READ_ERROR; - return 0; + if (r->stream_reader) { + if (rioReadStream(r, buf, bytes_to_read) == 0) return 0; + } else { + if (r->read(r, buf, bytes_to_read) == 0) { + r->flags |= RIO_FLAG_READ_ERROR; + return 0; + } + r->stream_processed_bytes += bytes_to_read; } if (r->update_cksum) r->update_cksum(r, buf, bytes_to_read); buf = (char *)buf + bytes_to_read; @@ -151,11 +206,18 @@ static inline size_t rioRead(rio *r, void *buf, size_t len) { } static inline off_t rioTell(rio *r) { + /* Stream readers report physical bytes consumed from the source, which + * drives file-loading progress for decoded streams. */ + if (r->stream_reader) return (off_t)r->stream_processed_bytes; return r->tell(r); } +static inline int rioFlushRaw(rio *r) { + return r->flush ? r->flush(r) : 1; +} + static inline int rioFlush(rio *r) { - return r->flush(r); + return rioFlushRaw(r); } static inline void rioCloseASAP(rio *r) { @@ -187,6 +249,10 @@ void rioInitWithFile(rio *r, FILE *fp); void rioInitWithBuffer(rio *r, sds s); void rioInitWithConn(rio *r, connection *conn, size_t read_limit); void rioInitWithFd(rio *r, int fd); +void rioAttachStreamWriter(rio *r, struct streamWriter *writer); +void rioDetachStreamWriter(rio *r); +void rioAttachStreamReader(rio *r, struct streamReader *reader); +void rioDetachStreamReader(rio *r); void rioFreeFd(rio *r); void rioFreeConn(rio *r, sds *out_remainingBufferedData); @@ -200,6 +266,7 @@ struct serverObject; int rioWriteBulkObject(rio *r, struct serverObject *obj); void rioGenericUpdateChecksum(rio *r, const void *buf, size_t len); +ssize_t rioReadRawPartial(rio *r, void *buf, size_t len); void rioSetAutoSync(rio *r, off_t bytes); void rioSetReclaimCache(rio *r, int enabled); uint8_t rioCheckType(rio *r); diff --git a/src/server.c b/src/server.c index 10a2c6b06d9..5324f5b29ca 100644 --- a/src/server.c +++ b/src/server.c @@ -47,6 +47,8 @@ #include "threads_mngr.h" #include "fmtargs.h" #include "io_threads.h" +#include "compression.h" +#include "compression_repl.h" #include "tls.h" #include "sds.h" #include "module.h" @@ -2823,6 +2825,10 @@ void resetServerStats(void) { server.stat_sync_full = 0; server.stat_sync_partial_ok = 0; server.stat_sync_partial_err = 0; + atomic_store_explicit(&server.repl_compression_errors, 0, memory_order_relaxed); + server.repl_decompression_errors = 0; + server.repl_decompression_time_usec = 0; + server.repl_decompressed_bytes_total = 0; server.stat_io_reads_processed = 0; server.stat_total_reads_processed = 0; server.stat_io_writes_processed = 0; @@ -6627,6 +6633,23 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "slave_priority:%d\r\n", server.replica_priority, "slave_read_only:%d\r\n", server.repl_replica_ro, "replica_announced:%d\r\n", server.replica_announced)); + if (server.repl_decompression_errors > 0) { + info = sdscatfmt(info, "repl_decompression_errors:%I\r\n", server.repl_decompression_errors); + } + if (server.repl_decompressed_bytes_total > 0) { + info = sdscatfmt(info, + "repl_decompression_time_usec:%I\r\n" + "repl_decompressed_bytes_total:%I\r\n", + server.repl_decompression_time_usec, + server.repl_decompressed_bytes_total); + } + } + + /* Aggregated across replicas; survives disconnects. Emitted when non-zero. */ + long long repl_compression_errors = + (long long)atomic_load_explicit(&server.repl_compression_errors, memory_order_relaxed); + if (repl_compression_errors > 0) { + info = sdscatfmt(info, "repl_compression_errors:%I\r\n", repl_compression_errors); } info = sdscatprintf(info, "connected_slaves:%lu\r\n", listLength(server.replicas)); @@ -6659,12 +6682,30 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { info = sdscatprintf(info, "slave%d:ip=%s,port=%d,state=%s," - "offset=%lld,lag=%ld,type=%s\r\n", + "offset=%lld,lag=%ld,type=%s", replica_id, replica_ip, replica->repl_data->replica_listening_port, state, replica->repl_data->repl_ack_off, lag, replica->flag.repl_rdb_channel ? "rdb-channel" : replica->repl_data->repl_state == REPLICA_STATE_BG_RDB_LOAD ? "main-channel" : "replica"); + if (replica->repl_data->repl_compressor) { + info = sdscatprintf(info, + ",compression=%s" + ",compressed_bytes=%lld" + ",uncompressed_bytes=%lld" + ",compression_ratio=%.2f" + ",compression_time_usec=%lld", + compressionAlgoName(replCompressorAlgo(replica->repl_data->repl_compressor)), + replica->repl_data->repl_compressed_bytes_total, + replica->repl_data->repl_uncompressed_bytes_total, + replica->repl_data->repl_uncompressed_bytes_total > 0 + ? (double)replica->repl_data->repl_compressed_bytes_total / + (double)replica->repl_data->repl_uncompressed_bytes_total + : 0.0, + atomic_load_explicit(&replica->repl_data->repl_compression_time_usec, + memory_order_relaxed)); + } + info = sdscat(info, "\r\n"); replica_id++; } } diff --git a/src/server.h b/src/server.h index 0ac55373d37..9990d0dd51e 100644 --- a/src/server.h +++ b/src/server.h @@ -452,9 +452,11 @@ typedef enum { #define REPLICA_CAPA_PSYNC2 (1 << 1) /* Supports PSYNC2 protocol. */ #define REPLICA_CAPA_DUAL_CHANNEL (1 << 2) /* Supports dual channel replication sync */ #define REPLICA_CAPA_SKIP_RDB_CHECKSUM (1 << 3) /* Supports skipping RDB checksum for sync requests. */ +#define REPLICA_CAPA_COMPRESSION (1 << 4) /* Supports replication compression. */ /* Replica capability strings */ #define REPLICA_CAPA_SKIP_RDB_CHECKSUM_STR "skip-rdb-checksum" /* Supports skipping RDB checksum for sync requests. */ +#define REPLICA_CAPA_COMPRESSION_STR "compression" /* Supports replication compression. */ /* Replica requirements */ #define REPLICA_REQ_NONE 0 @@ -605,6 +607,18 @@ typedef enum { RDB_VERSION_CHECK_RELAXED } rdb_version_check_type; +typedef enum { + RDB_COMPRESSION_NO = 0, /* Disable RDB compression. */ + RDB_COMPRESSION_YES, /* Use the default compression algorithm. */ + RDB_COMPRESSION_LZF, /* Pin legacy per-string LZF compression. */ + RDB_COMPRESSION_LZ4 /* Pin whole-stream LZ4 compression. */ +} rdb_compression_mode; + +typedef enum { + REPL_COMPRESSION_NO = 0, + REPL_COMPRESSION_LZ4 +} repl_compression_mode; + /* Structure representing a non-owning view of a buffer. * A stringRef struct does not manage the underlying memory, so its destruction * will not free the buffer. */ @@ -1221,6 +1235,9 @@ typedef struct ClientPubSubData { context of client side caching. */ } ClientPubSubData; +typedef struct replCompressor replCompressor; +typedef struct replDecompressor replDecompressor; + typedef struct ClientReplicationData { int repl_state; /* Replication state if this is a replica. */ int repl_start_cmd_stream_on_ack; /* Install replica write handler on first ACK. */ @@ -1252,6 +1269,16 @@ typedef struct ClientReplicationData { size_t ref_block_pos; /* Access position of referenced buffer block, i.e. the next offset to send. */ sds replica_nodeid; /* Node id in cluster mode. */ + /* Incremental replication compression state (primary side, per replica). */ + replCompressor *repl_compressor; /* Per-replica replication compressor (NULL if uncompressed). */ + _Atomic(int) compression_error; /* Set by the IO thread on compression failure, + * read by the main thread in postWriteToReplica. */ + unsigned int repl_compression_decided : 1; /* Compression decision frozen when the command stream + * started; put-online must not re-make it. */ + /* Compression metrics (primary side, per replica). */ + long long repl_compressed_bytes_total; /* Total compressed bytes sent (main thread only). */ + long long repl_uncompressed_bytes_total; /* Total raw bytes before compression (main thread only). */ + _Atomic(long long) repl_compression_time_usec; /* Cumulative wall-clock usec in the compression path. */ } ClientReplicationData; typedef struct ClientModuleData { @@ -2063,7 +2090,11 @@ struct valkeyServer { struct saveparam *saveparams; /* Save points array for RDB */ int saveparamslen; /* Number of saving points */ char *rdb_filename; /* Name of RDB file */ - int rdb_compression; /* Use compression in RDB? */ + int rdb_compression; /* RDB compression mode */ + int repl_compression; /* Replication compression mode */ + int repl_provisional_compression; /* Replica: compression advertised on the current upstream + * handshake. Gates decompressor setup (not the live config), + * so a mid-handshake config flip cannot desync the link. */ int rdb_checksum; /* Use RDB checksum? */ int rdb_del_sync_files; /* Remove RDB files used only for SYNC if the instance does not use persistence. */ @@ -2178,33 +2209,39 @@ struct valkeyServer { long long read_reploff; int dbid; } repl_provisional_primary; - client *cached_primary; /* Cached primary to be reused for PSYNC. */ - rio *loading_rio; /* Pointer to the rio object currently used for loading data. */ - int repl_syncio_timeout; /* Timeout for synchronous I/O calls */ - int repl_state; /* Replication status if the instance is a replica */ - int repl_rdb_channel_state; /* State of the replica's rdb channel during dual-channel-replication */ - off_t repl_transfer_size; /* Size of RDB to read from primary during sync. */ - off_t repl_transfer_read; /* Amount of RDB read from primary during sync. */ - off_t repl_transfer_last_fsync_off; /* Offset when we fsync-ed last time. */ - connection *repl_transfer_s; /* Replica -> Primary SYNC connection */ - connection *repl_rdb_transfer_s; /* Primary FULL SYNC connection (RDB download) */ - int repl_transfer_fd; /* Replica -> Primary SYNC temp file descriptor */ - char *repl_transfer_tmpfile; /* Replica-> Primary SYNC temp file name */ - _Atomic(time_t) repl_transfer_lastio; /* Unix time of the latest read, for timeout */ - int repl_serve_stale_data; /* Serve stale data when link is down? */ - int repl_replica_ro; /* Replica is read only? */ - int repl_replica_ignore_maxmemory; /* If true replicas do not evict. */ - time_t repl_down_since; /* Unix time at which link with primary went down */ - int repl_disable_tcp_nodelay; /* Disable TCP_NODELAY after SYNC? */ - int repl_mptcp; /* Use Multipath TCP for replica on client side */ - int replica_priority; /* Reported in INFO and used by Sentinel. */ - int replica_announced; /* If true, replica is announced by Sentinel */ - int replica_announce_port; /* Give the primary this listening port. */ - char *replica_announce_ip; /* Give the primary this ip address. */ - int propagation_error_behavior; /* Configures the behavior of the replica - * when it receives an error on the replication stream */ - int repl_ignore_disk_write_error; /* Configures whether replicas panic when unable to - * persist writes to AOF. */ + client *cached_primary; /* Cached primary to be reused for PSYNC. */ + rio *loading_rio; /* Pointer to the rio object currently used for loading data. */ + int repl_syncio_timeout; /* Timeout for synchronous I/O calls */ + int repl_state; /* Replication status if the instance is a replica */ + int repl_rdb_channel_state; /* State of the replica's rdb channel during dual-channel-replication */ + off_t repl_transfer_size; /* Size of RDB to read from primary during sync. */ + off_t repl_transfer_read; /* Amount of RDB read from primary during sync. */ + off_t repl_transfer_last_fsync_off; /* Offset when we fsync-ed last time. */ + connection *repl_transfer_s; /* Replica -> Primary SYNC connection */ + connection *repl_rdb_transfer_s; /* Primary FULL SYNC connection (RDB download) */ + int repl_transfer_fd; /* Replica -> Primary SYNC temp file descriptor */ + char *repl_transfer_tmpfile; /* Replica-> Primary SYNC temp file name */ + _Atomic(time_t) repl_transfer_lastio; /* Unix time of the latest read, for timeout */ + int repl_serve_stale_data; /* Serve stale data when link is down? */ + int repl_replica_ro; /* Replica is read only? */ + int repl_replica_ignore_maxmemory; /* If true replicas do not evict. */ + time_t repl_down_since; /* Unix time at which link with primary went down */ + int repl_disable_tcp_nodelay; /* Disable TCP_NODELAY after SYNC? */ + int repl_mptcp; /* Use Multipath TCP for replica on client side */ + int replica_priority; /* Reported in INFO and used by Sentinel. */ + int replica_announced; /* If true, replica is announced by Sentinel */ + int replica_announce_port; /* Give the primary this listening port. */ + char *replica_announce_ip; /* Give the primary this ip address. */ + int propagation_error_behavior; /* Configures the behavior of the replica + * when it receives an error on the replication stream */ + int repl_ignore_disk_write_error; /* Configures whether replicas panic when unable to + * persist writes to AOF. */ + replDecompressor *repl_decompressor; /* Replica-side replication decoder (NULL when inactive). */ + long long repl_decompression_errors; /* Decompression failures (replica side). */ + long long repl_decompression_time_usec; /* Cumulative wall-clock usec in the decompression path. */ + long long repl_decompressed_bytes_total; /* Total decompressed bytes processed (replica side). */ + /* Compression failures across all replicas; written from IO threads. */ + _Atomic(long long) repl_compression_errors; /* The following two fields is where we store primary PSYNC replid/offset * while the PSYNC is in progress. At the end we'll copy the fields into @@ -3027,6 +3064,9 @@ int getClientTypeByName(char *name); char *getClientTypeName(int client_class); void flushReplicasOutputBuffers(void); void disconnectReplicas(void); +void reconcileReplicaCompression(void); +void replDestroyCompression(client *c); +int replDecompressQueryBuf(client *c, size_t new_data_start, size_t *decoded); void evictClients(void); int listenToPort(connListener *fds); void pauseActions(pause_purpose purpose, mstime_t end, uint32_t actions); @@ -3055,6 +3095,7 @@ void protectClient(client *c); void unprotectClient(client *c); void initSharedQueryBuf(void); void freeSharedQueryBuf(void); +void clientUnshareQuerybufIfNeeded(client *c, size_t addlen); client *lookupClientByID(uint64_t id); int authRequired(client *c); void clientSetUser(client *c, user *u, int authenticated); diff --git a/src/unit/CMakeLists.txt b/src/unit/CMakeLists.txt index b4a23cbc5bd..ca12c8be0fc 100644 --- a/src/unit/CMakeLists.txt +++ b/src/unit/CMakeLists.txt @@ -142,6 +142,7 @@ target_link_libraries( PRIVATE valkeylib-gtest fpconv + lz4 lualib hdr_histogram valkey::valkey diff --git a/src/unit/Makefile b/src/unit/Makefile index 3c9ed44bbb0..f27417f461b 100644 --- a/src/unit/Makefile +++ b/src/unit/Makefile @@ -94,6 +94,7 @@ HIREDIS_LIB := ../../deps/libvalkey/lib/libvalkey.a JEMALLOC_LIB := ../../deps/jemalloc/lib/libjemalloc.a HDR_HISTOGRAM_LIB := ../../deps/hdr_histogram/libhdrhistogram.a FPCONV_LIB := ../../deps/fpconv/libfpconv.a +LZ4_LIB := ../../deps/lz4/liblz4.a TLS_LIBS := ../../deps/libvalkey/lib/libvalkey_tls.a -lssl -lcrypto GTEST_CFLAGS ?= GTEST_LIBS ?= @@ -174,7 +175,8 @@ LD_LIBS := $(HIREDIS_LIB) \ $(JEMALLOC_LIB) \ $(GTEST_LIBS) \ $(HDR_HISTOGRAM_LIB) \ - $(FPCONV_LIB) + $(FPCONV_LIB) \ + $(LZ4_LIB) # Add libbacktrace support if enabled ifeq ($(USE_LIBBACKTRACE),yes) diff --git a/src/unit/test_compression.cpp b/src/unit/test_compression.cpp new file mode 100644 index 00000000000..38e26f5adc3 --- /dev/null +++ b/src/unit/test_compression.cpp @@ -0,0 +1,1323 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "generated_wrappers.hpp" + +#include +#include + +extern "C" { +#include "compression.h" +#include "compression_stream.h" +#include "server.h" +#include "zmalloc.h" +} + +/* zmalloc.h defines helper macros (__str/__xstr) that collide with libstdc++ internals. + * Keep them local to C headers in this C++ translation unit. */ +#ifdef __xstr +#undef __xstr +#endif +#ifdef __str +#undef __str +#endif + +typedef struct { + const uint8_t *data; + size_t len; + size_t pos; + size_t max_chunk; /* 0 => unbounded */ + size_t fail_after_pos; + bool fail_after_reads; + int fail_after_success_reads; + int success_reads; + int calls; + int overread_on_call; +} MemReader; + +typedef struct { + int calls; + int fail_on_call; +} FailingEmitter; + +static streamReaderConfig makeReaderConfig(bool allow_passthrough, + size_t buffer_size, + bool skip_codec_checksum_validation) { + streamReaderConfig cfg = {}; + cfg.allow_passthrough = allow_passthrough; + cfg.skip_codec_checksum_validation = skip_codec_checksum_validation; + cfg.buffer_size = buffer_size; + return cfg; +} + +static ssize_t memReaderRead(void *ctx, void *buf, size_t len) { + MemReader *r = (MemReader *)ctx; + r->calls++; + if (r->overread_on_call > 0 && r->calls == r->overread_on_call) return (ssize_t)(len + 1); + if (r->fail_after_reads && r->success_reads >= r->fail_after_success_reads) return -1; + if (r->fail_after_pos > 0 && r->pos >= r->fail_after_pos) return -1; + if (r->pos >= r->len) return 0; + + size_t avail = r->len - r->pos; + size_t n = len < avail ? len : avail; + if (r->max_chunk && n > r->max_chunk) n = r->max_chunk; + if (r->fail_after_pos > 0) { + size_t until_failure = r->fail_after_pos - r->pos; + if (n > until_failure) n = until_failure; + } + + memcpy(buf, r->data + r->pos, n); + r->pos += n; + r->success_reads++; + return (ssize_t)n; +} + +static ssize_t noProgressDecompressorFeed(streamDecompressor *decompressor, + uint8_t *output, + size_t output_capacity, + const uint8_t *input, + size_t input_len, + size_t *input_consumed) { + (void)output; + (void)output_capacity; + (void)input; + (void)input_len; + decompressor->input_hint = 0; + *input_consumed = 0; + return 0; +} + +static ssize_t overconsumingDecompressorFeed(streamDecompressor *decompressor, + uint8_t *output, + size_t output_capacity, + const uint8_t *input, + size_t input_len, + size_t *input_consumed) { + (void)decompressor; + (void)output; + (void)output_capacity; + (void)input; + *input_consumed = input_len + 1; + return 0; +} + +static ssize_t overproducingDecompressorFeed(streamDecompressor *decompressor, + uint8_t *output, + size_t output_capacity, + const uint8_t *input, + size_t input_len, + size_t *input_consumed) { + (void)decompressor; + (void)output; + (void)input; + (void)input_len; + *input_consumed = 0; + return (ssize_t)(output_capacity + 1); +} + +static ssize_t decompressAll(streamDecompressor *decompressor, + const uint8_t *input, + size_t input_len, + uint8_t *output, + size_t output_capacity) { + size_t input_offset = 0; + size_t output_len = 0; + + while (input_offset < input_len && !decompressor->frame_done) { + size_t consumed = 0; + ssize_t produced = streamDecompressorFeed( + decompressor, output + output_len, output_capacity - output_len, + input + input_offset, input_len - input_offset, &consumed); + if (produced < 0 || consumed > input_len - input_offset || + (size_t)produced > output_capacity - output_len) { + return -1; + } + input_offset += consumed; + output_len += (size_t)produced; + if (consumed == 0 && produced == 0) return -1; + } + if (!decompressor->frame_done || input_offset != input_len) return -1; + return (ssize_t)output_len; +} + +/* =================================================================== + * Streaming compression/decompression tests + * =================================================================== */ + +TEST(CompressionTest, streamCompressorOutputBound) { + const size_t input_sizes[] = {0, 1, 1024, 64 * 1024}; + const compressFlushMode flush_modes[] = { + COMPRESS_FLUSH_CONTINUE, + COMPRESS_FLUSH_END, + }; + const size_t max_input_size = input_sizes[sizeof(input_sizes) / sizeof(input_sizes[0]) - 1]; + uint8_t *input = (uint8_t *)zmalloc(max_input_size); + for (size_t i = 0; i < max_input_size; i++) input[i] = (uint8_t)(i % 251); + + for (size_t i = 0; i < sizeof(input_sizes) / sizeof(input_sizes[0]); i++) { + for (size_t j = 0; j < sizeof(flush_modes) / sizeof(flush_modes[0]); j++) { + streamCompressor compressor; + ASSERT_EQ(streamCompressorInit(&compressor, ALGO_LZ4, 0, false), C_OK); + size_t bound = streamCompressorOutputBound(&compressor, input_sizes[i]); + uint8_t *output = (uint8_t *)zmalloc(bound); + + EXPECT_GE(streamCompressorFeed(&compressor, output, bound, + input, input_sizes[i], flush_modes[j]), + 0) + << "input size " << input_sizes[i] << ", flush mode " << flush_modes[j]; + + zfree(output); + streamCompressorFree(&compressor); + } + } + + zfree(input); +} + +TEST(CompressionTest, streamReaderClassifiesProbeInputs) { + static const uint8_t plain_input[] = {'H', 'E', 'L', 'L', 'O'}; + static const uint8_t strict_non_vcs[VCS_ENVELOPE_SIZE] = {'R', 'E', 'D', 'I', 'S', '0', '0'}; + + struct { + const char *name; + const uint8_t *input; + size_t input_len; + size_t max_chunk; + bool allow_passthrough; + bool expect_probe_ok; + streamReaderErrorKind expected_error; + size_t expected_read_len; + } cases[] = { + {"plain passthrough", + plain_input, + sizeof(plain_input), + 2, + true, + true, + STREAM_READER_ERROR_NONE, + sizeof(plain_input)}, + {"non-VCS with passthrough disabled", + strict_non_vcs, + sizeof(strict_non_vcs), + 0, + false, + false, + STREAM_READER_ERROR_INCOMPATIBLE, + 0}, + }; + + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + MemReader mr = {}; + mr.data = cases[i].input; + mr.len = cases[i].input_len; + mr.max_chunk = cases[i].max_chunk; + streamReaderConfig cfg = makeReaderConfig(cases[i].allow_passthrough, + STREAM_READER_BUFFER_SIZE_DEFAULT, + false); + streamReader t; + compressionAlgo algo = ALGO_NONE; + if (cases[i].expect_probe_ok) { + ASSERT_EQ(streamReaderInit(&t, &cfg, memReaderRead, &mr, &algo), C_OK) << cases[i].name; + ASSERT_EQ(algo, ALGO_NONE) << cases[i].name; + + uint8_t out[16] = {0}; + ASSERT_EQ(streamReaderRead(&t, out, cases[i].expected_read_len), (ssize_t)cases[i].expected_read_len) + << cases[i].name; + EXPECT_EQ(memcmp(out, cases[i].input, cases[i].expected_read_len), 0) << cases[i].name; + ASSERT_EQ(streamReaderRead(&t, out, sizeof(out)), 0) << cases[i].name; + ASSERT_EQ(streamReaderFinish(&t), C_OK) << cases[i].name; + ASSERT_EQ(t.state, STREAM_READER_STATE_FINISHED) << cases[i].name; + } else { + ASSERT_EQ(streamReaderInit(&t, &cfg, memReaderRead, &mr, &algo), C_ERR) << cases[i].name; + ASSERT_EQ(t.error_kind, cases[i].expected_error) << cases[i].name; + + uint8_t out[8] = {0}; + ASSERT_EQ(streamReaderRead(&t, out, sizeof(out)), -1) << cases[i].name; + } + + streamReaderFree(&t); + } +} + +TEST(CompressionTest, streamReaderRejectsEveryTruncatedVcsEnvelope) { + const uint8_t envelope[VCS_ENVELOPE_SIZE] = { + VCS_MAGIC_0, + VCS_MAGIC_1, + VCS_MAGIC_2, + VCS_VERSION, + VCS_CODEC_LZ4, + 0, + VCS_STREAM_RDB, + }; + + for (size_t prefix_len = 1; prefix_len < VCS_ENVELOPE_SIZE; prefix_len++) { + MemReader mr = {}; + mr.data = envelope; + mr.len = prefix_len; + mr.max_chunk = 1; + streamReaderConfig cfg = makeReaderConfig(true, STREAM_READER_BUFFER_SIZE_DEFAULT, false); + streamReader reader; + compressionAlgo algo = ALGO_NONE; + ASSERT_EQ(streamReaderInit(&reader, &cfg, memReaderRead, &mr, &algo), C_ERR) + << "accepted VCS prefix length " << prefix_len; + ASSERT_EQ(reader.error_kind, STREAM_READER_ERROR_INCOMPATIBLE) + << "VCS prefix length " << prefix_len; + streamReaderFree(&reader); + } + + MemReader empty = {}; + streamReaderConfig cfg = makeReaderConfig(true, STREAM_READER_BUFFER_SIZE_DEFAULT, false); + streamReader reader; + compressionAlgo algo = ALGO_LZ4; + ASSERT_EQ(streamReaderInit(&reader, &cfg, memReaderRead, &empty, &algo), C_OK); + ASSERT_EQ(algo, ALGO_NONE); + uint8_t out = 0; + ASSERT_EQ(streamReaderRead(&reader, &out, 1), 0); + streamReaderFree(&reader); +} + +TEST(CompressionTest, streamReaderClampsSmallBuffer) { + const uint8_t input[] = {'R', 'D', 'B'}; + MemReader source = {}; + source.data = input; + source.len = sizeof(input); + streamReaderConfig cfg = makeReaderConfig(true, 1, false); + streamReader reader; + ASSERT_EQ(streamReaderInit(&reader, &cfg, memReaderRead, &source, NULL), C_OK); + ASSERT_EQ(reader.buffer_size, (size_t)STREAM_READER_BUFFER_SIZE_MIN); + ASSERT_EQ(streamReaderFinish(&reader), C_OK); + streamReaderFree(&reader); +} + +TEST(CompressionTest, streamReaderClassifiesCodecContractViolationsAsInternalErrors) { + typedef ssize_t (*feedFn)(streamDecompressor *, uint8_t *, size_t, const uint8_t *, size_t, size_t *); + struct { + const char *name; + feedFn feed; + } cases[] = { + {"consumed input exceeds supplied input", overconsumingDecompressorFeed}, + {"produced output exceeds supplied capacity", overproducingDecompressorFeed}, + }; + const uint8_t input[] = { + VCS_MAGIC_0, + VCS_MAGIC_1, + VCS_MAGIC_2, + VCS_VERSION, + VCS_CODEC_LZ4, + 0, + VCS_STREAM_RDB, + 0, + }; + + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + MockValkey mock; + EXPECT_CALL(mock, streamDecompressorFeed(_, _, _, _, _, _)) + .WillOnce(Invoke(cases[i].feed)); + + MemReader source = {}; + source.data = input; + source.len = sizeof(input); + streamReaderConfig cfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_MIN, false); + streamReader reader; + ASSERT_EQ(streamReaderInit(&reader, &cfg, memReaderRead, &source, NULL), C_OK) << cases[i].name; + + uint8_t output; + EXPECT_EQ(streamReaderRead(&reader, &output, 1), -1) << cases[i].name; + EXPECT_EQ(reader.error_kind, STREAM_READER_ERROR_INTERNAL) << cases[i].name; + streamReaderFree(&reader); + } +} + +TEST(CompressionTest, streamReaderRejectsFullInputBufferWithoutCodecProgress) { + MockValkey mock; + EXPECT_CALL(mock, streamDecompressorFeed(_, _, _, _, _, _)) + .WillRepeatedly(Invoke(noProgressDecompressorFeed)); + + size_t input_len = VCS_ENVELOPE_SIZE + STREAM_READER_COMPRESSED_BUFFER_SIZE + 1; + uint8_t *input = (uint8_t *)zmalloc(input_len); + memset(input, 0, input_len); + const uint8_t envelope[VCS_ENVELOPE_SIZE] = { + VCS_MAGIC_0, + VCS_MAGIC_1, + VCS_MAGIC_2, + VCS_VERSION, + VCS_CODEC_LZ4, + 0, + VCS_STREAM_RDB, + }; + memcpy(input, envelope, sizeof(envelope)); + + MemReader source = {}; + source.data = input; + source.len = input_len; + streamReaderConfig cfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_DEFAULT, false); + streamReader reader; + ASSERT_EQ(streamReaderInit(&reader, &cfg, memReaderRead, &source, NULL), C_OK); + + uint8_t output; + EXPECT_EQ(streamReaderRead(&reader, &output, 1), -1); + EXPECT_EQ(reader.error_kind, STREAM_READER_ERROR_CORRUPT); + EXPECT_EQ(reader.compressed_buf_len, reader.compressed_buf_size); + EXPECT_LT(reader.compressed_buf_size, reader.buffer_size); + EXPECT_LT(source.pos, source.len); + + streamReaderFree(&reader); + zfree(input); +} + +/* =================================================================== + * Tests for stream writer API and rio decorators + * =================================================================== */ + +typedef struct { + uint8_t *data; +} DynamicBuf; + +static void dynamicBufInit(DynamicBuf *db) { + db->data = (uint8_t *)sdsempty(); +} + +static void dynamicBufFree(DynamicBuf *db) { + if (db->data) sdsfree((sds)db->data); + db->data = NULL; +} + +static int emitToDynamicBuf(void *ctx, const uint8_t *data, size_t len) { + DynamicBuf *db = (DynamicBuf *)ctx; + db->data = (uint8_t *)sdscatlen((sds)db->data, data, len); + return db->data != NULL ? C_OK : C_ERR; +} + +static int failSelectedEmit(void *ctx, const uint8_t *data, size_t len) { + (void)data; + (void)len; + FailingEmitter *emitter = (FailingEmitter *)ctx; + emitter->calls++; + return emitter->calls == emitter->fail_on_call ? C_ERR : C_OK; +} + +static ssize_t testRioBufferReadSome(rio *r, void *buf, size_t len) { + size_t buflen = sdslen(r->io.buffer.ptr); + if (r->io.buffer.pos < 0 || (size_t)r->io.buffer.pos >= buflen) return 0; + + size_t available = buflen - (size_t)r->io.buffer.pos; + size_t read_len = available < len ? available : len; + memcpy(buf, r->io.buffer.ptr + r->io.buffer.pos, read_len); + r->io.buffer.pos += read_len; + return (ssize_t)read_len; +} + +static void enableTestRioBufferPartialReads(rio *r) { + r->read_some = testRioBufferReadSome; +} + +static int initVcsRdbStreamReader(streamReader *reader, rio *r) { + enableTestRioBufferPartialReads(r); + return rdbInitStreamReader(r, reader, false, NULL) == RDB_STREAM_READER_INIT_OK ? C_OK : C_ERR; +} + +static size_t rio_update_calls = 0; + +static void countRioUpdateCalls(rio *r, const void *buf, size_t len) { + (void)r; + (void)buf; + (void)len; + rio_update_calls++; +} + +static int emitToRioBackend(void *ctx, const uint8_t *data, size_t len) { + return rioWriteRaw((rio *)ctx, data, len) ? C_OK : C_ERR; +} + +static int attachCompressionWriter(rio *r, streamWriter *writer) { + if (streamWriterInit(writer, ALGO_LZ4, true, emitToRioBackend, r) == C_ERR) return C_ERR; + rioAttachStreamWriter(r, writer); + return C_OK; +} + +static int finishCompressionWriter(rio *r, streamWriter *writer) { + if (streamWriterFinish(writer) == C_ERR) { + r->flags |= RIO_FLAG_WRITE_ERROR; + return C_ERR; + } + if (rioFlushRaw(r)) return C_OK; + r->flags |= RIO_FLAG_WRITE_ERROR; + return C_ERR; +} + +static void freeCompressionWriter(rio *r, streamWriter *writer) { + rioDetachStreamWriter(r); + streamWriterFree(writer); +} + +typedef enum { + TEST_COMPRESSION_LAYER_CODEC = 0, + TEST_COMPRESSION_LAYER_STREAM, + TEST_COMPRESSION_LAYER_RIO, +} testCompressionLayer; + +typedef struct { + const char *name; + testCompressionLayer writer_layer; + testCompressionLayer reader_layer; + size_t payload_len; + size_t reader_chunk; + size_t source_chunk; +} compressionRoundTripCase; + +static void fillRoundTripPayload(uint8_t *payload, size_t len) { + uint32_t state = 0x6d2b79f5; + for (size_t i = 0; i < len; i++) { + state = state * 1664525u + 1013904223u; + payload[i] = (uint8_t)(state >> 24); + } +} + +static int encodeRoundTripPayload(testCompressionLayer layer, + const uint8_t *payload, + size_t payload_len, + sds *encoded) { + *encoded = NULL; + + if (layer == TEST_COMPRESSION_LAYER_CODEC) { + streamCompressor compressor; + if (streamCompressorInit(&compressor, ALGO_LZ4, 0, true) == C_ERR) return C_ERR; + + size_t bound = streamCompressorOutputBound(&compressor, payload_len); + sds output = sdsMakeRoomFor(sdsempty(), bound); + ssize_t output_len = streamCompressorFeed(&compressor, (uint8_t *)output, bound, + payload, payload_len, COMPRESS_FLUSH_END); + streamCompressorFree(&compressor); + if (output_len <= 0) { + sdsfree(output); + return C_ERR; + } + sdsIncrLen(output, output_len); + *encoded = output; + return C_OK; + } + + if (layer == TEST_COMPRESSION_LAYER_STREAM) { + DynamicBuf output; + dynamicBufInit(&output); + streamWriter writer; + if (streamWriterInit(&writer, ALGO_LZ4, true, emitToDynamicBuf, &output) == C_ERR) { + dynamicBufFree(&output); + return C_ERR; + } + int result = streamWriterWrite(&writer, payload, payload_len); + if (result == C_OK && writer.state != STREAM_WRITER_STATE_ACTIVE) result = C_ERR; + if (result == C_OK) result = streamWriterFinish(&writer); + if (result == C_OK && writer.state != STREAM_WRITER_STATE_FINISHED) result = C_ERR; + streamWriterFree(&writer); + if (result == C_ERR) { + dynamicBufFree(&output); + return C_ERR; + } + *encoded = (sds)output.data; + return C_OK; + } + + sds output = sdsempty(); + rio buffer_rio; + rioInitWithBuffer(&buffer_rio, output); + streamWriter writer; + if (attachCompressionWriter(&buffer_rio, &writer) == C_ERR) { + streamWriterFree(&writer); + sdsfree(buffer_rio.io.buffer.ptr); + return C_ERR; + } + + int result = rioWrite(&buffer_rio, payload, payload_len) ? C_OK : C_ERR; + if (result == C_OK) result = finishCompressionWriter(&buffer_rio, &writer); + if (result == C_OK && buffer_rio.processed_bytes != payload_len) result = C_ERR; + if (result == C_OK && buffer_rio.stream_processed_bytes != sdslen(buffer_rio.io.buffer.ptr)) result = C_ERR; + freeCompressionWriter(&buffer_rio, &writer); + if (result == C_ERR) { + sdsfree(buffer_rio.io.buffer.ptr); + return C_ERR; + } + *encoded = buffer_rio.io.buffer.ptr; + return C_OK; +} + +static int decodeRoundTripPayload(testCompressionLayer layer, + const sds encoded, + bool has_envelope, + const uint8_t *payload, + size_t payload_len, + size_t reader_chunk, + size_t source_chunk) { + uint8_t *output = (uint8_t *)zmalloc(payload_len); + int result = C_ERR; + + if (layer == TEST_COMPRESSION_LAYER_CODEC) { + size_t offset = has_envelope ? VCS_ENVELOPE_SIZE : 0; + streamDecompressor decompressor; + if (streamDecompressorInit(&decompressor, ALGO_LZ4, false) == C_ERR) { + zfree(output); + return C_ERR; + } + ssize_t output_len = decompressAll(&decompressor, + (const uint8_t *)encoded + offset, + sdslen(encoded) - offset, + output, + payload_len); + streamDecompressorFree(&decompressor); + if (output_len == (ssize_t)payload_len && memcmp(output, payload, payload_len) == 0) result = C_OK; + zfree(output); + return result; + } + + if (layer == TEST_COMPRESSION_LAYER_STREAM) { + MemReader source = {}; + source.data = (const uint8_t *)encoded; + source.len = sdslen(encoded); + source.max_chunk = source_chunk; + streamReaderConfig cfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_MIN, false); + streamReader reader; + if (streamReaderInit(&reader, &cfg, memReaderRead, &source, NULL) == C_ERR) { + zfree(output); + return C_ERR; + } + + size_t total = 0; + while (total < payload_len) { + size_t chunk = reader_chunk == 0 ? payload_len - total : reader_chunk; + if (chunk > payload_len - total) chunk = payload_len - total; + ssize_t nread = streamReaderRead(&reader, output + total, chunk); + if (nread <= 0) break; + total += (size_t)nread; + } + if (total == payload_len && + memcmp(output, payload, payload_len) == 0 && + streamReaderRead(&reader, output, 1) == 0 && + streamReaderFinish(&reader) == C_OK) { + result = C_OK; + } + streamReaderFree(&reader); + zfree(output); + return result; + } + + sds input = sdsdup(encoded); + rio buffer_rio; + rioInitWithBuffer(&buffer_rio, input); + streamReader reader; + if (initVcsRdbStreamReader(&reader, &buffer_rio) == C_ERR) { + sdsfree(input); + zfree(output); + return C_ERR; + } + + size_t total = 0; + while (total < payload_len) { + size_t chunk = reader_chunk == 0 ? payload_len - total : reader_chunk; + if (chunk > payload_len - total) chunk = payload_len - total; + if (!rioRead(&buffer_rio, output + total, chunk)) break; + total += chunk; + } + if (total == payload_len && + memcmp(output, payload, payload_len) == 0 && + streamReaderFinish(&reader) == C_OK) { + result = C_OK; + } + rdbFreeStreamReader(&buffer_rio, &reader); + sdsfree(input); + zfree(output); + return result; +} + +TEST(CompressionTest, compressionLayersRoundTrip) { + const compressionRoundTripCase cases[] = { + {"codec", TEST_COMPRESSION_LAYER_CODEC, TEST_COMPRESSION_LAYER_CODEC, 64, 0, 0}, + {"stream writer", TEST_COMPRESSION_LAYER_STREAM, TEST_COMPRESSION_LAYER_CODEC, 256, 0, 0}, + {"large stream write", TEST_COMPRESSION_LAYER_STREAM, TEST_COMPRESSION_LAYER_STREAM, + (1024 * 1024) + 4096, 0, 0}, + {"small stream reads", TEST_COMPRESSION_LAYER_STREAM, TEST_COMPRESSION_LAYER_STREAM, + 256 * 1024, 17, 4096}, + {"rio writer", TEST_COMPRESSION_LAYER_RIO, TEST_COMPRESSION_LAYER_CODEC, 256, 0, 0}, + {"rio reader", TEST_COMPRESSION_LAYER_STREAM, TEST_COMPRESSION_LAYER_RIO, 256, 0, 0}, + {"large rio reader", TEST_COMPRESSION_LAYER_STREAM, TEST_COMPRESSION_LAYER_RIO, + 256 * 1024, 4096, 0}, + }; + const uint8_t expected_envelope[VCS_ENVELOPE_SIZE] = { + VCS_MAGIC_0, + VCS_MAGIC_1, + VCS_MAGIC_2, + VCS_VERSION, + VCS_CODEC_LZ4, + 0, + VCS_STREAM_RDB, + }; + + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + uint8_t *payload = (uint8_t *)zmalloc(cases[i].payload_len); + fillRoundTripPayload(payload, cases[i].payload_len); + sds encoded = NULL; + ASSERT_EQ(encodeRoundTripPayload(cases[i].writer_layer, payload, + cases[i].payload_len, &encoded), + C_OK) + << cases[i].name; + + bool has_envelope = cases[i].writer_layer != TEST_COMPRESSION_LAYER_CODEC; + if (has_envelope) { + ASSERT_GE(sdslen(encoded), (size_t)VCS_ENVELOPE_SIZE) << cases[i].name; + EXPECT_EQ(memcmp(encoded, expected_envelope, sizeof(expected_envelope)), 0) << cases[i].name; + } + EXPECT_EQ(decodeRoundTripPayload(cases[i].reader_layer, encoded, has_envelope, + payload, cases[i].payload_len, + cases[i].reader_chunk, cases[i].source_chunk), + C_OK) + << cases[i].name; + + sdsfree(encoded); + zfree(payload); + } +} + +TEST(CompressionTest, streamReaderClassifiesSourceCallbackFailuresAsIoErrors) { + MemReader failed_source = {}; + failed_source.fail_after_reads = true; + failed_source.fail_after_success_reads = 0; + streamReaderConfig failed_cfg = makeReaderConfig(true, STREAM_READER_BUFFER_SIZE_DEFAULT, false); + streamReader failed_reader; + ASSERT_EQ(streamReaderInit(&failed_reader, &failed_cfg, memReaderRead, &failed_source, NULL), C_ERR); + ASSERT_EQ(failed_reader.error_kind, STREAM_READER_ERROR_IO); + streamReaderFree(&failed_reader); + + DynamicBuf db; + dynamicBufInit(&db); + streamWriter writer; + ASSERT_EQ(streamWriterInit(&writer, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); + ASSERT_EQ(streamWriterWrite(&writer, "source callback contract", 24), C_OK); + ASSERT_EQ(streamWriterFinish(&writer), C_OK); + streamWriterFree(&writer); + + const int overread_calls[] = {1, 2, 3}; + for (size_t i = 0; i < sizeof(overread_calls) / sizeof(overread_calls[0]); i++) { + int overread_on_call = overread_calls[i]; + MemReader source = {}; + source.data = db.data; + source.len = sdslen((const char *)db.data); + source.overread_on_call = overread_on_call; + streamReaderConfig rcfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_DEFAULT, false); + streamReader reader; + int init_result = streamReaderInit(&reader, &rcfg, memReaderRead, &source, NULL); + if (overread_on_call <= 2) { + ASSERT_EQ(init_result, C_ERR) << "callback call " << overread_on_call; + ASSERT_EQ(reader.error_kind, STREAM_READER_ERROR_IO) + << "callback call " << overread_on_call; + streamReaderFree(&reader); + continue; + } + ASSERT_EQ(init_result, C_OK); + + uint8_t out[24]; + ASSERT_EQ(streamReaderRead(&reader, out, sizeof(out)), -1) << "callback call " << overread_on_call; + ASSERT_EQ(reader.error_kind, STREAM_READER_ERROR_IO) << "callback call " << overread_on_call; + ASSERT_EQ(streamReaderRead(&reader, out, sizeof(out)), -1) << "I/O error must remain sticky"; + streamReaderFree(&reader); + } + + dynamicBufFree(&db); +} + +TEST(CompressionTest, streamReaderRejectsInvalidEnvelopeFields) { + const uint8_t good[VCS_ENVELOPE_SIZE] = { + VCS_MAGIC_0, + VCS_MAGIC_1, + VCS_MAGIC_2, + VCS_VERSION, + VCS_CODEC_LZ4, + 0, + VCS_STREAM_RDB, + }; + struct { + const char *name; + size_t offset; + uint8_t value; + } cases[] = { + {"magic", 0, 'X'}, + {"version", VCS_OFFSET_VERSION, VCS_VERSION + 1}, + {"unknown codec", VCS_OFFSET_CODEC, 0x7f}, + {"reserved byte", VCS_OFFSET_RESERVED, 1}, + {"stream kind", VCS_OFFSET_STREAM_KIND, 0x7f}, + }; + + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + uint8_t mutated[VCS_ENVELOPE_SIZE]; + memcpy(mutated, good, sizeof(mutated)); + mutated[cases[i].offset] = cases[i].value; + + MemReader source = {}; + source.data = mutated; + source.len = sizeof(mutated); + streamReaderConfig cfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_DEFAULT, false); + streamReader reader; + ASSERT_EQ(streamReaderInit(&reader, &cfg, memReaderRead, &source, NULL), C_ERR) << cases[i].name; + ASSERT_EQ(reader.error_kind, STREAM_READER_ERROR_INCOMPATIBLE) << cases[i].name; + streamReaderFree(&reader); + } +} + +/* Regression for partial output followed by a source read error. The partial + * bytes are returned, but the error must remain sticky for the next read. */ +TEST(CompressionTest, streamReaderPartialThenErrorSetsErrored) { + const size_t payload_len = 256 * 1024; + uint8_t *payload = (uint8_t *)zmalloc(payload_len); + uint32_t x = 0x12345678u; + for (size_t i = 0; i < payload_len; i++) { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + payload[i] = (uint8_t)(x & 0xFF); + } + + DynamicBuf db; + dynamicBufInit(&db); + streamWriter w; + ASSERT_EQ(streamWriterInit(&w, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); + ASSERT_EQ(streamWriterWrite(&w, payload, payload_len), C_OK); + ASSERT_EQ(streamWriterFinish(&w), C_OK); + streamWriterFree(&w); + + MemReader fr = {}; + fr.data = db.data; + fr.len = sdslen((const char *)db.data); + fr.max_chunk = 4096; + fr.fail_after_pos = VCS_ENVELOPE_SIZE + STREAM_READER_BUFFER_SIZE_MIN; + streamReaderConfig rcfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_MIN, false); + streamReader r; + ASSERT_EQ(streamReaderInit(&r, &rcfg, memReaderRead, &fr, NULL), C_OK); + + const size_t out_len = payload_len; + uint8_t *out = (uint8_t *)zmalloc(out_len); + ssize_t n1 = streamReaderRead(&r, out, out_len); + ASSERT_GT(n1, 0) << "first read should return partial output"; + ASSERT_LT(n1, (ssize_t)out_len) << "injected read error should stop the first read early"; + EXPECT_EQ(memcmp(out, payload, (size_t)n1), 0); + ASSERT_EQ(r.error_kind, STREAM_READER_ERROR_IO) << "error must be latched before returning partial output"; + ASSERT_EQ(streamReaderRead(&r, out, out_len), -1) << "second read should fail immediately"; + ASSERT_EQ(r.error_kind, STREAM_READER_ERROR_IO); + + streamReaderFree(&r); + + /* Passthrough mode should also preserve partial bytes when source read + * fails after probe/prefix buffering, then latch sticky error state. */ + const uint8_t plain[] = "NOTVCS-passthrough-regression"; + MemReader fr_passthrough = {}; + fr_passthrough.data = plain; + fr_passthrough.len = sizeof(plain) - 1; + fr_passthrough.max_chunk = 0; + fr_passthrough.fail_after_reads = true; + fr_passthrough.fail_after_success_reads = 1; /* probe succeeds, next read fails */ + streamReaderConfig pass_cfg = makeReaderConfig(true, STREAM_READER_BUFFER_SIZE_DEFAULT, false); + streamReader rp; + ASSERT_EQ(streamReaderInit(&rp, &pass_cfg, memReaderRead, &fr_passthrough, NULL), C_OK); + + uint8_t pass_out[64]; + ssize_t p1 = streamReaderRead(&rp, pass_out, sizeof(pass_out)); + ASSERT_GT(p1, 0) << "passthrough first read should return partial output"; + EXPECT_EQ(memcmp(pass_out, plain, (size_t)p1), 0) << "passthrough partial bytes should match input prefix"; + ASSERT_EQ(streamReaderRead(&rp, pass_out, sizeof(pass_out)), -1) + << "passthrough second read should fail immediately"; + ASSERT_EQ(rp.error_kind, STREAM_READER_ERROR_IO); + streamReaderFree(&rp); + zfree(out); + + dynamicBufFree(&db); + zfree(payload); +} + +TEST(CompressionTest, streamWriterRejectsUnsupportedAlgorithms) { + const compressionAlgo algorithms[] = {ALGO_NONE, ALGO_LZF}; + DynamicBuf db; + dynamicBufInit(&db); + + for (size_t i = 0; i < sizeof(algorithms) / sizeof(algorithms[0]); i++) { + streamWriter writer; + EXPECT_EQ(streamWriterInit(&writer, algorithms[i], true, emitToDynamicBuf, &db), C_ERR) + << compressionAlgoName(algorithms[i]); + streamWriterFree(&writer); + } + + dynamicBufFree(&db); +} + +TEST(CompressionTest, streamWriterFinishProducesAValidEmptyStream) { + DynamicBuf db; + dynamicBufInit(&db); + streamWriter writer; + ASSERT_EQ(streamWriterInit(&writer, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); + + ASSERT_EQ(streamWriterWrite(&writer, NULL, 0), C_OK); + ASSERT_EQ(sdslen((const char *)db.data), 0u) << "empty writes must stay lazy"; + ASSERT_EQ(streamWriterFinish(&writer), C_OK); + ASSERT_GT(sdslen((const char *)db.data), (size_t)VCS_ENVELOPE_SIZE); + streamWriterFree(&writer); + + MemReader source = {}; + source.data = db.data; + source.len = sdslen((const char *)db.data); + source.max_chunk = 1; + streamReaderConfig rcfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_DEFAULT, false); + streamReader reader; + ASSERT_EQ(streamReaderInit(&reader, &rcfg, memReaderRead, &source, NULL), C_OK); + uint8_t out = 0; + ASSERT_EQ(streamReaderRead(&reader, &out, 1), 0); + ASSERT_EQ(streamReaderFinish(&reader), C_OK); + streamReaderFree(&reader); + dynamicBufFree(&db); +} + +TEST(CompressionTest, streamWriterSinkFailuresAreSticky) { + const int failing_calls[] = {1, 2}; + for (size_t i = 0; i < sizeof(failing_calls) / sizeof(failing_calls[0]); i++) { + int fail_on_call = failing_calls[i]; + FailingEmitter emitter = {0, fail_on_call}; + streamWriter writer; + ASSERT_EQ(streamWriterInit(&writer, ALGO_LZ4, true, failSelectedEmit, &emitter), C_OK); + + ASSERT_EQ(streamWriterWrite(&writer, "payload", 7), C_ERR) << "sink call " << fail_on_call; + ASSERT_EQ(writer.state, STREAM_WRITER_STATE_ERROR); + ASSERT_EQ(streamWriterWrite(&writer, "retry", 5), C_ERR); + ASSERT_EQ(streamWriterFinish(&writer), C_ERR); + ASSERT_EQ(emitter.calls, fail_on_call) << "an errored writer must not emit more bytes"; + streamWriterFree(&writer); + } + + FailingEmitter emitter = {0, 0}; + streamWriter writer; + ASSERT_EQ(streamWriterInit(&writer, ALGO_LZ4, true, failSelectedEmit, &emitter), C_OK); + ASSERT_EQ(streamWriterWrite(&writer, "payload", 7), C_OK); + emitter.fail_on_call = emitter.calls + 1; + ASSERT_EQ(streamWriterFinish(&writer), C_ERR); + int calls_after_failure = emitter.calls; + ASSERT_EQ(streamWriterFinish(&writer), C_ERR); + ASSERT_EQ(streamWriterWrite(&writer, "retry", 5), C_ERR); + ASSERT_EQ(emitter.calls, calls_after_failure) << "a failed finish must remain failed"; + streamWriterFree(&writer); +} + +TEST(CompressionTest, checksumBypassSkipsOnlyCodecVerification) { + const char *payload = "checksum bypass payload"; + DynamicBuf db; + dynamicBufInit(&db); + streamWriter writer; + ASSERT_EQ(streamWriterInit(&writer, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); + ASSERT_EQ(streamWriterWrite(&writer, payload, strlen(payload)), C_OK); + ASSERT_EQ(streamWriterFinish(&writer), C_OK); + streamWriterFree(&writer); + + ASSERT_GT(sdslen((const char *)db.data), (size_t)VCS_ENVELOPE_SIZE + 4); + db.data[sdslen((const char *)db.data) - 1] ^= 1; + + const bool skip_checksum_cases[] = {false, true}; + for (size_t i = 0; i < sizeof(skip_checksum_cases) / sizeof(skip_checksum_cases[0]); i++) { + bool skip_codec_checksum_validation = skip_checksum_cases[i]; + MemReader source = {}; + source.data = db.data; + source.len = sdslen((const char *)db.data); + streamReaderConfig rcfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_DEFAULT, + skip_codec_checksum_validation); + streamReader reader; + ASSERT_EQ(streamReaderInit(&reader, &rcfg, memReaderRead, &source, NULL), C_OK); + + char out[64] = {0}; + ASSERT_EQ(streamReaderRead(&reader, out, strlen(payload)), (ssize_t)strlen(payload)); + EXPECT_EQ(memcmp(out, payload, strlen(payload)), 0); + ASSERT_EQ(streamReaderFinish(&reader), skip_codec_checksum_validation ? C_OK : C_ERR); + streamReaderFree(&reader); + } + + MemReader truncated = {}; + truncated.data = db.data; + truncated.len = sdslen((const char *)db.data) - 5; + streamReaderConfig bypass = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_DEFAULT, true); + streamReader reader; + ASSERT_EQ(streamReaderInit(&reader, &bypass, memReaderRead, &truncated, NULL), C_OK); + char out[64] = {0}; + ASSERT_EQ(streamReaderRead(&reader, out, strlen(payload)), (ssize_t)strlen(payload)); + ASSERT_EQ(streamReaderFinish(&reader), C_ERR) + << "checksum bypass must not bypass exact frame-end validation"; + streamReaderFree(&reader); + + dynamicBufFree(&db); +} + +TEST(CompressionTest, streamReaderFinishAcceptsClosedFrame) { + const char *payload = "validate frame end payload"; + DynamicBuf db; + dynamicBufInit(&db); + streamWriter w; + ASSERT_EQ(streamWriterInit(&w, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); + ASSERT_EQ(streamWriterWrite(&w, payload, strlen(payload)), C_OK); + ASSERT_EQ(streamWriterFinish(&w), C_OK); + + MemReader reader_ctx = {}; + reader_ctx.data = db.data; + reader_ctx.len = sdslen((const char *)db.data); + reader_ctx.max_chunk = 7; + streamReaderConfig rcfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_DEFAULT, false); + streamReader reader; + ASSERT_EQ(streamReaderInit(&reader, &rcfg, memReaderRead, &reader_ctx, NULL), C_OK); + ASSERT_EQ(reader.state, STREAM_READER_STATE_COMPRESSED); + + char out[64]; + ASSERT_EQ(streamReaderRead(&reader, out, strlen(payload)), (ssize_t)strlen(payload)); + ASSERT_EQ(reader.state, STREAM_READER_STATE_COMPRESSED); + EXPECT_EQ(memcmp(out, payload, strlen(payload)), 0); + ASSERT_EQ(streamReaderFinish(&reader), C_OK); + ASSERT_EQ(reader.state, STREAM_READER_STATE_FINISHED); + ASSERT_EQ(streamReaderFinish(&reader), C_OK); + ASSERT_EQ(streamReaderRead(&reader, out, sizeof(out)), 0); + + streamReaderFree(&reader); + streamWriterFree(&w); + dynamicBufFree(&db); +} + +TEST(CompressionTest, streamReaderFinishRejectsUnreadDecodedBytes) { + const char *payload = "payload with unread decoded suffix"; + const size_t payload_len = strlen(payload); + DynamicBuf db; + dynamicBufInit(&db); + streamWriter w; + ASSERT_EQ(streamWriterInit(&w, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); + ASSERT_EQ(streamWriterWrite(&w, payload, payload_len), C_OK); + ASSERT_EQ(streamWriterFinish(&w), C_OK); + + MemReader reader_ctx = {}; + reader_ctx.data = db.data; + reader_ctx.len = sdslen((const char *)db.data); + streamReaderConfig rcfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_DEFAULT, false); + streamReader reader; + ASSERT_EQ(streamReaderInit(&reader, &rcfg, memReaderRead, &reader_ctx, NULL), C_OK); + + char out[8]; + ASSERT_EQ(streamReaderRead(&reader, out, sizeof(out)), (ssize_t)sizeof(out)); + EXPECT_EQ(memcmp(out, payload, sizeof(out)), 0); + ASSERT_EQ(streamReaderFinish(&reader), C_ERR); + ASSERT_EQ(reader.error_kind, STREAM_READER_ERROR_CORRUPT); + + streamReaderFree(&reader); + streamWriterFree(&w); + dynamicBufFree(&db); +} + +TEST(CompressionTest, rioCompressionWriterDoesNotOwnRdbChecksumPolicy) { + const bool checksum_cases[] = {false, true}; + for (size_t i = 0; i < sizeof(checksum_cases) / sizeof(checksum_cases[0]); i++) { + bool inner_skips_checksum = checksum_cases[i]; + sds buf = sdsempty(); + rio inner; + rioInitWithBuffer(&inner, buf); + if (inner_skips_checksum) inner.flags |= RIO_FLAG_SKIP_RDB_CHECKSUM; + + streamWriter writer; + ASSERT_EQ(attachCompressionWriter(&inner, &writer), 0); + ASSERT_TRUE(inner.update_cksum == NULL); + ASSERT_EQ((inner.flags & RIO_FLAG_SKIP_RDB_CHECKSUM) != 0, inner_skips_checksum); + ASSERT_NE(rioWrite(&inner, "checksum policy", 15), 0u); + ASSERT_EQ(inner.cksum, 0u); + ASSERT_EQ(finishCompressionWriter(&inner, &writer), 0); + + freeCompressionWriter(&inner, &writer); + sdsfree(inner.io.buffer.ptr); + } +} + +TEST(CompressionTest, rioStreamReaderTellTracksSourceProgress) { + DynamicBuf db; + dynamicBufInit(&db); + + char payload[4096]; + memset(payload, 'A', sizeof(payload)); + streamWriter t; + ASSERT_EQ(streamWriterInit(&t, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); + ASSERT_EQ(streamWriterWrite(&t, payload, sizeof(payload)), C_OK); + ASSERT_EQ(streamWriterFinish(&t), C_OK); + streamWriterFree(&t); + + sds comp_sds = sdsnewlen(db.data, sdslen((const char *)db.data)); + rio buffer_rio; + rioInitWithBuffer(&buffer_rio, comp_sds); + + streamReader reader; + ASSERT_EQ(initVcsRdbStreamReader(&reader, &buffer_rio), 0); + + char out[2048]; + ASSERT_NE(rioRead(&buffer_rio, out, sizeof(out)), 0u); + ASSERT_EQ((size_t)rioTell(&buffer_rio), buffer_rio.stream_processed_bytes); + ASSERT_LT((size_t)rioTell(&buffer_rio), sizeof(out)) + << "rio tell should track source bytes, not logical output bytes"; + + rdbFreeStreamReader(&buffer_rio, &reader); + sdsfree(comp_sds); + dynamicBufFree(&db); +} + +TEST(CompressionTest, rioStreamReaderHonorsMaxProcessingChunk) { + const size_t payload_len = 1024; + const size_t chunk_size = 128; + uint8_t payload[payload_len]; + for (size_t i = 0; i < payload_len; i++) { + payload[i] = (uint8_t)(i % 251); + } + + DynamicBuf db; + dynamicBufInit(&db); + streamWriter writer; + ASSERT_EQ(streamWriterInit(&writer, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); + ASSERT_EQ(streamWriterWrite(&writer, payload, payload_len), C_OK); + ASSERT_EQ(streamWriterFinish(&writer), C_OK); + streamWriterFree(&writer); + + sds compressed = sdsnewlen(db.data, sdslen((const char *)db.data)); + rio buffer_rio; + rioInitWithBuffer(&buffer_rio, compressed); + streamReader reader; + ASSERT_EQ(initVcsRdbStreamReader(&reader, &buffer_rio), 0); + + buffer_rio.max_processing_chunk = chunk_size; + buffer_rio.update_cksum = countRioUpdateCalls; + rio_update_calls = 0; + uint8_t result[payload_len]; + ASSERT_NE(rioRead(&buffer_rio, result, payload_len), 0u); + ASSERT_EQ(rio_update_calls, payload_len / chunk_size); + ASSERT_EQ(buffer_rio.processed_bytes, payload_len); + EXPECT_EQ(memcmp(result, payload, payload_len), 0); + + rdbFreeStreamReader(&buffer_rio, &reader); + sdsfree(compressed); + dynamicBufFree(&db); +} + +TEST(CompressionTest, rdbStreamReaderRewindsPlainFileAfterProbe) { + const char *payload = "REDIS001remaining data after prefix"; + size_t payload_len = strlen(payload); + FILE *fp = tmpfile(); + ASSERT_NE(fp, (FILE *)NULL); + ASSERT_EQ(fwrite(payload, 1, payload_len, fp), payload_len); + rewind(fp); + + rio file_rio; + rioInitWithFile(&file_rio, fp); + streamReader reader; + compressionAlgo algo = ALGO_NONE; + ASSERT_EQ(rdbInitStreamReader(&file_rio, &reader, false, &algo), RDB_STREAM_READER_INIT_OK); + ASSERT_EQ(algo, ALGO_NONE); + ASSERT_EQ(file_rio.stream_reader, (streamReader *)NULL); + ASSERT_EQ(file_rio.stream_processed_bytes, 0u); + ASSERT_EQ(ftello(fp), 0); + + char result[64]; + memset(result, 0, sizeof(result)); + ASSERT_NE(rioRead(&file_rio, result, payload_len), 0u); + EXPECT_EQ(memcmp(result, payload, payload_len), 0); + + rdbFreeStreamReader(&file_rio, &reader); + fclose(fp); +} + +TEST(CompressionTest, rdbStreamReaderFallsBackWhenPlainFileCannotRewind) { + const char *payload = "REDIS001remaining data after prefix"; + size_t payload_len = strlen(payload); + int pipe_fds[2]; + ASSERT_EQ(pipe(pipe_fds), 0); + ASSERT_EQ(write(pipe_fds[1], payload, payload_len), (ssize_t)payload_len); + ASSERT_EQ(close(pipe_fds[1]), 0); + + FILE *fp = fdopen(pipe_fds[0], "r"); + ASSERT_NE(fp, (FILE *)NULL); + rio file_rio; + rioInitWithFile(&file_rio, fp); + streamReader reader; + compressionAlgo algo = ALGO_NONE; + ASSERT_EQ(rdbInitStreamReader(&file_rio, &reader, false, &algo), RDB_STREAM_READER_INIT_OK); + ASSERT_EQ(algo, ALGO_NONE); + ASSERT_EQ(file_rio.stream_reader, &reader); + + char result[64]; + memset(result, 0, sizeof(result)); + ASSERT_NE(rioRead(&file_rio, result, payload_len), 0u); + EXPECT_EQ(memcmp(result, payload, payload_len), 0); + + rdbFreeStreamReader(&file_rio, &reader); + fclose(fp); +} + +TEST(CompressionTest, rioStreamReaderClassifiesInput) { + { + const char *payload = "REDIS001remaining data after prefix"; + size_t payload_len = strlen(payload); + sds buf = sdsnewlen(payload, payload_len); + rio buffer_rio; + rioInitWithBuffer(&buffer_rio, buf); + enableTestRioBufferPartialReads(&buffer_rio); + + streamReader reader; + compressionAlgo algo = ALGO_NONE; + ASSERT_EQ(rdbInitStreamReader(&buffer_rio, &reader, false, &algo), RDB_STREAM_READER_INIT_OK); + ASSERT_EQ(algo, ALGO_NONE) << "passthrough stream should not be compressed"; + ASSERT_EQ(buffer_rio.stream_reader, &reader) + << "non-file sources should retain the passthrough fallback"; + + char result[64]; + memset(result, 0, sizeof(result)); + ASSERT_NE(rioRead(&buffer_rio, result, payload_len), 0u) << "rioRead should succeed"; + EXPECT_EQ(memcmp(result, payload, payload_len), 0) << "payload should be replayed exactly"; + + rdbFreeStreamReader(&buffer_rio, &reader); + sdsfree(buf); + } + + { + const uint8_t malformed[VCS_ENVELOPE_SIZE] = { + VCS_MAGIC_0, VCS_MAGIC_1, VCS_MAGIC_2, 0, VCS_CODEC_LZ4, 0, VCS_STREAM_RDB}; + sds buf = sdsnewlen(malformed, sizeof(malformed)); + rio buffer_rio; + rioInitWithBuffer(&buffer_rio, buf); + enableTestRioBufferPartialReads(&buffer_rio); + + streamReader reader; + ASSERT_EQ(rdbInitStreamReader(&buffer_rio, &reader, false, NULL), + RDB_STREAM_READER_INIT_INCOMPATIBLE); + + sdsfree(buf); + } +} + +TEST(CompressionTest, rioCompressionWriterFinishIdempotent) { + sds buf = sdsempty(); + rio buffer_rio; + rioInitWithBuffer(&buffer_rio, buf); + + streamWriter writer; + ASSERT_EQ(attachCompressionWriter(&buffer_rio, &writer), 0); + + ASSERT_NE(rioWrite(&buffer_rio, "test", 4), 0u); + ASSERT_EQ(finishCompressionWriter(&buffer_rio, &writer), 0); + size_t len_after_first = sdslen(buffer_rio.io.buffer.ptr); + + ASSERT_EQ(finishCompressionWriter(&buffer_rio, &writer), 0); + size_t len_after_second = sdslen(buffer_rio.io.buffer.ptr); + ASSERT_EQ(len_after_first, len_after_second) << "second finish should not produce more output"; + + freeCompressionWriter(&buffer_rio, &writer); + sdsfree(buffer_rio.io.buffer.ptr); +} + +/* The reader stops at the frame boundary so callers can manage subsequent + * bytes on a long-lived stream. */ +TEST(CompressionTest, streamReaderFinishStopsAtFrameEndBeforeTrailingBytes) { + const char *payload = "stream-reader-frame-end"; + const size_t payload_len = strlen(payload); + const char *trailer = "TRAILER-BYTES-AFTER-FRAME"; + const size_t trailer_len = strlen(trailer); + + DynamicBuf db; + dynamicBufInit(&db); + streamWriter w; + ASSERT_EQ(streamWriterInit(&w, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); + ASSERT_EQ(streamWriterWrite(&w, payload, payload_len), C_OK); + ASSERT_EQ(streamWriterFinish(&w), C_OK); + streamWriterFree(&w); + size_t frame_len = sdslen((const char *)db.data); + + sds input = sdsnewlen(db.data, sdslen((const char *)db.data)); + input = sdscatlen(input, trailer, trailer_len); + + MemReader mr = {}; + mr.data = (const uint8_t *)input; + mr.len = sdslen(input); + mr.max_chunk = 0; + streamReaderConfig rcfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_MIN, false); + streamReader r; + ASSERT_EQ(streamReaderInit(&r, &rcfg, memReaderRead, &mr, NULL), C_OK); + + char out[128]; + memset(out, 0, sizeof(out)); + ASSERT_EQ(streamReaderRead(&r, out, payload_len), (ssize_t)payload_len); + EXPECT_EQ(memcmp(out, payload, payload_len), 0); + + ASSERT_EQ(streamReaderFinish(&r), C_OK); + ASSERT_EQ(r.error_kind, STREAM_READER_ERROR_NONE); + ASSERT_EQ(mr.pos, frame_len) << "streamReader must not consume bytes after the LZ4 frame"; + + streamReaderFree(&r); + sdsfree(input); + dynamicBufFree(&db); +} + +TEST(CompressionTest, streamReaderRejectsTruncatedFrameTrailer) { + const size_t payload_len = 256; + uint8_t payload[payload_len]; + for (size_t i = 0; i < payload_len; i++) { + payload[i] = (uint8_t)(i & 0xFF); + } + + DynamicBuf db; + dynamicBufInit(&db); + streamWriter w; + ASSERT_EQ(streamWriterInit(&w, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); + ASSERT_EQ(streamWriterWrite(&w, payload, payload_len), C_OK); + ASSERT_EQ(streamWriterFinish(&w), C_OK); + streamWriterFree(&w); + + ASSERT_GT(sdslen((const char *)db.data), (size_t)VCS_ENVELOPE_SIZE + 1); + MemReader mr = {}; + mr.data = db.data; + mr.len = sdslen((const char *)db.data) - 1; + mr.max_chunk = 7; + streamReaderConfig rcfg = makeReaderConfig(false, STREAM_READER_BUFFER_SIZE_MIN, false); + streamReader r; + ASSERT_EQ(streamReaderInit(&r, &rcfg, memReaderRead, &mr, NULL), C_OK); + + uint8_t out[payload_len]; + ASSERT_EQ(streamReaderRead(&r, out, payload_len), (ssize_t)payload_len); + EXPECT_EQ(memcmp(out, payload, payload_len), 0); + ASSERT_LT(streamReaderRead(&r, out, 1), 0) << "EOF before frame end should be treated as corruption"; + ASSERT_EQ(r.error_kind, STREAM_READER_ERROR_CORRUPT) + << "truncated compressed frame should latch corruption, not I/O"; + + streamReaderFree(&r); + dynamicBufFree(&db); +} + +TEST(CompressionTest, streamWriterWriteAfterFinish) { + DynamicBuf db; + dynamicBufInit(&db); + streamWriter t; + ASSERT_EQ(streamWriterInit(&t, ALGO_LZ4, true, emitToDynamicBuf, &db), C_OK); + + ASSERT_EQ(streamWriterWrite(&t, "hello", 5), C_OK); + ASSERT_EQ(streamWriterFinish(&t), C_OK); + size_t len_after_finish = sdslen((const char *)db.data); + + ASSERT_EQ(streamWriterWrite(&t, "world", 5), C_ERR); + ASSERT_EQ(sdslen((const char *)db.data), len_after_finish) << "write after finish should not produce output"; + + ASSERT_EQ(streamWriterFinish(&t), C_OK); + ASSERT_EQ(sdslen((const char *)db.data), len_after_finish) << "second finish should not produce output"; + + ASSERT_GT(sdslen((const char *)db.data), (size_t)VCS_ENVELOPE_SIZE); + streamDecompressor sd; + ASSERT_EQ(streamDecompressorInit(&sd, ALGO_LZ4, false), C_OK); + + uint8_t decompressed[64]; + uint8_t *cdata = db.data + VCS_ENVELOPE_SIZE; + size_t comp_len = sdslen((const char *)db.data) - VCS_ENVELOPE_SIZE; + ssize_t total = decompressAll(&sd, cdata, comp_len, decompressed, sizeof(decompressed)); + + ASSERT_EQ(total, 5); + EXPECT_EQ(memcmp(decompressed, "hello", 5), 0) << "should decompress to 'hello' only"; + + streamDecompressorFree(&sd); + streamWriterFree(&t); + dynamicBufFree(&db); +} diff --git a/src/unit/test_repl_compression.cpp b/src/unit/test_repl_compression.cpp new file mode 100644 index 00000000000..7b938e8c4a9 --- /dev/null +++ b/src/unit/test_repl_compression.cpp @@ -0,0 +1,359 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* Unit tests for replication compression configuration and capability constants. */ + +#include "generated_wrappers.hpp" + +extern "C" { +#include "compression.h" +#include "compression_repl.h" +#include "server.h" +} + +/* zmalloc.h defines helper macros that collide with libstdc++ internals. */ +#ifdef __xstr +#undef __xstr +#endif +#ifdef __str +#undef __str +#endif + +TEST(replCompression, capaCompressionBitNoConflict) { + /* Each REPLICA_CAPA_* must occupy a unique bit position. */ + int all_capas[] = { + REPLICA_CAPA_EOF, + REPLICA_CAPA_PSYNC2, + REPLICA_CAPA_DUAL_CHANNEL, + REPLICA_CAPA_SKIP_RDB_CHECKSUM, + REPLICA_CAPA_COMPRESSION, + }; + int count = sizeof(all_capas) / sizeof(all_capas[0]); + + for (int i = 0; i < count; i++) { + /* Each value must be a power of two (single bit set). */ + EXPECT_NE(all_capas[i], 0) << "capability " << i << " must be non-zero"; + EXPECT_EQ(all_capas[i] & (all_capas[i] - 1), 0) + << "capability " << i << " must be a power of two"; + + for (int j = i + 1; j < count; j++) { + EXPECT_EQ(all_capas[i] & all_capas[j], 0) + << "capabilities " << i << " and " << j << " must not share bits"; + } + } + + /* Verify the specific value. */ + EXPECT_EQ(REPLICA_CAPA_COMPRESSION, (1 << 4)); +} + +TEST(replCompression, resetBatchRetainsAllocationForIncompressibleBatch) { + replCompressor *rc = replCompressorCreate(ALGO_LZ4); + ASSERT_TRUE(rc != NULL); + /* ~1 MiB of incompressible (xorshift) bytes: compressed payload ~= input, + * exercising the greedy-SDS over-allocation path. Stays at/under the 1 MiB + * batch cap so the payload is within the retention bound. */ + const size_t n = 1000000; + unsigned char *buf = (unsigned char *)zmalloc(n); + uint32_t x = 0x9e3779b9u; + for (size_t i = 0; i < n; i++) { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + buf[i] = (unsigned char)(x >> 24); + } + ASSERT_EQ(replCompressorWrite(rc, buf, n), C_OK); + ASSERT_EQ(replCompressorFlush(rc), C_OK); + size_t alloc_before = sdsalloc(rc->out_buf); + size_t len_before = sdslen(rc->out_buf); + EXPECT_GT(len_before, (size_t)(900 * 1024)); /* ratio ~1: payload is large */ + EXPECT_GT(alloc_before, (size_t)(1024 * 1024)); /* greedy SDS over-allocates past 1 MiB */ + replCompressorResetBatch(rc); + EXPECT_EQ(sdslen(rc->out_buf), (size_t)0); /* cleared */ + EXPECT_EQ(sdsalloc(rc->out_buf), alloc_before); /* retained, not freed to empty */ + zfree(buf); + replCompressorDestroy(rc); +} + +/* ===== Decoder corruption and edge cases (crafted bytes) ===== */ + +/* Fill buf with incompressible xorshift bytes. */ +static void fillIncompressible(unsigned char *buf, size_t n, uint32_t seed) { + uint32_t x = seed; + for (size_t i = 0; i < n; i++) { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + buf[i] = (unsigned char)(x >> 24); + } +} + +/* LZ4 frame FLG byte (follows the 4-byte frame magic): bit 2 = content + * checksum present, bit 4 = block checksums present. Frozen wire format. */ +#define LZ4F_FLG_CONTENT_CHECKSUM 0x04 +#define LZ4F_FLG_BLOCK_CHECKSUM 0x10 + +TEST(replCompression, replFrameOmitsContentChecksum) { + /* A repl frame never ends, so its content checksum would be computed on + * every byte but never emitted or validated. It must be off in the frame + * header while block checksums stay on. */ + replCompressor *rc = replCompressorCreate(ALGO_LZ4); + ASSERT_TRUE(rc != NULL); + const char payload[] = "content-checksum-off-for-repl"; + ASSERT_EQ(replCompressorWrite(rc, payload, sizeof(payload)), C_OK); + ASSERT_EQ(replCompressorFlush(rc), C_OK); + const unsigned char *stream = (const unsigned char *)rc->out_buf; + ASSERT_GE(sdslen(rc->out_buf), (size_t)(VCS_ENVELOPE_SIZE + 5)); + /* LZ4 frame magic 0x184D2204 (little-endian) right after the envelope. */ + EXPECT_EQ(stream[VCS_ENVELOPE_SIZE + 0], 0x04); + EXPECT_EQ(stream[VCS_ENVELOPE_SIZE + 1], 0x22); + EXPECT_EQ(stream[VCS_ENVELOPE_SIZE + 2], 0x4D); + EXPECT_EQ(stream[VCS_ENVELOPE_SIZE + 3], 0x18); + unsigned char flg = stream[VCS_ENVELOPE_SIZE + 4]; + EXPECT_EQ(flg & LZ4F_FLG_CONTENT_CHECKSUM, 0x00); + EXPECT_EQ(flg & LZ4F_FLG_BLOCK_CHECKSUM, LZ4F_FLG_BLOCK_CHECKSUM); + + /* Round-trip: the decoder learns checksum presence from the frame header, + * so it needs no matching configuration. */ + replDecompressor *rd = replDecompressorCreate(); + ASSERT_TRUE(rd != NULL); + size_t out_len = 0; + ASSERT_EQ(replDecompressorDecode(rd, rc->out_buf, sdslen(rc->out_buf), 1024 * 1024, &out_len), + REPL_DECODE_OK); + ASSERT_EQ(out_len, sizeof(payload)); + EXPECT_EQ(memcmp(replDecompressorBuf(rd), payload, sizeof(payload)), 0); + + replDecompressorDestroy(rd); + replCompressorDestroy(rc); +} + +TEST(replCompression, rdbFrameKeepsContentChecksum) { + /* Contrast: the default (RDB) stream kind finishes its frame, so the + * content checksum stays on. */ + streamWriter writer; + sds sink = sdsempty(); + ASSERT_EQ(streamWriterInit(&writer, ALGO_LZ4, true, NULL, NULL), C_OK); + streamWriterSetSink(&writer, &sink); + ASSERT_EQ(streamWriterWrite(&writer, "rdb-bytes", 9), C_OK); + ASSERT_GE(sdslen(sink), (size_t)(VCS_ENVELOPE_SIZE + 5)); + unsigned char flg = ((const unsigned char *)sink)[VCS_ENVELOPE_SIZE + 4]; + EXPECT_EQ(flg & LZ4F_FLG_CONTENT_CHECKSUM, LZ4F_FLG_CONTENT_CHECKSUM); + EXPECT_EQ(flg & LZ4F_FLG_BLOCK_CHECKSUM, LZ4F_FLG_BLOCK_CHECKSUM); + streamWriterFree(&writer); + sdsfree(sink); +} + +TEST(replCompression, decodeFrameDoneOnLiveLink) { + replCompressor *rc = replCompressorCreate(ALGO_LZ4); + ASSERT_TRUE(rc != NULL); + const char payload[] = "frame-done-on-live-link"; + ASSERT_EQ(replCompressorWrite(rc, payload, sizeof(payload)), C_OK); + /* Finish ends the frame; a live replication link must never see that. */ + ASSERT_EQ(streamWriterFinish(&rc->writer), C_OK); + + replDecompressor *rd = replDecompressorCreate(); + ASSERT_TRUE(rd != NULL); + size_t out_len = 0; + EXPECT_EQ(replDecompressorDecode(rd, rc->out_buf, sdslen(rc->out_buf), 1024 * 1024, &out_len), + REPL_DECODE_FRAME_DONE); + + replDecompressorDestroy(rd); + replCompressorDestroy(rc); +} + +TEST(replCompression, decodeOverflowGuard) { + replCompressor *rc = replCompressorCreate(ALGO_LZ4); + ASSERT_TRUE(rc != NULL); + /* Incompressible payload so decoded output far exceeds the cap. */ + const size_t n = 64 * 1024; + unsigned char *buf = (unsigned char *)zmalloc(n); + fillIncompressible(buf, n, 0x12345678u); + ASSERT_EQ(replCompressorWrite(rc, buf, n), C_OK); + ASSERT_EQ(replCompressorFlush(rc), C_OK); /* frame stays open */ + + replDecompressor *rd = replDecompressorCreate(); + ASSERT_TRUE(rd != NULL); + size_t out_len = 0; + EXPECT_EQ(replDecompressorDecode(rd, rc->out_buf, sdslen(rc->out_buf), 1024, &out_len), + REPL_DECODE_OVERFLOW); + + replDecompressorDestroy(rd); + zfree(buf); + replCompressorDestroy(rc); +} + +TEST(replCompression, decodeEnvelopeSplitAcrossFeeds) { + replCompressor *rc = replCompressorCreate(ALGO_LZ4); + ASSERT_TRUE(rc != NULL); + const size_t n = 10 * 1024; + unsigned char *payload = (unsigned char *)zmalloc(n); + memset(payload, 'A', n); + ASSERT_EQ(replCompressorWrite(rc, payload, n), C_OK); + ASSERT_EQ(replCompressorFlush(rc), C_OK); + const unsigned char *stream = (const unsigned char *)rc->out_buf; + const size_t stream_len = sdslen(rc->out_buf); + ASSERT_GT(stream_len, (size_t)VCS_ENVELOPE_SIZE); + + replDecompressor *rd = replDecompressorCreate(); + ASSERT_TRUE(rd != NULL); + unsigned char *decoded = (unsigned char *)zmalloc(n); + size_t decoded_len = 0; + size_t out_len = 0; + + /* Byte 0 alone: probe cannot classify yet, nothing decodes. */ + ASSERT_EQ(replDecompressorDecode(rd, stream, 1, 1024 * 1024, &out_len), REPL_DECODE_OK); + EXPECT_EQ(out_len, (size_t)0); + memcpy(decoded + decoded_len, replDecompressorBuf(rd), out_len); + decoded_len += out_len; + + /* Bytes 1-2: magic complete, envelope still short. */ + ASSERT_EQ(replDecompressorDecode(rd, stream + 1, 2, 1024 * 1024, &out_len), REPL_DECODE_OK); + EXPECT_EQ(out_len, (size_t)0); + memcpy(decoded + decoded_len, replDecompressorBuf(rd), out_len); + decoded_len += out_len; + + /* Remainder: envelope parses and the payload decodes. */ + ASSERT_EQ(replDecompressorDecode(rd, stream + 3, stream_len - 3, 1024 * 1024, &out_len), + REPL_DECODE_OK); + ASSERT_LE(decoded_len + out_len, n); + memcpy(decoded + decoded_len, replDecompressorBuf(rd), out_len); + decoded_len += out_len; + + ASSERT_EQ(decoded_len, n); + EXPECT_EQ(memcmp(decoded, payload, n), 0); + + zfree(decoded); + zfree(payload); + replDecompressorDestroy(rd); + replCompressorDestroy(rc); +} + +TEST(replCompression, decodePassthroughReplaysPrefix) { + replDecompressor *rd = replDecompressorCreate(); + ASSERT_TRUE(rd != NULL); + size_t out_len = 0; + /* "V" alone could still open the VCS magic: buffered, nothing emitted. */ + ASSERT_EQ(replDecompressorDecode(rd, "V", 1, 1024, &out_len), REPL_DECODE_OK); + EXPECT_EQ(out_len, (size_t)0); + EXPECT_FALSE(replDecompressorIsPassthrough(rd)); + /* "X" rules out the magic: the buffered "V" replays ahead of the new bytes. */ + ASSERT_EQ(replDecompressorDecode(rd, "XYZ", 3, 1024, &out_len), REPL_DECODE_OK); + EXPECT_TRUE(replDecompressorIsPassthrough(rd)); + ASSERT_EQ(out_len, (size_t)4); + ASSERT_EQ(sdslen(replDecompressorBuf(rd)), (size_t)4); + EXPECT_EQ(memcmp(replDecompressorBuf(rd), "VXYZ", 4), 0); + replDecompressorDestroy(rd); +} + +TEST(replCompression, decodeRejectsBadCodec) { + replDecompressor *rd = replDecompressorCreate(); + ASSERT_TRUE(rd != NULL); + /* Valid magic/version/kind, unknown codec id 0xFF. */ + unsigned char stream[VCS_ENVELOPE_SIZE + 4] = {'V', 'C', 'S', VCS_VERSION, 0xFF, 0x00, + VCS_STREAM_REPL, 0xDE, 0xAD, 0xBE, 0xEF}; + size_t out_len = 0; + EXPECT_EQ(replDecompressorDecode(rd, stream, sizeof(stream), 1024, &out_len), REPL_DECODE_ERR); + replDecompressorDestroy(rd); +} + +TEST(replCompression, decodeRejectsNonzeroReserved) { + replDecompressor *rd = replDecompressorCreate(); + ASSERT_TRUE(rd != NULL); + /* Valid magic/version/codec/kind, nonzero reserved byte. */ + unsigned char stream[VCS_ENVELOPE_SIZE + 4] = {'V', 'C', 'S', VCS_VERSION, VCS_CODEC_LZ4, 0x01, + VCS_STREAM_REPL, 0xDE, 0xAD, 0xBE, 0xEF}; + size_t out_len = 0; + EXPECT_EQ(replDecompressorDecode(rd, stream, sizeof(stream), 1024, &out_len), REPL_DECODE_ERR); + replDecompressorDestroy(rd); +} + +TEST(replCompression, decodeErrOnCorruptPayload) { + /* A real compressor emits the envelope on first write; reuse those bytes. */ + replCompressor *rc = replCompressorCreate(ALGO_LZ4); + ASSERT_TRUE(rc != NULL); + ASSERT_EQ(replCompressorWrite(rc, "seed", 4), C_OK); + ASSERT_GE(sdslen(rc->out_buf), (size_t)VCS_ENVELOPE_SIZE); + + /* Valid envelope followed by garbage that LZ4F rejects. */ + unsigned char stream[VCS_ENVELOPE_SIZE + 64]; + memcpy(stream, rc->out_buf, VCS_ENVELOPE_SIZE); + memset(stream + VCS_ENVELOPE_SIZE, 0xFF, 64); + + replDecompressor *rd = replDecompressorCreate(); + ASSERT_TRUE(rd != NULL); + size_t out_len = 0; + EXPECT_EQ(replDecompressorDecode(rd, stream, sizeof(stream), 1024 * 1024, &out_len), + REPL_DECODE_ERR); + + replDecompressorDestroy(rd); + replCompressorDestroy(rc); +} + +TEST(replCompression, decodeDrainsBufferedOutputWithoutMoreInput) { + /* The writer emits 64KB LZ4 blocks while the decoder offers 16KB of room + * per iteration, so LZ4F decodes a compressed block into its internal + * buffer and can report the block's input consumed with output still + * undelivered. Once input runs out the decoder must keep draining with + * empty input; otherwise the tail is stranded inside the codec until + * later transport bytes arrive (worst case a 10s replication PING). The + * payload ends with a compressible run: a stored (incompressible) block + * streams straight to the caller's buffer and would not strand. */ + const size_t incompressible = 36 * 1024; + const size_t compressible = 64 * 1024; + const size_t n = incompressible + compressible; /* ~100KB: multiple blocks */ + unsigned char *payload = (unsigned char *)zmalloc(n); + fillIncompressible(payload, incompressible, 0xC0FFEE42u); + memset(payload + incompressible, 'A', compressible); + + replCompressor *rc = replCompressorCreate(ALGO_LZ4); + ASSERT_TRUE(rc != NULL); + ASSERT_EQ(replCompressorWrite(rc, payload, n), C_OK); + ASSERT_EQ(replCompressorFlush(rc), C_OK); /* frame stays open */ + + /* All compressed bytes in ONE call: no later input can push out whatever + * the codec buffered, so the decode itself must drain it. */ + replDecompressor *rd = replDecompressorCreate(); + ASSERT_TRUE(rd != NULL); + size_t out_len = 0; + ASSERT_EQ(replDecompressorDecode(rd, rc->out_buf, sdslen(rc->out_buf), 4 * 1024 * 1024, &out_len), + REPL_DECODE_OK); + EXPECT_EQ(out_len, n); + ASSERT_EQ(sdslen(replDecompressorBuf(rd)), n); + EXPECT_EQ(memcmp(replDecompressorBuf(rd), payload, n), 0); + + replDecompressorDestroy(rd); + replCompressorDestroy(rc); + zfree(payload); +} + +TEST(replCompression, decodeCallOutputStaysUnderCapAtMaxRatio) { + /* Feed paths hand the decoder at most PROTO_IOBUF_LEN (16KB) per call and + * LZ4 expansion is bounded, so one call's output stays far under the + * 16MB overflow cap even for maximally compressible input. */ + const size_t n = 4 * 1024 * 1024; /* 4MB of one byte: near-max ratio */ + unsigned char *payload = (unsigned char *)zmalloc(n); + memset(payload, 'Z', n); + + replCompressor *rc = replCompressorCreate(ALGO_LZ4); + ASSERT_TRUE(rc != NULL); + ASSERT_EQ(replCompressorWrite(rc, payload, n), C_OK); + ASSERT_EQ(replCompressorFlush(rc), C_OK); + + /* One decode call fed a single clamped read (16KB of wire bytes): output + * must stay far under the 16MB cap; a 255x bound on 16KB is ~4MB. */ + size_t chunk = sdslen(rc->out_buf); + if (chunk > (size_t)16 * 1024) chunk = (size_t)16 * 1024; + replDecompressor *rd = replDecompressorCreate(); + ASSERT_TRUE(rd != NULL); + size_t out_len = 0; + ASSERT_EQ(replDecompressorDecode(rd, rc->out_buf, chunk, 16 * 1024 * 1024, &out_len), + REPL_DECODE_OK); + EXPECT_GT(out_len, (size_t)1024 * 1024); /* high ratio actually exercised */ + EXPECT_LT(out_len, (size_t)16 * 1024 * 1024); + + replDecompressorDestroy(rd); + replCompressorDestroy(rc); + zfree(payload); +} diff --git a/src/unit/wrappers.h b/src/unit/wrappers.h index a576814c28c..203c6dc29fb 100644 --- a/src/unit/wrappers.h +++ b/src/unit/wrappers.h @@ -44,6 +44,7 @@ extern "C" { #define protected protected_ /* Avoid conflict with C++ 'protected' keyword */ #include "ae.h" +#include "compression.h" #include "server.h" /** @@ -60,6 +61,7 @@ extern "C" { * Example: serverLog(int level, const char *fmt, ...) should NOT be mocked. */ long long __wrap_aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, aeTimeProc *proc, void *clientData, aeEventFinalizerProc *finalizerProc); +ssize_t __wrap_streamDecompressorFeed(streamDecompressor *decompressor, uint8_t *output, size_t output_capacity, const uint8_t *input, size_t input_len, size_t *input_consumed); #undef protected #undef _Bool #undef typename diff --git a/src/valkey-check-rdb.c b/src/valkey-check-rdb.c index f84ff67adfb..12b46e38268 100644 --- a/src/valkey-check-rdb.c +++ b/src/valkey-check-rdb.c @@ -95,6 +95,21 @@ struct { char *stats_output; } rdbstate; +static unsigned long long rdbCheckOffset(void) { + if (!rdbstate.rio) return 0; + return (unsigned long long)rdbstate.rio->processed_bytes; +} + +static void rdbCheckPrintOffset(const char *msg) { + if (rdbstate.rio && (rdbstate.rio->flags & RIO_FLAG_STREAMING_COMPRESSION)) { + printf("[logical offset %llu, physical offset %llu] %s\n", + (unsigned long long)rdbstate.rio->processed_bytes, + (unsigned long long)rdbstate.rio->stream_processed_bytes, msg); + } else { + printf("[offset %llu] %s\n", rdbCheckOffset(), msg); + } +} + /* At every loading step try to remember what we were about to do, so that * we can log this information when an error is encountered. */ #define RDB_CHECK_DOING_START 0 @@ -523,7 +538,7 @@ void rdbCheckError(const char *fmt, ...) { va_end(ap); printf("--- RDB ERROR DETECTED ---\n"); - printf("[offset %llu] %s\n", (unsigned long long)(rdbstate.rio ? rdbstate.rio->processed_bytes : 0), msg); + rdbCheckPrintOffset(msg); printf("[additional info] While doing: %s\n", rdb_check_doing_string[rdbstate.doing]); if (rdbstate.key) printf("[additional info] Reading key '%s'\n", (char *)objectGetVal(rdbstate.key)); if (rdbstate.key_type != -1) @@ -552,7 +567,7 @@ void rdbCheckInfo(const char *fmt, ...) { va_end(ap); } - printf("[offset %llu] %s\n", (unsigned long long)(rdbstate.rio ? rdbstate.rio->processed_bytes : 0), msgbuf); + rdbCheckPrintOffset(msgbuf); if (msgbuf != msg) sdsfree(msgbuf); } @@ -603,7 +618,10 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { int type, rdbver; char buf[1024]; long long expiretime; - static rio rdb; /* Pointed by global struct riostate. */ + static rio file_rdb; + rio *rdb = &file_rdb; /* Pointed by global struct riostate. */ + streamReader stream_reader; + bool stream_reader_initialized = false; struct stat sb; now = mstime(); @@ -613,10 +631,25 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { if (fstat(fileno(fp), &sb) == -1) sb.st_size = 0; startLoadingFile(sb.st_size, rdbfilename, RDBFLAGS_NONE); - rioInitWithFile(&rdb, fp); - rdbstate.rio = &rdb; - rdb.update_cksum = rdbLoadProgressCallback; - if (rioRead(&rdb, buf, 9) == 0) goto eoferr; + rioInitWithFile(&file_rdb, fp); + + /* Support both plain RDB files and VCS-wrapped streaming-compressed RDBs. */ + rdbStreamReaderInitResult init_rc = rdbInitStreamReader(&file_rdb, &stream_reader, false, NULL); + if (init_rc == RDB_STREAM_READER_INIT_INCOMPATIBLE) { + rdbCheckError("Invalid or unsupported RDB stream envelope. " + "File may require a Valkey version with streaming RDB " + "compression support."); + goto err; + } + if (init_rc != RDB_STREAM_READER_INIT_OK) { + rdbCheckError("Failed to inspect RDB stream metadata"); + goto err; + } + stream_reader_initialized = true; + + rdbstate.rio = rdb; + rdb->update_cksum = rdbLoadProgressCallback; + if (rioRead(rdb, buf, 9) == 0) goto eoferr; buf[9] = '\0'; bool is_valkey_magic = false, is_redis_magic = false; if (memcmp(buf, "REDIS0", 6) == 0) { @@ -646,7 +679,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { /* Read type. */ rdbstate.doing = RDB_CHECK_DOING_READ_TYPE; - if ((type = rdbLoadType(&rdb)) == -1) goto eoferr; + if ((type = rdbLoadType(rdb)) == -1) goto eoferr; /* Handle special types. */ if (type == RDB_OPCODE_EXPIRETIME) { @@ -654,25 +687,25 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { /* EXPIRETIME: load an expire associated with the next key * to load. Note that after loading an expire we need to * load the actual type, and continue. */ - expiretime = rdbLoadTime(&rdb); + expiretime = rdbLoadTime(rdb); expiretime *= 1000; - if (rioGetReadError(&rdb)) goto eoferr; + if (rioGetReadError(rdb)) goto eoferr; continue; /* Read next opcode. */ } else if (type == RDB_OPCODE_EXPIRETIME_MS) { /* EXPIRETIME_MS: milliseconds precision expire times introduced * with RDB v3. Like EXPIRETIME but no with more precision. */ rdbstate.doing = RDB_CHECK_DOING_READ_EXPIRE; - expiretime = rdbLoadMillisecondTime(&rdb, rdbver); - if (rioGetReadError(&rdb)) goto eoferr; + expiretime = rdbLoadMillisecondTime(rdb, rdbver); + if (rioGetReadError(rdb)) goto eoferr; continue; /* Read next opcode. */ } else if (type == RDB_OPCODE_FREQ) { /* FREQ: LFU frequency. */ uint8_t byte; - if (rioRead(&rdb, &byte, 1) == 0) goto eoferr; + if (rioRead(rdb, &byte, 1) == 0) goto eoferr; continue; /* Read next opcode. */ } else if (type == RDB_OPCODE_IDLE) { /* IDLE: LRU idle time. */ - if (rdbLoadLen(&rdb, NULL) == RDB_LENERR) goto eoferr; + if (rdbLoadLen(rdb, NULL) == RDB_LENERR) goto eoferr; continue; /* Read next opcode. */ } else if (type == RDB_OPCODE_EOF) { /* EOF: End of file, exit the main loop. */ @@ -680,7 +713,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { } else if (type == RDB_OPCODE_SELECTDB) { /* SELECTDB: Select the specified database. */ rdbstate.doing = RDB_CHECK_DOING_READ_LEN; - if ((dbid = rdbLoadLen(&rdb, NULL)) == RDB_LENERR) goto eoferr; + if ((dbid = rdbLoadLen(rdb, NULL)) == RDB_LENERR) goto eoferr; rdbCheckInfo("Selecting DB ID %llu", (unsigned long long)dbid); selected_dbid = dbid; if (selected_dbid > rdbstate.databases) { @@ -693,18 +726,18 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { * selected data base, in order to avoid useless rehashing. */ uint64_t db_size, expires_size; rdbstate.doing = RDB_CHECK_DOING_READ_LEN; - if ((db_size = rdbLoadLen(&rdb, NULL)) == RDB_LENERR) goto eoferr; - if ((expires_size = rdbLoadLen(&rdb, NULL)) == RDB_LENERR) goto eoferr; + if ((db_size = rdbLoadLen(rdb, NULL)) == RDB_LENERR) goto eoferr; + if ((expires_size = rdbLoadLen(rdb, NULL)) == RDB_LENERR) goto eoferr; continue; /* Read type again. */ } else if (type == RDB_OPCODE_SLOT_INFO) { /* Hint used in foreign RDB versions. */ - if (rdbLoadLen(&rdb, NULL) == RDB_LENERR) goto eoferr; - if (rdbLoadLen(&rdb, NULL) == RDB_LENERR) goto eoferr; - if (rdbLoadLen(&rdb, NULL) == RDB_LENERR) goto eoferr; + if (rdbLoadLen(rdb, NULL) == RDB_LENERR) goto eoferr; + if (rdbLoadLen(rdb, NULL) == RDB_LENERR) goto eoferr; + if (rdbLoadLen(rdb, NULL) == RDB_LENERR) goto eoferr; continue; /* Read type again. */ } else if (type == RDB_OPCODE_SLOT_IMPORT) { robj *job_name; - if ((job_name = rdbLoadStringObject(&rdb)) == NULL) goto eoferr; + if ((job_name = rdbLoadStringObject(rdb)) == NULL) goto eoferr; if (sdslen(objectGetVal(job_name)) != CLUSTER_NAMELEN) { rdbCheckError("Invalid slot import job name length in RDB"); decrRefCount(job_name); @@ -712,10 +745,10 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { } decrRefCount(job_name); uint64_t num_slot_ranges; - if ((num_slot_ranges = rdbLoadLen(&rdb, NULL)) == RDB_LENERR) goto eoferr; + if ((num_slot_ranges = rdbLoadLen(rdb, NULL)) == RDB_LENERR) goto eoferr; for (uint64_t i = 0; i < num_slot_ranges; i++) { - if (rdbLoadLen(&rdb, NULL) == RDB_LENERR) goto eoferr; - if (rdbLoadLen(&rdb, NULL) == RDB_LENERR) goto eoferr; + if (rdbLoadLen(rdb, NULL) == RDB_LENERR) goto eoferr; + if (rdbLoadLen(rdb, NULL) == RDB_LENERR) goto eoferr; } continue; /* Read type again. */ } else if (type == RDB_OPCODE_AUX) { @@ -726,8 +759,8 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { * An AUX field is composed of two strings: key and value. */ robj *auxkey, *auxval; rdbstate.doing = RDB_CHECK_DOING_READ_AUX; - if ((auxkey = rdbLoadStringObject(&rdb)) == NULL) goto eoferr; - if ((auxval = rdbLoadStringObject(&rdb)) == NULL) { + if ((auxkey = rdbLoadStringObject(rdb)) == NULL) goto eoferr; + if ((auxval = rdbLoadStringObject(rdb)) == NULL) { decrRefCount(auxkey); goto eoferr; } @@ -743,9 +776,9 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { /* AUX: Auxiliary data for modules. */ uint64_t moduleid, when_opcode, when; rdbstate.doing = RDB_CHECK_DOING_READ_MODULE_AUX; - if ((moduleid = rdbLoadLen(&rdb, NULL)) == RDB_LENERR) goto eoferr; - if ((when_opcode = rdbLoadLen(&rdb, NULL)) == RDB_LENERR) goto eoferr; - if ((when = rdbLoadLen(&rdb, NULL)) == RDB_LENERR) goto eoferr; + if ((moduleid = rdbLoadLen(rdb, NULL)) == RDB_LENERR) goto eoferr; + if ((when_opcode = rdbLoadLen(rdb, NULL)) == RDB_LENERR) goto eoferr; + if ((when = rdbLoadLen(rdb, NULL)) == RDB_LENERR) goto eoferr; if (when_opcode != RDB_MODULE_OPCODE_UINT) { rdbCheckError("bad when_opcode"); goto err; @@ -755,7 +788,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { moduleTypeNameByID(name, moduleid); rdbCheckInfo("MODULE AUX for: %s", name); - robj *o = rdbLoadCheckModuleValue(&rdb, name); + robj *o = rdbLoadCheckModuleValue(rdb, name); decrRefCount(o); continue; /* Read type again. */ } else if (type == RDB_OPCODE_FUNCTION_PRE_GA) { @@ -764,7 +797,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { } else if (type == RDB_OPCODE_FUNCTION2) { sds err = NULL; rdbstate.doing = RDB_CHECK_DOING_READ_FUNCTIONS; - if (rdbFunctionLoad(&rdb, rdbver, NULL, 0, &err) != C_OK) { + if (rdbFunctionLoad(rdb, rdbver, NULL, 0, &err) != C_OK) { rdbCheckError("Failed loading library, %s", err); sdsfree(err); goto err; @@ -789,12 +822,12 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { /* Read key */ rdbstate.doing = RDB_CHECK_DOING_READ_KEY; - if ((key = rdbLoadStringObject(&rdb)) == NULL) goto eoferr; + if ((key = rdbLoadStringObject(rdb)) == NULL) goto eoferr; rdbstate.key = key; rdbstate.keys++; /* Read value */ rdbstate.doing = RDB_CHECK_DOING_READ_OBJECT_VALUE; - if ((val = rdbLoadObject(type, &rdb, objectGetVal(key), selected_dbid, NULL, RDBFLAGS_NONE, 0)) == NULL) goto eoferr; + if ((val = rdbLoadObject(type, rdb, objectGetVal(key), selected_dbid, NULL, RDBFLAGS_NONE, 0)) == NULL) goto eoferr; if (rdbCheckStats) { int max_stats_num = (rdbstate.databases + 1) * OBJ_TYPE_MAX; if (max_stats_num > rdbstate.stats_num) { @@ -813,23 +846,36 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { rdbstate.key_type = -1; expiretime = -1; } - /* Verify the checksum if RDB version is >= 5 */ - if (rdbver >= 5 && server.rdb_checksum) { - uint64_t cksum, expected = rdb.cksum; + /* Consume the checksum trailer for every RDB version that has one. Whether + * it is validated is controlled separately by the checksum policy. */ + if (rdbver >= 5) { + uint64_t cksum, expected = rdb->cksum; rdbstate.doing = RDB_CHECK_DOING_CHECK_SUM; - if (rioRead(&rdb, &cksum, 8) == 0) goto eoferr; - memrev64ifbe(&cksum); - if (cksum == 0) { - rdbCheckInfo("RDB file was saved with checksum disabled: no check performed."); - } else if (cksum != expected) { - rdbCheckError("RDB CRC error"); - goto err; - } else { - rdbCheckInfo("Checksum OK"); + if (rioRead(rdb, &cksum, 8) == 0) goto eoferr; + if (rdb->flags & RIO_FLAG_STREAMING_COMPRESSION) { + rdbCheckInfo("Logical RDB CRC64 skipped for streaming-compressed input."); + } else if (server.rdb_checksum) { + memrev64ifbe(&cksum); + if (rdb->flags & RIO_FLAG_SKIP_RDB_CHECKSUM) { + rdbCheckInfo("RDB file was saved with checksum disabled: skipped checksum for this transfer."); + } else if (cksum == 0) { + rdbCheckInfo("RDB file was saved with checksum disabled: no check performed."); + } else if (cksum != expected) { + rdbCheckError("RDB CRC error"); + goto err; + } else { + rdbCheckInfo("Checksum OK"); + } } } + if (stream_reader_initialized && streamReaderFinish(&stream_reader) == C_ERR) { + rdbCheckError("Compressed RDB stream did not end cleanly"); + goto err; + } + + if (stream_reader_initialized) rdbFreeStreamReader(&file_rdb, &stream_reader); if (closefile) fclose(fp); stopLoading(1); return 0; @@ -837,10 +883,15 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { eoferr: /* unexpected end of file is handled here with a fatal exit */ if (rdbstate.error_set) { rdbCheckError(rdbstate.error); + } else if (rdbRioHasInternalStreamReaderError(rdb)) { + rdbCheckError("Internal error decoding compressed RDB stream"); + } else if (rdbRioHasCorruptCompressedInput(rdb)) { + rdbCheckError("Corrupt compressed RDB stream"); } else { rdbCheckError("Unexpected EOF reading RDB file"); } err: + if (stream_reader_initialized) rdbFreeStreamReader(&file_rdb, &stream_reader); if (closefile) fclose(fp); stopLoading(0); return 1; diff --git a/tests/integration/block-repl.tcl b/tests/integration/block-repl.tcl index 52b4a53ead8..4358d11db55 100644 --- a/tests/integration/block-repl.tcl +++ b/tests/integration/block-repl.tcl @@ -11,7 +11,7 @@ proc stop_bg_block_op {handle} { catch {exec /bin/kill -9 $handle} } -start_server {tags {"repl" "external:skip"}} { +start_server {tags {"repl" "external:skip" repl-compression}} { start_server {overrides {save {}}} { set master [srv -1 client] set master_host [srv -1 host] diff --git a/tests/integration/dual-channel-replication.tcl b/tests/integration/dual-channel-replication.tcl index 2674a9d745e..62a901759dc 100644 --- a/tests/integration/dual-channel-replication.tcl +++ b/tests/integration/dual-channel-replication.tcl @@ -12,7 +12,7 @@ proc wait_and_resume_process idx { resume_process $pid } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set replica [srv 0 client] set replica_host [srv 0 host] set replica_port [srv 0 port] @@ -78,7 +78,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set replica [srv 0 client] set replica_host [srv 0 host] set replica_port [srv 0 port] @@ -112,7 +112,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set replica [srv 0 client] set replica_host [srv 0 host] set replica_port [srv 0 port] @@ -200,7 +200,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set replica [srv 0 client] set replica_host [srv 0 host] set replica_port [srv 0 port] @@ -335,7 +335,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set replica1 [srv 0 client] set replica1_host [srv 0 host] set replica1_port [srv 0 port] @@ -469,7 +469,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set replica [srv 0 client] set replica_host [srv 0 host] set replica_port [srv 0 port] @@ -506,7 +506,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set replica [srv 0 client] set replica_host [srv 0 host] set replica_port [srv 0 port] @@ -571,7 +571,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set replica1 [srv 0 client] set replica1_host [srv 0 host] set replica1_port [srv 0 port] @@ -645,7 +645,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set primary [srv 0 client] set primary_host [srv 0 host] set primary_port [srv 0 port] @@ -707,7 +707,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set primary [srv 0 client] set primary_host [srv 0 host] set primary_port [srv 0 port] @@ -768,7 +768,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set primary [srv 0 client] set primary_host [srv 0 host] set primary_port [srv 0 port] @@ -848,7 +848,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { $primary config set shutdown-timeout 0 } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set primary [srv 0 client] set primary_host [srv 0 host] set primary_port [srv 0 port] @@ -909,7 +909,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set primary [srv 0 client] set primary_host [srv 0 host] set primary_port [srv 0 port] @@ -999,7 +999,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set primary [srv 0 client] set primary_host [srv 0 host] set primary_port [srv 0 port] @@ -1124,7 +1124,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set primary [srv 0 client] set primary_host [srv 0 host] set primary_port [srv 0 port] @@ -1175,7 +1175,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set primary [srv 0 client] set primary_host [srv 0 host] set primary_port [srv 0 port] @@ -1239,7 +1239,7 @@ start_server {tags {"dual-channel-replication external:skip"}} { } } -start_server {tags {"dual-channel-replication external:skip"}} { +start_server {tags {"dual-channel-replication external:skip" repl-compression}} { set primary [srv 0 client] set primary_host [srv 0 host] set primary_port [srv 0 port] diff --git a/tests/integration/rdb-compression.tcl b/tests/integration/rdb-compression.tcl new file mode 100644 index 00000000000..2f0235f9a16 --- /dev/null +++ b/tests/integration/rdb-compression.tcl @@ -0,0 +1,493 @@ +source tests/support/aofmanifest.tcl + +tags {"rdb-compression external:skip needs:debug"} { + +proc read_binary_file_prefix {path count} { + set fd [open $path r] + fconfigure $fd -translation binary + set prefix [read $fd $count] + close $fd + return $prefix +} + +proc read_binary_file {path} { + set fd [open $path r] + fconfigure $fd -translation binary + set data [read $fd] + close $fd + return $data +} + +proc write_binary_file {path data} { + set fd [open $path w] + fconfigure $fd -translation binary + puts -nonewline $fd $data + close $fd +} + +proc dump_rdb_path {client} { + return [file join [lindex [$client config get dir] 1] dump.rdb] +} + +proc read_dump_rdb_header_bytes {client} { + return [read_binary_file_prefix [dump_rdb_path $client] 8] +} + +proc assert_lz4_rdb_envelope {client} { + binary scan [read_binary_file_prefix [dump_rdb_path $client] 7] cu* bytes + assert_equal [list 86 67 83 1 1 0 1] $bytes +} + +proc assert_lz4_rdb_checksum_flags {client expected} { + set frame_flg_offset [expr {7 + 4}] + binary scan [read_binary_file_prefix [dump_rdb_path $client] [expr {$frame_flg_offset + 1}]] cu* bytes + set frame_flg [lindex $bytes $frame_flg_offset] + set has_block_checksum [expr {($frame_flg & 0x10) != 0}] + set has_content_checksum [expr {($frame_flg & 0x04) != 0}] + assert_equal $expected $has_block_checksum + assert_equal $expected $has_content_checksum +} + +proc write_rdb_test_dataset {client prefix} { + $client flushall + for {set i 0} {$i < 12} {incr i} { + $client set "${prefix}:str:$i" [string repeat "${prefix}:value:$i " 16] + } + $client lpush "${prefix}:list" a b c d e + $client sadd "${prefix}:set" alpha beta gamma + $client zadd "${prefix}:zset" 1 one 2 two 3 three + $client hset "${prefix}:hash" f1 v1 f2 [string repeat "${prefix}:hash " 8] + $client xadd "${prefix}:stream" * f1 s1 f2 [string repeat "${prefix}:stream " 4] + $client xadd "${prefix}:stream" * f1 s2 f2 tail +} + +proc assert_rdb_test_dataset {client prefix} { + assert_equal [string repeat "${prefix}:value:0 " 16] [$client get "${prefix}:str:0"] + assert_equal 17 [$client dbsize] + assert_equal 5 [$client llen "${prefix}:list"] + assert_equal 3 [$client scard "${prefix}:set"] + assert_equal 3 [$client zcard "${prefix}:zset"] + assert_equal v1 [$client hget "${prefix}:hash" f1] + assert_equal 2 [$client xlen "${prefix}:stream"] +} + +start_server {overrides {save "" enable-debug-command local}} { + test {RDB save and load round-trip with LZ4 compression} { + set prefix "lz4-round-trip" + r config set rdbcompression lz4 + write_rdb_test_dataset r $prefix + assert_rdb_test_dataset r $prefix + set digest [debug_digest] + + assert_equal "OK" [r save] + assert_lz4_rdb_envelope r + assert_lz4_rdb_checksum_flags r 1 + set loglines [count_log_lines 0] + assert_equal "OK" [r debug reload nosave] + verify_log_message 0 "*Logical RDB CRC64 skipped for streaming-compressed input*" $loglines + r config rewrite + restart_server 0 true false + + assert_equal "lz4" [lindex [r config get rdbcompression] 1] + set newdigest [debug_digest] + assert {$digest eq $newdigest} + assert_rdb_test_dataset r $prefix + } + + test {Empty LZ4-compressed RDB saves and loads correctly} { + r config set rdbcompression lz4 + r flushall + + assert_equal 0 [r dbsize] + assert_equal "OK" [r save] + r config rewrite + assert_lz4_rdb_envelope r + + restart_server 0 true false + + assert_equal "lz4" [lindex [r config get rdbcompression] 1] + assert_equal 0 [r dbsize] + } + + test {RDB save with LZF (default) round-trips correctly} { + set prefix "lzf-round-trip" + r config set rdbcompression yes + write_rdb_test_dataset r $prefix + assert_rdb_test_dataset r $prefix + + set digest [debug_digest] + assert_equal "OK" [r save] + r config rewrite + restart_server 0 true false + + assert_equal "yes" [lindex [r config get rdbcompression] 1] + set newdigest [debug_digest] + assert {$digest eq $newdigest} + assert_rdb_test_dataset r $prefix + } + + test {Uncompressed RDB files load correctly (backward compat)} { + set prefix "plain-rdb" + r config set rdbcompression no + write_rdb_test_dataset r $prefix + assert_rdb_test_dataset r $prefix + + set digest [debug_digest] + assert_equal "OK" [r save] + r config rewrite + restart_server 0 true false + + assert_equal "no" [lindex [r config get rdbcompression] 1] + set newdigest [debug_digest] + assert {$digest eq $newdigest} + assert_rdb_test_dataset r $prefix + } + + test {Changing compression config during active BGSAVE does not affect the in-flight save} { + r config set rdbcompression lz4 + r config set rdb-key-save-delay 10000 + r flushall + for {set i 0} {$i < 128} {incr i} { + r set "bgsave-race:$i" [string repeat "payload:$i " 128] + } + + assert_match {*Background saving started*} [r bgsave] + wait_for_condition 200 10 { + [s rdb_bgsave_in_progress] eq 1 + } else { + r config set rdb-key-save-delay 0 + fail "BGSAVE did not start in time" + } + + # The child must keep the compression setting inherited at fork. + r config set rdbcompression yes + + wait_for_condition 500 10 { + [s rdb_bgsave_in_progress] eq 0 + } else { + r config set rdb-key-save-delay 0 + fail "BGSAVE did not finish in time" + } + r config set rdb-key-save-delay 0 + + assert_equal "yes" [lindex [r config get rdbcompression] 1] + assert_lz4_rdb_envelope r + + assert_equal "OK" [r save] + assert_equal "VALKEY" [string range [read_dump_rdb_header_bytes r] 0 5] + + r config set rdbcompression lz4 + } + + test {Switching from LZ4 to LZF preserves data} { + set prefix "lz4-to-lzf" + r config set rdbcompression lz4 + write_rdb_test_dataset r $prefix + assert_rdb_test_dataset r $prefix + set digest [debug_digest] + + # Save with LZ4, then restart with LZF and load the existing file. + assert_equal "OK" [r save] + r config set rdbcompression lzf + r config rewrite + restart_server 0 true false + + assert_equal "lzf" [lindex [r config get rdbcompression] 1] + set newdigest [debug_digest] + assert {$digest eq $newdigest} + assert_rdb_test_dataset r $prefix + } + + test {Switching from LZF to LZ4 preserves data} { + set prefix "lzf-to-lz4" + r config set rdbcompression lzf + write_rdb_test_dataset r $prefix + assert_rdb_test_dataset r $prefix + set digest [debug_digest] + + # Save with LZF, then restart with LZ4 and load the existing file. + assert_equal "OK" [r save] + r config set rdbcompression lz4 + r config rewrite + restart_server 0 true false + + assert_equal "lz4" [lindex [r config get rdbcompression] 1] + set newdigest [debug_digest] + assert {$digest eq $newdigest} + assert_rdb_test_dataset r $prefix + } + + test {Invalid compression config is rejected} { + set previous [lindex [r config get rdbcompression] 1] + assert_error "*argument(s) must be one of the following: no, yes, lzf, lz4*" { + r config set rdbcompression snappy + } + assert_equal $previous [lindex [r config get rdbcompression] 1] + } + + test {Truncated LZ4 frame is rejected on load} { + r config set rdbcompression lz4 + r flushall + set noisy_payload "" + for {set j 0} {$j < 32768} {incr j} { + append noisy_payload [format %c [expr {(($j * 31) + 17) % 94 + 33}]] + } + for {set i 0} {$i < 128} {incr i} { + r set "partial:$i" "${noisy_payload}:$i" + } + + assert_equal "OK" [r save] + set rdbfile [dump_rdb_path r] + assert_lz4_rdb_envelope r + + set truncated [read_binary_file_prefix $rdbfile [expr {[file size $rdbfile] / 2}]] + set fd [open $rdbfile w] + fconfigure $fd -translation binary + puts -nonewline $fd $truncated + close $fd + + set failed [catch {r debug reload nosave} err] + assert_equal 1 $failed + assert_match "*Error trying to load the RDB*" $err + + r debug set-skip-checksum-validation 1 + set failed [catch {r debug reload nosave} err] + r debug set-skip-checksum-validation 0 + assert_equal 1 $failed + assert_match "*Error trying to load the RDB*" $err + } + + test {LZ4 compressed RDB detects a content checksum mismatch} { + r config set rdbcompression lz4 + assert_equal "yes" [lindex [r config get rdbchecksum] 1] + r flushall + for {set i 0} {$i < 100} {incr i} { + r set "footer:$i" [string repeat "payload$i " 100] + } + + r save + set rdbfile [file join [lindex [r config get dir] 1] dump.rdb] + set fd [open $rdbfile r+] + fconfigure $fd -translation binary + seek $fd -1 end + binary scan [read $fd 1] cu checksum_byte + seek $fd -1 end + puts -nonewline $fd [binary format c [expr {$checksum_byte ^ 1}]] + close $fd + + set failed [catch {r debug reload nosave} err] + assert_equal 1 $failed + assert_match "*Error trying to load the RDB*" $err + + set loglines [count_log_lines 0] + r debug set-skip-checksum-validation 1 + assert_equal "OK" [r debug reload nosave] + r debug set-skip-checksum-validation 0 + verify_log_message 0 "*Logical RDB CRC64 skipped for streaming-compressed input*" $loglines + } + + test {RDB loader rejects incompatible VCS envelope fields without changing data} { + r config set rdbcompression lz4 + r flushall + r set incompatible-envelope:key [string repeat "payload " 100] + + assert_equal "OK" [r save] + set digest [debug_digest] + set rdbfile [dump_rdb_path r] + set original [read_binary_file $rdbfile] + + foreach case { + {version 3 2} + {codec 4 127} + {reserved-byte 5 1} + {stream-kind 6 127} + } { + lassign $case field offset value + set mutated [string replace $original $offset $offset [binary format c $value]] + write_binary_file $rdbfile $mutated + set loglines [count_log_lines 0] + + set failed [catch {r debug reload nosave} err] + assert_equal 1 $failed "VCS $field should be rejected" + assert_match "*Error trying to load the RDB*" $err + verify_log_message 0 "*Invalid or unsupported RDB stream envelope*" $loglines + assert_equal $digest [debug_digest] + } + + write_binary_file $rdbfile $original + } + + test {RDB loader ignores trailing data after an LZ4 frame like a plain RDB} { + r config set rdbcompression lz4 + r flushall + r set trailing-data:key value + assert_equal "OK" [r save] + + set rdbfile [dump_rdb_path r] + set fd [open $rdbfile a] + fconfigure $fd -translation binary + puts -nonewline $fd "trailing-data" + close $fd + + assert_equal "OK" [r debug reload nosave] + assert_equal value [r get trailing-data:key] + } + + test {LZ4 compressed RDB detects corruption in compressed payload} { + r config set rdbcompression lz4 + r flushall + for {set i 0} {$i < 100} {incr i} { + r set "corrupt:$i" [string repeat "testdata$i " 100] + } + + assert_equal "OK" [r save] + + set rdbfile [file join [lindex [r config get dir] 1] dump.rdb] + + set fd [open $rdbfile r+] + fconfigure $fd -translation binary + set data [read $fd] + set len [string length $data] + set pos [expr {$len / 2}] + set byte [string index $data $pos] + binary scan $byte c val + set newval [expr {($val + 1) & 0xFF}] + set newbyte [binary format c $newval] + set data [string replace $data $pos $pos $newbyte] + seek $fd 0 + puts -nonewline $fd $data + close $fd + + set failed [catch {r debug reload nosave} err] + assert_equal 1 $failed + assert_match "*Error trying to load the RDB*" $err + } + +} + +start_server {config "minimal.conf" args {"--rdbcompression lz4"}} { + test {Startup accepts valid LZ4 compression config} { + assert_equal "lz4" [lindex [r config get rdbcompression] 1] + } +} + +start_server {overrides {save "" enable-debug-command local rdbchecksum no}} { + test {rdbchecksum controls LZ4 frame checksums} { + r config set rdbcompression lz4 + r flushall + for {set i 0} {$i < 50} {incr i} { + r set "nocksum:$i" [string repeat "data$i " 100] + } + + r save + assert_lz4_rdb_envelope r + assert_lz4_rdb_checksum_flags r 0 + set rdbfile [dump_rdb_path r] + set digest [debug_digest] + set loglines [count_log_lines 0] + assert_equal "OK" [r debug reload nosave] + verify_log_message 0 "*Logical RDB CRC64 skipped for streaming-compressed input*" $loglines + + restart_server 0 true false + set newdigest [debug_digest] + assert {$digest eq $newdigest} + assert_equal [string repeat "data10 " 100] [r get nocksum:10] + } +} + +start_server {overrides {save "" appendonly yes aof-use-rdb-preamble yes rdbcompression lz4}} { + test {AOF rewrite RDB preamble remains plain with LZ4 stream snapshots} { + r set aof-lz4:key [string repeat "aof-lz4-value " 100] + set digest [debug_digest] + + r bgrewriteaof + waitForBgrewriteaof r + + set base_aof [get_base_aof_path r] + assert {[file exists $base_aof]} + assert_equal "VALKEY" [string range [read_binary_file_prefix $base_aof 7] 0 5] + + restart_server 0 true false + assert_equal $digest [debug_digest] + assert_equal [string repeat "aof-lz4-value " 100] [r get aof-lz4:key] + } +} + +start_server {tags {"rdb-compression repl external:skip"}} { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + + start_server {overrides {save "" enable-debug-command local}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Disk-based full sync writes plain RDB when rdbcompression is lz4} { + $primary config set rdbcompression lz4 + $primary config set repl-diskless-sync no + $primary config set rdb-del-sync-files no + $primary flushall + for {set i 0} {$i < 500} {incr i} { + $primary set "repl:$i" [string repeat "payload$i " 40] + } + + $replica replicaof $primary_host $primary_port + wait_for_sync $replica + # wait_for_sync only checks master_link_status; the replica may + # still be loading the RDB. Wait until loading completes before + # comparing digests. + wait_done_loading $replica + + wait_for_condition 50 100 { + [status $replica master_link_status] eq "up" && + [$primary debug digest] eq [$replica debug digest] + } else { + fail "Replica digest mismatch after LZ4 RDB full sync" + } + + assert_equal [string repeat "payload42 " 40] [$replica get repl:42] + assert {[file exists [dump_rdb_path $primary]]} + assert_equal "VALKEY" [string range [read_dump_rdb_header_bytes $primary] 0 5] + + $primary set repl:post-sync "after-sync" + wait_for_condition 50 100 { + [$replica get repl:post-sync] eq "after-sync" + } else { + fail "Replica did not receive post-sync write" + } + } + + test {Diskless full sync remains compatible when rdbcompression is lz4} { + $replica replicaof no one + $primary config set repl-diskless-sync yes + $primary config set repl-diskless-sync-delay 0 + $replica config set repl-diskless-load swapdb + $primary flushall + for {set i 0} {$i < 300} {incr i} { + $primary set "diskless:$i" [string repeat "payload$i " 60] + } + + $replica replicaof $primary_host $primary_port + wait_for_sync $replica + # wait_for_sync only checks master_link_status; the replica may + # still be loading the RDB. Wait until loading completes before + # comparing digests. + wait_done_loading $replica + + wait_for_condition 50 100 { + [status $replica master_link_status] eq "up" && + [$primary debug digest] eq [$replica debug digest] + } else { + fail "Replica digest mismatch after diskless LZ4 full sync" + } + + assert_equal [string repeat "payload42 " 60] [$replica get diskless:42] + $primary config set repl-diskless-sync no + } + + $replica replicaof no one + } +} + +} diff --git a/tests/integration/repl-compression.tcl b/tests/integration/repl-compression.tcl new file mode 100644 index 00000000000..56805f0fbf4 --- /dev/null +++ b/tests/integration/repl-compression.tcl @@ -0,0 +1,1205 @@ +tags {"repl repl-compression external:skip"} { + +# uncompressed_bytes= from the replica line of the primary's INFO replication. +proc replica_line_uncompressed_bytes {primary} { + set info [$primary info replication] + assert {[regexp {uncompressed_bytes=([0-9]+)} $info -> ub]} + return $ub +} + +# ============================================================ +# Config CRUD — single-server tests, no replication needed +# ============================================================ + +start_server {overrides {save "" repl-compression no}} { + + test {Repl compression config defaults are correct} { + assert_equal "no" [lindex [r config get repl-compression] 1] + } + + test {repl-compression can be toggled on and off} { + r config set repl-compression lz4 + assert_equal "lz4" [lindex [r config get repl-compression] 1] + r config set repl-compression no + assert_equal "no" [lindex [r config get repl-compression] 1] + } + + test {Repl compression configs survive CONFIG REWRITE and restart} { + r config set repl-compression lz4 + r config rewrite + + restart_server 0 true false + + assert_equal "lz4" [lindex [r config get repl-compression] 1] + + # Restore default + r config set repl-compression no + } +} + +# ============================================================ +# Replication handshake behavior — primary + replica tests +# ============================================================ + +start_server {tags {"repl"} overrides {save ""}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Replica with repl-compression no does NOT send capa compression} { + start_server {overrides {save "" repl-compression no}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + + # Full sync completes normally without compression capability + assert_equal {up} [s 0 master_link_status] + + $replica replicaof no one + } + } + + test {Replica with repl-compression lz4 and diskless load sends capa compression} { + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + + set info [$primary info replication] + assert_match "*slave0:*" $info + assert_equal {up} [s 0 master_link_status] + + $replica replicaof no one + } + } + + test {Replica with repl-compression lz4 and disk-backed load also negotiates compression} { + $primary config set repl-compression lz4 + set _code [catch { + start_server {overrides {save "" repl-compression lz4 repl-diskless-load disabled}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + + # Disk-backed full sync completes, then compression activates for the + # post-sync incremental stream (the full-sync RDB itself is never + # compressed by this capability). + wait_for_condition 50 100 { + [regexp -all "compression=lz4" [$primary info replication]] >= 1 + } else { + fail "Compression not negotiated for disk-backed replica" + } + + # Exercise the compressed incremental stream over a disk-backed link. + for {set i 0} {$i < 100} {incr i} { + $primary set "diskbacked:$i" [string repeat "v" 50] + } + wait_for_condition 50 100 { + [$replica get "diskbacked:99"] eq [string repeat "v" 50] + } else { + fail "Disk-backed replica did not receive compressed incremental stream" + } + assert_equal [$primary debug digest] [$replica debug digest] + + $replica replicaof no one + } + } _res _opts] + $primary config set repl-compression no + return -options $_opts $_res + } + + test {Primary receiving capa compression still completes full sync correctly (no-op)} { + $primary flushall + for {set i 0} {$i < 100} {incr i} { + $primary set "noop:$i" [string repeat "value$i " 10] + } + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + wait_for_sync $replica + + wait_for_condition 50 100 { + [status $replica master_link_status] eq "up" + } else { + fail "Replica did not complete full sync" + } + + # Data integrity check — primary records capa but takes no action + assert_equal [string repeat "value42 " 10] [$replica get noop:42] + assert_equal 100 [$replica dbsize] + + $replica replicaof no one + } + } + + test {Backward compatibility - older replica without capa compression connects successfully} { + start_server {overrides {save ""}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + + assert_equal {up} [s 0 master_link_status] + + $replica replicaof no one + } + } + + test {Toggling repl-compression mid-runtime affects the next handshake} { + # First sync with compression disabled + start_server {overrides {save "" repl-compression no repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started with repl-compression no" + } + + assert_equal {up} [s 0 master_link_status] + + # Toggle compression on at runtime + $replica config set repl-compression lz4 + + # Disconnect and reconnect to trigger a new handshake + $replica replicaof no one + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started after toggling repl-compression lz4" + } + + assert_equal {up} [s 0 master_link_status] + + $replica replicaof no one + } + } + + test {Compressed incremental replication delivers correct data} { + $primary config set repl-compression lz4 + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + + # Write data on primary AFTER full sync (goes through incremental stream) + for {set i 0} {$i < 50} {incr i} { + $primary set "compressed:$i" [string repeat "payload$i " 20] + } + + # Wait for replica to catch up + wait_for_condition 50 100 { + [$replica dbsize] == [$primary dbsize] + } else { + fail "Replica did not catch up: replica=[$replica dbsize] primary=[$primary dbsize]" + } + + # Verify data integrity + for {set i 0} {$i < 50} {incr i} { + assert_equal [string repeat "payload$i " 20] [$replica get "compressed:$i"] + } + + $replica replicaof no one + } + + $primary config set repl-compression no + } + + test {Compressed incremental replication handles values larger than the batch limit} { + # A value past REPL_COMPRESSION_BATCH_LIMIT (1 MB) is compressed across + # multiple dispatches; verify it round-trips intact. + $primary config set repl-compression lz4 + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + + # ~4 MB value (well past the 1 MB batch limit) written after full sync. + set bigval [string repeat "abcdefghij0123456789" 209715] + $primary set bigkey $bigval + + wait_for_condition 50 200 { + [$replica get bigkey] eq $bigval + } else { + fail "Large value did not replicate intact under compression" + } + assert_equal [string length $bigval] [string length [$replica get bigkey]] + + $replica replicaof no one + } + + $primary config set repl-compression no + } + + test {Compressed incremental replication handles incompressible values larger than the batch limit} { + # A pseudo-random (incompressible) value past REPL_COMPRESSION_BATCH_LIMIT + # (1 MB) exercises the codec's ratio~1 expansion path across multiple + # dispatches; verify it round-trips intact with no compression errors. + $primary config set repl-compression lz4 + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + + # Deterministic pseudo-random binary payload >= 1.5 MiB, built in + # 4096-byte chunks from a seeded PRNG so failures are reproducible. + expr {srand(424242)} + set payload "" + while {[string length $payload] < 1572864} { + set chunk "" + for {set i 0} {$i < 4096} {incr i} { + append chunk [format %c [expr {int(rand()*256)}]] + } + append payload $chunk + } + $primary set incompressible_key $payload + + wait_for_condition 50 200 { + [$replica get incompressible_key] eq $payload + } else { + fail "Incompressible value did not replicate intact under compression" + } + assert_equal {up} [s 0 master_link_status] + + set info [$primary info replication] + assert_match "*compression=lz4*" $info + # Errors now live in the server-global repl_compression_errors, + # emitted only when non-zero: absence of both tokens means zero. + assert_equal 0 [string match "*compression_errors=*" $info] + assert_equal 0 [string match "*repl_compression_errors:*" $info] + + $replica replicaof no one + } + + $primary config set repl-compression no + } + + test {Backlog cursor stays pinned until the compressed batch fully drains} { + # The cursor advances only on full out_buf drain: pause the replica and + # uncompressed_bytes must freeze while master_repl_offset grows. + $primary config set repl-compression lz4 + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + set replica_pid [srv 0 pid] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + wait_for_condition 50 200 { + [string match {*state=online*compression=lz4*} [$primary info replication]] + } else { + fail "Compression not active on replica" + } + + $primary set pin:baseline baseline_val + wait_for_ofs_sync $primary $replica + set base_ub [replica_line_uncompressed_bytes $primary] + set base_off [status $primary master_repl_offset] + # The link must survive on the same connection (no resync). + set sync_full_before [status $primary sync_full] + set sync_partial_before [status $primary sync_partial_ok] + + pause_process $replica_pid + + # Pseudo-random 100KB block (past LZ4's 64KB window): ratio ~1 + # overfills the socket buffers and leaves out_buf mid-batch. + expr {srand(51555)} + set payload "" + while {[string length $payload] < 102400} { + set chunk "" + for {set i 0} {$i < 4096} {incr i} { + append chunk [format %c [expr {int(rand()*256)}]] + } + append payload $chunk + } + for {set i 0} {$i < 200} {incr i} { + $primary set "pin:burst:$i" $payload + } + + # Wait for the residual kernel-buffer drain to settle. + set prev [replica_line_uncompressed_bytes $primary] + set settled 0 + for {set i 0} {$i < 100} {incr i} { + after 100 + set cur [replica_line_uncompressed_bytes $primary] + if {$cur == $prev} { + set settled 1 + break + } + set prev $cur + } + assert {$settled == 1} + + # Frozen cursor: two samples with writes in between must be equal. + set ub1 [replica_line_uncompressed_bytes $primary] + set off1 [status $primary master_repl_offset] + for {set i 0} {$i < 20} {incr i} { + $primary set "pin:tick:$i" tick_val + } + after 300 + set ub2 [replica_line_uncompressed_bytes $primary] + set off2 [status $primary master_repl_offset] + + assert {$off2 > $off1} + assert_equal $ub1 $ub2 + # The 3/4 margin absorbs socket buffer sizing, no exact values. + assert {($ub2 - $base_ub) * 4 < ($off2 - $base_off) * 3} + + # Resume: pinned batches drain, cursor advances, data intact. + resume_process $replica_pid + wait_for_ofs_sync $primary $replica + assert {[$replica get pin:burst:199] eq $payload} + assert_equal tick_val [$replica get pin:tick:19] + assert {[replica_line_uncompressed_bytes $primary] > $ub2} + assert_equal $sync_full_before [status $primary sync_full] + assert_equal $sync_partial_before [status $primary sync_partial_ok] + + $replica replicaof no one + } + + $primary config set repl-compression no + } + + test {Partial resync with compression delivers correct data} { + $primary config set repl-compression lz4 + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + + # Write initial data + $primary set key1 value1 + + wait_for_condition 50 100 { + [$replica get key1] eq {value1} + } else { + fail "Initial replication failed" + } + + # Break the replication link from the primary side. The replica keeps + # its cached primary + offset and auto-reconnects, which exercises the + # compressed *partial* resync path (REPLICAOF NO ONE would force a full + # resync instead). + set partial_before [status $primary sync_partial_ok] + $primary client kill type replica + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Reconnection failed" + } + + # Confirm a partial resync actually happened (not a silent full sync) + wait_for_condition 50 100 { + [status $primary sync_partial_ok] > $partial_before + } else { + fail "Expected a partial resync after reconnect, but none occurred" + } + + # Write more data after partial resync + $primary set key2 value2 + $primary set key3 [string repeat "x" 1000] + + wait_for_condition 50 100 { + [$replica get key3] eq [string repeat "x" 1000] + } else { + fail "Post-partial-resync replication failed" + } + + assert_equal value1 [$replica get key1] + assert_equal value2 [$replica get key2] + + $replica replicaof no one + } + + $primary config set repl-compression no + } + + test {Replica with repl-compression lz4 handles a plaintext primary (passthrough)} { + # Primary has compression OFF, replica ON: the replica advertises the + # capability but the primary sends plaintext, so the replica must pass + # the stream through untouched rather than expecting a VCS envelope. + $primary config set repl-compression no + $primary flushall + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started (primary plaintext, replica compression on)" + } + + # Incremental writes arrive as plaintext; passthrough must deliver them. + for {set i 0} {$i < 50} {incr i} { + $primary set "pt:$i" [string repeat "payload$i " 20] + } + wait_for_condition 50 100 { + [$replica get pt:49] eq [string repeat "payload49 " 20] + } else { + fail "Replica did not receive plaintext data via passthrough" + } + assert_equal [$primary dbsize] [$replica dbsize] + + # Primary never compressed (its config is off). + assert_equal 0 [string match {*compression=lz4*} [$primary info replication]] + + $replica replicaof no one + } + } + + test {Replica repl-compression flip renegotiates upstream without manual reconnect} { + $primary config set repl-compression lz4 + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + [string match {*compression=lz4*} [$primary info replication]] + } else { + fail "Compressed replication not established" + } + + # Flipping compression OFF on the replica must, on its own, drop and + # reconnect the upstream link so it re-advertises capa without + # compression. No manual replicaof is issued. + $replica config set repl-compression no + + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} && + ![string match {*compression=lz4*} [$primary info replication]] + } else { + fail "Replica did not renegotiate to plaintext after flip" + } + + # Data still flows after the renegotiation. + $primary set flipkey flipval + wait_for_condition 50 100 { + [$replica get flipkey] eq {flipval} + } else { + fail "Data not replicated after replica-side flip" + } + + $replica replicaof no one + } + } + + test {CONFIG SET repl-compression no disconnects compressed replicas} { + $primary config set repl-compression lz4 + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + + # Wait for compression to be active (replica must be state=online) + wait_for_condition 50 200 { + [string match {*state=online*compression=lz4*} [$primary info replication]] + } else { + fail "Compression not active on replica" + } + + # Disable compression on primary — should disconnect compressed replicas + $primary config set repl-compression no + + # Replica should disconnect and reconnect + wait_for_condition 50 200 { + [s 0 master_link_status] eq {up} + } else { + fail "Replica did not reconnect after repl-compression disabled" + } + + # After reconnect, compression should NOT be active + set info [$primary info replication] + if {[string match "*compression=lz4*" $info]} { + fail "Compression still active after disable" + } + + $replica replicaof no one + } + } + + test {Multiple compressed replicas receive replication correctly} { + # Verifies that multiple compressed replicas can connect to the same + # primary and all receive replication data on the main-thread write + # path (io-threads multi-replica coverage lives in a later test). + $primary config set repl-compression lz4 + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica1 [srv 0 client] + $replica1 replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replica 1 not started" + } + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica2 [srv 0 client] + $replica2 replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replica 2 not started" + } + + # Write data and verify both replicas receive it + for {set i 0} {$i < 30} {incr i} { + $primary set "multi_repl:$i" "value_$i" + } + + wait_for_condition 50 100 { + [$replica1 get "multi_repl:29"] eq {value_29} && + [$replica2 get "multi_repl:29"] eq {value_29} + } else { + fail "Not all replicas caught up" + } + + # Verify both have compression active + set info [$primary info replication] + set matches [regexp -all "compression=lz4" $info] + assert {$matches >= 2} + + $replica2 replicaof no one + } + $replica1 replicaof no one + } + $primary config set repl-compression no + } + + test {Compressed replication works with io-threads enabled on the replica} { + $primary config set repl-compression lz4 + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb io-threads 4 io-threads-always-active yes}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started" + } + + # The link is compressed while the replica runs io threads; the + # primary-link decode stays on the replica's main thread. + wait_for_condition 50 200 { + [string match {*state=online*compression=lz4*} [$primary info replication]] + } else { + fail "Compression not active on replica" + } + + $primary set io_test_key "io_test_value" + wait_for_condition 50 100 { + [$replica get io_test_key] eq {io_test_value} + } else { + fail "Initial replication failed" + } + + for {set i 0} {$i < 20} {incr i} { + $primary set "io_threads:$i" "value_$i" + } + wait_for_condition 50 100 { + [$replica get "io_threads:19"] eq {value_19} + } else { + fail "Replication with replica io-threads failed" + } + + $replica replicaof no one + } + $primary config set repl-compression no + } + + test {Dual-channel full sync with compression delivers writes made during load} { + $primary config set repl-compression lz4 + $primary config set dual-channel-replication-enabled yes + $primary config set rdb-key-save-delay 100 + $primary flushall + $primary debug populate 10000 dc: 100 + + start_server {overrides {save "" repl-compression lz4 dual-channel-replication-enabled yes}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + # rdb-key-save-delay stretches the RDB stage; catch the sync window. + wait_for_condition 500 10 { + [s 0 master_sync_in_progress] == 1 + } else { + fail "Dual-channel sync did not start" + } + + # Writes made during load reach the replica via the compressed main + # channel: +CONTINUE starts compression, put-online must not restart + # it (a second init would emit a new envelope mid-frame). + for {set i 0} {$i < 200} {incr i} { + $primary set "during_load:$i" "value_$i" + } + + wait_for_condition 100 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not up after dual-channel sync" + } + + wait_for_condition 50 100 { + [$replica get "during_load:199"] eq {value_199} + } else { + fail "Writes made during load did not reach the replica" + } + for {set i 0} {$i < 200} {incr i} { + assert_equal "value_$i" [$replica get "during_load:$i"] + } + assert_match "*compression=lz4*" [$primary info replication] + wait_for_ofs_sync $primary $replica + + # A double init would emit a second envelope mid-frame on the first + # post-online write, corrupting the replica and forcing a resync. + # Stable sync counters prove the link survived that first write. + set sync_full_before [s -1 sync_full] + set sync_partial_before [s -1 sync_partial_ok] + $primary set post_online_probe delivered + wait_for_condition 50 100 { + [$replica get post_online_probe] eq {delivered} + } else { + fail "Post-online write did not reach the replica" + } + assert_equal $sync_full_before [s -1 sync_full] + assert_equal $sync_partial_before [s -1 sync_partial_ok] + + $replica replicaof no one + } + $primary config set rdb-key-save-delay 0 + $primary config set dual-channel-replication-enabled no + $primary config set repl-compression no + } + + test {Enabling repl-compression while a dual-channel replica loads converges after put-online} { + $primary config set repl-compression no + $primary config set dual-channel-replication-enabled yes + $primary config set rdb-key-save-delay 100 + $primary flushall + $primary debug populate 10000 midload: 100 + + start_server {overrides {save "" repl-compression lz4 dual-channel-replication-enabled yes}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + # Catch the window where the RDB is still streaming but the main + # channel already got +CONTINUE (its command stream is plaintext). + wait_for_condition 500 10 { + [s 0 master_sync_in_progress] == 1 && + [string match "*state=bg_transfer*" [$primary info replication]] + } else { + fail "Dual-channel sync window not reached" + } + + # Plaintext flows on the main channel first, so the replica probe + # latches passthrough. Only then flip the config: the link's + # decision is frozen at +CONTINUE, so the sync completes plaintext + # and put-online reconnects the link to renegotiate. + for {set i 0} {$i < 20} {incr i} { + $primary set "during_load:$i" "value_$i" + } + # This sync's full resync and main-channel +CONTINUE are already + # counted (bg_transfer implies the handshake finished), so any + # later movement comes from the renegotiation alone. + set sync_full_before [s -1 sync_full] + set sync_partial_before [s -1 sync_partial_ok] + $primary config set repl-compression lz4 + + # Put-online sees the frozen decision diverging from the config + # and drops the link; the replica renegotiates via partial resync + # (no second RDB load). + wait_for_condition 100 100 { + [s -1 sync_partial_ok] == $sync_partial_before + 1 + } else { + fail "Renegotiation partial resync did not happen" + } + assert_equal $sync_full_before [s -1 sync_full] + + wait_for_condition 100 100 { + [s 0 master_link_status] eq {up} && + [string match "*compression=lz4*" [$primary info replication]] + } else { + fail "Reconnected link did not come up compressed" + } + + # Continuity: pre-flip traffic survived the reconnect and the + # renegotiated compressed link delivers new writes. + wait_for_condition 50 100 { + [$replica get "during_load:19"] eq {value_19} + } else { + fail "Pre-flip writes did not reach the replica" + } + $primary set midload_probe delivered + wait_for_condition 50 100 { + [$replica get midload_probe] eq {delivered} + } else { + fail "Post-reconnect write did not reach the replica" + } + + # Exactly one renegotiation, and the RDB load was not redone. + assert_equal [expr {$sync_partial_before + 1}] [s -1 sync_partial_ok] + assert_equal $sync_full_before [s -1 sync_full] + + $replica replicaof no one + } + $primary config set rdb-key-save-delay 0 + $primary config set dual-channel-replication-enabled no + $primary config set repl-compression no + } + + test {Disabling repl-compression while a dual-channel replica loads converges after put-online} { + $primary config set repl-compression lz4 + $primary config set dual-channel-replication-enabled yes + $primary config set rdb-key-save-delay 100 + $primary flushall + $primary debug populate 10000 midload: 100 + + start_server {overrides {save "" repl-compression lz4 dual-channel-replication-enabled yes}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + + # Catch the window where the RDB is still streaming but the main + # channel already got +CONTINUE (its command stream is compressed). + wait_for_condition 500 10 { + [s 0 master_sync_in_progress] == 1 && + [string match "*state=bg_transfer*" [$primary info replication]] + } else { + fail "Dual-channel sync window not reached" + } + + # Compressed frames flow on the main channel first. Only then flip + # the config: the link's decision is frozen at +CONTINUE, so the + # sync completes compressed and put-online reconnects the link. + for {set i 0} {$i < 20} {incr i} { + $primary set "during_load:$i" "value_$i" + } + # This sync's full resync and main-channel +CONTINUE are already + # counted (bg_transfer implies the handshake finished), so any + # later movement comes from the renegotiation alone. + set sync_full_before [s -1 sync_full] + set sync_partial_before [s -1 sync_partial_ok] + $primary config set repl-compression no + + # The still-loading replica keeps its compressed stream; put-online + # sees the frozen decision diverging from the config and drops the + # link; the replica renegotiates via partial resync (no second RDB + # load). + wait_for_condition 100 100 { + [s -1 sync_partial_ok] == $sync_partial_before + 1 + } else { + fail "Renegotiation partial resync did not happen" + } + assert_equal $sync_full_before [s -1 sync_full] + + wait_for_condition 100 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Reconnected link did not come up" + } + # The renegotiated link is plaintext. + assert_equal 0 [string match "*compression=lz4*" [$primary info replication]] + + # Continuity: pre-flip traffic survived the reconnect and the + # renegotiated plaintext link delivers new writes. + wait_for_condition 50 100 { + [$replica get "during_load:19"] eq {value_19} + } else { + fail "Pre-flip writes did not reach the replica" + } + for {set i 0} {$i < 20} {incr i} { + assert_equal "value_$i" [$replica get "during_load:$i"] + } + $primary set midload_probe delivered + wait_for_condition 50 100 { + [$replica get midload_probe] eq {delivered} + } else { + fail "Post-reconnect write did not reach the replica" + } + assert_equal [$primary dbsize] [$replica dbsize] + + # Exactly one renegotiation, and the RDB load was not redone. + assert_equal [expr {$sync_partial_before + 1}] [s -1 sync_partial_ok] + assert_equal $sync_full_before [s -1 sync_full] + + $replica replicaof no one + } + $primary config set rdb-key-save-delay 0 + $primary config set dual-channel-replication-enabled no + $primary config set repl-compression no + } +} + +# ============================================================ +# Multi-replica compressed replication tests +# ============================================================ + +# Disabling repl-compression at runtime disconnects compressed replicas. The +# disconnect is deferred until after CONFIG SET commits (so a rolled-back +# multi-option CONFIG SET drops nothing), then the replica reconnects plaintext. +start_server {tags {"repl"} overrides {save "" repl-compression lz4}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Disabling repl-compression disconnects compressed replicas} { + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + wait_for_sync $replica + + wait_for_condition 50 100 { + [regexp -all "compression=lz4" [$primary info replication]] >= 1 + } else { + fail "Compression not active before disable" + } + + $primary config set repl-compression no + + # The compressed replica is dropped, then reconnects as plaintext, so + # the link no longer reports compression=lz4. + wait_for_condition 50 100 { + [regexp -all "compression=lz4" [$primary info replication]] == 0 + } else { + fail "Compressed replica was not disconnected after repl-compression no" + } + wait_for_condition 50 100 { + [status $replica master_link_status] eq "up" + } else { + fail "Replica did not reconnect after compression disabled" + } + + $replica replicaof no one + } + $primary config set repl-compression lz4 + } +} + +# Enabling repl-compression at runtime disconnects capable-but-plaintext replicas +# so they reconnect compressed (symmetric with the disable case; deferred to +# CONFIG SET commit so a rolled-back command reconnects nothing). +start_server {tags {"repl"} overrides {save "" repl-compression no}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Enabling repl-compression reconnects capable replicas compressed} { + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica [srv 0 client] + $replica replicaof $primary_host $primary_port + wait_for_sync $replica + + # Primary compression is off, so although the replica advertised the + # capability the negotiated link is plaintext. + wait_for_condition 50 100 { + [status $replica master_link_status] eq "up" + } else { + fail "Replica did not sync" + } + assert_equal 0 [regexp -all "compression=lz4" [$primary info replication]] + + # Enable on the primary: the capable replica is dropped and reconnects + # over a compressed stream. + $primary config set repl-compression lz4 + wait_for_condition 50 100 { + [regexp -all "compression=lz4" [$primary info replication]] >= 1 + } else { + fail "Capable replica did not reconnect compressed after enable" + } + + # Data flows correctly over the new compressed link. + $primary set enabled_key enabled_val + wait_for_condition 50 100 { + [$replica get enabled_key] eq "enabled_val" + } else { + fail "Compressed replication did not deliver after enable" + } + assert_equal [$primary debug digest] [$replica debug digest] + + $replica replicaof no one + } + } +} + +# Test 4: Multiple replicas distribute across threads and stay in sync. +# io-threads-always-active starts off during the handshakes: an offloaded +# REPLCONF reply can leave pending output that makes the primary reject PSYNC, +# and the legacy-SYNC fallback suppresses the ACK that diskless sync waits for +# (upstream race). It is enabled once all replicas are streaming. +start_server {tags {"repl"} overrides {save "" io-threads 4 repl-compression lz4}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Multiple replicas all stay in sync under load} { + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica1 [srv 0 client] + $replica1 replicaof $primary_host $primary_port + wait_for_sync $replica1 + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica2 [srv 0 client] + $replica2 replicaof $primary_host $primary_port + wait_for_sync $replica2 + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica3 [srv 0 client] + $replica3 replicaof $primary_host $primary_port + wait_for_sync $replica3 + + # Wait for all replicas to have compression active + wait_for_condition 50 200 { + [regexp -all "compression=lz4" [$primary info replication]] >= 3 + } else { + fail "Not all replicas have compression active" + } + + # Handshakes are done; run the load phase on IO threads. + $primary config set io-threads-always-active yes + + # Generate sustained load + for {set i 0} {$i < 500} {incr i} { + $primary set "multi_repl:$i" [string repeat "x" 100] + } + + # Wait for all replicas to catch up + wait_for_condition 100 200 { + [$replica1 dbsize] == [$primary dbsize] && + [$replica2 dbsize] == [$primary dbsize] && + [$replica3 dbsize] == [$primary dbsize] + } else { + fail "Not all replicas caught up: r1=[$replica1 dbsize] r2=[$replica2 dbsize] r3=[$replica3 dbsize] primary=[$primary dbsize]" + } + + # Verify data integrity + set primary_digest [$primary debug digest] + assert_equal $primary_digest [$replica1 debug digest] + assert_equal $primary_digest [$replica2 debug digest] + assert_equal $primary_digest [$replica3 debug digest] + + $replica3 replicaof no one + } + $replica2 replicaof no one + } + $replica1 replicaof no one + } + } +} + +# Test 5: Compressed replication survives replica disconnect/reconnect +start_server {tags {"repl"} overrides {save "" io-threads 4 io-threads-always-active yes repl-compression lz4}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Compressed replication survives replica disconnect and reconnect} { + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica1 [srv 0 client] + $replica1 replicaof $primary_host $primary_port + wait_for_sync $replica1 + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set replica2 [srv 0 client] + $replica2 replicaof $primary_host $primary_port + wait_for_sync $replica2 + + # Drive traffic, verify both in sync + for {set i 0} {$i < 200} {incr i} { + $primary set "pre_disconnect:$i" [string repeat "x" 50] + } + + wait_for_condition 50 200 { + [$replica1 dbsize] == [$primary dbsize] && + [$replica2 dbsize] == [$primary dbsize] + } else { + fail "Replicas not in sync before disconnect" + } + + # Disconnect replica1 + $replica1 replicaof no one + + # Drive more traffic — replica2 should stay connected and in sync + for {set i 0} {$i < 200} {incr i} { + $primary set "post_disconnect:$i" [string repeat "x" 50] + } + + wait_for_condition 50 200 { + [$replica2 get "post_disconnect:199"] eq [string repeat "x" 50] + } else { + fail "Replica2 did not stay in sync after replica1 disconnect" + } + + # Reconnect replica1 + $replica1 replicaof $primary_host $primary_port + + wait_for_condition 50 200 { + [status $replica1 master_link_status] eq "up" + } else { + fail "Replica1 did not reconnect" + } + + # Wait for replica1 to catch up + wait_for_condition 50 200 { + [$replica1 dbsize] == [$primary dbsize] + } else { + fail "Replica1 did not re-sync: replica1=[$replica1 dbsize] primary=[$primary dbsize]" + } + + # Verify compression is re-negotiated on replica1 + wait_for_condition 50 200 { + [regexp -all "compression=lz4" [$primary info replication]] >= 2 + } else { + fail "Compression not re-negotiated after reconnect" + } + + # Verify data integrity + assert_equal [$primary debug digest] [$replica1 debug digest] + assert_equal [$primary debug digest] [$replica2 debug digest] + + $replica2 replicaof no one + } + $replica1 replicaof no one + } + } +} + +# Chained replication: each hop negotiates compression independently, and the +# middle node simultaneously decodes its primary link on the main thread while +# encoding for its own replica on IO threads. +start_server {tags {"repl"} overrides {save "" repl-compression lz4}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test {Chained replication compresses each hop independently} { + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb io-threads 4 io-threads-always-active yes}} { + set middle [srv 0 client] + set middle_host [srv 0 host] + set middle_port [srv 0 port] + + start_server {overrides {save "" repl-compression lz4 repl-diskless-load swapdb}} { + set leaf [srv 0 client] + + $middle replicaof $primary_host $primary_port + wait_for_sync $middle + $leaf replicaof $middle_host $middle_port + wait_for_sync $leaf + + # Both hops negotiated compression. + wait_for_condition 100 200 { + [regexp -all "compression=lz4" [$primary info replication]] >= 1 && + [regexp -all "compression=lz4" [$middle info replication]] >= 1 + } else { + fail "Compression not active on both hops" + } + + # Writes flow primary -> middle -> leaf across two compressed hops. + for {set i 0} {$i < 200} {incr i} { + $primary set "chain:$i" "chain_value_$i" + } + wait_for_condition 100 200 { + [$leaf dbsize] == [$primary dbsize] + } else { + fail "Leaf did not catch up: leaf=[$leaf dbsize] primary=[$primary dbsize]" + } + assert_equal "chain_value_0" [$leaf get chain:0] + assert_equal "chain_value_99" [$leaf get chain:99] + assert_equal "chain_value_199" [$leaf get chain:199] + + # Flip compression off on the leaf only and force a clean + # reconnect: hop2 renegotiates plaintext, hop1 stays compressed. + $leaf config set repl-compression no + $leaf replicaof no one + $leaf replicaof $middle_host $middle_port + wait_for_sync $leaf + + wait_for_condition 100 200 { + [regexp -all "compression=lz4" [$middle info replication]] == 0 + } else { + fail "Hop2 still compressed after leaf disabled repl-compression" + } + assert {[regexp -all "compression=lz4" [$primary info replication]] >= 1} + + # Data still flows end-to-end over mixed hops. + $primary set chain:final final_val + wait_for_condition 100 200 { + [$leaf get chain:final] eq {final_val} + } else { + fail "Write did not reach leaf after hop2 renegotiated plaintext" + } + + $leaf replicaof no one + } + $middle replicaof no one + } + } +} + + +} diff --git a/tests/integration/replication-2.tcl b/tests/integration/replication-2.tcl index c18ff24fc43..6215349da50 100644 --- a/tests/integration/replication-2.tcl +++ b/tests/integration/replication-2.tcl @@ -1,4 +1,4 @@ -start_server {tags {"repl external:skip"}} { +start_server {tags {"repl external:skip" repl-compression}} { start_server {} { test {First server should have role slave after SLAVEOF} { r -1 slaveof [srv 0 host] [srv 0 port] diff --git a/tests/integration/replication-3.tcl b/tests/integration/replication-3.tcl index f53a05abce1..e1bcb3914f0 100644 --- a/tests/integration/replication-3.tcl +++ b/tests/integration/replication-3.tcl @@ -1,4 +1,4 @@ -start_server {tags {"repl external:skip"}} { +start_server {tags {"repl external:skip" repl-compression}} { start_server {} { test {First server should have role slave after SLAVEOF} { r -1 slaveof [srv 0 host] [srv 0 port] diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl index d5b55b3b570..76880d2a42a 100644 --- a/tests/integration/replication-4.tcl +++ b/tests/integration/replication-4.tcl @@ -1,4 +1,4 @@ -start_server {tags {"repl network external:skip singledb:skip"} overrides {save {}}} { +start_server {tags {"repl network external:skip singledb:skip" repl-compression} overrides {save {}}} { start_server { overrides {save {}}} { set master [srv -1 client] @@ -52,7 +52,7 @@ start_server {tags {"repl network external:skip singledb:skip"} overrides {save } } -start_server {tags {"repl external:skip"}} { +start_server {tags {"repl external:skip" repl-compression}} { start_server {} { set master [srv -1 client] set master_host [srv -1 host] @@ -118,7 +118,7 @@ start_server {tags {"repl external:skip"}} { } } -start_server {tags {"repl external:skip"}} { +start_server {tags {"repl external:skip" repl-compression}} { start_server {} { set master [srv -1 client] set master_host [srv -1 host] @@ -168,7 +168,7 @@ start_server {tags {"repl external:skip"}} { } } -start_server {tags {"repl external:skip"}} { +start_server {tags {"repl external:skip" repl-compression}} { start_server {} { set master [srv -1 client] set master_host [srv -1 host] @@ -249,7 +249,7 @@ start_server {tags {"repl external:skip"}} { } } -start_server {tags {"repl external:skip"}} { +start_server {tags {"repl external:skip" repl-compression}} { start_server {} { set master [srv -1 client] set master_host [srv -1 host] diff --git a/tests/integration/replication-aof-sync.tcl b/tests/integration/replication-aof-sync.tcl index 14113bd0708..893fa976aa6 100644 --- a/tests/integration/replication-aof-sync.tcl +++ b/tests/integration/replication-aof-sync.tcl @@ -17,7 +17,7 @@ proc get_aof_manifest_path {r} { return [file join $dir $appenddirname $appendfilename$::manifest_suffix] } -tags {"repl external:skip"} { +tags {"repl external:skip" repl-compression} { # Test 1: Disk-based full sync with aof-use-rdb-preamble yes should # reuse the RDB file as AOF base file diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 038560469fd..30ecd1daf26 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -5,7 +5,7 @@ proc log_file_matches {log pattern} { string match $pattern $content } -start_server {tags {"repl network external:skip"}} { +start_server {tags {"repl network external:skip" repl-compression}} { set slave [srv 0 client] set slave_host [srv 0 host] set slave_port [srv 0 port] @@ -59,7 +59,7 @@ start_server {tags {"repl network external:skip"}} { } } -start_server {tags {"repl external:skip"}} { +start_server {tags {"repl external:skip" repl-compression}} { set A [srv 0 client] set A_host [srv 0 host] set A_port [srv 0 port] @@ -228,7 +228,7 @@ start_server {tags {"repl external:skip"}} { } } -start_server {tags {"repl external:skip"}} { +start_server {tags {"repl external:skip" repl-compression}} { r set mykey foo start_server {} { @@ -429,7 +429,7 @@ foreach mdl {no yes} dualchannel {no yes} { } } -start_server {tags {"repl external:skip"} overrides {save {}}} { +start_server {tags {"repl external:skip" repl-compression} overrides {save {}}} { set master [srv 0 client] set master_host [srv 0 host] set master_port [srv 0 port] @@ -892,7 +892,7 @@ proc compute_cpu_usage {start end} { # test diskless rdb pipe with multiple replicas, which may drop half way -start_server {tags {"repl external:skip"} overrides {save ""}} { +start_server {tags {"repl external:skip" repl-compression} overrides {save ""}} { set master [srv 0 client] $master config set repl-diskless-sync yes $master config set repl-diskless-sync-delay 5 @@ -1472,7 +1472,7 @@ test {replica can handle EINTR if use diskless load} { } } {} {external:skip} -start_server {tags {"repl" "external:skip"}} { +start_server {tags {"repl" "external:skip" repl-compression}} { test "replica do not write the reply to the replication link - SYNC (_addReplyToBufferOrList)" { set rd [valkey_deferring_client] set lines [count_log_lines 0] @@ -1554,7 +1554,7 @@ start_server {tags {"repl" "external:skip"}} { } } -start_server {tags {"repl external:skip"}} { +start_server {tags {"repl external:skip" repl-compression}} { set master [srv 0 client] set master_host [srv 0 host] set master_port [srv 0 port] @@ -1642,7 +1642,7 @@ foreach dualchannel {yes no} { } {} {external:skip} } -start_server {tags {"repl external:skip"}} { +start_server {tags {"repl external:skip" repl-compression}} { set replica [srv 0 client] $replica set replica_key replica_value diff --git a/tests/integration/skip-rdb-checksum.tcl b/tests/integration/skip-rdb-checksum.tcl index 324096c5a20..2b01451a541 100644 --- a/tests/integration/skip-rdb-checksum.tcl +++ b/tests/integration/skip-rdb-checksum.tcl @@ -26,7 +26,7 @@ proc test_skip_rdb_checksum {primary primary_host primary_port primary_skipped_r } } -start_server {tags {"repl tls cluster:skip external:skip"} overrides {save {}}} { +start_server {tags {"repl tls cluster:skip external:skip" repl-compression} overrides {save {}}} { set primary [srv 0 client] set primary_host [srv 0 host] set primary_port [srv 0 port] diff --git a/tests/integration/valkey-check-rdb.tcl b/tests/integration/valkey-check-rdb.tcl index 05753ca9e39..264578a1532 100644 --- a/tests/integration/valkey-check-rdb.tcl +++ b/tests/integration/valkey-check-rdb.tcl @@ -2,6 +2,21 @@ proc get_function_code {args} { return [format "#!%s name=%s\nserver.register_function('%s', function(KEYS, ARGV)\n %s \nend)" [lindex $args 0] [lindex $args 1] [lindex $args 2] [lindex $args 3]] } +proc check_rdb_read_binary_file {path} { + set fd [open $path r] + fconfigure $fd -translation binary + set data [read $fd] + close $fd + return $data +} + +proc check_rdb_write_binary_file {path data} { + set fd [open $path w] + fconfigure $fd -translation binary + puts -nonewline $fd $data + close $fd +} + tags {"check-rdb external:skip logreqres:skip"} { test {Check old valid RDB} { catch { @@ -49,6 +64,94 @@ tags {"check-rdb external:skip logreqres:skip"} { tags {"check-rdb network external:skip logreqres:skip"} { start_server {} { + test "valkey-check-rdb validates the contents of an LZ4-compressed RDB" { + r flushall + r config set rdbcompression lz4 + r set lz4:key [string repeat "payload " 200] + r save + + set dump_rdb [file join [lindex [r config get dir] 1] dump.rdb] + set failed [catch { + exec $::VALKEY_CHECK_RDB_BIN $dump_rdb --stats --format info + } result] + r config set rdbcompression yes + + assert_equal 0 $failed + assert_match {*RDB looks OK!*} $result + assert_match {*\[logical offset *, physical offset *\] Logical RDB CRC64 skipped for streaming-compressed input*} $result + assert_match {*type.string.keys.total:1*} $result + assert_no_match {*Checksum OK*} $result + } + + test "valkey-check-rdb rejects an incompatible VCS envelope" { + r flushall + r config set rdbcompression lz4 + r set lz4:incompatible payload + r save + + set dir [lindex [r config get dir] 1] + set dump_rdb [file join $dir dump.rdb] + set incompatible_rdb [file join $dir incompatible-vcs.rdb] + set data [check_rdb_read_binary_file $dump_rdb] + set data [string replace $data 3 3 [binary format c 2]] + check_rdb_write_binary_file $incompatible_rdb $data + + set failed [catch { + exec $::VALKEY_CHECK_RDB_BIN $incompatible_rdb + } result] + file delete -force $incompatible_rdb + r config set rdbcompression yes + + assert_equal 1 $failed + assert_match {*Invalid or unsupported RDB stream envelope*} $result + assert_no_match {*RDB looks OK*} $result + } + + test "valkey-check-rdb rejects a compressed RDB with a truncated frame trailer" { + r flushall + r config set rdbcompression lz4 + r set lz4:truncated [string repeat "payload " 200] + r save + + set dir [lindex [r config get dir] 1] + set dump_rdb [file join $dir dump.rdb] + set truncated_rdb [file join $dir truncated-vcs.rdb] + set data [check_rdb_read_binary_file $dump_rdb] + check_rdb_write_binary_file $truncated_rdb [string range $data 0 end-1] + + set failed [catch { + exec $::VALKEY_CHECK_RDB_BIN $truncated_rdb + } result] + file delete -force $truncated_rdb + r config set rdbcompression yes + + assert_equal 1 $failed + assert_match {*\[logical offset *, physical offset *\] Compressed RDB stream did not end cleanly*} $result + assert_no_match {*RDB looks OK*} $result + } + + test "valkey-check-rdb ignores trailing data after a compressed RDB" { + r flushall + r config set rdbcompression lz4 + r set lz4:trailing payload + r save + + set dir [lindex [r config get dir] 1] + set dump_rdb [file join $dir dump.rdb] + set trailing_rdb [file join $dir trailing-vcs.rdb] + set data [check_rdb_read_binary_file $dump_rdb] + check_rdb_write_binary_file $trailing_rdb "${data}trailing-data" + + set failed [catch { + exec $::VALKEY_CHECK_RDB_BIN $trailing_rdb + } result] + file delete -force $trailing_rdb + r config set rdbcompression yes + + assert_equal 0 $failed + assert_match {*RDB looks OK*} $result + } + test "test valkey-check-rdb stats with empty RDB" { r flushall r save @@ -142,3 +245,23 @@ tags {"check-rdb network external:skip logreqres:skip"} { } } } + +tags {"check-rdb network external:skip logreqres:skip"} { + start_server {overrides {save "" rdbchecksum no}} { + test "valkey-check-rdb accepts compressed RDBs created with rdbchecksum no" { + r flushall + r config set rdbcompression lz4 + r set lz4:no-cksum [string repeat "payload " 200] + r save + + set dump_rdb [file join [lindex [r config get dir] 1] dump.rdb] + set failed [catch { + exec $::VALKEY_CHECK_RDB_BIN $dump_rdb + } result] + assert_equal 0 $failed + assert_match {*Logical RDB CRC64 skipped for streaming-compressed input*} $result + assert_match {*RDB looks OK!*} $result + assert_no_match {*Checksum OK*} $result + } + } +} diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 53c55c87a36..149c93865e6 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -2,7 +2,7 @@ set ::global_overrides {} set ::tags {} set ::valgrind_errors {} # Tags that are only allowed at the top level (not in nested blocks) -set ::toplevel_only_tags {large-memory needs:other-server compatible-redis network} +set ::toplevel_only_tags {large-memory needs:other-server compatible-redis network repl-compression} proc start_server_error {executable config_file error} { set err {} diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 0eae584faca..151a5a5f570 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -811,7 +811,7 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } else { # Validate that allowtags only use top-level tags if {[lsearch -exact $::toplevel_only_tags $tag] < 0} { - puts "Error: --tags allowlist can only use top-level-only tags: large-memory, needs:other-server, compatible-redis, network" + puts "Error: --tags allowlist can only use top-level-only tags: $::toplevel_only_tags" puts "Invalid tag: $tag" exit 1 } diff --git a/valkey.conf b/valkey.conf index 7cfbf2b2099..ac0ee296ad1 100644 --- a/valkey.conf +++ b/valkey.conf @@ -594,19 +594,33 @@ locale-collate "" # permissions, and so forth. stop-writes-on-bgsave-error yes -# Compress string objects using LZF when dump .rdb databases? -# By default compression is enabled as it's almost always a win. -# If you want to save some CPU in the saving child set it to 'no' but -# the dataset will likely be bigger if you have compressible values or keys. +# Control compression when dumping .rdb databases. +# Supported values: +# yes - use the default compression algorithm (currently lzf) +# no - no RDB compression +# lzf - per-string LZF compression inside the RDB payload +# lz4 - streaming LZ4 frame compression for the entire RDB file. +# Supported by Valkey 9.2 and later. +# +# Streaming compression currently applies only to on-disk snapshots. +# Replication full synchronization continues to use the plain RDB format until +# compressed full-sync capability negotiation is implemented. +# +# Before downgrading a server that may load an existing lz4 snapshot, +# rewrite the snapshot in the legacy format and verify it with the older binary: +# CONFIG SET rdbcompression yes +# CONFIG REWRITE +# SAVE +# +# By default compression is enabled as it's almost always a win. If you want to +# save some CPU in the saving child set it to 'no' but the dataset will likely +# be bigger if you have compressible values or keys. rdbcompression yes -# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. -# This makes the format more resistant to corruption but there is a performance -# hit to pay (around 10%) when saving and loading RDB files, so you can disable it -# for maximum performances. -# -# RDB files created with checksum disabled have a checksum of zero that will -# tell the loading code to skip the check. +# Since version 5, plain RDB files carry a CRC64 checksum at the end. With +# rdbcompression lz4, the logical CRC64 trailer is zero and this setting instead +# controls the LZ4 block and content checksums that protect the compressed +# stream. rdbchecksum yes # Valkey can try to load an RDB dump produced by a future version of Valkey. @@ -816,6 +830,26 @@ repl-diskless-sync-max-replicas 0 # lose all your data. repl-diskless-load disabled +# Compression for the replication transport between primary and replica. +# no - no compression (default) +# lz4 - compress the incremental replication stream with LZ4 +# When set to a compressing mode on a replica, the replica advertises the +# "compression" capability during the replication handshake (REPLCONF capa). +# The primary records this capability and compresses the incremental +# replication stream sent to that replica (only if the primary itself has +# repl-compression enabled). +# +# Changing this at runtime reconnects only the replicas that negotiated the +# compression capability, so they renegotiate the transport (a live link cannot +# switch between plaintext and compressed mid-stream). Replicas that never +# advertised the capability are left connected and unaffected. +# On a replica, changing the value also drops and renegotiates its own link to +# the primary. If the setting changes while a replica is completing a full +# sync, the link finishes with its original negotiation to keep the stream +# consistent, and the primary reconnects it right after it comes online so it +# converges to the new setting via a partial resync. +repl-compression no + # This dual channel replication sync feature optimizes the full synchronization process # between a primary and its replicas. When enabled, it reduces both memory and CPU load # on the primary server.