roshkhatri · roshkhatri · Jul 28, 2026 · Jul 28, 2026 · Jul 30, 2026 · Jul 31, 2026
diff --git a/.config/typos.toml b/.config/typos.toml
@@ -14,6 +14,7 @@ exat = "exat"
 optin = "optin"
 smove = "smove"
 Parth = "Parth" # seems like the spellchecker does not like it is similar to "Path"
+Collet = "Collet" # LZ4 author Yann Collet
 nd = "nd"
 
 [default]

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -197,6 +197,39 @@ jobs:
       - name: test-tls-builtin
         run: ./runtest --verbose --single unit/tls --dump-logs --tls
 
+  test-replication-compression:
+    # Re-runs the replication suite with repl-compression=lz4 applied
+    # globally via --config. Test files opt in by carrying the top-level "repl-compression"
+    # tag (see tests/support/server.tcl), so this job selects them with
+    # --tags repl-compression instead of a hardcoded filename list that would
+    # silently drift if a file is renamed. Exercises the streaming-compression
+    # transport across the replication surface (full sync, dual-channel, buffer
+    # management, AOF-sync, etc.) to catch regressions that only surface under
+    # compressed replication.
+    # Not tagged for this job: integration/replication-psync (start_server lives
+    # inside a proc, top-level tags cannot apply) and the replication-buffer /
+    # dual-channel buffer-memory tests (assert exact byte volumes that compression changes).
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install libbacktrace
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          repository: ianlancetaylor/libbacktrace
+          ref: b9e40069c0b47a722286b94eb5231f7f05c08713
+          path: libbacktrace
+      - run: cd libbacktrace && ./configure && make && sudo make install
+      - name: Checkout Valkey
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: make
+        run: make -j4 SERVER_CFLAGS='-Werror' USE_LIBBACKTRACE=yes
+      - name: install test dependencies
+        run: sudo apt-get install -y tcl8.6 tclx
+      - name: replication tests with streaming compression enabled
+        run: |
+          ./runtest --verbose --dump-logs \
+            --tags "repl-compression -slow" \
+            --config repl-compression lz4
+
   build-debian-old:
     runs-on: ubuntu-latest
     container: debian:bullseye

diff --git a/REUSE.toml b/REUSE.toml
@@ -90,6 +90,12 @@ precedence = "aggregate"
 SPDX-FileCopyrightText = ["2010-2016, Redis Ltd.", "2010-2013, Pieter Noordhuis <pcnoordhuis@gmail.com>", "2024-present, Valkey contributors"]
 SPDX-License-Identifier = "BSD-2-Clause"
 
+[[annotations]]
+path = "deps/lz4/**"
+precedence = "aggregate"
+SPDX-FileCopyrightText = "2011-2023, Yann Collet"
+SPDX-License-Identifier = "BSD-2-Clause"
+
 [[annotations]]
 path = "deps/hdr_histogram/**"
 precedence = "aggregate"

diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake
@@ -121,7 +121,11 @@ set(VALKEY_SERVER_SRCS
     ${CMAKE_SOURCE_DIR}/src/vset.c
     ${CMAKE_SOURCE_DIR}/src/fifo.c
     ${CMAKE_SOURCE_DIR}/src/mutexqueue.c
-    ${CMAKE_SOURCE_DIR}/src/queues.c)
+    ${CMAKE_SOURCE_DIR}/src/queues.c
+    ${CMAKE_SOURCE_DIR}/src/compression.c
+    ${CMAKE_SOURCE_DIR}/src/compression_lz4.c
+    ${CMAKE_SOURCE_DIR}/src/compression_stream.c
+    ${CMAKE_SOURCE_DIR}/src/compression_repl.c)
 
 
 # valkey-cli

diff --git a/cmake/Modules/ValkeySetup.cmake b/cmake/Modules/ValkeySetup.cmake
@@ -282,6 +282,7 @@ include_directories("${CMAKE_SOURCE_DIR}/src/modules/lua")
 include_directories("${CMAKE_SOURCE_DIR}/deps/linenoise")
 include_directories("${CMAKE_SOURCE_DIR}/deps/hdr_histogram")
 include_directories("${CMAKE_SOURCE_DIR}/deps/fpconv")
+include_directories("${CMAKE_SOURCE_DIR}/deps/lz4")
 
 add_subdirectory("${CMAKE_SOURCE_DIR}/deps")
 

diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt
@@ -33,6 +33,7 @@ add_subdirectory(linenoise)
 add_subdirectory(fpconv)
 add_subdirectory(hdr_histogram)
 add_subdirectory(fast_float)
+add_subdirectory(lz4)
 
 # Clear any cached variables passed to libvalkey from the cache
 unset(BUILD_SHARED_LIBS CACHE)

diff --git a/deps/Makefile b/deps/Makefile
@@ -42,6 +42,7 @@ distclean:
 	-(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true
 	-(cd hdr_histogram && $(MAKE) clean) > /dev/null || true
 	-(cd fpconv && $(MAKE) clean) > /dev/null || true
+	-(cd lz4 && $(MAKE) clean) > /dev/null || true
 	-(rm -f .make-*)
 
 .PHONY: distclean
@@ -132,3 +133,9 @@ gtest-parallel: .make-prerequisites
 		rm -rf gtest-parallel; \
 		git clone --depth 1 https://github.com/google/gtest-parallel.git gtest-parallel; \
 	fi
+
+lz4: .make-prerequisites
+	@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
+	cd lz4 && $(MAKE) CFLAGS="$(CFLAGS)"
+
+.PHONY: lz4
diff --git a/deps/README.md b/deps/README.md
@@ -5,6 +5,7 @@ should be provided by the operating system.
 * **libvalkey** is the official C client library for Valkey. It is used by valkey-cli, valkey-benchmark and Valkey Sentinel. It is managed in a separate project and updated as needed.
 * **linenoise** is a readline replacement. It is developed by the same authors of Valkey but is managed as a separated project and updated as needed.
 * **lua** is Lua 5.1 with minor changes for security and additional libraries.
+* **LZ4** is the v1.10.0 streaming compression library used for whole-file RDB compression.
 * **hdr_histogram** Used for per-command latency tracking histograms.
 * **ffc.h** is a C99 port of the fast_float library, used as a replacement for strtod to convert strings to floats efficiently.
 * **gtest-parallel** is a script for running googletest tests in parallel.
@@ -62,6 +63,24 @@ following:
 1. Remove the linenoise directory.
 2. Substitute it with the new linenoise source tree.
 
+LZ4
+---
+
+LZ4 is imported from the upstream release archive. The vendored copy currently
+uses version 1.10.0. `Makefile` and `CMakeLists.txt` are maintained locally for
+the Valkey build, and xxHash symbols are namespaced to avoid conflicts with
+modules loaded by Valkey.
+
+To upgrade LZ4:
+
+1. Download the new release archive from https://github.com/lz4/lz4/releases.
+2. Replace `lz4.c`, `lz4.h`, `lz4hc.c`, `lz4hc.h`, `lz4frame.c`,
+   `lz4frame.h`, `xxhash.c`, `xxhash.h`, and `LICENSE` with the versions from
+   the release's `lib` directory.
+3. Preserve the local `Makefile` and `CMakeLists.txt`, including the
+   `XXH_NAMESPACE` definition.
+4. Update the version recorded above.
+
 Lua
 ---
 

diff --git a/deps/lz4/CMakeLists.txt b/deps/lz4/CMakeLists.txt
@@ -0,0 +1,15 @@
+project(lz4)
+
+set(SRCS
+    "${CMAKE_CURRENT_LIST_DIR}/lz4.c"
+    "${CMAKE_CURRENT_LIST_DIR}/lz4.h"
+    "${CMAKE_CURRENT_LIST_DIR}/lz4hc.c"
+    "${CMAKE_CURRENT_LIST_DIR}/lz4hc.h"
+    "${CMAKE_CURRENT_LIST_DIR}/lz4frame.c"
+    "${CMAKE_CURRENT_LIST_DIR}/lz4frame.h"
+    "${CMAKE_CURRENT_LIST_DIR}/xxhash.c"
+    "${CMAKE_CURRENT_LIST_DIR}/xxhash.h")
+
+add_library(lz4 STATIC ${SRCS})
+target_compile_definitions(lz4 PRIVATE XXH_NAMESPACE=VALKEY_LZ4_)
+set_target_properties(lz4 PROPERTIES POSITION_INDEPENDENT_CODE ON)
diff --git a/deps/lz4/LICENSE b/deps/lz4/LICENSE
@@ -0,0 +1,24 @@
+LZ4 Library
+Copyright (c) 2011-2020, Yann Collet
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/deps/lz4/Makefile b/deps/lz4/Makefile
@@ -0,0 +1,29 @@
+STD=
+WARN= -Wall
+OPT= -Os
+
+R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) -DXXH_NAMESPACE=VALKEY_LZ4_
+R_LDFLAGS= $(LDFLAGS)
+DEBUG= -g
+
+R_CC=$(CC) $(R_CFLAGS)
+R_LD=$(CC) $(R_LDFLAGS)
+
+AR= ar
+ARFLAGS= rcs
+
+liblz4.a: lz4.o lz4hc.o lz4frame.o xxhash.o
+	$(AR) $(ARFLAGS) $@ $^
+
+lz4.o: lz4.c lz4.h
+lz4hc.o: lz4hc.c lz4hc.h lz4.h
+lz4frame.o: lz4frame.c lz4frame.h lz4.h lz4hc.h xxhash.h
+xxhash.o: xxhash.c xxhash.h
+
+.c.o:
+	$(R_CC) -fPIC -c $<
+
+.PHONY: clean
+clean:
+	rm -f *.o
+	rm -f *.a