Add exact hash provider

python/sglang/srt/environ.py

@@ -364,6 +364,12 @@ class Envs:
     SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL = EnvBool(False)
     # KL tests: skip the cache-hit count assertion (e.g. when alloc failure reduces hits)
     SGLANG_TEST_SKIP_CACHE_HIT_ASSERT = EnvBool(False)
+    # Fuzzy-match E2E tests: force every chunk fingerprint to the same
+    # constant, so two genuinely different chunks collide by construction —
+    # exercises the mandatory token-ID equality-check fallback at the real
+    # system level (through the live scheduler), which can't otherwise be
+    # forced without an astronomically unlikely real hash collision.
+    SGLANG_TEST_FUZZY_FORCE_HASH_COLLISION = EnvBool(False)
     SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY = EnvInt(0)
     SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_IDLE = EnvBool(True)
     # Physical KV-page checks: committed<=allocated + no page alias.

python/sglang/srt/mem_cache/fuzzy_match/README.md

@@ -1,12 +1,21 @@
 # Fuzzy KV Cache Reuse (`--radix-cache-backend fuzzy_match`)
 
-Semantic KV cache reuse for prompts that share meaning but not tokens.
-When exact prefix matching leaves part of a prompt uncovered, a pluggable
-`FuzzyMatchProvider` may nominate donor KV from a previously finished
-request; the donor KV is position-corrected (RoPE) into recipient-owned
-slots before the forward pass. Reuse follows the `|exact|fuzzy|miss|`
-prompt decomposition: one contiguous fuzzy span anchored at the exact
-prefix boundary.
+KV cache reuse for prompts whose content reappears at a different offset
+than where it was originally computed. When exact prefix matching leaves
+part of a prompt uncovered, a pluggable `FuzzyMatchProvider` may nominate
+donor KV from a previously finished request; the donor KV is
+position-corrected (RoPE) into recipient-owned slots before the forward
+pass. Reuse follows the `|exact|fuzzy|miss|` prompt decomposition: one
+contiguous fuzzy span anchored at the exact prefix boundary.
+
+Two providers are available:
+- `SemanticEmbedding` (default): matches merely-similar content by cosine
+  similarity. Requires the `semblend` package. Not lossless — reuse changes
+  model outputs by construction (see Scope and guarantees below).
+- `ExactHash`: matches content that is byte-identical to the current
+  prompt's unmatched tail but sits at a different offset (content-defined
+  chunking, no external dependency). Lossless: every hash hit is confirmed
+  by a token-ID equality check before being served.
 
 ## Enabling
 
@@ -20,6 +29,15 @@ python -m sglang.launch_server \
   --fuzzy-model-arch qwen2.5-7b
 ```
 
+Or, with no external dependency:
+
+```bash
+python -m sglang.launch_server \
+  --model-path Qwen/Qwen2.5-7B-Instruct-AWQ \
+  --radix-cache-backend fuzzy_match \
+  --fuzzy-match-provider ExactHash
+```
+
 Selecting the backend enables the feature; the default provider is
 `SemanticEmbedding`. Reuse markers in the server log: `fuzzy match success`
 (match accepted), `[FUZZY] Realized N fuzzy tokens` (KV copied +
@@ -98,7 +116,7 @@ detectably stale — never dangling.
 | Flag | Default | Why it exists |
 |---|---|---|
 | `--radix-cache-backend fuzzy_match` | off | The enable switch; registers nothing and costs nothing when unset. |
-| `--fuzzy-match-provider` | `SemanticEmbedding` | Provider selection; the interface admits out-of-tree providers. |
+| `--fuzzy-match-provider` | `SemanticEmbedding` | `SemanticEmbedding` or `ExactHash`; the interface admits out-of-tree providers too. |
 | `--fuzzy-semantic-threshold` | `0.60` | Precision knob: cosine floor for accepting a donor. |
 | `--fuzzy-min-reuse-ratio` | `0.50` | Hit gate: donors covering less of the prompt are rejected. |
 | `--fuzzy-min-match-length` | `16` | Skips fuzzy lookup behind weak partial exact anchors. |
@@ -115,10 +133,13 @@ flags.
   the missed suffix. The default backend path is byte-identical when the
   backend is not selected (two seams: one `MatchResult` field, one no-op
   hook in `cache_finished_req`).
-- Reuse changes model outputs by construction: donor K/V attended to the
-  donor's context. The provider's quality gates plus the per-layer
-  zero-out mask bound the drift; accuracy methodology and results are in
-  the PR description.
-- Not yet supported: MLA-style KV pools, EAGLE speculative decoding,
+- `SemanticEmbedding` reuse changes model outputs by construction: donor
+  K/V attended to the donor's context. The provider's quality gates plus
+  the per-layer zero-out mask bound the drift. `ExactHash` reuse is
+  lossless by construction — matched content is byte-identical to the
+  current prompt, confirmed by a mandatory token-ID equality check on
+  every hit. Accuracy methodology and results for both are in the PR
+  descriptions that introduced them.
+- Not yet supported (both providers): MLA-style KV pools, EAGLE speculative decoding,
   multi-region (`|exact|miss|fuzzy|miss|...`) reuse, hierarchical (host)
   cache interaction. Each is rejected explicitly rather than silently.

python/sglang/srt/mem_cache/fuzzy_match/chunker.py

@@ -0,0 +1,176 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Content-defined chunking (CDC) over token-ID sequences.
+
+Boundaries depend only on the last ``WINDOW_SIZE`` tokens, not on absolute
+position or anything upstream — the property RadixAttention's chained hash
+lacks by design. Used by ``ExactHashProvider`` to find donor spans
+independent of where they sit in the current prompt.
+
+The chunking parameters below (window size, boundary mask, chunk-size
+clamp, sink carve-out) follow the methodology in Ma, Eitzinger, and
+Köstler, "Irminsul: MLA-Native Position-Independent Caching for Agentic
+LLM Serving" (arXiv:2605.05696), Section 4 and Appendix B. That paper's
+own reuse mechanism (RoPE delta-rotation over MLA's decomposed KV) is
+unrelated to this module — only its CDC/chunking layer is used here.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import List, NamedTuple, Sequence
+
+_MASK64 = (1 << 64) - 1
+
+# Gear-hash rolling window, in tokens. A boundary decision only ever depends
+# on the last WINDOW_SIZE tokens (Gear hash's left-shift makes older tokens'
+# contributions fall off the top of a 64-bit accumulator automatically —
+# no explicit "remove the oldest byte" step needed, unlike Rabin-Karp).
+WINDOW_SIZE = 64
+
+# Tuned so a boundary fires with probability 2^-BOUNDARY_BITS per token,
+# i.e. an expected chunk size of 2**BOUNDARY_BITS tokens. k=7 is the value
+# the cited paper's mask-exponent ablation (Appendix B) settled on — wider
+# masks push expected chunk length above MAX_CHUNK_TOKENS, which clamps
+# chunks to fixed-size and degenerates back to the fixed-block hashing this
+# design exists to avoid.
+BOUNDARY_BITS = 7  # 2**7 = 128
+_BOUNDARY_MASK = (1 << BOUNDARY_BITS) - 1
+
+MIN_CHUNK_TOKENS = 32
+MAX_CHUNK_TOKENS = 512
+
+# Content before this position is never chunked/cached — attention-sink
+# territory: early positions absorb a disproportionate, content-independent
+# share of attention regardless of what's actually there, so a content hash
+# match in that zone isn't a trustworthy signal. 32 tokens matches the
+# cited paper's own sink-extent measurement (Section 7.1).
+SINK_TOKENS = 32
+
+# Fixed seed: the Gear table must be identical across the write side
+# (cache_on_request_finished) and the read side (match_on_prefix_miss),
+# and across process restarts, or chunk boundaries silently stop lining up.
+_GEAR_SEED = 0x516D5A4F2F3A7C5C
+_GEAR_TABLE_SIZE = 1 << 16  # token_id % this size indexes the table
+
+
+def _build_gear_table() -> List[int]:
+    rng = random.Random(_GEAR_SEED)
+    return [rng.getrandbits(64) for _ in range(_GEAR_TABLE_SIZE)]
+
+
+_GEAR_TABLE = _build_gear_table()
+
+try:
+    import xxhash
+
+    def _real_fingerprint(token_ids: Sequence[int]) -> int:
+        h = xxhash.xxh64(seed=0)
+        h.update(_tokens_to_bytes(token_ids))
+        return h.intdigest()
+
+except ImportError:
+    import hashlib
+
+    def _real_fingerprint(token_ids: Sequence[int]) -> int:
+        # xxhash isn't installed in this environment; blake2b is stdlib,
+        # fast enough for a fingerprint (not a security boundary), and
+        # gives the same "cheap, non-cryptographic-role" fingerprint
+        # xxHash64 would — the token-ID equality check downstream is what
+        # actually guards correctness, not this hash's collision odds.
+        digest = hashlib.blake2b(_tokens_to_bytes(token_ids), digest_size=8).digest()
+        return int.from_bytes(digest, "little")
+
+
+def _fingerprint(token_ids: Sequence[int]) -> int:
+    # Test-only: force every chunk to the same fingerprint so a real E2E
+    # test can exercise the mandatory token-ID equality-check fallback
+    # (never trust the hash alone) without needing an astronomically
+    # unlikely real collision. Off by default in production.
+    from sglang.srt.environ import envs
+
+    if envs.SGLANG_TEST_FUZZY_FORCE_HASH_COLLISION.get():
+        return 0
+    return _real_fingerprint(token_ids)
+
+
+def _tokens_to_bytes(token_ids: Sequence[int]) -> bytes:
+    return b"".join(t.to_bytes(4, "little", signed=False) for t in token_ids)
+
+
+class Chunk(NamedTuple):
+    """One content-defined chunk: token IDs plus its offset within the
+    sequence that was chunked (not an absolute sequence position — callers
+    add their own base offset)."""
+
+    start: int  # offset into the chunked sequence, inclusive
+    end: int  # offset into the chunked sequence, exclusive
+    token_ids: List[int]
+    fingerprint: int
+
+
+def chunk_tokens(token_ids: Sequence[int]) -> List[Chunk]:
+    """Split ``token_ids`` into content-defined chunks.
+
+    Boundaries are declared where a Gear-hash rolling hash over the last
+    ``WINDOW_SIZE`` tokens has its low ``BOUNDARY_BITS`` bits all zero,
+    clamped to ``[MIN_CHUNK_TOKENS, MAX_CHUNK_TOKENS]``. The same content at
+    a different position in a different call produces the same boundaries
+    and the same per-chunk fingerprint, as long as ``WINDOW_SIZE`` tokens of
+    identical context precede each boundary.
+    """
+    n = len(token_ids)
+    chunks: List[Chunk] = []
+    chunk_start = 0
+    h = 0
+    for i, tok in enumerate(token_ids):
+        h = ((h << 1) + _GEAR_TABLE[tok % _GEAR_TABLE_SIZE]) & _MASK64
+        chunk_len = i - chunk_start + 1
+        if chunk_len < MIN_CHUNK_TOKENS:
+            continue
+        at_boundary = (i - chunk_start + 1 >= WINDOW_SIZE) and (h & _BOUNDARY_MASK == 0)
+        if at_boundary or chunk_len >= MAX_CHUNK_TOKENS:
+            span = token_ids[chunk_start : i + 1]
+            chunks.append(
+                Chunk(
+                    start=chunk_start,
+                    end=i + 1,
+                    token_ids=list(span),
+                    fingerprint=_fingerprint(span),
+                )
+            )
+            chunk_start = i + 1
+            h = 0
+
+    # Trailing remainder: the rolling hash may never hit a boundary before
+    # the input runs out, regardless of how much is left — this is not
+    # bounded to be small. If what's left is at least MIN_CHUNK_TOKENS,
+    # emit it as a final chunk (standard CDC practice: the last chunk of
+    # any given input is "whatever's left" once you hit the end); only drop
+    # it when it's genuinely too small to amortize a lookup. This only
+    # affects the *final* chunk of whatever span is being chunked — content
+    # that isn't at the tail end still gets purely content-defined
+    # boundaries, unaffected by this.
+    if len(token_ids) - chunk_start >= MIN_CHUNK_TOKENS:
+        span = token_ids[chunk_start:]
+        chunks.append(
+            Chunk(
+                start=chunk_start,
+                end=len(token_ids),
+                token_ids=list(span),
+                fingerprint=_fingerprint(span),
+            )
+        )
+
+    return chunks

python/sglang/srt/mem_cache/fuzzy_match/config.py

@@ -75,9 +75,9 @@ def __post_init__(self):
                 f"got {self.fuzzy_semantic_threshold}"
             )
 
-        if self.fuzzy_match_provider not in ("SemanticEmbedding",):
+        if self.fuzzy_match_provider not in ("SemanticEmbedding", "ExactHash"):
             raise ValueError(
-                f"fuzzy_match_provider must be 'SemanticEmbedding', "
+                f"fuzzy_match_provider must be 'SemanticEmbedding' or 'ExactHash', "
                 f"got {self.fuzzy_match_provider}"
             )