From a2c06f4260fd855c5fa12dde020a32c3b8b2529f Mon Sep 17 00:00:00 2001 From: Luke Merrett Date: Fri, 31 Jul 2026 08:15:09 +0100 Subject: [PATCH] Add generate_jittered_key_between for concurrent inserts generate_key_between is deterministic, so two writers concurrently inserting between the same pair of keys generate the same key - and a later insert between two equal keys raises FIError. The new function appends jitter_length (default 3) random digits so simultaneous identical inserts diverge with probability ~1 - 62^-3. Design notes: - Separate function rather than a parameter, so the core parity surface stays signature-identical to the JS reference and deterministic. - The suffix is skipped when the generated key is a prefix of the effective upper bound (e.g. 'a5' between 'a4' and 'a52'), where appending anything could push the key past the bound. Bounds are normalised first since they may be passed in either order. - The suffix's last character is never the zero digit (order keys may not end in zero). - Optional rng (random.Random) for reproducible tests; the default is a module-level Random. Collision avoidance only, not security. - No jittered generate_n_keys_between: its results contain keys that are prefixes of their neighbours, so suffixes could reorder them, and bulk generation is single-writer anyway. Co-Authored-By: Claude Fable 5 --- README.md | 29 ++++++++++++++++ fractional_indexing.py | 69 ++++++++++++++++++++++++++++++++++++++ tests.py | 75 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+) diff --git a/README.md b/README.md index eec033d..7cce373 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,29 @@ except FIError as e: ``` +### Jittered keys for concurrent inserts + +`generate_key_between()` is deterministic: two devices concurrently inserting between the same pair of keys generate +the *same* key, and a later insert between two equal keys raises `FIError`. When concurrent writers are possible, use +`generate_jittered_key_between()`, which appends a few random digits (default 3) so simultaneous identical inserts +diverge with high probability: + +```python +import random + +from fractional_indexing import generate_jittered_key_between + + +key = generate_jittered_key_between('a0', 'a1') # e.g. 'a0Vk2P' +key = generate_jittered_key_between('a0', 'a1', jitter_length=4) # longer suffix, lower collision odds +key = generate_jittered_key_between('a0', 'a1', rng=random.Random(42)) # seeded for reproducible tests + +``` + +The result is always a valid order key strictly between the bounds. In one corner case (the generated key is a prefix +of the upper bound) the suffix is skipped and the plain key returned. There is deliberately no jittered variant of +`generate_n_keys_between()` — bulk generation is single-writer, and suffixes could reorder prefix-related neighbours. + ### Use custom base digits By default, this library uses Base62 character encoding. To use a different set of digits, pass them in as the `digits` @@ -149,6 +172,12 @@ This is a Python port of the original JavaScript implementation by [@rocicorp](h ## Changelog +### 1.1.0 + +- New `generate_jittered_key_between()`: appends `jitter_length` (default 3) random digits so concurrent inserts + between the same pair of keys diverge instead of colliding. Takes an optional seeded `rng` for reproducibility. + Skips the suffix in the one corner case where the generated key is a prefix of the upper bound. + ### 1.0.1 - `generate_n_keys_between()` now raises `FIError` for a negative `n`, instead of silently returning a single key diff --git a/fractional_indexing.py b/fractional_indexing.py index 0021e80..2069041 100644 --- a/fractional_indexing.py +++ b/fractional_indexing.py @@ -11,6 +11,7 @@ """ from __future__ import annotations +import random from functools import lru_cache from math import floor from typing import List, Optional @@ -24,6 +25,7 @@ 'BASE_62_DIGITS', 'BASE_52_DIGITS', 'FIError', + 'generate_jittered_key_between', 'generate_key_between', 'generate_n_keys_between', 'validate_order_key', @@ -347,6 +349,10 @@ def generate_key_between( makes the keys self-headed; only omitting `digits` entirely yields the A-Z/a-z heads. + This function is deterministic: concurrent writers inserting between the + same pair of keys will generate the same key. If that can happen in your + application, consider `generate_jittered_key_between()`. + >>> generate_key_between(None, None) 'a0' >>> generate_key_between(None, None, '0123456789') @@ -402,6 +408,69 @@ def generate_key_between( return ia + _midpoint(fa, None, digits, lookup) +# Default RNG for generate_jittered_key_between(). Jitter is collision +# avoidance, not security, so a plain seeded-from-OS Random is fine. +_jitter_rng = random.Random() + + +def generate_jittered_key_between( + a: Optional[str], + b: Optional[str], + digits: Optional[str] = None, + int_digits: Optional[str] = None, + jitter_length: int = 3, + rng: Optional[random.Random] = None, +) -> str: + """ + Like `generate_key_between()`, but appends `jitter_length` random digits to + the generated key. + + `generate_key_between()` is deterministic, so two writers concurrently + inserting between the same pair of keys generate the *same* key - and a + later insert between two equal keys raises FIError. The random suffix makes + simultaneous identical inserts diverge with probability roughly + 1 - len(digits) ** -jitter_length, at the cost of `jitter_length` extra + characters per key. + + The result is a valid order key strictly between `a` and `b`. In one corner + case the suffix is skipped and the plain key returned unchanged: when the + generated key is a prefix of the upper bound (e.g. "a5" between "a4" and + "a52"), where appending anything could push the key past the bound. + + `rng` is the random source (a `random.Random`); pass a seeded instance for + reproducible output. Jitter is collision avoidance, not security - do not + use keys as unguessable tokens. + + There is deliberately no jittered variant of `generate_n_keys_between()`: + its results routinely contain keys that are prefixes of their neighbours, + so suffixes could reorder them - and bulk generation is a single-writer + operation with no concurrent-collision problem to solve. + + >>> generate_jittered_key_between(None, None, rng=random.Random(42)) + 'a0e72' + + """ + digits, int_digits = _resolve_alphabets(digits, int_digits) + if jitter_length < 0: + raise FIError(f'jitter_length must be >= 0: {jitter_length}') + key = generate_key_between(a, b, digits, int_digits) + if jitter_length == 0: + return key + # generate_key_between() accepts its bounds in either order, so normalise + # before checking against the effective upper bound. + if a is not None and b is not None and a > b: + a, b = b, a + if b is not None and b.startswith(key): + return key + if rng is None: + rng = _jitter_rng + # The suffix extends the key's fractional part, so its last character must + # not be the zero digit (order keys may not end in zero). Any character + # sequence is valid in the middle. + suffix = ''.join(rng.choice(digits) for _ in range(jitter_length - 1)) + return key + suffix + rng.choice(digits[1:]) + + def generate_n_keys_between( a: Optional[str], b: Optional[str], diff --git a/tests.py b/tests.py index 8d6893c..1ddf790 100644 --- a/tests.py +++ b/tests.py @@ -1,3 +1,4 @@ +import random from typing import Optional import pytest @@ -6,6 +7,7 @@ BASE_52_DIGITS, BASE_62_DIGITS, FIError, + generate_jittered_key_between, generate_key_between, generate_n_keys_between, validate_order_key, @@ -261,6 +263,79 @@ def test_negative_n_rejected(a: Optional[str], b: Optional[str]) -> None: assert e.value.args[0] == 'n must be >= 0: -1' +def test_jittered_key_appends_suffix() -> None: + rng = random.Random(42) + key = generate_jittered_key_between(None, None, rng=rng) + base = generate_key_between(None, None) + assert key.startswith(base) + assert len(key) == len(base) + 3 + validate_order_key(key) + + +def test_jittered_key_seeded_rng_is_reproducible() -> None: + first = generate_jittered_key_between('a0', 'a1', rng=random.Random(7)) + second = generate_jittered_key_between('a0', 'a1', rng=random.Random(7)) + assert first == second + assert 'a0' < first < 'a1' + + +def test_jittered_key_default_rng() -> None: + key = generate_jittered_key_between('a0', 'a1') + assert 'a0' < key < 'a1' + validate_order_key(key) + + +def test_jittered_key_length_zero_matches_plain_key() -> None: + assert generate_jittered_key_between('a0', 'a1', jitter_length=0) == generate_key_between('a0', 'a1') + + +def test_jittered_key_negative_length_rejected() -> None: + with pytest.raises(FIError) as e: + generate_jittered_key_between(None, None, jitter_length=-1) + assert e.value.args[0] == 'jitter_length must be >= 0: -1' + + +@pytest.mark.parametrize(['a', 'b'], [ + ('a4', 'a52'), + # bounds may be passed in either order; the guard must apply to the + # effective upper bound + ('a52', 'a4'), +]) +def test_jittered_key_skips_jitter_when_key_prefixes_upper_bound(a: str, b: str) -> None: + # generate_key_between('a4', 'a52') returns 'a5', a prefix of the upper + # bound: appending any suffix could push the key past 'a52', so the plain + # key must be returned unchanged. + assert generate_jittered_key_between(a, b, rng=random.Random(1)) == 'a5' + + +def test_jittered_key_custom_alphabet_avoids_trailing_zero() -> None: + # base 2: with only '0' and '1' available, the suffix's last character must + # always be '1' (order keys may not end in the zero digit). + for seed in range(20): + key = generate_jittered_key_between(None, None, '01', BASE_52_DIGITS, rng=random.Random(seed)) + assert key[-1] == '1' + validate_order_key(key, '01', BASE_52_DIGITS) + + +@pytest.mark.parametrize('jitter_length', [1, 3, 8]) +def test_jittered_key_ordering_property(jitter_length: int) -> None: + # Same insertion-hammering property test as test_ordering, but through the + # jittered function: every result must stay strictly within its bounds and + # the list must remain lexicographically sorted. + rnd = LCG() + rng = random.Random(99) + keys = [] + for _ in range(1000): + pos = int(rnd.random() * (len(keys) + 1)) + a = keys[pos - 1] if pos > 0 else None + b = keys[pos] if pos < len(keys) else None + k = generate_jittered_key_between(a, b, jitter_length=jitter_length, rng=rng) + assert (a is None or a < k) and (b is None or k < b), f'out of range: {a} < {k} < {b}' + validate_order_key(k) + keys.insert(pos, k) + assert keys == sorted(keys) + + def test_readme_examples_single_key(): # Insert at the beginning first = generate_key_between(None, None)