diff --git a/README.md b/README.md index eec033d..7cce373 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,29 @@ except FIError as e: ``` +### Jittered keys for concurrent inserts + +`generate_key_between()` is deterministic: two devices concurrently inserting between the same pair of keys generate +the *same* key, and a later insert between two equal keys raises `FIError`. When concurrent writers are possible, use +`generate_jittered_key_between()`, which appends a few random digits (default 3) so simultaneous identical inserts +diverge with high probability: + +```python +import random + +from fractional_indexing import generate_jittered_key_between + + +key = generate_jittered_key_between('a0', 'a1') # e.g. 'a0Vk2P' +key = generate_jittered_key_between('a0', 'a1', jitter_length=4) # longer suffix, lower collision odds +key = generate_jittered_key_between('a0', 'a1', rng=random.Random(42)) # seeded for reproducible tests + +``` + +The result is always a valid order key strictly between the bounds. In one corner case (the generated key is a prefix +of the upper bound) the suffix is skipped and the plain key returned. There is deliberately no jittered variant of +`generate_n_keys_between()` — bulk generation is single-writer, and suffixes could reorder prefix-related neighbours. + ### Use custom base digits By default, this library uses Base62 character encoding. To use a different set of digits, pass them in as the `digits` @@ -149,6 +172,12 @@ This is a Python port of the original JavaScript implementation by [@rocicorp](h ## Changelog +### 1.1.0 + +- New `generate_jittered_key_between()`: appends `jitter_length` (default 3) random digits so concurrent inserts + between the same pair of keys diverge instead of colliding. Takes an optional seeded `rng` for reproducibility. + Skips the suffix in the one corner case where the generated key is a prefix of the upper bound. + ### 1.0.1 - `generate_n_keys_between()` now raises `FIError` for a negative `n`, instead of silently returning a single key diff --git a/fractional_indexing.py b/fractional_indexing.py index 0021e80..2069041 100644 --- a/fractional_indexing.py +++ b/fractional_indexing.py @@ -11,6 +11,7 @@ """ from __future__ import annotations +import random from functools import lru_cache from math import floor from typing import List, Optional @@ -24,6 +25,7 @@ 'BASE_62_DIGITS', 'BASE_52_DIGITS', 'FIError', + 'generate_jittered_key_between', 'generate_key_between', 'generate_n_keys_between', 'validate_order_key', @@ -347,6 +349,10 @@ def generate_key_between( makes the keys self-headed; only omitting `digits` entirely yields the A-Z/a-z heads. + This function is deterministic: concurrent writers inserting between the + same pair of keys will generate the same key. If that can happen in your + application, consider `generate_jittered_key_between()`. + >>> generate_key_between(None, None) 'a0' >>> generate_key_between(None, None, '0123456789') @@ -402,6 +408,69 @@ def generate_key_between( return ia + _midpoint(fa, None, digits, lookup) +# Default RNG for generate_jittered_key_between(). Jitter is collision +# avoidance, not security, so a plain seeded-from-OS Random is fine. +_jitter_rng = random.Random() + + +def generate_jittered_key_between( + a: Optional[str], + b: Optional[str], + digits: Optional[str] = None, + int_digits: Optional[str] = None, + jitter_length: int = 3, + rng: Optional[random.Random] = None, +) -> str: + """ + Like `generate_key_between()`, but appends `jitter_length` random digits to + the generated key. + + `generate_key_between()` is deterministic, so two writers concurrently + inserting between the same pair of keys generate the *same* key - and a + later insert between two equal keys raises FIError. The random suffix makes + simultaneous identical inserts diverge with probability roughly + 1 - len(digits) ** -jitter_length, at the cost of `jitter_length` extra + characters per key. + + The result is a valid order key strictly between `a` and `b`. In one corner + case the suffix is skipped and the plain key returned unchanged: when the + generated key is a prefix of the upper bound (e.g. "a5" between "a4" and + "a52"), where appending anything could push the key past the bound. + + `rng` is the random source (a `random.Random`); pass a seeded instance for + reproducible output. Jitter is collision avoidance, not security - do not + use keys as unguessable tokens. + + There is deliberately no jittered variant of `generate_n_keys_between()`: + its results routinely contain keys that are prefixes of their neighbours, + so suffixes could reorder them - and bulk generation is a single-writer + operation with no concurrent-collision problem to solve. + + >>> generate_jittered_key_between(None, None, rng=random.Random(42)) + 'a0e72' + + """ + digits, int_digits = _resolve_alphabets(digits, int_digits) + if jitter_length < 0: + raise FIError(f'jitter_length must be >= 0: {jitter_length}') + key = generate_key_between(a, b, digits, int_digits) + if jitter_length == 0: + return key + # generate_key_between() accepts its bounds in either order, so normalise + # before checking against the effective upper bound. + if a is not None and b is not None and a > b: + a, b = b, a + if b is not None and b.startswith(key): + return key + if rng is None: + rng = _jitter_rng + # The suffix extends the key's fractional part, so its last character must + # not be the zero digit (order keys may not end in zero). Any character + # sequence is valid in the middle. + suffix = ''.join(rng.choice(digits) for _ in range(jitter_length - 1)) + return key + suffix + rng.choice(digits[1:]) + + def generate_n_keys_between( a: Optional[str], b: Optional[str], diff --git a/tests.py b/tests.py index 8d6893c..1ddf790 100644 --- a/tests.py +++ b/tests.py @@ -1,3 +1,4 @@ +import random from typing import Optional import pytest @@ -6,6 +7,7 @@ BASE_52_DIGITS, BASE_62_DIGITS, FIError, + generate_jittered_key_between, generate_key_between, generate_n_keys_between, validate_order_key, @@ -261,6 +263,79 @@ def test_negative_n_rejected(a: Optional[str], b: Optional[str]) -> None: assert e.value.args[0] == 'n must be >= 0: -1' +def test_jittered_key_appends_suffix() -> None: + rng = random.Random(42) + key = generate_jittered_key_between(None, None, rng=rng) + base = generate_key_between(None, None) + assert key.startswith(base) + assert len(key) == len(base) + 3 + validate_order_key(key) + + +def test_jittered_key_seeded_rng_is_reproducible() -> None: + first = generate_jittered_key_between('a0', 'a1', rng=random.Random(7)) + second = generate_jittered_key_between('a0', 'a1', rng=random.Random(7)) + assert first == second + assert 'a0' < first < 'a1' + + +def test_jittered_key_default_rng() -> None: + key = generate_jittered_key_between('a0', 'a1') + assert 'a0' < key < 'a1' + validate_order_key(key) + + +def test_jittered_key_length_zero_matches_plain_key() -> None: + assert generate_jittered_key_between('a0', 'a1', jitter_length=0) == generate_key_between('a0', 'a1') + + +def test_jittered_key_negative_length_rejected() -> None: + with pytest.raises(FIError) as e: + generate_jittered_key_between(None, None, jitter_length=-1) + assert e.value.args[0] == 'jitter_length must be >= 0: -1' + + +@pytest.mark.parametrize(['a', 'b'], [ + ('a4', 'a52'), + # bounds may be passed in either order; the guard must apply to the + # effective upper bound + ('a52', 'a4'), +]) +def test_jittered_key_skips_jitter_when_key_prefixes_upper_bound(a: str, b: str) -> None: + # generate_key_between('a4', 'a52') returns 'a5', a prefix of the upper + # bound: appending any suffix could push the key past 'a52', so the plain + # key must be returned unchanged. + assert generate_jittered_key_between(a, b, rng=random.Random(1)) == 'a5' + + +def test_jittered_key_custom_alphabet_avoids_trailing_zero() -> None: + # base 2: with only '0' and '1' available, the suffix's last character must + # always be '1' (order keys may not end in the zero digit). + for seed in range(20): + key = generate_jittered_key_between(None, None, '01', BASE_52_DIGITS, rng=random.Random(seed)) + assert key[-1] == '1' + validate_order_key(key, '01', BASE_52_DIGITS) + + +@pytest.mark.parametrize('jitter_length', [1, 3, 8]) +def test_jittered_key_ordering_property(jitter_length: int) -> None: + # Same insertion-hammering property test as test_ordering, but through the + # jittered function: every result must stay strictly within its bounds and + # the list must remain lexicographically sorted. + rnd = LCG() + rng = random.Random(99) + keys = [] + for _ in range(1000): + pos = int(rnd.random() * (len(keys) + 1)) + a = keys[pos - 1] if pos > 0 else None + b = keys[pos] if pos < len(keys) else None + k = generate_jittered_key_between(a, b, jitter_length=jitter_length, rng=rng) + assert (a is None or a < k) and (b is None or k < b), f'out of range: {a} < {k} < {b}' + validate_order_key(k) + keys.insert(pos, k) + assert keys == sorted(keys) + + def test_readme_examples_single_key(): # Insert at the beginning first = generate_key_between(None, None)