diff --git a/README.md b/README.md index c780620..6be5ca2 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ Cachew gives the best of two worlds and makes it both **easy and efficient**. Th # How it works -- first your objects get [converted](src/cachew/marshall/cachew.py#L29) into a simpler JSON-like representation +- first your objects get [converted](src/cachew/marshall/cachew.py#L31) into a simpler JSON-like representation - after that, they are mapped into byte blobs via [`orjson`](https://github.com/ijl/orjson). When the function is called, cachew [computes the hash of your function's arguments ](src/cachew/__init__.py#L390) @@ -140,18 +140,18 @@ and compares it against the previously stored hash value. -* automatic schema inference: [1](src/cachew/tests/test_cachew.py#L321), [2](src/cachew/tests/test_cachew.py#L378) +* automatic schema inference: [1](src/cachew/tests/test_cachew.py#L353), [2](src/cachew/tests/test_cachew.py#L410) * supported types: * primitive: `str`, `int`, `float`, `bool`, `datetime`, `date`, `Exception` - See [tests.test_types](src/cachew/tests/test_cachew.py#L742), [tests.test_primitive](src/cachew/tests/test_cachew.py#L780), [tests.test_dates](src/cachew/tests/test_cachew.py#L692), [tests.test_exceptions](src/cachew/tests/test_cachew.py#L1420) - * [@dataclass and NamedTuple](src/cachew/tests/test_cachew.py#L657) - * [Optional](src/cachew/tests/test_cachew.py#L584) types - * [Union](src/cachew/tests/test_cachew.py#L887) types - * [nested datatypes](src/cachew/tests/test_cachew.py#L423) + See [tests.test_types](src/cachew/tests/test_cachew.py#L848), [tests.test_primitive](src/cachew/tests/test_cachew.py#L886), [tests.test_dates](src/cachew/tests/test_cachew.py#L798), [tests.test_exceptions](src/cachew/tests/test_cachew.py#L1536) + * [@dataclass and NamedTuple](src/cachew/tests/test_cachew.py#L763) + * [Optional](src/cachew/tests/test_cachew.py#L690) types + * [Union](src/cachew/tests/test_cachew.py#L993) types + * [nested datatypes](src/cachew/tests/test_cachew.py#L455) -* detects [datatype schema changes](src/cachew/tests/test_cachew.py#L453) and discards old data automatically +* detects [datatype schema changes](src/cachew/tests/test_cachew.py#L479) and discards old data automatically # Performance @@ -170,7 +170,7 @@ You can also use [extensive unit tests](src/cachew/tests/test_cachew.py#L1) as a Some useful (but optional) arguments of `@cachew` decorator: -* `cache_path` can be a directory, or a callable that [returns a path](src/cachew/tests/test_cachew.py#L400) and depends on function's arguments. +* `cache_path` can be a directory, or a callable that [returns a path](src/cachew/tests/test_cachew.py#L432) and depends on function's arguments. By default, `settings.DEFAULT_CACHEW_DIR` is used. diff --git a/src/cachew/__init__.py b/src/cachew/__init__.py index 95f46b9..0adf658 100644 --- a/src/cachew/__init__.py +++ b/src/cachew/__init__.py @@ -44,7 +44,7 @@ def orjson_dumps(*args: Any, **kwargs: Any) -> bytes: # type: ignore[misc] from .backend.sqlite_raw import SqliteRawBackend from .common import DEPENDENCIES, CacheReadError, CachewException, CacheWriteError, SourceHash from .logging_helper import make_logger -from .marshall.cachew import CachewMarshall +from .marshall.cachew import CachewMarshall, SchemaFingerprint # in case of changes in the way cachew stores data, this should be changed to discard old caches CACHEW_VERSION: str = importlib.metadata.version(__name__) @@ -387,7 +387,7 @@ def resolve_cache_path(self, /, *args: P.args, **kwargs: P.kwargs) -> Path | Non self.logger.debug(f'using {self.backend}:{resolved_path} for cache') return resolved_path - def composite_hash(self, *args, **kwargs) -> dict[str, Any]: + def composite_hash(self, schema_fingerprint: SchemaFingerprint, /, *args, **kwargs) -> dict[str, Any]: fsig = inspect.signature(self.func) # defaults wouldn't be passed in kwargs, but they can be an implicit dependency (especially inbetween program runs) defaults = { @@ -403,10 +403,9 @@ def composite_hash(self, *args, **kwargs) -> dict[str, Any]: if k in hsig.parameters or 'kwargs' in hsig.parameters } # fmt: skip kwargs = {**defaults, **kwargs} - schema = str(self.cls_) hash_parts = { 'cachew' : CACHEW_VERSION, - 'schema' : schema, + 'schema' : schema_fingerprint, DEPENDENCIES: str(self.depends_on(*args, **kwargs)), } # fmt: skip synthetic_key = self.synthetic_key @@ -601,10 +600,10 @@ def cachew_wrapper[**P, ItemT]( session: CacheSession[ItemT] | None = None try: BackendCls = BACKENDS[C.backend] - new_hash_d = C.composite_hash(*args, **kwargs) + marshall: CachewMarshall[ItemT] = CachewMarshall(Type_=C.cls_) + new_hash_d = C.composite_hash(marshall.schema_fingerprint, *args, **kwargs) new_hash: SourceHash = json.dumps(new_hash_d) logger.debug(f'new hash: {new_hash}') - marshall: CachewMarshall[ItemT] = CachewMarshall(Type_=C.cls_) backend = BackendCls(cache_path=resolved_cache_path, logger=logger) session = CacheSession( diff --git a/src/cachew/marshall/cachew.py b/src/cachew/marshall/cachew.py index 567034a..190038d 100644 --- a/src/cachew/marshall/cachew.py +++ b/src/cachew/marshall/cachew.py @@ -11,9 +11,11 @@ Any, Dict, List, + Literal, NamedTuple, Optional, Tuple, + TypedDict, Union, get_args, get_origin, @@ -28,7 +30,7 @@ class CachewMarshall[T](AbstractMarshall[T]): def __init__(self, Type_: type[T]) -> None: - self.schema = build_schema(Type_) + self.schema, self.schema_fingerprint = _build_schema(Type_) def dump(self, obj: T) -> Json: return self.schema.dump(obj) @@ -289,7 +291,63 @@ def load(self, dct: str): } -def build_schema(Type) -> Schema: +class _LeafSchemaFingerprint(TypedDict): + kind: Literal['primitive', 'exception', 'datetime', 'date'] + type: str + + +class _SchemaFieldFingerprint(TypedDict): + name: str + schema: SchemaFingerprint + + +class _RecordSchemaFingerprint(TypedDict): + kind: Literal['dataclass', 'namedtuple'] + type: str + fields: list[_SchemaFieldFingerprint] + + +class _CollectionSchemaFingerprint(TypedDict): + kind: Literal['list', 'sequence'] + item: SchemaFingerprint + + +class _TupleSchemaFingerprint(TypedDict): + kind: Literal['tuple'] + items: list[SchemaFingerprint] + + +class _UnionSchemaFingerprint(TypedDict): + kind: Literal['union'] + args: list[SchemaFingerprint] + + +class _DictSchemaFingerprint(TypedDict): + kind: Literal['dict'] + key: SchemaFingerprint + value: SchemaFingerprint + + +type SchemaFingerprint = ( + _LeafSchemaFingerprint + | _RecordSchemaFingerprint + | _CollectionSchemaFingerprint + | _TupleSchemaFingerprint + | _UnionSchemaFingerprint + | _DictSchemaFingerprint +) + + +def _type_identifier(Type: Any) -> str: + return f'{Type.__module__}.{Type.__qualname__}' + + +def build_schema(Type: Any) -> Schema: + schema, _ = _build_schema(Type) + return schema + + +def _build_schema(Type: Any) -> tuple[Schema, SchemaFingerprint]: # just to avoid confusion in case of weirdness with stringish type annotations assert not isinstance(Type, str), Type @@ -297,7 +355,7 @@ def build_schema(Type) -> Schema: ptype = PRIMITIVES.get(Type) if ptype is not None: - return SPrimitive(type=ptype) + return SPrimitive(type=ptype), {'kind': 'primitive', 'type': _type_identifier(Type)} origin = get_origin(Type) # origin is 'unsubscripted/erased' version of type @@ -305,13 +363,13 @@ def build_schema(Type) -> Schema: if origin is None: if issubclass(Type, Exception): - return SException(type=Type) + return SException(type=Type), {'kind': 'exception', 'type': _type_identifier(Type)} if issubclass(Type, datetime): - return SDatetime(type=Type) + return SDatetime(type=Type), {'kind': 'datetime', 'type': _type_identifier(Type)} if issubclass(Type, date): - return SDate(type=Type) + return SDate(type=Type), {'kind': 'date', 'type': _type_identifier(Type)} if not (is_dataclass(Type) or is_namedtuple(Type)): raise TypeNotSupported(type_=Type, reason='unknown type') @@ -321,10 +379,23 @@ def build_schema(Type) -> Schema: # this can happen for instance on 3.9 if pipe syntax was used for Union types # would be nice to provide a friendlier error though raise TypeNotSupported(type_=Type, reason='failed to get type hints') from te - fields = tuple((k, build_schema(t)) for k, t in hints.items()) - return SDataclass( - type=Type, - fields=fields, + schema_fields: list[tuple[str, Schema]] = [] + fingerprint_fields: list[_SchemaFieldFingerprint] = [] + for name, field_type in hints.items(): + field_schema, field_fingerprint = _build_schema(field_type) + schema_fields.append((name, field_schema)) + fingerprint_fields.append({'name': name, 'schema': field_fingerprint}) + kind: Literal['namedtuple', 'dataclass'] = 'namedtuple' if is_namedtuple(Type) else 'dataclass' + return ( + SDataclass( + type=Type, + fields=tuple(schema_fields), + ), + { + 'kind': kind, + 'type': _type_identifier(Type), + 'fields': fingerprint_fields, + }, ) args = get_args(Type) @@ -333,24 +404,32 @@ def build_schema(Type) -> Schema: if is_union: # We 'erasing' types (since generic types don't work with isinstance checks). # So we need to make sure the types are unique to make sure we can deserialise them. - schemas = [build_schema(a) for a in args] + union_schemas_and_fingerprints = [_build_schema(a) for a in args] + schemas = [schema for schema, _ in union_schemas_and_fingerprints] union_types = [s.type for s in schemas if s.type is not Real] if len(set(union_types)) != len(union_types): raise TypeNotSupported(type_=Type, reason=f'runtime union arguments are not unique: {union_types}') - return SUnion( - type=origin, - args=tuple( - (tidx, s) - for tidx, s in enumerate(schemas) + return ( + SUnion( + type=origin, + args=tuple((tidx, s) for tidx, s in enumerate(schemas)), ), - ) # fmt: skip + { + 'kind': 'union', + 'args': [fingerprint for _, fingerprint in union_schemas_and_fingerprints], + }, + ) is_listish = origin is list if is_listish: (t,) = args - return SList( - type=origin, - arg=build_schema(t), + item_schema, item_fingerprint = _build_schema(t) + return ( + SList( + type=origin, + arg=item_schema, + ), + {'kind': 'list', 'item': item_fingerprint}, ) # hmm check for is typing.Sequence doesn't pass for some reason @@ -362,28 +441,46 @@ def build_schema(Type) -> Schema: # before python 3.11, get_args for that gives ((),) instead of an empty tuple () as one might expect if args == ((),): args = () - return STuple( - type=origin, - args=tuple(build_schema(a) for a in args), + tuple_schemas_and_fingerprints = tuple(_build_schema(a) for a in args) + return ( + STuple( + type=origin, + args=tuple(schema for schema, _ in tuple_schemas_and_fingerprints), + ), + { + 'kind': 'tuple', + 'items': [fingerprint for _, fingerprint in tuple_schemas_and_fingerprints], + }, ) else: (t,) = args - return SSequence( - type=origin, - arg=build_schema(t), + item_schema, item_fingerprint = _build_schema(t) + return ( + SSequence( + type=origin, + arg=item_schema, + ), + {'kind': 'sequence', 'item': item_fingerprint}, ) is_dictish = origin is dict if is_dictish: (ft, tt) = args - fts = build_schema(ft) + fts, ft_fingerprint = _build_schema(ft) if not isinstance(fts, SPrimitive): raise TypeNotSupported(type_=Type, reason='dictionary key type must be primitive') - tts = build_schema(tt) - return SDict( - type=origin, - ft=fts, - tt=tts, + tts, tt_fingerprint = _build_schema(tt) + return ( + SDict( + type=origin, + ft=fts, + tt=tts, + ), + { + 'kind': 'dict', + 'key': ft_fingerprint, + 'value': tt_fingerprint, + }, ) raise TypeNotSupported(type_=Type, reason=f'generic type with origin {origin} is unsupported') @@ -425,6 +522,53 @@ def normalise(x): ## +@dataclass +class _FingerprintChild: + ratio: float + + +@dataclass +class _FingerprintRecord: + count: int + child: _FingerprintChild | None + timestamps: list[datetime] + + +def test_schema_fingerprint() -> None: + marshall = CachewMarshall(_FingerprintRecord) + + assert marshall.schema_fingerprint == { + 'kind': 'dataclass', + 'type': f'{__name__}._FingerprintRecord', + 'fields': [ + {'name': 'count', 'schema': {'kind': 'primitive', 'type': 'builtins.int'}}, + { + 'name': 'child', + 'schema': { + 'kind': 'union', + 'args': [ + { + 'kind': 'dataclass', + 'type': f'{__name__}._FingerprintChild', + 'fields': [ + { + 'name': 'ratio', + 'schema': {'kind': 'primitive', 'type': 'builtins.float'}, + } + ], + }, + {'kind': 'primitive', 'type': 'builtins.NoneType'}, + ], + }, + }, + { + 'name': 'timestamps', + 'schema': {'kind': 'list', 'item': {'kind': 'datetime', 'type': 'datetime.datetime'}}, + }, + ], + } + + # TODO customise with cattrs def test_serialize_and_deserialize() -> None: import pytest diff --git a/src/cachew/tests/test_cachew.py b/src/cachew/tests/test_cachew.py index a61d57b..f9cd5ff 100644 --- a/src/cachew/tests/test_cachew.py +++ b/src/cachew/tests/test_cachew.py @@ -476,32 +476,106 @@ def get_data(): assert list(get_data()) == [d1, d2] -class BBv2(NamedTuple): - xx: int - yy: int - zz: float +def test_schema_change(tmp_path: Path) -> None: + """ + A changed recursive schema must invalidate a cache even when the outer type identity is unchanged. + """ + cache_path = tmp_path / 'schema_change.sqlite' + @dataclass + class ChildV1: + value: int -def test_schema_change(tmp_path: Path) -> None: + @dataclass + class ItemV1: + child: ChildV1 + + @dataclass + class ChildV2: + value: float + + @dataclass + class ItemV2: + child: ChildV2 + + ChildV2.__name__ = ChildV1.__name__ + ChildV2.__qualname__ = ChildV1.__qualname__ + ItemV2.__name__ = ItemV1.__name__ + ItemV2.__qualname__ = ItemV1.__qualname__ + assert str(ChildV1) == str(ChildV2) # precondition + assert str(ItemV1) == str(ItemV2) # precondition + + calls_v1 = 0 + item_v1 = ItemV1(child=ChildV1(value=1)) + + @cachew(cache_path=cache_path, force_file=True, cls=ItemV1) + def get_data_v1(): + nonlocal calls_v1 + calls_v1 += 1 + return [item_v1] + + assert list(get_data_v1()) == [item_v1] + assert list(get_data_v1()) == [item_v1] + assert calls_v1 == 1 + + calls_v2 = 0 + item_v2 = ItemV2(child=ChildV2(value=1.5)) + + @cachew(cache_path=cache_path, force_file=True, cls=ItemV2) + def get_data_v2(): + nonlocal calls_v2 + calls_v2 += 1 + return [item_v2] + + assert list(get_data_v2()) == [item_v2] + assert list(get_data_v2()) == [item_v2] + assert calls_v2 == 1 + + +def test_schema_change_from_required_to_optional(tmp_path: Path) -> None: """ - Should discard cache on schema change (BB to BBv2) in this example + Changing a field to Optional must invalidate the old cache before decoding its incompatible rows. + This is a regression for https://github.com/karlicoss/cachew/issues/59. """ - b = BB(xx=2, yy=3) + cache_path = tmp_path / 'optional_schema_change.sqlite' - @cachew(cache_path=tmp_path, cls=BB) - def get_data(): - return [b] + @dataclass + class ItemV1: + value: float + + @dataclass + class ItemV2: + value: float | None + + ItemV2.__name__ = ItemV1.__name__ + ItemV2.__qualname__ = ItemV1.__qualname__ + assert str(ItemV1) == str(ItemV2) # precondition + + calls_v1 = 0 + item_v1 = ItemV1(value=1.5) - assert list(get_data()) == [b] + @cachew(cache_path=cache_path, force_file=True, cls=ItemV1) + def get_data_v1(): + nonlocal calls_v1 + calls_v1 += 1 + return [item_v1] - # TODO make type part of key? - b2 = BBv2(xx=3, yy=4, zz=5.0) + assert list(get_data_v1()) == [item_v1] + assert list(get_data_v1()) == [item_v1] + assert calls_v1 == 1 - @cachew(cache_path=tmp_path, cls=BBv2) + calls_v2 = 0 + item_v2 = ItemV2(value=None) + + @cachew(cache_path=cache_path, force_file=True, cls=ItemV2) def get_data_v2(): - return [b2] + nonlocal calls_v2 + calls_v2 += 1 + return [item_v2] - assert list(get_data_v2()) == [b2] + assert list(get_data_v2()) == [item_v2] + assert list(get_data_v2()) == [item_v2] + assert calls_v2 == 1 def test_transaction(tmp_path: Path) -> None: @@ -1172,37 +1246,37 @@ def test_defensive_read_error_after_yield_raises_cache_read_error( settings.THROW_ON_ERROR = False calls = 0 + cache_path = tmp_path / 'partially_corrupted_cache' class Item(NamedTuple): - value: Any + value: int - first_loose = Item(value=[1]) - second_loose = Item(value=2) + first = Item(value=1) + second = Item(value=2) - # First populate the cache with a looser schema. - @cachew(tmp_path) + @cachew(cache_path, force_file=True) def fun() -> Iterator[Item]: nonlocal calls calls += 1 - yield first_loose - yield second_loose + yield first + yield second - assert list(fun()) == [first_loose, second_loose] + assert list(fun()) == [first, second] assert calls == 1 - class Item(NamedTuple): # type: ignore[no-redef] - value: list[int] - - first_strict = Item(value=[1]) - second_strict = Item(value=[2]) - - # Then reuse the same function and type names, so the cache hash still matches, but the second cached item no longer loads. - @cachew(tmp_path) # type: ignore[no-redef] - def fun() -> Iterator[Item]: - nonlocal calls - calls += 1 - yield first_strict - yield second_strict + # Corrupt only the second blob so the cache read emits one valid item before failing. + if settings.DEFAULT_BACKEND in _SQLITE_BACKENDS: + with sqlite3.connect(cache_path) as connection: + changed = connection.execute( + 'UPDATE cache SET data = ? WHERE rowid = (SELECT rowid FROM cache ORDER BY rowid LIMIT 1 OFFSET 1)', + (b'not-json',), + ).rowcount + assert changed == 1 + else: + lines = cache_path.read_bytes().splitlines(keepends=True) + assert len(lines) == 3, lines + lines[2] = b'not-json\n' + cache_path.write_bytes(b''.join(lines)) # Previous buggy behavior was [first, first, second]: one item loaded from cache, then full fallback. # Expected behavior is a hard cache read error, even when THROW_ON_ERROR is false.