diff --git a/CHANGELOG.md b/CHANGELOG.md index 3562419..58ed508 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [0.4.4] 2026-01-09 +### Added +- `SizedGenerator` class wrapper. +- `executor_kwds` argument to `ThreadPoolExecutorHelper` class. +- `cache_fname_fmt` argument in `disk_cache` now supports inputs arguments values to name the cache file. + ## [0.4.3] 2025-12-13 ### Added - `as_builtin` now supports `datetime.date` instances. diff --git a/CITATION.cff b/CITATION.cff index b4608e1..52054c3 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -17,5 +17,5 @@ keywords: - tools - utilities license: MIT -version: 0.4.3 -date-released: '2025-12-13' +version: 0.4.4 +date-released: '2026-01-09' diff --git a/LICENSE b/LICENSE index 8ac214a..cfadb11 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2025 Labbeti +Copyright (c) 2026 Labbeti Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index b782c9c..c5596c6 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ This library has been tested on all Python versions **3.8 - 3.14**, requires onl ### Typing -Check generic types with ìsinstance_generic` : +Check generic types with `isinstance_generic` : ```python >>> import pythonwrench as pw diff --git a/docs/index.rst b/docs/index.rst index 7fdbe86..bb7c8c3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -171,4 +171,4 @@ Contact Maintainer: -- `Étienne Labbé`_ "Labbeti": labbeti.pub@gmail.com +- `Étienne Labbé `_ "Labbeti": labbeti.pub@gmail.com diff --git a/src/pythonwrench/__init__.py b/src/pythonwrench/__init__.py index 029a897..8a6420e 100644 --- a/src/pythonwrench/__init__.py +++ b/src/pythonwrench/__init__.py @@ -9,7 +9,7 @@ __license__ = "MIT" __maintainer__ = "Étienne Labbé (Labbeti)" __status__ = "Development" -__version__ = "0.4.3" +__version__ = "0.4.4" # Re-import for language servers @@ -18,6 +18,7 @@ from . import cast as cast from . import checksum as checksum from . import collections as collections +from . import concurrent as concurrent from . import csv as csv from . import dataclasses as dataclasses from . import datetime as datetime diff --git a/src/pythonwrench/collections/__init__.py b/src/pythonwrench/collections/__init__.py index b873880..986f08d 100644 --- a/src/pythonwrench/collections/__init__.py +++ b/src/pythonwrench/collections/__init__.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from .collections import ( + SizedGenerator, contained, dict_list_to_list_dict, dump_dict, diff --git a/src/pythonwrench/collections/collections.py b/src/pythonwrench/collections/collections.py index f3d0c9b..1671d8f 100644 --- a/src/pythonwrench/collections/collections.py +++ b/src/pythonwrench/collections/collections.py @@ -10,8 +10,10 @@ Callable, Dict, Generator, + Generic, Hashable, Iterable, + Iterator, List, Literal, Mapping, @@ -47,6 +49,21 @@ Order = Literal["left", "right"] +class SizedGenerator(Generic[T]): + """Wraps a generator and size to provide a sized iterable object.""" + + def __init__(self, generator: Generator[T, None, None], size: int) -> None: + super().__init__() + self._generator = generator + self._size = size + + def __iter__(self) -> Iterator[T]: + yield from self._generator + + def __len__(self) -> int: + return self._size + + def contained( x: T, include: Optional[Iterable[T]] = None, @@ -497,7 +514,7 @@ def list_dict_to_dict_list( """Convert list of dicts to dict of lists. Args: - lst: The list of dict to merge. + lst: The list of dict to merge. Cannot be a Generator. key_mode: Can be "same" or "intersect". \ - If "same", all the dictionaries must contains the same keys otherwise a ValueError will be raised. \ - If "intersect", only the intersection of all keys will be used in output. \ @@ -507,6 +524,10 @@ def list_dict_to_dict_list( default_val_fn: Function to return the default value according to a specific key. defaults to None. list_fn: Optional function to build the values. defaults to identity. """ + if isinstance(lst, Generator): + msg = f"Invalid argument type {type(lst)}. (expected any Iterable except Generator)" + raise TypeError(msg) + try: item0 = next(iter(lst)) except StopIteration: diff --git a/src/pythonwrench/concurrent.py b/src/pythonwrench/concurrent.py index 35779b0..f0663af 100644 --- a/src/pythonwrench/concurrent.py +++ b/src/pythonwrench/concurrent.py @@ -3,7 +3,7 @@ import logging from concurrent.futures import Future, ThreadPoolExecutor -from typing import Callable, Generic, List, Optional, TypeVar +from typing import Any, Callable, Dict, Generic, Iterable, List, Optional, TypeVar from typing_extensions import ParamSpec @@ -15,16 +15,32 @@ class ThreadPoolExecutorHelper(Generic[P, T]): - def __init__(self, fn: Callable[P, T], **default_kwargs) -> None: + # Note: use commas for typing because Future is not generic in older python versions + + def __init__( + self, + fn: Callable[P, T], + *, + executor_kwds: Optional[Dict[str, Any]] = None, + executor: Optional[ThreadPoolExecutor] = None, + futures: "Iterable[Future[T]]" = (), + **default_fn_kwds, + ) -> None: + futures = list(futures) + super().__init__() self.fn = fn - self.default_kwargs = default_kwargs - self.executor: Optional[ThreadPoolExecutor] = None - self.futures: list[Future[T]] = [] + self.executor_kwds = executor_kwds + self.executor = executor + self.futures = futures + self.default_kwargs = default_fn_kwds - def submit(self, *args: P.args, **kwargs: P.kwargs) -> Future[T]: + def submit(self, *args: P.args, **kwargs: P.kwargs) -> "Future[T]": if self.executor is None: - self.executor = ThreadPoolExecutor() + executor_kwds = self.executor_kwds + if executor_kwds is None: + executor_kwds = {} + self.executor = ThreadPoolExecutor(**executor_kwds) kwargs = self.default_kwargs | kwargs # type: ignore future = self.executor.submit(self.fn, *args, **kwargs) @@ -39,9 +55,8 @@ def wait_all(self, shutdown: bool = True, verbose: bool = True) -> List[T]: futures = tqdm.tqdm(futures, disable=not verbose) except ImportError: - logger.warning( - "Cannot display verbose bar because tqdm is not installed." - ) + msg = "Cannot display verbose bar because tqdm is not installed." + logger.warning(msg) results = [future.result() for future in futures] self.futures.clear() diff --git a/src/pythonwrench/disk_cache.py b/src/pythonwrench/disk_cache.py index 3419a5f..17b9bed 100644 --- a/src/pythonwrench/disk_cache.py +++ b/src/pythonwrench/disk_cache.py @@ -5,6 +5,7 @@ import os import shutil import time +import warnings from functools import wraps from pathlib import Path from typing import ( @@ -25,7 +26,7 @@ from pythonwrench.checksum import checksum_any from pythonwrench.datetime import get_now -from pythonwrench.inspect import get_fullname +from pythonwrench.inspect import get_argnames, get_fullname T = TypeVar("T") P = ParamSpec("P") @@ -218,6 +219,10 @@ def _disk_cache_impl( ) -> Callable[[Callable[P, T]], Callable[P, T]]: # for backward compatibility if cache_fname_fmt is None: + warnings.warn( + f"Deprecated argument value {cache_fname_fmt=}. (use default instead)", + DeprecationWarning, + ) cache_fname_fmt = "{fn_name}_{csum}{suffix}" if cache_saving_backend == "pickle": @@ -270,13 +275,19 @@ def _disk_cache_impl_fn(fn: Callable[P, T]) -> Callable[P, T]: ) load_start_msg = f"[{fn_name}] Loading cache..." load_end_msg = f"[{fn_name}] Cache loaded." + argnames = get_argnames(fn) @wraps(fn) def _disk_cache_wrapper(*args: P.args, **kwargs: P.kwargs) -> T: checksum_args = fn, args, kwargs csum = cache_checksum_fn(checksum_args) + inputs = dict(zip(argnames, args)) + inputs.update(kwargs) cache_fname = cache_fname_fmt.format( - fn_name=fn_name, csum=csum, suffix=suffix + fn_name=fn_name, + csum=csum, + suffix=suffix, + **inputs, ) cache_fpath = cache_fn_dpath.joinpath(cache_fname) diff --git a/src/pythonwrench/jsonl.py b/src/pythonwrench/jsonl.py index 28e1fc6..34dae5e 100644 --- a/src/pythonwrench/jsonl.py +++ b/src/pythonwrench/jsonl.py @@ -18,6 +18,15 @@ from pythonwrench.semver import Version from pythonwrench.warnings import warn_once +__all__ = [ + "dump_jsonl", + "dumps_jsonl", + "save_jsonl", + "load_jsonl", + "loads_jsonl", + "read_jsonl", +] + # -- Dump / Save / Serialize content to JSONL -- diff --git a/tests/test_disk_cache.py b/tests/test_disk_cache.py index 8f10c8a..563f11d 100644 --- a/tests/test_disk_cache.py +++ b/tests/test_disk_cache.py @@ -23,7 +23,7 @@ def heavy_processing(x: float): def test_disk_cache_example_2(self) -> None: @pw.disk_cache_decorator( - cache_fname_fmt="{fn_name}_{csum}.json", + cache_fname_fmt="{fn_name}_{csum}_x={x}.json", cache_load_fn=pw.load_json, cache_dump_fn=pw.dump_json, )