diff --git a/.github/workflows/test-python-package.yml b/.github/workflows/test-python-package.yml index a3864ae..3397627 100644 --- a/.github/workflows/test-python-package.yml +++ b/.github/workflows/test-python-package.yml @@ -19,21 +19,58 @@ permissions: jobs: build: - + name: Build distribution runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 with: - python-version: "3.10" + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + test: + name: >- + Testing distribution + needs: + - build + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - python -m pip install -e ./ + for w in ./dist/*.whl; do python -m pip install $w; done - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/ChangeLog.md b/ChangeLog.md index 957cbd5..4656fe3 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,6 +1,12 @@ # [saveable-objects](README.md) Change Log -## Release 1.0.0 +## Release v1.1.0 + +This release adds support for earlier python versions that 3.10. Specifically the changes in this release are: +- Support was added for python versions 3.7, 3.8, and 3.9 +- Minor updates to the documentation. + +## Release v1.0.0 This is the initial release. Future changes to this release will be documented above. \ No newline at end of file diff --git a/README.md b/README.md index f0d810e..10ee730 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,15 @@ The python package can be installed with pip as follows: pip install saveable-objects ``` +### Requirements: + +Python >= 3.7 + +Packages: + +- [NumPy](https://numpy.org/) +- [cloudpickle](https://github.com/cloudpipe/cloudpickle) + ## Documentation Documentation including worked examples can be found at: [https://saveable-objects.readthedocs.io/](https://saveable-objects.readthedocs.io/). @@ -19,5 +28,5 @@ Source code can be found at: [https://github.com/Christopher-K-Long/saveable-obj ## Version and Changes -The current version is [`1.0.0`](ChangeLog.md#release-100). Please see the [Change Log](ChangeLog.md) for more +The current version is [`1.1.0`](ChangeLog.md#release-110). Please see the [Change Log](ChangeLog.md) for more details. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 79c9e96..6c56f32 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,8 +3,8 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -saveable-objects documentation -============================== +saveable-objects +================ .. include:: ../README.md :parser: myst_parser.sphinx_ diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 3c524c5..a876af1 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -1,5 +1,5 @@ API Reference -============================= +============= .. autosummary:: :toctree: _autosummary diff --git a/pyproject.toml b/pyproject.toml index dcba10a..4bace66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "saveable-objects" -version = "1.0.0" +version = "1.1.0" authors = [ { name="Christopher_K._Long", email="ckl45@cam.ac.uk" }, ] @@ -14,7 +14,7 @@ maintainers = [ description = "A python package for checkpointing, saving, and loading objects." keywords = ["save", "saving", "saveable", "object", "checkpoint", "checkpointing", "load", "loading"] readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.7" classifiers = [ "Programming Language :: Python :: 3", "Operating System :: OS Independent", diff --git a/src/saveable_objects/__init__.py b/src/saveable_objects/__init__.py index 13d3296..91dac11 100644 --- a/src/saveable_objects/__init__.py +++ b/src/saveable_objects/__init__.py @@ -2,4 +2,9 @@ A python package for checkpointing, saving, and loading objects. """ -from ._saveable_object import SaveableObject \ No newline at end of file +import sys + +if sys.version_info[1] >= 10: # check if python is version 3.8 or later + from ._saveable_object import SaveableObject +else: # import from the python version 3.7 compatable file + from ._saveable_object_3_7 import SaveableObject \ No newline at end of file diff --git a/src/saveable_objects/_saveable_object.py b/src/saveable_objects/_saveable_object.py index 42cd140..7c5dd63 100644 --- a/src/saveable_objects/_saveable_object.py +++ b/src/saveable_objects/_saveable_object.py @@ -89,7 +89,8 @@ def _save(self, path: str, write_mode: Literal["w", "wb", "a", "ab", "x", "xb"] as `mode` for ``open``. By default ``"wb"``. """ if not os.path.exists(path): - if (dirname := os.path.dirname(path)) != '': + dirname = os.path.dirname(path) + if dirname != '': os.makedirs(dirname, exist_ok=True) with open(path, write_mode) as file: cpkl.dump(self, file, pkl.HIGHEST_PROTOCOL) @@ -343,7 +344,8 @@ def loadif(cls, *args, **kwargs) -> Tuple["SaveableObject", bool]: path = bound_args.arguments["path"] except KeyError: path = None - if instance := cls.tryload(path): + instance = cls.tryload(path) + if instance: return instance, True return cls(*args, **kwargs), False @classmethod diff --git a/src/saveable_objects/_saveable_object_3_7.py b/src/saveable_objects/_saveable_object_3_7.py new file mode 100644 index 0000000..283e2b5 --- /dev/null +++ b/src/saveable_objects/_saveable_object_3_7.py @@ -0,0 +1,407 @@ +import os +import inspect +import numpy as np +import pickle as pkl +import cloudpickle as cpkl +from typing import Optional, Union, IO, Tuple + +from ._meta_class import SaveAfterInitMetaClass + +class SaveableObject(metaclass=SaveAfterInitMetaClass): + """A utility class for saving objects to pickles and checkpointing. + """ + def __init__(self, path: Optional[str] = None): + """Initialises an instance of the :class:`SaveableObject`. + If a path is specified the saveable object is automatically saved after + initialisation. + + Parameters + ---------- + path : str, optional + The file :attr:`path` to save the object to. If ``None`` then the + object is not saved. By default ``None``. + + Notes + ----- + If no file extension is provided for `path` then the class name and the + ``.pkl`` extension are appended to the file name. + """ + self.path = path + @property + def path(self) -> Optional[str]: + """The current file path of the object. + + Notes + ----- + On setting the value, if no file extension is provided then the class + name and the ``.pkl`` extension are appended to the file name. + """ + return self._path + @path.setter + def path(self, value: Optional[str]): + """Set the current file path of the object. + + Parameters + ---------- + value : str, optional + The new file path of the object. By default ``None``. + + Notes + ----- + If no file extension is provided then the class name and the ``.pkl`` + extension are appended to the file name. + """ + self._path = self._updatepathroot(value) + @classmethod + def _get_name(cls, path: Optional[str]) -> Optional[str]: + """Returns the file name of the specified path (without the file + extension). + + Parameters + ---------- + path : str, optional + The path to obtain the file name for. + + Returns + ------- + str, optional + The file name (without the file extension). + """ + if path is None: + return None + return os.path.split(os.path.splitext(cls._updatepathroot(path))[0])[-1] + @property + def name(self) -> Optional[str]: + """The file name of the object (without the file extension). Note that + `name` is read only. + """ + return self._get_name(self._path) + + def _save(self, path: str, write_mode = "wb"): + """Saves the object to `path` using `write_mode`. + + Parameters + ---------- + path : str + The path to save the object to. + write_mode : Literal["w", "wb", "a", "ab", "x", "xb"], optional + The mode with which to open the file to write to. These are the same + as `mode` for ``open``. By default ``"wb"``. + """ + if not os.path.exists(path): + dirname = os.path.dirname(path) + if dirname != '': + os.makedirs(dirname, exist_ok=True) + with open(path, write_mode) as file: + cpkl.dump(self, file, pkl.HIGHEST_PROTOCOL) + def _getpath(self, path: Optional[str]) -> str: + """Returns the specified or saved :attr:`path`. + + Parameters + ---------- + path : str, optional + Specified path. + + Returns + ------- + str + Returns the specified or saved :attr:`path`. + + Raises + ------ + ValueError + No save path provided. Raised if no path is saved or specified. + """ + path = path if path is not None or not hasattr(self, 'path') else self.path + if path is None: + raise ValueError("No save path provided.") + path = self._updatepathroot(path) + return path + @classmethod + def _updatepathroot(cls, path: Optional[str]) -> Optional[str]: + """If no file extension is provided then the class name and the ``.pkl`` + extension are appended to the file name. + + Parameters + ---------- + path : str, optional + The file path. + + Returns + ------- + str, optional + The modified file path. + """ + if path is None: + return None + split = os.path.splitext(path) + if len(split[1]) == 0: + file_name = os.path.split(split[0])[-1] + prefix = "_" if len(file_name) != 0 and file_name[-1] != "_" else "" + path += prefix + cls.__name__ + ".pkl" + return path + def save(self, path: Optional[str] = None): + """Pickles the current instance. + + Parameters + ---------- + path : str, optional + The path to pickle the instance to. If ``None`` is specified + then the attribute :attr:`path` is used instead. + By default ``None``. + + Raises + ------ + ValueError + Raised if no path specified either by the parameter `path` or the + attribute :attr:`path`. + + Notes + ----- + If no file extension is provided then the class name and the ``.pkl`` + extension are appended to the file name. + """ + self.path = self._getpath(path) + self._save(self.path) + def update_save(self, path: Optional[str] = None) -> bool: + """Pickles the current instance and retains the saved arguments if + they exist. + + Parameters + ---------- + path : str, optional + The path to pickle the instance to. If ``None`` is specified + then the attribute :attr:`path` is used instead. By default + ``None``. + + Returns + ------- + bool + ``True`` if there was an argument pickle to retain. ``False`` if + there was not an argument pickle to retain. + + Raises + ------ + ValueError + Raised if no path specified. + + Notes + ----- + If no file extension is provided then the class name and the ``.pkl`` + extension are appended to the file name. + """ + self.path = self._getpath(path) + file = open(self.path, "rb") + # Throw away the prior save: + try: + type(self)._load(file) + except: + pass + # Retain the parameters: + try: + params = pkl.load(file) + except EOFError: + # Close the file before writing to it + file.close() + self._save(self.path) + return False + else: + # Close the file before writing to it + file.close() + self._save(self.path) + SaveableObject._save(params, self.path, write_mode="ab") + return True + + @classmethod + def _load(cls, file: IO, new_path: Optional[str] = None, strict_typing: bool = True) -> "SaveableObject": + """Loads an instance from the `file`. + + Parameters + ---------- + file : IO + The file to load the instance from. + new_path : str, optional + The path to replace the previous path with. If ``None`` the `path` + is not replaced. By default ``None``. + strict_typing : bool, optional + If ``True`` then the loaded instance must be an instance of `cls`. + By default ``True``. + + Returns + ------- + `cls` + The loaded instance. + + Raises + ------ + TypeError + If `strict_typing` and the loaded instance is not an instance of + `cls`. + + Notes + ----- + ``strict_typing=True`` acts as a safety guard. Setting + ``strict_typing=False`` may increase the probability of unexpected or + uncaught errors. + """ + instance = pkl.load(file) + if strict_typing and not isinstance(instance, cls): + raise TypeError(f"The loaded instance is not an instance of {cls}.") + if new_path is not None: + instance.path = new_path + return instance + @classmethod + def load(cls, path: str, new_path: Optional[str] = None, strict_typing: bool = True) -> "SaveableObject": + """Loads a pickled instance. + + Parameters + ---------- + path : str + The path of the pickle. + new_path : str, optional + The path to replace the previous path with. If ``None`` the `path` + is not replaced. By default ``None``. + strict_typing : bool, optional + If ``True`` then the loaded instance must be an instance of `cls`. + By default ``True``. + + Returns + ------- + SaveableObject + The loaded instance. + + Raises + ------ + TypeError + If `strict_typing` and the loaded instance is not an instance of + `cls`. + + Notes + ----- + ``strict_typing=True`` acts as a safety guard. Setting + ``strict_typing=False`` may increase the probability of unexpected or + uncaught errors. + """ + path = cls._updatepathroot(path) + with open(path, "rb") as file: + return cls._load(file, new_path, strict_typing) + @classmethod + def tryload(cls, path: Optional[str], new_path: Optional[str] = None, strict_typing: bool = True) -> Union["SaveableObject", bool]: + """Attempts to :meth:`load` from the specified `path`. If the loading + fails then ``False`` is returned. + + Parameters + ---------- + path : str, optional + The path of the pickle. If ``None`` then ``False`` is returned. + new_path : str, optional + The path to replace the previous path with. If ``None`` the `path` + is not replaced. By default ``None``. + strict_typing : bool, optional + If ``True`` then the loaded instance must be an instance of `cls`. + By default ``True``. + + Returns + ------- + SaveableObject | Literal[False] + If succeeded the loaded instance, else False. + + Notes + ----- + ``strict_typing=True`` acts as a safety guard. Setting + ``strict_typing=False`` may increase the probability of unexpected or + uncaught errors. + """ + try: + return cls.load(path, new_path, strict_typing) + except (FileNotFoundError, TypeError): + return False + @classmethod + def loadif(cls, *args, **kwargs) -> Tuple["SaveableObject", bool]: + """Attempts to load from a specified `path`. If the loading fails or no + `path` is specified then a new instance of the object is generated with + the specified `*args` and `**kwargs`. + + Parameters + ---------- + *args + The arguments to pass to the initialisation on a failed + :meth:`load`. + path : str, optional + The path of the pickle, by default the parameter is not specified. + **kwargs + The keyword arguments to pass to the initialisation on a failed + :meth:`load`. + + Returns + ------- + (SaveableObject, bool) + The loaded or initialised instance followed by ``True`` if the + instance was loaded and ``False`` if the instance was initialised. + """ + bound_args = inspect.signature(cls.__init__).bind(..., *args, **kwargs) + try: + path = bound_args.arguments["path"] + except KeyError: + path = None + instance = cls.tryload(path) + if instance: + return instance, True + return cls(*args, **kwargs), False + @classmethod + def loadifparams(cls, *args, dependencies: dict = {}, **kwargs) -> Tuple["SaveableObject", bool]: + """Attempts to :meth:`load` from a specified `path`. If the loading + fails or no `path` is specified or the parameters do not match the saved + parameters then a new instance of the object is generated with the + specified `*args` and `**kwargs`. + + Parameters + ---------- + *args + The arguments to pass to the initialisation on a failed + :meth:`load`. + path : str, optional + The path of the pickle, by default the parameter is not specified. + dependencies : dict, optional, must be specified as a keyword argument + A dictionary of additional dependencies to check. + **kwargs + The keyword arguments to pass to the initialisation on a failed + :meth:`load`. + + Returns + ------- + (SaveableObject, bool) + The loaded or initialised instance followed by ``True`` if the + instance was loaded and ``False`` if the instance was initialised. + """ + bound_args = inspect.signature(cls.__init__).bind(..., *args, **kwargs) + try: + path = bound_args.arguments["path"] + except KeyError: + path = None + path = cls._updatepathroot(path) + duplicates = [] + for key in dependencies.keys(): + if key in bound_args.arguments.keys(): + duplicates.append(key) + if len(duplicates) != 0: + raise TypeError(f"The dependencies {duplicates} are also arguments. They must have different names.") + arguments = {**bound_args.arguments, **dependencies} + try: + with open(path, "rb") as file: + instance = cls._load(file) + params = pkl.load(file) + for key, value in arguments.items(): + comparison = params[key] != value + if isinstance(comparison, bool): + if comparison: + raise ValueError + else: + if not np.array_equal(params[key], value): + raise ValueError + return instance, True + except (FileNotFoundError, EOFError, ValueError, TypeError, KeyError): + instance = cls(*args, **kwargs) + if path is not None: + SaveableObject._save(arguments, path, write_mode="ab") + return instance, False \ No newline at end of file diff --git a/src/saveable_objects/checkpointing/__init__.py b/src/saveable_objects/checkpointing/__init__.py index 968b14b..4bd144d 100644 --- a/src/saveable_objects/checkpointing/__init__.py +++ b/src/saveable_objects/checkpointing/__init__.py @@ -2,4 +2,9 @@ A collection of functions for checkpointing objects. """ -from ._checkpointing import failed, succeeded \ No newline at end of file +import sys + +if sys.version_info[1] >= 10: # check if python is version 3.8 or later + from ._checkpointing import failed, succeeded +else: # import from the python version 3.7 compatable file + from ._checkpointing_3_7 import failed, succeeded \ No newline at end of file diff --git a/src/saveable_objects/checkpointing/_checkpointing.py b/src/saveable_objects/checkpointing/_checkpointing.py index d11a8c1..8e0c1e0 100644 --- a/src/saveable_objects/checkpointing/_checkpointing.py +++ b/src/saveable_objects/checkpointing/_checkpointing.py @@ -1,6 +1,6 @@ from typing import Tuple -from .._saveable_object import SaveableObject +from .. import SaveableObject def failed(load_attempt: SaveableObject | bool | Tuple[SaveableObject, bool]) -> bool: """Determines if a :class:`SaveableObject ` diff --git a/src/saveable_objects/checkpointing/_checkpointing.pyi b/src/saveable_objects/checkpointing/_checkpointing.pyi index 6a5e30b..b745907 100644 --- a/src/saveable_objects/checkpointing/_checkpointing.pyi +++ b/src/saveable_objects/checkpointing/_checkpointing.pyi @@ -1,6 +1,6 @@ from typing import Tuple -from .._saveable_object import SaveableObject +from .. import SaveableObject def failed(load_attempt: SaveableObject | bool | Tuple[SaveableObject, bool]) -> bool: """Determines if a ``SaveableObject`` ``.load()``, ``.tryload()``, diff --git a/src/saveable_objects/checkpointing/_checkpointing_3_7.py b/src/saveable_objects/checkpointing/_checkpointing_3_7.py new file mode 100644 index 0000000..869563a --- /dev/null +++ b/src/saveable_objects/checkpointing/_checkpointing_3_7.py @@ -0,0 +1,73 @@ +from typing import Tuple, Union + +from .. import SaveableObject + +def failed(load_attempt: Union[SaveableObject, bool, Tuple[SaveableObject, bool]]) -> bool: + """Determines if a :class:`SaveableObject ` + :meth:`.load() `, + :meth:`.tryload() `, + :meth:`.loadif() `, or + :meth:`.loadifparams() ` attempt + fails. + + Parameters + ---------- + load_attempt : SaveableObject | bool | (SaveableObject, bool) + The output of :meth:`load() `, + :meth:`tryload() `, + :meth:`loadif() `, or + :meth:`loadifparams() `. + + Returns + ------- + bool + Returns ``True`` is the the `load_attempt` failed, else ``False``. + + Notes + ----- + Example use: + + .. code-block:: python + + if failed(obj := SaveableObject.loadif(*args, path="filename.pkl", **kwargs)): + ... # code that generates obj + ... # code that uses obj + """ + try: + return not load_attempt[1] + except: + return not load_attempt + +def succeeded(load_attempt: Union[SaveableObject, bool, Tuple[SaveableObject, bool]]) -> bool: + """Determines if a :class:`SaveableObject ` + :meth:`.load() `, + :meth:`.tryload() `, + :meth:`.loadif() `, or + :meth:`.loadifparams() ` attempt + succeeds. + + Parameters + ---------- + load_attempt : SaveableObject | bool | (SaveableObject, bool) + The output of :meth:`load() `, + :meth:`tryload() `, + :meth:`loadif() `, or + :meth:`loadifparams() `. + + Returns + ------- + bool + Returns ``True`` is the the `load_attempt` succeeded, else ``False``. + + Notes + ----- + Example use: + + .. code-block:: python + + if succeeded(obj := SaveableObject.loadif(*args, path="filename.pkl", **kwargs)): + ... # code that uses a successfully loaded obj + else: + ... # code to run if obj failed to load + """ + return not failed(load_attempt) \ No newline at end of file diff --git a/src/saveable_objects/extensions/_extensions.py b/src/saveable_objects/extensions/_extensions.py index eb8292d..d59a497 100644 --- a/src/saveable_objects/extensions/_extensions.py +++ b/src/saveable_objects/extensions/_extensions.py @@ -1,6 +1,6 @@ from typing import Optional, TypeVar, Generic -from .._saveable_object import SaveableObject +from .. import SaveableObject from .._meta_class import SaveAfterInitMetaClass T = TypeVar("T") diff --git a/src/saveable_objects/extensions/_extensions.pyi b/src/saveable_objects/extensions/_extensions.pyi index f3b5e77..0a58f35 100644 --- a/src/saveable_objects/extensions/_extensions.pyi +++ b/src/saveable_objects/extensions/_extensions.pyi @@ -1,6 +1,6 @@ from typing import TypeVar, Generic -from .._saveable_object import SaveableObject +from .. import SaveableObject from .._meta_class import SaveAfterInitMetaClass T = TypeVar("T")