transitioned existing code into vpd.legacy and added vpd.next code for in-memory relationship graphs and kubernetes state management

Author: dbotwinick

README.md

@@ -1,6 +1,48 @@
 # python-vpd
+
 VirtualPathDictChains - Hierarchical Settings Management using YaML
 
-This is an amalgamation of existing code that has been separated into its own package. Documentation and examples coming soon. 
+This is an amalgamation of existing code that has been separated into its own package.
 
 Hosted on GitHub: https://github.com/dbotwinick/python-vpd
+
+As of version 0.9.5+, python 2.7 support is being phased out and targeting 3.9+ for python support. The legacy code
+is actually still python 2.7 compatible; however, the vpd.next code is not guaranteed to support python versions
+less than 3.9.
+
+The base legacy code for VirtualPathDictChains is still useful and provides a mechanism to find data in "chains" of
+yaml. For example, if you had a dict: "{"test": {"v1": "v1", "v2":"v2"}}", using the VPD/VirtualPathDictChain approach,
+you could query for "test/v1" and get the result "v1". This was originally designed for complex settings or
+preferences management in python applications.
+
+The "chain" part is that if a value is not found, the next source/VirtualPathDict would be searched. By having a chain,
+settings could be "merged" into a single queryable view. String values in the dicts also supported default arg
+substitution such that if a query result contained a text value of "{test/v2}", the bracketed expression
+would be used as a lookup to find that value--allowing references--which is also really useful for managing complex
+settings or preferences for an application etc.
+
+The legacy code is maintained at vpd.legacy and backwards-compatible stubs are provided in the package root. Therefore,
+the following packages still work:
+
+- vpd.arguments
+- vpd.cid
+- vpd.cmp
+- vpd.iterable
+- vpd.yaml_dict
+
+The newer generation code expands on this base concept to allow modeling arbitrary relationships among data "types"
+(expected to be yaml or yaml-like) to be able to create novel use-cases. So the next generation mechanism can also
+be used to model settings and use references to share settings/preferences. It's basically something like a simple,
+non-indexed, in-memory graph database for modeling relationships that do not require much complexity such that a
+real graph solution is warranted; however, the problem at hand benefits from describing relationships first and
+then lazily calculating some result following along the data relationships.
+
+This newer code is in "vpd.next.graph".
+
+As an additional utility basis, vpd.next.k8s provides utility mechanisms for Kubernetes. These are meant to ease
+tasks managing state (via ConfigMaps) and secrets for python applications. These can be combined with the legacy
+VirtualPathDict mechanisms as well as the vpd.next.graph mechanisms as part of maintaining complex applications
+intended to operate in a Kubernetes context. Note that the official Kubernetes python client is required for Kubernetes
+functionality.
+
+More documentation to come...

setup.py

@@ -7,7 +7,7 @@
 
 setuptools.setup(
     name="vpd",
-    version="0.9.4",
+    version="0.9.10",
     author="Drew Botwinick",
     author_email="foss@drewbotwinick.com",
     description="VirtualPathDictChains. Hierarchical, Addressable Dicts, potentially using YaML",
@@ -19,10 +19,9 @@
     classifiers=[
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',
-        'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 3',
         'License :: OSI Approved :: BSD License',
         "Operating System :: OS Independent",
     ],
-    python_requires='>=2.7, >=3.6',
+    python_requires='>=3.9',
 )

vpd/__init__.py

@@ -1,7 +1,11 @@
 # author: Drew Botwinick, Botwinick Innovations
 # license: 3-clause BSD
 
-from .arguments import arg_substitute
-from .cid import CaseInsensitiveDict
-from .iterable import flatten, is_iterable
-from .yaml_dict import VirtualPathDictChain, get_data, read_yaml, vpd_chain, vpd_data, vpd_get
+# this is a stub to provide backwards compatibility while transitioning code
+
+from .legacy.arguments import arg_substitute
+from .legacy.cid import CaseInsensitiveDict
+from .legacy.iterable import flatten, is_iterable
+from .legacy.yaml_dict import VirtualPathDictChain, get_data, vpd_chain, vpd_data, vpd_get
+
+from .next.util import read_yaml

vpd/arguments.py

@@ -1,87 +1,7 @@
 # author: Drew Botwinick, Botwinick Innovations
 # license: 3-clause BSD
-import operator
-import re
-from functools import reduce
 
-from six import iteritems, string_types
+# this is a stub to provide backwards compatibility while transitioning code
 
-_find_arg_re = re.compile(r"\{(.*?)\}")
-
-
-# region Argument Interpretation
-
-def arg_substitute(arg, data):
-    if isinstance(arg, list):
-        return [arg_substitute(a, data) for a in arg]
-    elif not isinstance(arg, string_types):
-        return None
-
-    def _replace_match(m):
-        r = data.get(m.group().strip("}{"))
-        return str(r) if r is not None else m.group()  # put back the original if we don't have a match
-
-    return _find_arg_re.sub(_replace_match, arg)
-
-
-def variable_substitute(arg, data, flatten=False):
-    if isinstance(arg, list) and flatten:
-        # TODO: switch to chain? maybe just remove flatten altogether?
-        return reduce(operator.add, (variable_substitute(a, data, flatten) for a in arg))
-    elif isinstance(arg, list):
-        return [variable_substitute(a, data, flatten) for a in arg]
-    elif not isinstance(arg, string_types):
-        return None
-
-    matches = _find_arg_re.findall(arg)
-    subs = [data.get(m) for m in matches] if matches else None
-    if matches and subs:
-        ref_type = type(subs[0])
-        types_match = all(type(s) == ref_type for s in subs)
-        if types_match and isinstance(subs[0], (list, string_types)):
-            if len(subs) > 1:
-                return reduce(operator.add, subs)
-            else:
-                return subs[0]
-    return None
-
-
-def interpret_args_yaml(args, data, default_arg_sep=' ', sub_fn=arg_substitute):
-    if args:
-        result = []
-        for s in args:
-            if isinstance(s, dict):
-                arg_sep = s.pop('SEPARATOR', default_arg_sep)  # separator between arguments
-                key_join = s.pop('KEY_JOIN', default_arg_sep)  # separator between complex arg key and resolved value
-                for k, v in iteritems(s):
-                    if isinstance(v, list):
-                        t = interpret_args_yaml(v, data, default_arg_sep)
-                        if t and arg_sep != ' ':
-                            t = arg_sep.join(t)
-                            if key_join != ' ':
-                                result.append(k + key_join + t)
-                            else:  # if the separator is ' ' then we're going to pop them on as separate tokens
-                                if k:
-                                    result.append(k)
-                                result.append(t)
-                        elif t:
-                            result += t  # t is definitely a list at this point...
-                    else:
-                        t = sub_fn(v, data)
-                        if v != t and len(t) > 0:
-                            if arg_sep != ' ':
-                                result.append(k + arg_sep + t)
-                            else:  # if the separator is ' ' then we're going to pop them on as separate tokens
-                                if k:
-                                    result.append(k)
-                                result.append(t)
-            else:
-                a = sub_fn(s, data)
-                if isinstance(a, list):
-                    result.extend(a)
-                else:
-                    result.append(a)
-        return result
-    return []
-
-# endregion
+# noinspection PyUnresolvedReferences
+from .legacy.arguments import *

vpd/cid.py

@@ -1,108 +1,7 @@
-"""
-This implementation of CaseInsensitiveDict is taken from the python requests library.
+# author: Drew Botwinick, Botwinick Innovations
+# license: 3-clause BSD
 
-Although some modifications were made, they were generally trivial. Given the ubiquitous
-nature of requests, it would've been reasonable to just depend on the upstream code, but
-it seemed inappropriate to risk upstream changes breaking code with changes across versions.
-This seemed extremely relevant because the previous version of this code that was used in
-a few places was from an older implementation that used upper-cased keys instead of
-lower-cased keys. This type of change could break augmenting code. Therefore, it made
-sense to reproduce it here to prevent version-related breaking changes.
+# this is a stub to provide backwards compatibility while transitioning code
 
-The original license and copyright are:
-
-(c) 2017 by Kenneth Reitz.
-License: Apache 2.0
-
-"""
-
-from collections import OrderedDict
-
-try:
-    from collections import Mapping, MutableMapping
-except ImportError:
-    from collections.abc import Mapping, MutableMapping
-
-
-class CaseInsensitiveDict(MutableMapping):
-    """A case-insensitive ``dict``-like object.
-
-    Implements all methods and operations of
-    ``MutableMapping`` as well as dict's ``copy``. Also
-    provides ``lower_items``.
-
-    All keys are expected to be strings. The structure remembers the
-    case of the last key to be set, and ``iter(instance)``,
-    ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()``
-    will contain case-sensitive keys. However, querying and contains
-    testing is case insensitive::
-
-        cid = CaseInsensitiveDict()
-        cid['Accept'] = 'application/json'
-        cid['aCCEPT'] == 'application/json'  # True
-        list(cid) == ['Accept']  # True
-
-    For example, ``headers['content-encoding']`` will return the
-    value of a ``'Content-Encoding'`` response header, regardless
-    of how the header name was originally stored.
-
-    If the constructor, ``.update``, or equality comparison
-    operations are given keys that have equal ``.lower()``s, the
-    behavior is undefined.
-    """
-
-    def __init__(self, data=None, **kwargs):
-        self._store = OrderedDict()
-        if data is None:
-            data = {}
-        self.update(data, **kwargs)
-
-    def __setitem__(self, key, value):
-        # Use the lowercased key for lookups, but store the actual key alongside the value.
-        self._store[key.lower()] = (key, value)
-
-    def __getitem__(self, key):
-        return self._store[key.lower()][1]
-
-    def __delitem__(self, key):
-        del self._store[key.lower()]
-
-    def __iter__(self):
-        return (cased_key for cased_key, mapped_value in self._store.values())
-
-    def __len__(self):
-        return len(self._store)
-
-    def lower_items(self):
-        """Like iteritems(), but with all lowercase keys."""
-        return ((lower_key, key_val[1]) for (lower_key, key_val) in self._store.items())
-
-    def get_item(self, key, default_key=None, default_value=None):
-        """
-        Alternative version of get(key, default) for a case-insensitive dict. This always return a (key, value) tuple.
-        The key will be returned in its original cased form. This can be useful when filtering user input to lookup
-        data. The user provided key may not match the case of the backing dict data. This allows retrieval of the
-        "authoritative" key and associated value.
-
-        :param key: key of item to retrieve
-        :param default_key: default key to respond with if key not in dict
-        :param default_value: default value to respond with if key not in dict
-        :return: (key, value) tuple
-        """
-        kl = key.lower()
-        return self._store[kl] if kl in self._store else (default_key, default_value)
-
-    def __eq__(self, other):
-        if isinstance(other, Mapping):
-            other = CaseInsensitiveDict(other)
-        else:
-            return NotImplemented
-        # Compare insensitively
-        return dict(self.lower_items()) == dict(other.lower_items())
-
-    # Copy is required
-    def copy(self):
-        return CaseInsensitiveDict(self._store.values())
-
-    def __repr__(self):
-        return str(dict(self.items()))
+# noinspection PyUnresolvedReferences
+from .legacy.cid import CaseInsensitiveDict

vpd/cmp.py

@@ -1,22 +1,7 @@
 # author: Drew Botwinick, Botwinick Innovations
 # license: 3-clause BSD
 
-try:
-    from collections import Mapping
-except ImportError:
-    from collections.abc import Mapping
+# this is a stub to provide backwards compatibility while transitioning code
 
-from .iterable import is_iterable
-
-
-def tuplize(obj, iterable_exclusions=None):
-    result = []
-    if obj is not None:
-        if isinstance(obj, Mapping):
-            result.append(tuplize(obj.items(), iterable_exclusions))
-        elif not is_iterable(obj, excluded_types=iterable_exclusions):
-            result.append(((), obj))
-        else:
-            for o in obj:
-                result.extend(tuplize(o, iterable_exclusions))
-    return tuple(result)
+# noinspection PyUnresolvedReferences
+from .legacy.cmp import tuplize