Switch from pykwalify to yamale

.pre-commit-config.yaml

@@ -9,6 +9,7 @@ repos:
     -   id: end-of-file-fixer
         exclude: ^doc/static/uml/styles/plantuml-c4/
     -   id: check-yaml
+        args: ["--allow-multiple-documents"]
     -   id: check-added-large-files
 -   repo: local
     hooks:

CHANGELOG.rst

@@ -20,6 +20,7 @@ Release 0.11.0 (unreleased)
 * Add more tests and documentation for patching (#888)
 * Restrict ``src`` to string only in schema (#889)
 * Don't consider ignored files for determining local changes (#350)
+* Switch from pykwalify to yamale for yaml validation
 
 Release 0.10.0 (released 2025-03-12)
 ====================================

dfetch/manifest/manifest.py

@@ -412,12 +412,12 @@ class ManifestDumper(yaml.SafeDumper):  # pylint: disable=too-many-ancestors
 
     def write_line_break(self, data: Any = None) -> None:
         """Write a line break."""
-        super().write_line_break(data)  # type: ignore[unused-ignore]
+        super().write_line_break(data)  # type: ignore[unused-ignore, no-untyped-call]
 
         if len(self.indents) == 2 and getattr(self.event, "value", "") != "version":
-            super().write_line_break()  # type: ignore[unused-ignore]
+            super().write_line_break()  # type: ignore[unused-ignore, no-untyped-call]
 
         if len(self.indents) == 3 and self._last_additional_break != 2:
-            super().write_line_break()  # type: ignore[unused-ignore]
+            super().write_line_break()  # type: ignore[unused-ignore, no-untyped-call]
 
         self._last_additional_break = len(self.indents)

dfetch/manifest/validate.py

@@ -1,27 +1,73 @@
 """Validate manifests."""
 
-import logging
+from typing import Any
 
-import pykwalify
-from pykwalify.core import Core, SchemaError
-from yaml.scanner import ScannerError
+import yamale
+from yamale.validators.constraints import Constraint
+from yamale.validators.validators import DefaultValidators, List
 
 import dfetch.resources
 
 
+class UniqueItemsByProperty(Constraint):  # type: ignore
+    """Ensure a certain property is unique."""
+
+    keywords = {"unique_property": str}
+
+    def __init__(self, value_type: Any, kwargs: Any):
+        """Create Unique Items by Property constraint.
+
+        inspired by https://github.com/23andMe/Yamale/issues/201
+        """
+        super().__init__(value_type, kwargs)
+        self.fail = ""
+
+    def _is_valid(self, value: Any) -> bool:
+
+        unique_property = getattr(self, "unique_property", None)
+
+        if not value or not unique_property:
+            return True
+
+        seen: dict[str, set[Any]] = {
+            property_name.strip(): set() for property_name in unique_property.split(",")
+        }
+
+        for item in value:
+            for prop_name, prev_values in seen.items():
+
+                if prop_name in item.keys():
+                    property_ = item[prop_name]
+
+                    if property_ in prev_values:
+                        self.fail = (
+                            f"Property '{prop_name}' is not unique."
+                            f" Duplicate value '{property_}'"
+                        )
+                        return False
+
+                    prev_values.add(property_)
+        return True
+
+    def _fail(self, value: Any) -> str:
+        return self.fail
+
+
 def validate(path: str) -> None:
     """Validate the given manifest."""
-    logging.getLogger(pykwalify.__name__).setLevel(logging.CRITICAL)
-
     with dfetch.resources.schema_path() as schema_path:
-        try:
-            validator = Core(source_file=path, schema_files=[str(schema_path)])
-        except ScannerError as err:
-            raise RuntimeError(f"{schema_path} is not a valid YAML file!") from err
-
-    try:
-        validator.validate(raise_exception=True)
-    except SchemaError as err:
-        raise RuntimeError(
-            str(err.msg)  # pyright: ignore[reportAttributeAccessIssue, reportCallIssue]
-        ) from err
+
+        validators = DefaultValidators.copy()
+        validators[List.tag].constraints.append(UniqueItemsByProperty)
+
+        validation_result = yamale.validate(
+            schema=yamale.make_schema(str(schema_path), validators=validators),
+            data=yamale.make_data(path),
+            _raise_error=False,
+        )
+
+        for result in validation_result:
+            if result.errors:
+                raise RuntimeError(
+                    "Schema validation failed:\n- " + "\n- ".join(result.errors)
+                )

dfetch/resources/schema.yaml

@@ -1,39 +1,24 @@
 # Schema file (schema.yaml)
-type: map
-mapping:
-  "manifest":
-    type: map
-    required: True
-    mapping:
-      "version": { type: number, required: True}
-      "remotes":
-          required: False
-          type: seq
-          sequence:
-             - type: map
-               mapping:
-                   "name": { type: str, required: True, unique: True}
-                   "url-base": { type: str, required: True}
-                   "default": { type: bool }
-      "projects":
-          required: True
-          type: seq
-          sequence:
-             - type: map
-               mapping:
-                   "name": { type: str, required: True, unique: True}
-                   "dst": { type: str, unique: True }
-                   "branch": { type: str }
-                   "tag": { type: str }
-                   "revision": { type: str }
-                   "url": { type: str }
-                   "repo-path": { type: str }
-                   "remote": { type: str }
-                   "patch": { type: str }
-                   "vcs": { type: str, enum: ['git', 'svn'] }
-                   "src": { type: str }
-                   "ignore":
-                        required: False
-                        type: seq
-                        sequence:
-                          - type: str
+manifest:
+    version: enum('0.0', 0.0)
+    remotes: list(include('remote'), unique_property='name', required=False)
+    projects: list(include('project'), unique_property='name, dst')
+---
+remote:
+    name: str()
+    url-base: str()
+    default: bool(required=False)
+
+project:
+    name: str()
+    dst: str(required=False)
+    branch: str(required=False)
+    tag: str(required=False)
+    revision: str(required=False)
+    url: str(required=False)
+    repo-path: str(required=False)
+    remote: str(required=False)
+    patch: str(required=False)
+    vcs: enum('git', 'svn', required=False)
+    src: str(required=False)
+    ignore: list(str(), required=False)

doc/legal.rst

@@ -106,36 +106,35 @@ python-coloredlogs
 
 .. _`Colored logs`: https://coloredlogs.readthedocs.io/en/latest/
 
-pykwalify
+yamale
 ~~~~~~~~~
-`pykwalify`_ is used for validating manifests.
+`yamale`_ is used for validating manifests.
 
 ::
 
-    Copyright (c) 2013-2021 Johan Andersson
+    The MIT License
 
-    Permission is hereby granted, free of charge, to any person
-    obtaining a copy of this software and associated documentation
-    files (the "Software"), to deal in the Software without
-    restriction, including without limitation the rights to use,
-    copy, modify, merge, publish, distribute, sublicense, and/or sell
-    copies of the Software, and to permit persons to whom the
-    Software is furnished to do so, subject to the following
-    conditions:
+    Copyright (c) 23andMe, http://23andme.com
 
-    The above copyright notice and this permission notice shall be
-    included in all copies or substantial portions of the Software.
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
 
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-    OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-    HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-    WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-    OTHER DEALINGS IN THE SOFTWARE.
-
-.. _`pykwalify`: https://github.com/Grokzen/pykwalify
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+
+.. _`yamale`: https://github.com/23andMe/Yamale
 
 Colorama
 ~~~~~~~~

example/dfetch.yaml

@@ -26,7 +26,7 @@ manifest:
     repo-path: tortoisesvn/code
 
   - name: tortoise-svn-tag
-    dst: Tests/tortoise-svn-tag/
+    dst: Tests/tortoise-svn-branch-rev/
     remote: sourceforge
     tag: version-1.13.1
     src: src/*.txt

features/updated-project-has-dependencies.feature

@@ -90,7 +90,7 @@ Feature: Updated project has dependencies
             Dfetch (0.10.0)
               SomeProject         : Fetched v1
             SomeProject/dfetch.yaml: Schema validation failed:
-             - Value 'very-invalid-manifest' is not a dict. Value path: ''.
+            -  : 'very-invalid-manifest' is not a map
             """
         And 'MyProject' looks like:
             """

features/validate-manifest.feature

@@ -44,6 +44,6 @@ Feature: Validate a manifest
             """
             Dfetch (0.10.0)
             Schema validation failed:
-             - Cannot find required key 'manifest'. Path: ''.
-             - Key 'manifest-wrong' was not defined. Path: ''.
+            - manifest-wrong: Unexpected element
+            - manifest: Required field missing
             """

pyproject.toml

@@ -41,7 +41,7 @@ classifiers = [
 dependencies = [
     "PyYAML==6.0.3",
     "coloredlogs==15.0.1",
-    "pykwalify==1.8.0",
+    "yamale==6.1.0",
     "halo==0.0.31",
     "colorama==0.4.6",
     "typing-extensions==4.15.0",