From a0e8c1707e243cbe21b74ecc328b47a3a1ed774a Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 1 Jan 2026 22:41:30 +0000 Subject: [PATCH] Switch from pykwalify to yamale --- .pre-commit-config.yaml | 1 + CHANGELOG.rst | 1 + dfetch/manifest/manifest.py | 6 +- dfetch/manifest/validate.py | 80 +++++++++++++++---- dfetch/resources/schema.yaml | 61 ++++++-------- doc/legal.rst | 45 +++++------ example/dfetch.yaml | 2 +- .../updated-project-has-dependencies.feature | 2 +- features/validate-manifest.feature | 4 +- pyproject.toml | 2 +- 10 files changed, 118 insertions(+), 86 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 740157c5..37f61e3a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: - id: end-of-file-fixer exclude: ^doc/static/uml/styles/plantuml-c4/ - id: check-yaml + args: ["--allow-multiple-documents"] - id: check-added-large-files - repo: local hooks: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bee8b814..42ff5671 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,7 @@ Release 0.11.0 (unreleased) * Add more tests and documentation for patching (#888) * Restrict ``src`` to string only in schema (#889) * Don't consider ignored files for determining local changes (#350) +* Switch from pykwalify to yamale for yaml validation Release 0.10.0 (released 2025-03-12) ==================================== diff --git a/dfetch/manifest/manifest.py b/dfetch/manifest/manifest.py index 40e10b4e..1e98b3a4 100644 --- a/dfetch/manifest/manifest.py +++ b/dfetch/manifest/manifest.py @@ -412,12 +412,12 @@ class ManifestDumper(yaml.SafeDumper): # pylint: disable=too-many-ancestors def write_line_break(self, data: Any = None) -> None: """Write a line break.""" - super().write_line_break(data) # type: ignore[unused-ignore] + super().write_line_break(data) # type: ignore[unused-ignore, no-untyped-call] if len(self.indents) == 2 and getattr(self.event, "value", "") != "version": - super().write_line_break() # type: ignore[unused-ignore] + super().write_line_break() # type: ignore[unused-ignore, no-untyped-call] if len(self.indents) == 3 and self._last_additional_break != 2: - super().write_line_break() # type: ignore[unused-ignore] + super().write_line_break() # type: ignore[unused-ignore, no-untyped-call] self._last_additional_break = len(self.indents) diff --git a/dfetch/manifest/validate.py b/dfetch/manifest/validate.py index d22ce60a..7202affa 100644 --- a/dfetch/manifest/validate.py +++ b/dfetch/manifest/validate.py @@ -1,27 +1,73 @@ """Validate manifests.""" -import logging +from typing import Any -import pykwalify -from pykwalify.core import Core, SchemaError -from yaml.scanner import ScannerError +import yamale +from yamale.validators.constraints import Constraint +from yamale.validators.validators import DefaultValidators, List import dfetch.resources +class UniqueItemsByProperty(Constraint): # type: ignore + """Ensure a certain property is unique.""" + + keywords = {"unique_property": str} + + def __init__(self, value_type: Any, kwargs: Any): + """Create Unique Items by Property constraint. + + inspired by https://github.com/23andMe/Yamale/issues/201 + """ + super().__init__(value_type, kwargs) + self.fail = "" + + def _is_valid(self, value: Any) -> bool: + + unique_property = getattr(self, "unique_property", None) + + if not value or not unique_property: + return True + + seen: dict[str, set[Any]] = { + property_name.strip(): set() for property_name in unique_property.split(",") + } + + for item in value: + for prop_name, prev_values in seen.items(): + + if prop_name in item.keys(): + property_ = item[prop_name] + + if property_ in prev_values: + self.fail = ( + f"Property '{prop_name}' is not unique." + f" Duplicate value '{property_}'" + ) + return False + + prev_values.add(property_) + return True + + def _fail(self, value: Any) -> str: + return self.fail + + def validate(path: str) -> None: """Validate the given manifest.""" - logging.getLogger(pykwalify.__name__).setLevel(logging.CRITICAL) - with dfetch.resources.schema_path() as schema_path: - try: - validator = Core(source_file=path, schema_files=[str(schema_path)]) - except ScannerError as err: - raise RuntimeError(f"{schema_path} is not a valid YAML file!") from err - - try: - validator.validate(raise_exception=True) - except SchemaError as err: - raise RuntimeError( - str(err.msg) # pyright: ignore[reportAttributeAccessIssue, reportCallIssue] - ) from err + + validators = DefaultValidators.copy() + validators[List.tag].constraints.append(UniqueItemsByProperty) + + validation_result = yamale.validate( + schema=yamale.make_schema(str(schema_path), validators=validators), + data=yamale.make_data(path), + _raise_error=False, + ) + + for result in validation_result: + if result.errors: + raise RuntimeError( + "Schema validation failed:\n- " + "\n- ".join(result.errors) + ) diff --git a/dfetch/resources/schema.yaml b/dfetch/resources/schema.yaml index 16142203..4ea25992 100644 --- a/dfetch/resources/schema.yaml +++ b/dfetch/resources/schema.yaml @@ -1,39 +1,24 @@ # Schema file (schema.yaml) -type: map -mapping: - "manifest": - type: map - required: True - mapping: - "version": { type: number, required: True} - "remotes": - required: False - type: seq - sequence: - - type: map - mapping: - "name": { type: str, required: True, unique: True} - "url-base": { type: str, required: True} - "default": { type: bool } - "projects": - required: True - type: seq - sequence: - - type: map - mapping: - "name": { type: str, required: True, unique: True} - "dst": { type: str, unique: True } - "branch": { type: str } - "tag": { type: str } - "revision": { type: str } - "url": { type: str } - "repo-path": { type: str } - "remote": { type: str } - "patch": { type: str } - "vcs": { type: str, enum: ['git', 'svn'] } - "src": { type: str } - "ignore": - required: False - type: seq - sequence: - - type: str +manifest: + version: enum('0.0', 0.0) + remotes: list(include('remote'), unique_property='name', required=False) + projects: list(include('project'), unique_property='name, dst') +--- +remote: + name: str() + url-base: str() + default: bool(required=False) + +project: + name: str() + dst: str(required=False) + branch: str(required=False) + tag: str(required=False) + revision: str(required=False) + url: str(required=False) + repo-path: str(required=False) + remote: str(required=False) + patch: str(required=False) + vcs: enum('git', 'svn', required=False) + src: str(required=False) + ignore: list(str(), required=False) diff --git a/doc/legal.rst b/doc/legal.rst index 70cafc9f..397efb0a 100644 --- a/doc/legal.rst +++ b/doc/legal.rst @@ -106,36 +106,35 @@ python-coloredlogs .. _`Colored logs`: https://coloredlogs.readthedocs.io/en/latest/ -pykwalify +yamale ~~~~~~~~~ -`pykwalify`_ is used for validating manifests. +`yamale`_ is used for validating manifests. :: - Copyright (c) 2013-2021 Johan Andersson + The MIT License - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: + Copyright (c) 23andMe, http://23andme.com - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - -.. _`pykwalify`: https://github.com/Grokzen/pykwalify + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +.. _`yamale`: https://github.com/23andMe/Yamale Colorama ~~~~~~~~ diff --git a/example/dfetch.yaml b/example/dfetch.yaml index 7f950abd..5af149b7 100644 --- a/example/dfetch.yaml +++ b/example/dfetch.yaml @@ -26,7 +26,7 @@ manifest: repo-path: tortoisesvn/code - name: tortoise-svn-tag - dst: Tests/tortoise-svn-tag/ + dst: Tests/tortoise-svn-branch-rev/ remote: sourceforge tag: version-1.13.1 src: src/*.txt diff --git a/features/updated-project-has-dependencies.feature b/features/updated-project-has-dependencies.feature index efce5311..b8f3f4e0 100644 --- a/features/updated-project-has-dependencies.feature +++ b/features/updated-project-has-dependencies.feature @@ -90,7 +90,7 @@ Feature: Updated project has dependencies Dfetch (0.10.0) SomeProject : Fetched v1 SomeProject/dfetch.yaml: Schema validation failed: - - Value 'very-invalid-manifest' is not a dict. Value path: ''. + - : 'very-invalid-manifest' is not a map """ And 'MyProject' looks like: """ diff --git a/features/validate-manifest.feature b/features/validate-manifest.feature index 517bcf3c..74466cff 100644 --- a/features/validate-manifest.feature +++ b/features/validate-manifest.feature @@ -44,6 +44,6 @@ Feature: Validate a manifest """ Dfetch (0.10.0) Schema validation failed: - - Cannot find required key 'manifest'. Path: ''. - - Key 'manifest-wrong' was not defined. Path: ''. + - manifest-wrong: Unexpected element + - manifest: Required field missing """ diff --git a/pyproject.toml b/pyproject.toml index b2a782a5..c5bfc613 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ classifiers = [ dependencies = [ "PyYAML==6.0.3", "coloredlogs==15.0.1", - "pykwalify==1.8.0", + "yamale==6.1.0", "halo==0.0.31", "colorama==0.4.6", "typing-extensions==4.15.0",