diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..e1a1300 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,87 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances of + any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, + without their explicit permission +- Other conduct that could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct. They will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +info@limebit.de. +All complaints will be reviewed and investigated and will result in an +appropriate response. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..4567271 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,54 @@ +name: 🐛 Bug report +description: Report a bug to help us improve 💪 +labels: [bug, needs triage] + +body: + - type: checkboxes + id: checks + attributes: + label: Preliminary checks + options: + - label: I have checked that this issue has not already been reported. + required: true + - label: I have checked that this issue has not been fixed in the latest version. + required: true + + - type: textarea + id: bug + attributes: + label: Describe the bug + description: A clear and concise description of what the bug is. + validations: + required: true + + - type: textarea + id: reproduce + attributes: + label: To Reproduce + description: Steps to reproduce the behavior or a simple, minimal, self-contained script that can immediately reproduce the issue. + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected behavior + description: A clear and concise description of what you expected to happen. + validations: + required: true + + - type: textarea + id: actual + attributes: + label: Actual behavior + description: A clear and concise description of what actually happens when the bug occurs. + validations: + required: true + + - type: textarea + id: context + attributes: + label: Additional context + description: Add any other context about the problem here, such as the Python version, the operating system, and installed packages. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..d383394 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: 💬 Ask a question + url: https://github.com/limebit/transformplan/discussions + about: Please ask and answer questions here. 🙋‍♀️ diff --git a/.github/ISSUE_TEMPLATE/documentation_improvement.yml b/.github/ISSUE_TEMPLATE/documentation_improvement.yml new file mode 100644 index 0000000..bdd2c34 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation_improvement.yml @@ -0,0 +1,28 @@ +name: 📚 Documentation improvement +description: Suggest an improvement for this project's documentation 📖 +labels: [documentation, needs triage] + +body: + - type: textarea + id: issue + attributes: + label: 📝 Describe the issue with the documentation + description: A clear and concise description of what the issue is. + validations: + required: true + + - type: textarea + id: suggestion + attributes: + label: 💡 Suggest what to add or change + description: Describe what you think should be added or changed in the documentation. + validations: + required: true + + - type: textarea + id: context + attributes: + label: 📎 Additional context + description: Add any other context or screenshots about the feature request here. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..0e89c4c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,36 @@ +name: 🚀 Feature request +description: Suggest an idea for this project 💡 +labels: [feat, needs triage] + +body: + - type: textarea + id: problem + attributes: + label: Is your feature request related to a problem? Please describe. + description: A clear and concise description of the problem, e.g., I'm always frustrated when [...] + validations: + required: true + + - type: textarea + id: solution + attributes: + label: Describe the solution you'd like + description: A clear and concise description of what you want to happen. + validations: + required: true + + - type: textarea + id: code + attributes: + label: Code Example (if applicable) + description: If possible, provide a code example that demonstrates your proposed solution. + validations: + required: false + + - type: textarea + id: context + attributes: + label: Additional context + description: Add any other context or screenshots about the feature request here. + validations: + required: false diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b458216..3b93e0d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,7 +16,7 @@ concurrency: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -25,10 +25,10 @@ jobs: with: enable-cache: true cache-dependency-glob: pyproject.toml - python-version: "3.12" + python-version: "3.10" - name: Install dependencies - run: uv sync --group dev + run: uv sync --group docs - name: Build docs run: uv run mkdocs build @@ -40,7 +40,7 @@ jobs: deploy: needs: build - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml index a8431f9..4e91909 100644 --- a/.github/workflows/formatting.yml +++ b/.github/workflows/formatting.yml @@ -5,7 +5,7 @@ on: jobs: format: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index d57d5ce..f05e1bf 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -5,7 +5,7 @@ on: jobs: lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml new file mode 100644 index 0000000..dc0f86a --- /dev/null +++ b/.github/workflows/pr-comment.yml @@ -0,0 +1,33 @@ +name: Write PR Comment + +on: + pull_request: + types: [opened] + +jobs: + comment: + runs-on: ubuntu-latest + permissions: + pull-requests: write + + steps: + - name: Comment on PR + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const message = ` + Thank you for your pull request! 🎉 + + While you're waiting for a review, please ensure that: + + * All required checks have passed. + * Documentation is updated if necessary. + `; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: message + }); diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..9e92dc7 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,26 @@ +name: Publish Package + +on: + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: pyproject.toml + python-version: "3.10" + + - name: Build package + run: uv build + + - name: Publish to PyPI + run: uv publish + env: + UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a796f54..cd7a24c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,10 +7,10 @@ on: jobs: test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v5 @@ -23,14 +23,14 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: uv sync --group dev + run: uv sync --group tests - name: Run tests with coverage run: uv run pytest -vv -W error --cov=transformplan --cov-report=xml --cov-report=term coverage-badge: needs: test - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: contents: write @@ -43,10 +43,10 @@ jobs: with: enable-cache: true cache-dependency-glob: pyproject.toml - python-version: "3.12" + python-version: "3.10" - name: Install dependencies - run: uv sync --group dev + run: uv sync --group tests - name: Generate coverage report run: uv run pytest --cov=transformplan --cov-report=xml --cov-report=term diff --git a/.gitignore b/.gitignore index 887ef2e..a36bb65 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,10 @@ sandbox/ */.ipynb_checkpoints .DS_Store *.egg-info -.vscode/*.log +.vscode +.devcontainer __pycache__ *.pyc +*.log .coverage site/ diff --git a/.vscode/extensions.json b/.vscode/extensions.json deleted file mode 100644 index bdc7719..0000000 --- a/.vscode/extensions.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "recommendations": [ - "ms-python.python", - "charliermarsh.ruff", - "ms-vscode-remote.remote-containers", - "EditorConfig.EditorConfig", - "eamodio.gitlens", - "ms-vsliveshare.vsliveshare", - "ms-vscode.makefile-tools", - "njpwerner.autodocstring", - "usernamehw.errorlens" - ] -} diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 1c408a2..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "editor.formatOnSave": true, - "editor.formatOnPaste": true, - "editor.autoIndent": "advanced", - "files.associations": { - "setup.cfg": "ini", - }, - "editor.rulers": [88], - "files.trimTrailingWhitespace": true, - "files.insertFinalNewline": true, - "python.terminal.activateEnvironment": true, - "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python", - "python.terminal.activateEnvInCurrentTerminal": true, - "autoDocstring.docstringFormat": "google", - "python.analysis.typeCheckingMode": "strict", - "[python]": { - "editor.defaultFormatter": "charliermarsh.ruff" - } -} diff --git a/README.md b/README.md index 344ff54..557e7bd 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

TransformPlan: Auditable Data Transformation Pipelines

- Python 3.10+ + Python 3.10+ Coverage
@@ -87,14 +87,14 @@ Total time: 0.0247s ## Available Operations -| Category | Description | Examples | -|----------|-------------|----------| -| **col_** | Column operations | `col_rename`, `col_drop`, `col_cast`, `col_add`, `col_select` | -| **math_** | Arithmetic & scaling | `math_add`, `math_multiply`, `math_standardize`, `math_minmax`, `math_clamp` | -| **rows_** | Row filtering & reshaping | `rows_filter`, `rows_drop_nulls`, `rows_sort`, `rows_unique`, `rows_pivot` | -| **str_** | String operations | `str_lower`, `str_upper`, `str_strip`, `str_replace`, `str_split` | -| **dt_** | Datetime operations | `dt_year`, `dt_month`, `dt_parse`, `dt_age_years`, `dt_diff_days` | -| **map_** | Value mapping & encoding | `map_values`, `map_discretize`, `map_onehot`, `map_ordinal` | +| Category | Description | Examples | +| ---------- | ------------------------- | ---------------------------------------------------------------------------- | +| **col\_** | Column operations | `col_rename`, `col_drop`, `col_cast`, `col_add`, `col_select` | +| **math\_** | Arithmetic & scaling | `math_add`, `math_multiply`, `math_standardize`, `math_minmax`, `math_clamp` | +| **rows\_** | Row filtering & reshaping | `rows_filter`, `rows_drop_nulls`, `rows_sort`, `rows_unique`, `rows_pivot` | +| **str\_** | String operations | `str_lower`, `str_upper`, `str_strip`, `str_replace`, `str_split` | +| **dt\_** | Datetime operations | `dt_year`, `dt_month`, `dt_parse`, `dt_age_years`, `dt_diff_days` | +| **map\_** | Value mapping & encoding | `map_values`, `map_discretize`, `map_onehot`, `map_ordinal` | ## Installation @@ -111,7 +111,7 @@ uv add transformplan ## Development Setup ```bash -make install-dev # Install with dev dependencies and pre-commit hooks +make install-dev # Install with dev dependencies make test # Run the test suite make lint # Run ruff linting and pyright type checking make format # Fix import sorting and format code diff --git a/pyproject.toml b/pyproject.toml index 09f37f7..5147aa6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,35 +1,44 @@ [project] name = "transformplan" -version = "0.0.1" +version = "0.1.0" description = "Safe, reproducible data transformations with built-in auditing and validation" readme = "README.md" requires-python = ">=3.10" -license = {text = "MIT"} +license = { text = "MIT" } keywords = ["data", "transformation", "polars", "pandas", "audit", "validation"] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", + "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Scientific/Engineering", + "Topic :: Software Development", + "Typing :: Typed", ] -dependencies = ["polars","pandas","pyarrow"] +dependencies = ["polars", "pandas", "pyarrow"] [project.urls] Homepage = "https://github.com/limebit/transformplan" Repository = "https://github.com/limebit/transformplan" [dependency-groups] -dev = [ - "pytest>=8.2.2", - "ruff>=0.4.9", - "pyright>=1.1.370,<1.1.374", +tests = ["pytest>=8.2.2", "pytest-cov>=7.0.0"] +docs = [ "mkdocs>=1.6.0", "mkdocs-material>=9.5.0", "mkdocstrings[python]>=0.24.0", - "pytest-cov>=7.0.0", +] +dev = [ + { include-group = "docs" }, + { include-group = "tests" }, + "ruff>=0.4.9", + "pyright==1.1.408", ] [build-system] diff --git a/transformplan/__init__.py b/transformplan/__init__.py index 7a91fc3..70c0515 100644 --- a/transformplan/__init__.py +++ b/transformplan/__init__.py @@ -27,11 +27,15 @@ >>> protocol.print() """ -from .chunking import ChunkedProtocol, ChunkingError, ChunkValidationResult -from .filters import Col, Filter -from .plan import TransformPlan -from .protocol import Protocol, frame_hash -from .validation import DryRunResult, SchemaValidationError, ValidationResult +from transformplan.chunking import ChunkedProtocol, ChunkingError, ChunkValidationResult +from transformplan.filters import Col, Filter +from transformplan.plan import TransformPlan +from transformplan.protocol import Protocol, frame_hash +from transformplan.validation import ( + DryRunResult, + SchemaValidationError, + ValidationResult, +) __all__ = [ "ChunkValidationResult", diff --git a/transformplan/core.py b/transformplan/core.py index 48c4d91..b3e675a 100644 --- a/transformplan/core.py +++ b/transformplan/core.py @@ -22,15 +22,20 @@ import polars as pl -from .chunking import ( +from transformplan.chunking import ( ChunkedProtocol, ChunkInfo, ChunkingError, ChunkValidationResult, validate_chunked_pipeline, ) -from .protocol import Protocol, frame_hash -from .validation import DryRunResult, ValidationResult, dry_run_schema, validate_schema +from transformplan.protocol import Protocol, frame_hash +from transformplan.validation import ( + DryRunResult, + ValidationResult, + dry_run_schema, + validate_schema, +) if TYPE_CHECKING: from collections.abc import Generator, Mapping @@ -475,7 +480,7 @@ def process_chunked( schema = dict(lazy_frame.collect_schema()) if validate: - from .validation import validate_schema + from transformplan.validation import validate_schema schema_validation = validate_schema(self._operations, schema) schema_validation.raise_if_invalid() diff --git a/transformplan/ops/__init__.py b/transformplan/ops/__init__.py index 2575702..219b696 100644 --- a/transformplan/ops/__init__.py +++ b/transformplan/ops/__init__.py @@ -16,12 +16,12 @@ provide the complete transformation API. """ -from .column import ColumnOps -from .datetime import DatetimeOps -from .map import MapOps -from .math import MathOps -from .rows import RowOps -from .string import StrOps +from transformplan.ops.column import ColumnOps +from transformplan.ops.datetime import DatetimeOps +from transformplan.ops.map import MapOps +from transformplan.ops.math import MathOps +from transformplan.ops.rows import RowOps +from transformplan.ops.string import StrOps __all__ = [ "ColumnOps", diff --git a/transformplan/ops/rows.py b/transformplan/ops/rows.py index 84936b5..db93262 100644 --- a/transformplan/ops/rows.py +++ b/transformplan/ops/rows.py @@ -38,7 +38,7 @@ import polars as pl -from ..filters import Filter +from transformplan.filters import Filter if TYPE_CHECKING: from typing import Any, Callable diff --git a/transformplan/plan.py b/transformplan/plan.py index a75b99a..e0e74d1 100644 --- a/transformplan/plan.py +++ b/transformplan/plan.py @@ -19,8 +19,8 @@ >>> result, protocol = plan.process(df) """ -from .core import TransformPlanBase -from .ops import ColumnOps, DatetimeOps, MapOps, MathOps, RowOps, StrOps +from transformplan.core import TransformPlanBase +from transformplan.ops import ColumnOps, DatetimeOps, MapOps, MathOps, RowOps, StrOps class TransformPlan( diff --git a/transformplan/.gitkeep b/transformplan/py.typed similarity index 100% rename from transformplan/.gitkeep rename to transformplan/py.typed diff --git a/uv.lock b/uv.lock index d1ab366..9403308 100644 --- a/uv.lock +++ b/uv.lock @@ -993,14 +993,15 @@ wheels = [ [[package]] name = "pyright" -version = "1.1.373" +version = "1.1.408" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nodeenv" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3b/3a/80b92c0a7e8e44de32d76607383a27ec61a157cc86b3ffc486f2d2180810/pyright-1.1.373.tar.gz", hash = "sha256:f41bcfc8b9d1802b09921a394d6ae1ce19694957b628bc657629688daf8a83ff", size = 17480, upload-time = "2024-07-24T08:50:37.474Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/b2/5db700e52554b8f025faa9c3c624c59f1f6c8841ba81ab97641b54322f16/pyright-1.1.408.tar.gz", hash = "sha256:f28f2321f96852fa50b5829ea492f6adb0e6954568d1caa3f3af3a5f555eb684", size = 4400578, upload-time = "2026-01-08T08:07:38.795Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/20/2dde33b6171ac7d7d1a018ac49f899c5ad10ad1d66e7127cca0beab65813/pyright-1.1.373-py3-none-any.whl", hash = "sha256:b805413227f2c209f27b14b55da27fe5e9fb84129c9f1eb27708a5d12f6f000e", size = 18226, upload-time = "2024-07-24T08:50:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/0c/82/a2c93e32800940d9573fb28c346772a14778b84ba7524e691b324620ab89/pyright-1.1.408-py3-none-any.whl", hash = "sha256:090b32865f4fdb1e0e6cd82bf5618480d48eecd2eb2e70f960982a3d9a4c17c1", size = 6399144, upload-time = "2026-01-08T08:07:37.082Z" }, ] [[package]] @@ -1238,7 +1239,7 @@ wheels = [ [[package]] name = "transformplan" -version = "0.0.1" +version = "0.1.0" source = { editable = "." } dependencies = [ { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -1257,6 +1258,15 @@ dev = [ { name = "pytest-cov" }, { name = "ruff" }, ] +docs = [ + { name = "mkdocs" }, + { name = "mkdocs-material" }, + { name = "mkdocstrings", extra = ["python"] }, +] +tests = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] [package.metadata] requires-dist = [ @@ -1270,11 +1280,20 @@ dev = [ { name = "mkdocs", specifier = ">=1.6.0" }, { name = "mkdocs-material", specifier = ">=9.5.0" }, { name = "mkdocstrings", extras = ["python"], specifier = ">=0.24.0" }, - { name = "pyright", specifier = ">=1.1.370,<1.1.374" }, + { name = "pyright", specifier = "==1.1.408" }, { name = "pytest", specifier = ">=8.2.2" }, { name = "pytest-cov", specifier = ">=7.0.0" }, { name = "ruff", specifier = ">=0.4.9" }, ] +docs = [ + { name = "mkdocs", specifier = ">=1.6.0" }, + { name = "mkdocs-material", specifier = ">=9.5.0" }, + { name = "mkdocstrings", extras = ["python"], specifier = ">=0.24.0" }, +] +tests = [ + { name = "pytest", specifier = ">=8.2.2" }, + { name = "pytest-cov", specifier = ">=7.0.0" }, +] [[package]] name = "typing-extensions"