RFC: fix CI, add PyPI publish action, poetry 1.6.1 (#9)

* pypi publish action

* add testpypi

* Update .github/workflows/publish.yml

Co-authored-by: martín <MartinCura@users.noreply.github.com>

* chore(lockfile): lock with poetry 1.6.1

* ci(publish): complete test publish

* ci(publish): more specific python version

* ci(publish): complete publishing jobs

...maybe missing API token for prod publish if not doing Trusted
Publishing

* ci(publish): typo

* chore: use poetry 1.6.1

* chore: bump & changelog

* ci: small fixes

* docs(CHANGELOG): nit

* rollback version bump

* bump: 1.4.24

* ci(validate): fix check_version

* ci: rename job per ✨ feedback ✨

* ci: has no pipx

* ci(validate): fix check_version

* ci(validate): fix check_version

---------

Co-authored-by: martín <MartinCura@users.noreply.github.com>
Co-authored-by: ten <martin.cura@muttdata.ai>

Author: leo-mansini

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.4.23
+current_version = 1.4.24
 commit = False
 tag = False
 

.github/workflows/publish.yml

@@ -0,0 +1,64 @@
+name: publish
+on: push
+
+env:
+  PYTHON_VERSION: 3.10.10
+  POETRY_VERSION: 1.6.1
+
+jobs:
+  pypi-publish:
+    name: Publish Python 🐍 distribution 📦 to PyPI
+    if: ${{ startsWith(github.ref, 'refs/tags') }}
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/muttlib
+    permissions:
+      id-token: write # Mandatory for trusted publishing
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install Poetry
+        run: pipx install poetry==${{ env.POETRY_VERSION }}
+
+      - name: Install dependencies
+        run: poetry install
+      - name: Build package
+        run: poetry build
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  testpypi-publish:
+    name: Test publish Python 🐍 distribution 📦 to TestPyPI
+    if: ${{ ! startsWith(github.ref, 'refs/tags') }}
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/muttlib
+    permissions:
+      id-token: write # Mandatory for trusted publishing
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install Poetry
+        run: pipx install poetry==${{ env.POETRY_VERSION }}
+
+      - name: Install dependencies
+        run: poetry install
+      - name: Build package
+        run: poetry build
+
+      - name: Publish distribution 📦 to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip-existing: true
+          repository-url: https://test.pypi.org/legacy/

.github/workflows/test.yml

@@ -2,6 +2,7 @@ name: test
 on: [push]
 
 env:
+  POETRY_VERSION: 1.6.1
   MIN_LINE_RATE: 0.2
 
 jobs:
@@ -14,8 +15,8 @@ jobs:
         run: |
           apt update
           apt install -qy libkrb5-dev libsasl2-dev
-          pip install poetry==1.3.2
-      - uses: actions/checkout@v2
+          pip install poetry==$POETRY_VERSION
+      - uses: actions/checkout@v3
       - name: Install packages and extras
         run: poetry install --extras "forecast gsheets gdrive" --with dev
       - name: Test base imports
@@ -29,7 +30,7 @@ jobs:
       - name: Test gdrive
         run: |
           poetry run python -c 'from muttlib import gdrive'
-      - name: Run nox
+      - name: Run tests with nox
         run: |
           poetry run nox --envdir $GITHUB_WORKSPACE/.nox --sessions tests
 
@@ -42,11 +43,11 @@ jobs:
         run: |
           apt update
           apt install -y libkrb5-dev libsasl2-dev
-          pip install poetry==1.3.2
-      - uses: actions/checkout@v2
+          pip install poetry==$POETRY_VERSION
+      - uses: actions/checkout@v3
       - name: Install package
         run: poetry install -vvv --with dev -E "gdrive gsheets forecast"
-      - name: Run nox
+      - name: Run pre-commit hooks with nox
         run: poetry run nox --envdir $GITHUB_WORKSPACE/.nox --sessions precommit_hooks
 
   docstr-cov:
@@ -58,8 +59,8 @@ jobs:
         run: |
           apt update
           apt install -y libkrb5-dev libsasl2-dev
-          pip install poetry==1.3.2
-      - uses: actions/checkout@v2
+          pip install poetry==$POETRY_VERSION
+      - uses: actions/checkout@v3
       - name: Install package
         run: poetry install -vvv --with dev
       - name: Run interrogate
@@ -74,8 +75,8 @@ jobs:
         run: |
           apt update
           apt install -y libkrb5-dev libsasl2-dev
-          pip install poetry==1.3.2
-      - uses: actions/checkout@v2
+          pip install poetry==$POETRY_VERSION
+      - uses: actions/checkout@v3
       - name: Install package
         run: poetry install -vvv --with dev
       - name: Run nox
@@ -90,8 +91,8 @@ jobs:
         run: |
           apt update
           apt install -y libkrb5-dev libsasl2-dev
-          pip install poetry==1.3.2
-      - uses: actions/checkout@v2
+          pip install poetry==$POETRY_VERSION
+      - uses: actions/checkout@v3
       - name: Install package
         run: poetry install -vvv --with dev
       - name: Run mypy and check lines

.github/workflows/validate.yml

@@ -1,6 +1,9 @@
 name: validate
 on: [pull_request]
 
+env:
+  POETRY_VERSION: 1.6.1
+
 jobs:
   check_version:
     if: github.base_ref == 'master'
@@ -12,8 +15,8 @@ jobs:
         run: |
           apt update
           apt install -y libkrb5-dev libsasl2-dev
-          pip install poetry==1.3.2
-      - uses: actions/checkout@v2
+          pip install poetry==$POETRY_VERSION
+      - uses: actions/checkout@v3
       - name: Install dependencies
         run: pip install packaging
       - name: Fetch source ref
@@ -23,8 +26,10 @@ jobs:
       - name: Check version
         run: |
           lib_ver=$(git diff origin/$GITHUB_HEAD_REF origin/$GITHUB_BASE_REF -- .bumpversion.cfg | grep "current_version" | cut -d = -f 2 | xargs)
-          echo "lib_ver=$lib_ver"
-          python -c "import sys; from packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver
+          ver_new=$(echo $lib_ver | cut -f1 -d' ')
+          ver_old=$(echo $lib_ver | cut -f2 -d' ')
+          echo "new version: $ver_new (vs) old version: $ver_old"
+          python -c "import sys; from packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $ver_new $ver_old
           exit_status=$?
           if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi;
           exit $exit_status
@@ -39,13 +44,13 @@ jobs:
         run: |
           apt update
           apt install -y libkrb5-dev libsasl2-dev
-          pip install poetry==1.3.2
-      - uses: actions/checkout@v2
+          pip install poetry==$POETRY_VERSION
+      - uses: actions/checkout@v3
       - name: Fetch source ref
         run: git fetch origin $GITHUB_HEAD_REF
       - name: Fetch target ref
         run: git fetch origin $GITHUB_BASE_REF
       - name: Check changed lines
         run: |
           added_lines=$(git diff --numstat origin/$GITHUB_BASE_REF origin/$GITHUB_HEAD_REF -- CHANGELOG.md | awk '{print $1}')
-          if [ -z $added_lines ] || [ $added_lines -eq 0 ]; then echo "Changelog has not been modified" && exit 1; else echo "Changelog has been modified" && exit 0; fi;
+          if [ -z $added_lines ] || [ $added_lines -eq 0 ]; then echo "Changelog has NOT been modified" && exit 1; else echo "Changelog has been modified" && exit 0; fi;

.github/workflows/version.yml

@@ -5,8 +5,11 @@ on:
       - closed
     branches: [master]
 
+env:
+  POETRY_VERSION: 1.6.1
+
 jobs:
-  bump_version:
+  create_github_release:
     if: github.event.pull_request.merged == true
     runs-on: [self-hosted]
     container:
@@ -16,8 +19,8 @@ jobs:
         run: |
           apt update
           apt install -y libkrb5-dev libsasl2-dev
-          pip install poetry==1.3.2
-      - uses: actions/checkout@v2
+          pip install poetry==$POETRY_VERSION
+      - uses: actions/checkout@v3
       - name: Set tag output
         id: set-tag
         run: echo "::set-output name=tag_name::v$(grep current_version .bumpversion.cfg | cut -d= -f2 | xargs)"

.gitlab/issue_templates/.TODO_migrate_to_github

@@ -5,7 +5,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.4.23] - 2023-06-09
+## [1.4.24] - 2023-10-19
+
+### CI
+
+- Automatic publishing to PyPI through GitHub Actions when a tag is pushed.
+
+### Chore
+
+- Use poetry 1.6.1.
+
+## [1.4.23] - 2023-10-10
 
 ### Added
 

CHANGELOG.md

@@ -76,7 +76,7 @@ On Fedora/CentOS:
 
 #### Poetry
 
-`muttlib` is packaged and developed using [poetry](https://python-poetry.org). Follow [these steps](https://python-poetry.org/docs/#installation) to install it and get familiar with the [quickstart](https://python-poetry.org/docs/basic-usage/). Make sure to install version `1.3.2` or higher.
+`muttlib` is packaged and developed using [poetry](https://python-poetry.org). Follow [these steps](https://python-poetry.org/docs/#installation) to install it and get familiar with the [quickstart](https://python-poetry.org/docs/basic-usage/). Make sure to install the version currently specified at the start of the lockfile (e.g. `1.6.1`).
 
 ### Installation
 
@@ -188,7 +188,7 @@ Alternatively you can see the docs for the `master` branch [here.](https://mutt_
 
 ## Versioning
 
-`muttlib` uses [SemVer](https://semver.org). To keep things easy, we've included [bump2version](https://github.com/c4urself/bump2version/) as a dev dependency. You can use `poetry run bump2version minor` to increase the minor number.
+`muttlib` uses [SemVer](https://semver.org). To keep things easy, we've included [bump2version](https://github.com/c4urself/bump2version/) as a dev dependency. For example, you can use `poetry run bump2version minor` to increase the minor number.
 
 Please remember to bump the version when submitting your PR!
 

CONTRIBUTING.md

@@ -1,7 +1,12 @@
 # muttlib 🐶📚
 
-[![pypi version](https://img.shields.io/pypi/v/muttlib?color=blue)](https://pypi.org/project/muttlib/)[![pipeline status](https://gitlab.com/mutt_data/muttlib/badges/master/pipeline.svg)](https://gitlab.com/mutt_data/muttlib/-/commits/master)[![coverage report](https://gitlab.com/mutt_data/muttlib/badges/master/coverage.svg)](https://gitlab.com/mutt_data/muttlib/-/commits/master)[![docstring report](https://gitlab.com/mutt_data/muttlib/-/jobs/artifacts/master/raw/docs_coverage.svg?job=docstr-cov)](https://interrogate.readthedocs.io/en/latest/)[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
-
+[![pypi version](https://img.shields.io/pypi/v/muttlib?color=blue)](https://pypi.org/project/muttlib/)
+[![pipeline status](https://gitlab.com/mutt_data/muttlib/badges/master/pipeline.svg)](https://gitlab.com/mutt_data/muttlib/-/commits/master)
+[![coverage report](https://gitlab.com/mutt_data/muttlib/badges/master/coverage.svg)](https://gitlab.com/mutt_data/muttlib/-/commits/master)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+<!-- TODO: fix for github -->
+<!-- [![docstring report](https://gitlab.com/mutt_data/muttlib/-/jobs/artifacts/master/raw/docs_coverage.svg?job=docstr-cov)](https://interrogate.readthedocs.io/en/latest/) -->
 
 ## Description
 
@@ -30,28 +35,32 @@ Current modules:
   - [Contributing](#contributing)
   - [License](#license)
 
-### Installing from PyPi
+### Installing from PyPI
 
-_Note:_ `muttlib` is packaged and developed using [poetry](https://python-poetry.org) v1.3.2.
+_Note:_ `muttlib` is packaged and developed using [poetry](https://python-poetry.org).
 
 Base lib:
+
 ```bash
 poetry add muttlib
 ```
 
 Check below for available extras.
 
 Parquet and Feather support:
+
 ```bash
 poetry add muttlib -E pyarrow
 ```
 
 Forecast:
+
 ```bash
 poetry add muttlib -E forecast
 ```
 
 Misc DB support for dbconn:
+
 ```bash
 poetry add muttlib -E oracle
 poetry add muttlib -E hive
@@ -67,11 +76,13 @@ _Note:_ the `ibis` extra requires installing binary packages. Check [CONTRIBUTIN
 ### Installing custom branches from the repos
 
 From GitHub mirror:
+
 ```bash
 poetry add -e git+https://github.com/MuttData/muttlib.git@AWESOME_FEATURE_BRANCH#egg=muttlib
 ```
 
 From Gitlab main repo:
+
 ```bash
 poetry add -e git+https://gitlab.com/mutt_data/muttlib.git@AWESOME_FEATURE_BRANCH#egg=muttlib
 ```
@@ -85,4 +96,5 @@ See the [documentation](https://mutt_data.gitlab.io/muttlib/) to get started wit
 We appreciate for considering to help out maintaining this project. If you'd like to contribute please read our [contributing guidelines](https://mutt_data.gitlab.io/muttlib/contributing.html).
 
 ## License
+
 `muttlib` is licensed under the [Apache License 2.0](https://github.com/MuttData/muttlib/blob/master/LICENCE).