General Python code standards

.github/workflows/annotate-modified-PY.yml

@@ -0,0 +1,47 @@
+# this will run a linter (pylint) on all new or modified Python files
+# and output annotations for linting errors to the files in GitHub
+
+name: Lint all new or modified Python files
+
+on: workflow_call
+
+jobs:
+  lint-changed-python:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: "actions/checkout@v3"
+
+      - run: | # download the config file from this repo to the repo the action is being run in
+          curl -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' \
+          -H 'Accept: application/vnd.github.v4.raw' \
+          -O \
+          -L https://api.github.com/repos/Toronto-Big-Data-Innovation-Team/.github/contents/pylint.toml
+
+      - uses: "actions/setup-python@v4"
+        with:
+          python-version: "3.10"
+
+      - name: Install pylint
+        run: "pip install pylint~=2.17.0" # i.e. the latest version - upgrade as necessary
+
+      - name: Find changed .py files # they appear as ${{ steps.files.outputs }} in a later step (a space-separated list of filepaths)
+        id: files
+        uses: Ana06/get-changed-files@v2.2.0 # there are several forks of this; I picked the one that seemed most up to date; others may suffice as well
+        with:
+          filter: '*\.py'
+
+      - name: Setup Pylint annotations
+        uses: rodrigo-nogues/pylint-annotations@v1
+
+      - name: Lint
+        if: steps.files.outputs.modified != '' || steps.files.outputs.added != ''
+        shell: bash -l {0}
+        run: pylint --output-format=json:py-annotations.json --rcfile pylint.toml ${{ steps.files.outputs.modified }} ${{ steps.files.outputs.added }}
+
+      - name: Annotate # reads that JSON file and does specal GitHub magic
+        if: steps.files.outputs.modified != '' || steps.files.outputs.added != ''
+        uses: yuzutech/annotations-action@v0.4.0
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"
+          title: "SQLFluff Lint"
+          input: "./py-annotations.json"

pylint.toml

@@ -0,0 +1,126 @@
+###############################################################################
+#                                  PYLINT                                     #
+###############################################################################
+[tool.pylint.main]
+# Return non-zero exit code if any of these messages/categories are detected.
+# fail-on =
+
+# Specify a score threshold under which the program will exit with error.
+fail-under = 7
+
+# Files or directories to be skipped. They should be base names, not paths.
+ignore = [".ipynb_checkpoints"]
+# ignore-patterns = ["^\\.#"]
+# ignore-paths =
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use, and will cap the count on Windows to
+# avoid hangs.
+jobs = 0
+
+# Minimum Python version to use for version dependent checks. Will default to the
+# version used to run pylint.
+py-version = "3.9"
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode = true
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+init-hook = "import sys; sys.path.append('/home/airflow/airflow_venv/lib/python3.9/site-packages')"
+
+[tool.pylint.basic]
+# Naming style matching correct argument names.
+argument-naming-style = "snake_case"
+
+# Naming style matching correct attribute names.
+attr-naming-style = "snake_case"
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names = ["foo", "bar", "baz", "toto", "tutu", "tata"]
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style = "any"
+
+# Naming style matching correct class constant names.
+class-const-naming-style = "UPPER_CASE"
+
+# Naming style matching correct class names.
+class-naming-style = "PascalCase"
+
+# Naming style matching correct constant names.
+const-naming-style = "UPPER_CASE"
+
+# Naming style matching correct function names.
+function-naming-style = "snake_case"
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names = ["geoprocessing", "ds", "df", "dt", "datatype"]
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint = true
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style = "any"
+
+# Naming style matching correct method names.
+method-naming-style = "snake_case"
+
+# Naming style matching correct module names.
+module-naming-style = "snake_case"
+
+# Naming style matching correct variable names.
+variable-naming-style = "snake_case"
+
+[tool.pylint.design]
+# Maximum number of arguments for function / method.
+max-args = 5
+
+# Maximum number of locals for function / method body.
+max-locals = 20
+
+[tool.pylint.exceptions]
+# Exceptions that will emit a warning when caught.
+overgeneral-exceptions = ["builtins.BaseException", "builtins.Exception"]
+
+[tool.pylint.format]
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines = "^\\s*(# )?<?https?://\\S+>?$"
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren = 4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string = "    "
+
+# Maximum number of characters on a single line.
+max-line-length = 80
+
+[tool.pylint."messages control"]
+# Disable the message, report, category or checker with the given id(s).
+disable = ["raw-checker-failed", "bad-inline-option", "locally-disabled", "file-ignored", "suppressed-message", "useless-suppression", "deprecated-pragma", "use-symbolic-message-instead", "unrecognized-option"]
+
+[tool.pylint.miscellaneous]
+# List of note tags to take in consideration, separated by a comma.
+notes = ["FIXME", "XXX", "TODO"]
+
+[tool.pylint.spelling]
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions = 4
+
+# List of comma separated words that should be considered directives if they
+# appear at the beginning of a comment and should not be checked.
+spelling-ignore-comment-directives = "fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:"
+
+# List of comma separated words that should not be checked.
+# spelling-ignore-words =
+
+[tool.pylint.variables]
+# A regular expression matching the name of dummy variables (i.e. expected to not
+# be used).
+dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_"
+
+# Argument names that match this expression will be ignored.
+ignored-argument-names = "_.*|^ignored_|^unused_"