From 02cdb1d6e41f9b9461cf30aeb35dfe06cf6e1187 Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Fri, 30 Jan 2026 16:12:08 -0500 Subject: [PATCH 1/3] ci: consolidate workflows and add pyright type checking - Consolidate pylint, yapf, and ruff into a single 'lint' job - Combine mypy into 'type-check' job and add pyright alongside it - Fix exit code masking by adding 'set -o pipefail' in integration tests - Add workflow_call trigger to main.yml for reuse - Simplify release.yml by calling main.yml instead of duplicating jobs - Update codecov-action to v5 --- .github/workflows/main.yml | 140 ++++++++++++++++++---------------- .github/workflows/release.yml | 108 ++++---------------------- 2 files changed, 92 insertions(+), 156 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21c0fe8..a812d9a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,6 +7,8 @@ on: pull_request: schedule: - cron: '0 0 * * *' + # Allow release.yml to call this workflow + workflow_call: jobs: # @@ -29,13 +31,47 @@ jobs: # - run: python3 --version # - # Verify "pylint" runs successfully. + # Consolidated linting job: pylint + ruff + yapf # - # Note, we need to have "drgn" installed in order to run "pylint". - # Thus, prior to running "pylint" we have to clone, build, and install + # All linters run on a single Python version since they check code quality, + # not runtime behavior. + # + lint: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - run: ./.github/scripts/install-drgn.sh + - run: python3 -m pip install pylint pytest ruff yapf + # + # Pylint checks + # + - name: Run pylint on sdb + run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring sdb + - name: Run pylint on tests + run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring tests + # + # Ruff checks (fast Python linter) + # + - name: Run ruff + run: ruff check sdb tests + # + # YAPF formatting check + # + - name: Check formatting with yapf (sdb) + run: yapf --diff --style google --recursive sdb + - name: Check formatting with yapf (tests) + run: yapf --diff --style google --recursive tests + # + # Type checking with mypy and pyright + # + # Note, we need to have "drgn" installed in order to run type checkers. + # Thus, prior to running them we have to clone, build, and install # the "drgn" from source (there's no package currently available). # - pylint: + type-check: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -43,10 +79,22 @@ jobs: with: python-version: '3.10' - run: ./.github/scripts/install-drgn.sh - - run: python3 -m pip install pylint pytest - - run: python3 -m pip install . - - run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring sdb - - run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring tests + - run: python3 -m pip install mypy pyright pytest + # + # Mypy checks + # Note: --ignore-missing-imports for tests because pytest doesn't provide stubs + # + - name: Run mypy on sdb + run: python3 -m mypy --strict --show-error-codes -p sdb + - name: Run mypy on tests + run: python3 -m mypy --strict --ignore-missing-imports --show-error-codes -p tests + # + # Pyright checks + # + - name: Run pyright on sdb + run: pyright sdb + - name: Run pyright on tests + run: pyright tests # # Verify "pytest" runs successfully on unit tests. # @@ -66,15 +114,11 @@ jobs: python-version: ${{ matrix.python-version }} - run: python3 -m pip install pytest pytest-cov - run: ./.github/scripts/install-drgn.sh - - run: python3 -m pip install . - run: pytest -v --cov sdb --cov-report xml tests/unit - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - fail_ci_if_error: false - verbose: true # # Verify "pytest" runs successfully on integration tests with crash dumps. # @@ -96,62 +140,30 @@ jobs: - run: python3 -m pip install pytest pytest-cov - run: ./.github/scripts/install-libkdumpfile.sh - run: ./.github/scripts/install-drgn.sh - - run: python3 -m pip install . - run: ./.github/scripts/download-dumps-from-gdrive.sh - run: ./.github/scripts/extract-dump.sh dump.201912060006.tar.lzma - run: ./.github/scripts/extract-dump.sh dump.202303131823.tar.gz - - run: pytest -v --cov sdb --cov-report xml tests/integration + # + # Run integration tests. Use -s flag to show size comparison output + # for record-replay tests. Use set -o pipefail to ensure test failures + # are properly reported even when piping output. + # + - name: Run integration tests + run: | + set -o pipefail + pytest -v -s --cov sdb --cov-report xml tests/integration 2>&1 | tee test_output.txt + # + # Extract and display record-replay size comparison in GitHub summary + # + - name: Add size comparison to summary + if: always() + run: | + echo "## Record-Replay Size Comparison" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -A 8 "DUMP SIZE COMPARISON" test_output.txt >> $GITHUB_STEP_SUMMARY || echo "No size comparison data found" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - fail_ci_if_error: false - verbose: true - # - # Verify "yapf" runs successfully. - # - yapf: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - run: python3 -m pip install yapf - - run: yapf --diff --style google --recursive sdb - - run: yapf --diff --style google --recursive tests - # - # Verify "ruff" runs successfully. - # - ruff: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - run: python3 -m pip install ruff - - run: ruff check sdb tests - # - # Verify "mypy" runs successfully. - # - # Note, we need to have "drgn" installed in order to run "mypy". - # Thus, prior to running "mypy" we have to clone, build, and install - # the "drgn" from source (there's no package currently available). - # - # Also note that we supply --ignore-missing-imports to the tests package - # because pytest doesn't provide stubs on typeshed. - # - mypy: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - run: ./.github/scripts/install-drgn.sh - - run: python3 -m pip install mypy pytest - - run: python3 -m pip install . - - run: python3 -m mypy --strict --show-error-codes -p sdb - - run: python3 -m mypy --strict --ignore-missing-imports --show-error-codes -p tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 168bf59..23441c9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,8 +1,8 @@ # -# Release workflow - triggered on version tag push (e.g., v0.2.0) or manually +# Release workflow - triggered on version tag push (e.g., v0.2.0) # # This workflow: -# 1. Runs all CI checks (lint, type-check, tests) +# 1. Runs all CI checks via the main workflow # 2. Builds the package (sdist + wheel) # 3. Publishes to PyPI using Trusted Publishing (OIDC) # 4. Creates a GitHub Release with auto-generated notes @@ -12,18 +12,7 @@ name: Release on: push: tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+-*' # Allow pre-release tags like v0.2.0-beta1 - workflow_dispatch: - inputs: - tag: - description: 'Tag to release (e.g., v0.2.0)' - required: true - type: string - -env: - # Use input tag for workflow_dispatch, or ref_name for tag push - RELEASE_TAG: ${{ inputs.tag || github.ref_name }} + - 'v*' # Required for PyPI Trusted Publishing (OIDC) permissions: @@ -32,87 +21,21 @@ permissions: jobs: # - # Run all CI checks before releasing + # Run all CI checks by calling the main workflow # - lint: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.tag || github.ref_name }} - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - run: ./.github/scripts/install-drgn.sh - - run: python3 -m pip install pylint pytest ruff yapf - - run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring sdb - - run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring tests - - run: ruff check sdb tests - - run: yapf --diff --style google --recursive sdb - - run: yapf --diff --style google --recursive tests - - type-check: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.tag || github.ref_name }} - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - run: ./.github/scripts/install-drgn.sh - - run: python3 -m pip install mypy pytest - - run: python3 -m mypy --strict --show-error-codes -p sdb - - run: python3 -m mypy --strict --ignore-missing-imports --show-error-codes -p tests - - test-unit: - runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: ['3.10', '3.12'] - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.tag || github.ref_name }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - run: python3 -m pip install pytest pytest-cov - - run: ./.github/scripts/install-drgn.sh - - run: pytest -v --cov sdb --cov-report xml tests/unit - - test-integration: - runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: ['3.10', '3.12'] - env: - AWS_DEFAULT_REGION: 'us-west-2' - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.tag || github.ref_name }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - run: python3 -m pip install pytest pytest-cov - - run: ./.github/scripts/install-libkdumpfile.sh - - run: ./.github/scripts/install-drgn.sh - - run: ./.github/scripts/download-dumps-from-gdrive.sh - - run: ./.github/scripts/extract-dump.sh dump.201912060006.tar.lzma - - run: ./.github/scripts/extract-dump.sh dump.202303131823.tar.gz - - run: pytest -v --cov sdb --cov-report xml tests/integration + ci: + uses: ./.github/workflows/main.yml + secrets: inherit # # Build the package # build: - needs: [lint, type-check, test-unit, test-integration] + needs: ci runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.tag || github.ref_name }} fetch-depth: 0 # Required for setuptools_scm to get version from tags - uses: actions/setup-python@v5 with: @@ -154,18 +77,19 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.tag || github.ref_name }} fetch-depth: 0 # Required for generating release notes - name: Download build artifacts uses: actions/download-artifact@v4 with: name: dist path: dist/ + - name: Get version from tag + id: version + run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + name: sdb ${{ steps.version.outputs.VERSION }} + generate_release_notes: true env: - GITHUB_TOKEN: ${{ github.token }} - run: | - gh release create '${{ env.RELEASE_TAG }}' \ - --title 'sdb ${{ env.RELEASE_TAG }}' \ - --generate-notes \ - dist/* + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From cdea942cd2da8b277a2639bdabef4f5ce6285f3f Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Fri, 30 Jan 2026 16:20:37 -0500 Subject: [PATCH 2/3] fix: add missing pip install . to install sdb and dependencies The kdumpling dependency is installed via pip install . which was missing from the lint, type-check, pytest-unit, and pytest-integration jobs in the consolidated workflow. --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a812d9a..f6feced 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,6 +45,7 @@ jobs: python-version: '3.10' - run: ./.github/scripts/install-drgn.sh - run: python3 -m pip install pylint pytest ruff yapf + - run: python3 -m pip install . # # Pylint checks # @@ -80,6 +81,7 @@ jobs: python-version: '3.10' - run: ./.github/scripts/install-drgn.sh - run: python3 -m pip install mypy pyright pytest + - run: python3 -m pip install . # # Mypy checks # Note: --ignore-missing-imports for tests because pytest doesn't provide stubs @@ -114,6 +116,7 @@ jobs: python-version: ${{ matrix.python-version }} - run: python3 -m pip install pytest pytest-cov - run: ./.github/scripts/install-drgn.sh + - run: python3 -m pip install . - run: pytest -v --cov sdb --cov-report xml tests/unit - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 @@ -140,6 +143,7 @@ jobs: - run: python3 -m pip install pytest pytest-cov - run: ./.github/scripts/install-libkdumpfile.sh - run: ./.github/scripts/install-drgn.sh + - run: python3 -m pip install . - run: ./.github/scripts/download-dumps-from-gdrive.sh - run: ./.github/scripts/extract-dump.sh dump.201912060006.tar.lzma - run: ./.github/scripts/extract-dump.sh dump.202303131823.tar.gz From 89f6a910e8d465c8e3618e0524e97cf22d6985ff Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Fri, 30 Jan 2026 16:44:36 -0500 Subject: [PATCH 3/3] fix: remove install job, fix pyright errors, add pyright config - Remove redundant 'install' job from CI (all other jobs now install sdb) - Export 'error' and 'target' submodules in sdb/__init__.py for proper module access (fixes pyright errors like sdb.target.create_object()) - Add pyright configuration to pyproject.toml with appropriate settings for drgn interoperability - Fix pyright errors: - Add None checks before iterating over .members and .enumerators - Fix return type annotations on generator functions (spa.py, zio.py) - Fix variable shadowing bug in metaslab.py (vdev -> vdev_id) - Use assert statements for guaranteed non-None after if checks --- .github/workflows/main.yml | 19 ------------------- pyproject.toml | 14 ++++++++++++++ sdb/__init__.py | 6 ++++++ sdb/command.py | 11 ++++++----- sdb/commands/spl/internal/kmem_helpers.py | 5 +++-- sdb/commands/zfs/arc.py | 4 +++- sdb/commands/zfs/metaslab.py | 5 +++-- sdb/commands/zfs/range_tree.py | 4 +++- sdb/commands/zfs/spa.py | 2 +- sdb/commands/zfs/zio.py | 2 +- sdb/session.py | 2 ++ 11 files changed, 42 insertions(+), 32 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f6feced..0716aa5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,25 +12,6 @@ on: jobs: # - # Verify the build and installation of SDB. - # - install: - runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: ['3.10', '3.12'] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - run: python3 -m pip install --upgrade pip setuptools wheel - - run: python3 -m pip install . - # - # The statement below is used for debugging the Github job. - # - - run: python3 --version - # # Consolidated linting job: pylint + ruff + yapf # # All linters run on a single Python version since they check code quality, diff --git a/pyproject.toml b/pyproject.toml index 1efc2ca..fdb052c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,20 @@ ignore_missing_imports = true module = "sdb._version" ignore_missing_imports = true +# Pyright configuration +[tool.pyright] +pythonVersion = "3.10" +typeCheckingMode = "basic" +# drgn.Object is designed to be used interchangeably with int in many contexts +# These are intentional design patterns, not bugs +reportArgumentType = "warning" +reportReturnType = "warning" +# Dynamic attributes added by decorators (e.g., input_typename_handled) +reportFunctionMemberAccess = "none" +reportAttributeAccessIssue = "warning" +# Operator type issues in tests (drgn.Type.size can be None but is always int for our types) +reportOperatorIssue = "warning" + # Pytest configuration [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/sdb/__init__.py b/sdb/__init__.py index 29dca16..7b6c1d0 100644 --- a/sdb/__init__.py +++ b/sdb/__init__.py @@ -34,6 +34,10 @@ # the modules are imported and attempt to have a cleaner # separation of concerns between modules. # +# Export submodules for direct access (e.g., sdb.target.create_object()) +from sdb import error +from sdb import target + from sdb.error import ( Error, CommandNotFoundError, @@ -88,6 +92,8 @@ '__version__', 'Address', 'All', + 'error', + 'target', 'Cast', 'Command', 'CommandArgumentsError', diff --git a/sdb/command.py b/sdb/command.py index fb4d44f..52cd4f5 100644 --- a/sdb/command.py +++ b/sdb/command.py @@ -171,9 +171,9 @@ def _init_parser(cls, name: str) -> argparse.ArgumentParser: # may be calling splitlines() for None. # if cls.__doc__: - summary = ( - inspect.getdoc( # type: ignore[union-attr] - cls).splitlines()[0].strip()) + docstring = inspect.getdoc(cls) + assert docstring is not None # guaranteed by if cls.__doc__ + summary = docstring.splitlines()[0].strip() else: summary = None return argparse.ArgumentParser(prog=name, description=summary) @@ -293,8 +293,9 @@ def help(cls, name: str) -> None: # already be included in the parser description. The second # line should be empty. Thus, we skip these two lines. # - for line in inspect.getdoc( # type: ignore[union-attr] - cls).splitlines()[2:]: + docstring = inspect.getdoc(cls) + assert docstring is not None # guaranteed by if cls.__doc__ + for line in docstring.splitlines()[2:]: print(f"{line}") print() diff --git a/sdb/commands/spl/internal/kmem_helpers.py b/sdb/commands/spl/internal/kmem_helpers.py index 7eda7ca..ea9b84f 100644 --- a/sdb/commands/spl/internal/kmem_helpers.py +++ b/sdb/commands/spl/internal/kmem_helpers.py @@ -71,8 +71,9 @@ def slab_linux_cache_source(cache: drgn.Object) -> str: def for_each_slab_flag_in_cache(cache: drgn.Object) -> Iterable[str]: assert sdb.type_canonical_name(cache.type_) == 'struct spl_kmem_cache *' flag = cache.skc_flags.value_() - for enum_entry, enum_entry_bit in cache.prog_.type( - 'enum kmc_bit').enumerators: + enum_type = cache.prog_.type('enum kmc_bit') + assert enum_type.enumerators is not None + for enum_entry, enum_entry_bit in enum_type.enumerators: if flag & (1 << enum_entry_bit): yield enum_entry.replace('_BIT', '') diff --git a/sdb/commands/zfs/arc.py b/sdb/commands/zfs/arc.py index abe0008..253a8b1 100644 --- a/sdb/commands/zfs/arc.py +++ b/sdb/commands/zfs/arc.py @@ -28,7 +28,9 @@ class ARCStats(sdb.Locator, sdb.PrettyPrinter): @staticmethod def print_stats(obj: drgn.Object) -> None: - names = [memb.name for memb in sdb.get_type('struct arc_stats').members] + arc_stats_type = sdb.get_type('struct arc_stats') + assert arc_stats_type.members is not None + names = [memb.name for memb in arc_stats_type.members] for name in names: print(f"{name:32} = {int(obj.member_(name).value.ui64)}") diff --git a/sdb/commands/zfs/metaslab.py b/sdb/commands/zfs/metaslab.py index 9546be3..3c69725 100644 --- a/sdb/commands/zfs/metaslab.py +++ b/sdb/commands/zfs/metaslab.py @@ -187,10 +187,11 @@ def from_vdev(self, vdev: drgn.Object) -> Iterable[drgn.Object]: for i in self.args.metaslab_ids: if i >= vdev.vdev_ms_count: ms_count = int(vdev.vdev_ms_count) - vdev = int(vdev.vdev_id) + vdev_id = int(vdev.vdev_id) raise sdb.CommandError( self.name, f"metaslab id {i} not valid; " - f"there are only {ms_count} metaslabs in vdev {vdev}") + f"there are only {ms_count} metaslabs in vdev {vdev_id}" + ) yield vdev.vdev_ms[i] else: for i in range(int(vdev.vdev_ms_count)): diff --git a/sdb/commands/zfs/range_tree.py b/sdb/commands/zfs/range_tree.py index 001ec56..dfdd6bb 100644 --- a/sdb/commands/zfs/range_tree.py +++ b/sdb/commands/zfs/range_tree.py @@ -73,7 +73,9 @@ class RangeSeg(sdb.Locator): @sdb.InputHandler('range_tree_t *') def from_range_tree(self, rt: drgn.Object) -> Iterable[drgn.Object]: - enum_dict = dict(sdb.get_type('enum range_seg_type').enumerators) + enum_type = sdb.get_type('enum range_seg_type') + assert enum_type.enumerators is not None + enum_dict: dict[str, int] = dict(enum_type.enumerators) # pyright: ignore range_seg_type_to_type = { enum_dict['RANGE_SEG32']: 'range_seg32_t*', enum_dict['RANGE_SEG64']: 'range_seg64_t*', diff --git a/sdb/commands/zfs/spa.py b/sdb/commands/zfs/spa.py index 4b19057..f050a22 100644 --- a/sdb/commands/zfs/spa.py +++ b/sdb/commands/zfs/spa.py @@ -78,7 +78,7 @@ def pretty_print(self, objs: Iterable[drgn.Object]) -> None: vdevs = sdb.execute_pipeline([spa], [Vdev()]) Vdev(self.arg_list).print_indented(vdevs, 5) - def no_input(self) -> drgn.Object: + def no_input(self) -> Iterable[drgn.Object]: spas = sdb.execute_pipeline( [sdb.get_object("spa_namespace_avl").address_of_()], [Avl(), sdb.Cast(["spa_t *"])], diff --git a/sdb/commands/zfs/zio.py b/sdb/commands/zfs/zio.py index 1891053..83e5114 100644 --- a/sdb/commands/zfs/zio.py +++ b/sdb/commands/zfs/zio.py @@ -128,7 +128,7 @@ def zio_has_parents(zio: drgn.Object) -> bool: return True return False - def no_input(self) -> drgn.Object: + def no_input(self) -> Iterable[drgn.Object]: if self.args.parents: raise sdb.CommandInvalidInputError( self.name, "command argument -p is not applicable " + diff --git a/sdb/session.py b/sdb/session.py index d143c3b..b261500 100644 --- a/sdb/session.py +++ b/sdb/session.py @@ -412,6 +412,8 @@ def _capture_pointer(self, obj: drgn.Object, depth: int) -> None: def _capture_struct_pointers(self, obj: drgn.Object, depth: int) -> None: """Capture pointer members of a struct.""" + if obj.type_.members is None: + return for member in obj.type_.members: if member.name: try: