diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7871bbe..5b6c477 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -12,8 +12,8 @@ jobs: continue-on-error: ${{ matrix.experimental }} strategy: matrix: - os: [ubuntu-latest] - python-version: ["3.10", "3.11", "3.12"] + os: [windows-latest] + python-version: ["3.10"] experimental: [false] # Include experimental or bleeding-edge releases. # Windows is not included as it can be unreliable, e.g. @@ -30,12 +30,6 @@ jobs: - os: ubuntu-latest python-version: "3.13" experimental: true - - os: macos-latest - python-version: "3.13" - experimental: true - - os: windows-latest - python-version: "3.13" - experimental: true steps: - name: "check out repository" uses: "actions/checkout@v4" diff --git a/src/demystify/demystify.py b/src/demystify/demystify.py index 0de2d7f..2d6e8ef 100644 --- a/src/demystify/demystify.py +++ b/src/demystify/demystify.py @@ -1,8 +1,4 @@ -# -*- coding: utf-8 -*- - -# pylint: disable=R0917 - -"""Demystify +"""Demystify. Demystify is a utility for static analysis of file format identification tool results. The utility analyses all aspects of a format @@ -27,6 +23,8 @@ rsync style lists for filtering on disk. """ +# pylint: disable=R0917 + import argparse import configparser as ConfigParser import logging @@ -35,18 +33,80 @@ import sqlite3 import sys import time +from pathlib import Path + + +sys.dont_write_bytecode = True + +if os.name != "nt": + try: + from denylist_template import denylist_template + from libs import version + from libs.DemystifyAnalysisClass import AnalysisError, DemystifyAnalysis + from libs.HandleDenylistClass import HandleDenylist + from libs.IdentifyDatabase import IdentifyDB + from libs.outputhandlers.htmloutputclass import FormatAnalysisHTMLOutput + from libs.outputhandlers.roguesgalleryoutputclass import rogueoutputclass + from libs.outputhandlers.textoutputclass import FormatAnalysisTextOutput + except ModuleNotFoundError: + try: + from src.demystify.denylist_template import denylist_template + from src.demystify.libs import version + from src.demystify.libs.DemystifyAnalysisClass import ( + AnalysisError, + DemystifyAnalysis, + ) + from src.demystify.libs.HandleDenylistClass import HandleDenylist + from src.demystify.libs.IdentifyDatabase import IdentifyDB + from src.demystify.libs.outputhandlers.htmloutputclass import ( + FormatAnalysisHTMLOutput, + ) + from src.demystify.libs.outputhandlers.roguesgalleryoutputclass import ( + rogueoutputclass, + ) + from src.demystify.libs.outputhandlers.textoutputclass import ( + FormatAnalysisTextOutput, + ) + except ModuleNotFoundError: + from demystify.denylist_template import denylist_template + from demystify.libs import version + from demystify.libs.DemystifyAnalysisClass import ( + AnalysisError, + DemystifyAnalysis, + ) + from demystify.libs.HandleDenylistClass import HandleDenylist + from demystify.libs.IdentifyDatabase import IdentifyDB + from demystify.libs.outputhandlers.htmloutputclass import ( + FormatAnalysisHTMLOutput, + ) + from demystify.libs.outputhandlers.roguesgalleryoutputclass import ( + rogueoutputclass, + ) + from demystify.libs.outputhandlers.textoutputclass import ( + FormatAnalysisTextOutput, + ) + + # pylint: disable=E0401; unable to import (not needed for local tests). + # pylint: disable=C0413; import not at top of file. + sys.path.insert(0, str(Path("./src/demystify/sqlitefid/src/sqlitefid/"))) + print( + "PATH: %s %s", + os.path.exists(str(Path("./src/demystify/sqlitefid/src/sqlitefid/"))), + file=sys.stderr, + ) + import sqlitefid # noqa: E402 + +else: + from .denylist_template import denylist_template + from .libs import version + from .libs.DemystifyAnalysisClass import AnalysisError, DemystifyAnalysis + from .libs.HandleDenylistClass import HandleDenylist + from .libs.IdentifyDatabase import IdentifyDB + from .libs.outputhandlers.htmloutputclass import FormatAnalysisHTMLOutput + from .libs.outputhandlers.roguesgalleryoutputclass import rogueoutputclass + from .libs.outputhandlers.textoutputclass import FormatAnalysisTextOutput + from .sqlitefid.src.sqlitefid import sqlitefid -from .denylist_template import denylist_template -from .libs import version -from .libs.DemystifyAnalysisClass import AnalysisError, DemystifyAnalysis -from .libs.HandleDenylistClass import HandleDenylist -from .libs.IdentifyDatabase import IdentifyDB - -# Custom output handlers -from .libs.outputhandlers.htmloutputclass import FormatAnalysisHTMLOutput -from .libs.outputhandlers.roguesgalleryoutputclass import rogueoutputclass -from .libs.outputhandlers.textoutputclass import FormatAnalysisTextOutput -from .sqlitefid.src.sqlitefid import sqlitefid logging.basicConfig( format="%(asctime)-15s %(levelname)s :: %(filename)s:%(lineno)s:%(funcName)s() :: %(message)s", diff --git a/src/demystify/denylist_template.py b/src/demystify/denylist_template.py index e03001c..2f9dccf 100644 --- a/src/demystify/denylist_template.py +++ b/src/demystify/denylist_template.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- - +"""Denylist template.""" denylist_template = """######################################################################## # Denylist and Rogues output configuration for demystify. diff --git a/src/demystify/i18n/internationalstrings.py b/src/demystify/i18n/internationalstrings.py index c7bf2a4..1fc3b4b 100644 --- a/src/demystify/i18n/internationalstrings.py +++ b/src/demystify/i18n/internationalstrings.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +"""Analysis strings for i18n.""" class AnalysisStringsEN: diff --git a/src/demystify/libs/AnalysisQueriesClass.py b/src/demystify/libs/AnalysisQueriesClass.py index 506e38e..dc7ae36 100644 --- a/src/demystify/libs/AnalysisQueriesClass.py +++ b/src/demystify/libs/AnalysisQueriesClass.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- - -"""Analysis Queries to be used by Demystify to extract as much +"""Analysis Queries to be used by Demystify to extract as much information as possible from a format identification extract. """ diff --git a/src/demystify/libs/AnalysisResultsClass.py b/src/demystify/libs/AnalysisResultsClass.py index 2308023..d9a0d47 100644 --- a/src/demystify/libs/AnalysisResultsClass.py +++ b/src/demystify/libs/AnalysisResultsClass.py @@ -1,8 +1,17 @@ -# -*- coding: utf-8 -*- - """Analysis results module.""" -from .version import get_version +import os + +if os.name != "nt": + try: + from libs import version + except ModuleNotFoundError: + try: + from src.demystify.libs import version + except ModuleNotFoundError: + from demystify.libs import version +else: + from .version import version class AnalysisResults: @@ -119,4 +128,4 @@ def __init__(self): @staticmethod def __version__(): - return get_version() + return version.get_version() diff --git a/src/demystify/libs/DemystifyAnalysisClass.py b/src/demystify/libs/DemystifyAnalysisClass.py index 50b7248..84471b0 100644 --- a/src/demystify/libs/DemystifyAnalysisClass.py +++ b/src/demystify/libs/DemystifyAnalysisClass.py @@ -1,23 +1,58 @@ -# -*- coding: utf-8 -*- +"""Demystify analysis class for collecting demystify results.""" # pylint: disable=R0917; too-many positional args. import logging +import os +import sys from collections import Counter from configparser import NoOptionError - -try: - # Required for imports calling from repository root. - from src.demystify.pathlesstaken.src.pathlesstaken import pathlesstaken -except ModuleNotFoundError: - # Required for Pypi install. - from ..pathlesstaken.src.pathlesstaken import pathlesstaken - -from . import AnalysisResultsClass, version -from .AnalysisQueriesClass import AnalysisQueries -from .DenylistQueriesClass import DenylistQueries -from .HandleDenylistClass import HandleDenylist -from .RoguesQueriesClass import RogueQueries +from pathlib import Path + +if os.name != "nt": + try: + import AnalysisResultsClass + import version + from AnalysisQueriesClass import AnalysisQueries + from DenylistQueriesClass import DenylistQueries + from HandleDenylistClass import HandleDenylist + from RoguesQueriesClass import RogueQueries + except ModuleNotFoundError: + try: + from src.demystify.libs import AnalysisResultsClass, version + from src.demystify.libs.AnalysisQueriesClass import AnalysisQueries + from src.demystify.libs.DenylistQueriesClass import DenylistQueries + from src.demystify.libs.HandleDenylistClass import HandleDenylist + from src.demystify.libs.RoguesQueriesClass import RogueQueries + except ModuleNotFoundError: + from demystify.libs import AnalysisResultsClass, version + from demystify.libs.AnalysisQueriesClass import AnalysisQueries + from demystify.libs.DenylistQueriesClass import DenylistQueries + from demystify.libs.HandleDenylistClass import HandleDenylist + from demystify.libs.RoguesQueriesClass import RogueQueries + + # pylint: disable=E0401; unable to import (not needed for local tests). + # pylint: disable=C0413; import not at top of file. + sys.path.insert(0, str(Path("./src/demystify/pathlesstaken/src/pathlesstaken/"))) + print( + "PATH: %s %s", + os.path.exists(str(Path("./src/demystify/pathlesstaken/src/pathlesstaken/"))), + file=sys.stderr, + ) + import pathlesstaken # noqa: E402 +else: + try: + # Required for imports calling from repository root. + from src.demystify.pathlesstaken.src.pathlesstaken import pathlesstaken + except ModuleNotFoundError: + # Required for Pypi install. + from ..pathlesstaken.src.pathlesstaken import pathlesstaken + + from . import AnalysisResultsClass, version + from .AnalysisQueriesClass import AnalysisQueries + from .DenylistQueriesClass import DenylistQueries + from .HandleDenylistClass import HandleDenylist + from .RoguesQueriesClass import RogueQueries class AnalysisError(Exception): diff --git a/src/demystify/libs/DenylistQueriesClass.py b/src/demystify/libs/DenylistQueriesClass.py index ae71f22..27b325e 100644 --- a/src/demystify/libs/DenylistQueriesClass.py +++ b/src/demystify/libs/DenylistQueriesClass.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Denylist queries for filtering denylist items from a format identification extract. """ diff --git a/src/demystify/libs/HandleDenylistClass.py b/src/demystify/libs/HandleDenylistClass.py index d869a71..a7fb927 100644 --- a/src/demystify/libs/HandleDenylistClass.py +++ b/src/demystify/libs/HandleDenylistClass.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Denylist module.""" # [denylist] diff --git a/src/demystify/libs/IdentifyDatabase.py b/src/demystify/libs/IdentifyDatabase.py index 6645af7..6487c64 100644 --- a/src/demystify/libs/IdentifyDatabase.py +++ b/src/demystify/libs/IdentifyDatabase.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- - -"""IdentifyDatabase +"""IdentifyDatabase Enables identification of a sqlite3 database to ensure that the input data for the calling application is correct. @@ -14,7 +12,7 @@ class IdentifyDB: UNKNOWN_DB = "unknown" # 'SQLite format 3' - sqlite3 = b"\x53\x51\x4C\x69\x74\x65\x20\x66\x6F\x72\x6D\x61\x74\x20\x33" + sqlite3 = b"\x53\x51\x4c\x69\x74\x65\x20\x66\x6f\x72\x6d\x61\x74\x20\x33" def identify_export(self, database_path): """Reads an input file and checks whether it is an sqlite3 diff --git a/src/demystify/libs/RoguesQueriesClass.py b/src/demystify/libs/RoguesQueriesClass.py index e162cc4..02fca2d 100644 --- a/src/demystify/libs/RoguesQueriesClass.py +++ b/src/demystify/libs/RoguesQueriesClass.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Rogue Queries for extracting information about Rogues and Heroes from a format identification extract. """ diff --git a/src/demystify/libs/outputhandlers/htmloutputclass.py b/src/demystify/libs/outputhandlers/htmloutputclass.py index d71ccb4..50c1015 100644 --- a/src/demystify/libs/outputhandlers/htmloutputclass.py +++ b/src/demystify/libs/outputhandlers/htmloutputclass.py @@ -1,17 +1,32 @@ -# -*- coding: utf-8 -*- +"""HTML Output Class for HTML formatted results.""" # pylint: disable=R0917; too-many positional args. import logging +import os import re -try: - from src.demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN - from src.demystify.libs import DemystifyAnalysisClass -except ModuleNotFoundError: - # Needed to run from root dir. - from demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN - from demystify.libs import DemystifyAnalysisClass +if os.name != "nt": + try: + from i18n.internationalstrings import AnalysisStringsEN as IN_EN + from libs import DemystifyAnalysisClass + except ModuleNotFoundError: + try: + from src.demystify.i18n.internationalstrings import ( + AnalysisStringsEN as IN_EN, + ) + from src.demystify.libs import DemystifyAnalysisClass + except ModuleNotFoundError: + from demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN + from demystify.libs import DemystifyAnalysisClass +else: + try: + from src.demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN + from src.demystify.libs import DemystifyAnalysisClass + except ModuleNotFoundError: + # Needed to run from root dir. + from demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN + from demystify.libs import DemystifyAnalysisClass # NONE_REPLACE_DEBUG is a logging prompt to help us to understand what # needs changing around 'None'/null values from the database. These diff --git a/src/demystify/libs/outputhandlers/roguesgalleryoutputclass.py b/src/demystify/libs/outputhandlers/roguesgalleryoutputclass.py index 19abc4b..4ddf50e 100644 --- a/src/demystify/libs/outputhandlers/roguesgalleryoutputclass.py +++ b/src/demystify/libs/outputhandlers/roguesgalleryoutputclass.py @@ -1,13 +1,23 @@ -# -*- coding: utf-8 -*- +"""Rogue output.""" import configparser as ConfigParser import logging +import os -try: - from src.demystify.libs.HandleDenylistClass import HandleDenylist -except ModuleNotFoundError: - # Needed for PyPi import. - from demystify.libs.HandleDenylistClass import HandleDenylist +if os.name != "nt": + try: + from libs.HandleDenylistClass import HandleDenylist + except ModuleNotFoundError: + try: + from src.demystify.libs.HandleDenylistClass import HandleDenylist + except ModuleNotFoundError: + from demystify.libs.HandleDenylistClass import HandleDenylist +else: + try: + from src.demystify.libs.HandleDenylistClass import HandleDenylist + except ModuleNotFoundError: + # Needed for PyPi import. + from demystify.libs.HandleDenylistClass import HandleDenylist class rogueoutputclass: diff --git a/src/demystify/libs/outputhandlers/textoutputclass.py b/src/demystify/libs/outputhandlers/textoutputclass.py index f5d02e9..c389711 100644 --- a/src/demystify/libs/outputhandlers/textoutputclass.py +++ b/src/demystify/libs/outputhandlers/textoutputclass.py @@ -1,15 +1,31 @@ -# -*- coding: utf-8 -*- +"""Text output for output formatted as plain-text.""" import logging +import os import re -try: - from src.demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN - from src.demystify.libs import DemystifyAnalysisClass -except ModuleNotFoundError: - # Needed to run from root dir. - from demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN - from demystify.libs import DemystifyAnalysisClass +if os.name != "nt": + try: + from i18n.internationalstrings import AnalysisStringsEN as IN_EN + from libs import DemystifyAnalysisClass + except ModuleNotFoundError: + try: + from src.demystify.i18n.internationalstrings import ( + AnalysisStringsEN as IN_EN, + ) + from src.demystify.libs import DemystifyAnalysisClass + except ModuleNotFoundError: + # Needed to run from root dir. + from demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN + from demystify.libs import DemystifyAnalysisClass +else: + try: + from src.demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN + from src.demystify.libs import DemystifyAnalysisClass + except ModuleNotFoundError: + # Needed to run from root dir. + from demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN + from demystify.libs import DemystifyAnalysisClass # NONE_REPLACE_DEBUG is a logging prompt to help us to understand what # needs changing around 'None'/null values from the database. These diff --git a/src/demystify/libs/version.py b/src/demystify/libs/version.py index f04e893..eb08880 100644 --- a/src/demystify/libs/version.py +++ b/src/demystify/libs/version.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Version.py Helper class to return the version of this application to the caller. diff --git a/src/demystify/pathlesstaken b/src/demystify/pathlesstaken index be8c2ae..bfe8dc6 160000 --- a/src/demystify/pathlesstaken +++ b/src/demystify/pathlesstaken @@ -1 +1 @@ -Subproject commit be8c2ae8d23da13ae3fc915f00dc78eaede54fbe +Subproject commit bfe8dc606494a5044a7aeaadb62125addedcfe14 diff --git a/src/demystify/sqlitefid b/src/demystify/sqlitefid index 6745d7d..bd9060c 160000 --- a/src/demystify/sqlitefid +++ b/src/demystify/sqlitefid @@ -1 +1 @@ -Subproject commit 6745d7de73a9414fc866f863f6284519cf32fe11 +Subproject commit bd9060cac97fb78c0cb3f2829595a5bcfc28f84a diff --git a/tox.ini b/tox.ini index db29427..9dde62b 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ commands = pre-commit run --all-files [testenv:linting-show] basepython = python3 deps = pre-commit -commands = pre-commit run --all-files --show-diff-on-failure +commands = pre-commit run --all-files [testenv:coverage] deps = -r requirements/local.txt