Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 2 additions & 8 deletions .github/workflows/unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ jobs:
continue-on-error: ${{ matrix.experimental }}
strategy:
matrix:
os: [ubuntu-latest]
python-version: ["3.10", "3.11", "3.12"]
os: [windows-latest]
python-version: ["3.10"]
experimental: [false]
# Include experimental or bleeding-edge releases.
# Windows is not included as it can be unreliable, e.g.
Expand All @@ -30,12 +30,6 @@ jobs:
- os: ubuntu-latest
python-version: "3.13"
experimental: true
- os: macos-latest
python-version: "3.13"
experimental: true
- os: windows-latest
python-version: "3.13"
experimental: true
steps:
- name: "check out repository"
uses: "actions/checkout@v4"
Expand Down
92 changes: 76 additions & 16 deletions src/demystify/demystify.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
# -*- coding: utf-8 -*-

# pylint: disable=R0917

"""Demystify
"""Demystify.

Demystify is a utility for static analysis of file format identification
tool results. The utility analyses all aspects of a format
Expand All @@ -27,6 +23,8 @@
rsync style lists for filtering on disk.
"""

# pylint: disable=R0917

import argparse
import configparser as ConfigParser
import logging
Expand All @@ -35,18 +33,80 @@
import sqlite3
import sys
import time
from pathlib import Path


sys.dont_write_bytecode = True

if os.name != "nt":
try:
from denylist_template import denylist_template
from libs import version
from libs.DemystifyAnalysisClass import AnalysisError, DemystifyAnalysis
from libs.HandleDenylistClass import HandleDenylist
from libs.IdentifyDatabase import IdentifyDB
from libs.outputhandlers.htmloutputclass import FormatAnalysisHTMLOutput
from libs.outputhandlers.roguesgalleryoutputclass import rogueoutputclass
from libs.outputhandlers.textoutputclass import FormatAnalysisTextOutput
except ModuleNotFoundError:
try:
from src.demystify.denylist_template import denylist_template
from src.demystify.libs import version
from src.demystify.libs.DemystifyAnalysisClass import (
AnalysisError,
DemystifyAnalysis,
)
from src.demystify.libs.HandleDenylistClass import HandleDenylist
from src.demystify.libs.IdentifyDatabase import IdentifyDB
from src.demystify.libs.outputhandlers.htmloutputclass import (
FormatAnalysisHTMLOutput,
)
from src.demystify.libs.outputhandlers.roguesgalleryoutputclass import (
rogueoutputclass,
)
from src.demystify.libs.outputhandlers.textoutputclass import (
FormatAnalysisTextOutput,
)
except ModuleNotFoundError:
from demystify.denylist_template import denylist_template
from demystify.libs import version
from demystify.libs.DemystifyAnalysisClass import (
AnalysisError,
DemystifyAnalysis,
)
from demystify.libs.HandleDenylistClass import HandleDenylist
from demystify.libs.IdentifyDatabase import IdentifyDB
from demystify.libs.outputhandlers.htmloutputclass import (
FormatAnalysisHTMLOutput,
)
from demystify.libs.outputhandlers.roguesgalleryoutputclass import (
rogueoutputclass,
)
from demystify.libs.outputhandlers.textoutputclass import (
FormatAnalysisTextOutput,
)

# pylint: disable=E0401; unable to import (not needed for local tests).
# pylint: disable=C0413; import not at top of file.
sys.path.insert(0, str(Path("./src/demystify/sqlitefid/src/sqlitefid/")))
print(
"PATH: %s %s",
os.path.exists(str(Path("./src/demystify/sqlitefid/src/sqlitefid/"))),
file=sys.stderr,
)
import sqlitefid # noqa: E402

else:
from .denylist_template import denylist_template
from .libs import version
from .libs.DemystifyAnalysisClass import AnalysisError, DemystifyAnalysis
from .libs.HandleDenylistClass import HandleDenylist
from .libs.IdentifyDatabase import IdentifyDB
from .libs.outputhandlers.htmloutputclass import FormatAnalysisHTMLOutput
from .libs.outputhandlers.roguesgalleryoutputclass import rogueoutputclass
from .libs.outputhandlers.textoutputclass import FormatAnalysisTextOutput
from .sqlitefid.src.sqlitefid import sqlitefid

from .denylist_template import denylist_template
from .libs import version
from .libs.DemystifyAnalysisClass import AnalysisError, DemystifyAnalysis
from .libs.HandleDenylistClass import HandleDenylist
from .libs.IdentifyDatabase import IdentifyDB

# Custom output handlers
from .libs.outputhandlers.htmloutputclass import FormatAnalysisHTMLOutput
from .libs.outputhandlers.roguesgalleryoutputclass import rogueoutputclass
from .libs.outputhandlers.textoutputclass import FormatAnalysisTextOutput
from .sqlitefid.src.sqlitefid import sqlitefid

logging.basicConfig(
format="%(asctime)-15s %(levelname)s :: %(filename)s:%(lineno)s:%(funcName)s() :: %(message)s",
Expand Down
3 changes: 1 addition & 2 deletions src/demystify/denylist_template.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-

"""Denylist template."""

denylist_template = """########################################################################
# Denylist and Rogues output configuration for demystify.
Expand Down
2 changes: 1 addition & 1 deletion src/demystify/i18n/internationalstrings.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
"""Analysis strings for i18n."""


class AnalysisStringsEN:
Expand Down
4 changes: 1 addition & 3 deletions src/demystify/libs/AnalysisQueriesClass.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
# -*- coding: utf-8 -*-

"""Analysis Queries to be used by Demystify to extract as much
"""Analysis Queries to be used by Demystify to extract as much
information as possible from a format identification extract.
"""

Expand Down
17 changes: 13 additions & 4 deletions src/demystify/libs/AnalysisResultsClass.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
# -*- coding: utf-8 -*-

"""Analysis results module."""

from .version import get_version
import os

if os.name != "nt":
try:
from libs import version
except ModuleNotFoundError:
try:
from src.demystify.libs import version
except ModuleNotFoundError:
from demystify.libs import version
else:
from .version import version


class AnalysisResults:
Expand Down Expand Up @@ -119,4 +128,4 @@ def __init__(self):

@staticmethod
def __version__():
return get_version()
return version.get_version()
63 changes: 49 additions & 14 deletions src/demystify/libs/DemystifyAnalysisClass.py
Original file line number Diff line number Diff line change
@@ -1,23 +1,58 @@
# -*- coding: utf-8 -*-
"""Demystify analysis class for collecting demystify results."""

# pylint: disable=R0917; too-many positional args.

import logging
import os
import sys
from collections import Counter
from configparser import NoOptionError

try:
# Required for imports calling from repository root.
from src.demystify.pathlesstaken.src.pathlesstaken import pathlesstaken
except ModuleNotFoundError:
# Required for Pypi install.
from ..pathlesstaken.src.pathlesstaken import pathlesstaken

from . import AnalysisResultsClass, version
from .AnalysisQueriesClass import AnalysisQueries
from .DenylistQueriesClass import DenylistQueries
from .HandleDenylistClass import HandleDenylist
from .RoguesQueriesClass import RogueQueries
from pathlib import Path

if os.name != "nt":
try:
import AnalysisResultsClass
import version
from AnalysisQueriesClass import AnalysisQueries
from DenylistQueriesClass import DenylistQueries
from HandleDenylistClass import HandleDenylist
from RoguesQueriesClass import RogueQueries
except ModuleNotFoundError:
try:
from src.demystify.libs import AnalysisResultsClass, version
from src.demystify.libs.AnalysisQueriesClass import AnalysisQueries
from src.demystify.libs.DenylistQueriesClass import DenylistQueries
from src.demystify.libs.HandleDenylistClass import HandleDenylist
from src.demystify.libs.RoguesQueriesClass import RogueQueries
except ModuleNotFoundError:
from demystify.libs import AnalysisResultsClass, version
from demystify.libs.AnalysisQueriesClass import AnalysisQueries
from demystify.libs.DenylistQueriesClass import DenylistQueries
from demystify.libs.HandleDenylistClass import HandleDenylist
from demystify.libs.RoguesQueriesClass import RogueQueries

# pylint: disable=E0401; unable to import (not needed for local tests).
# pylint: disable=C0413; import not at top of file.
sys.path.insert(0, str(Path("./src/demystify/pathlesstaken/src/pathlesstaken/")))
print(
"PATH: %s %s",
os.path.exists(str(Path("./src/demystify/pathlesstaken/src/pathlesstaken/"))),
file=sys.stderr,
)
import pathlesstaken # noqa: E402
else:
try:
# Required for imports calling from repository root.
from src.demystify.pathlesstaken.src.pathlesstaken import pathlesstaken
except ModuleNotFoundError:
# Required for Pypi install.
from ..pathlesstaken.src.pathlesstaken import pathlesstaken

from . import AnalysisResultsClass, version
from .AnalysisQueriesClass import AnalysisQueries
from .DenylistQueriesClass import DenylistQueries
from .HandleDenylistClass import HandleDenylist
from .RoguesQueriesClass import RogueQueries


class AnalysisError(Exception):
Expand Down
2 changes: 0 additions & 2 deletions src/demystify/libs/DenylistQueriesClass.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-

"""Denylist queries for filtering denylist items from a format
identification extract.
"""
Expand Down
2 changes: 0 additions & 2 deletions src/demystify/libs/HandleDenylistClass.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-

"""Denylist module."""

# [denylist]
Expand Down
6 changes: 2 additions & 4 deletions src/demystify/libs/IdentifyDatabase.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
# -*- coding: utf-8 -*-

"""IdentifyDatabase
"""IdentifyDatabase

Enables identification of a sqlite3 database to ensure that the input
data for the calling application is correct.
Expand All @@ -14,7 +12,7 @@ class IdentifyDB:
UNKNOWN_DB = "unknown"

# 'SQLite format 3'
sqlite3 = b"\x53\x51\x4C\x69\x74\x65\x20\x66\x6F\x72\x6D\x61\x74\x20\x33"
sqlite3 = b"\x53\x51\x4c\x69\x74\x65\x20\x66\x6f\x72\x6d\x61\x74\x20\x33"

def identify_export(self, database_path):
"""Reads an input file and checks whether it is an sqlite3
Expand Down
2 changes: 0 additions & 2 deletions src/demystify/libs/RoguesQueriesClass.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-

"""Rogue Queries for extracting information about Rogues and Heroes from
a format identification extract.
"""
Expand Down
31 changes: 23 additions & 8 deletions src/demystify/libs/outputhandlers/htmloutputclass.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,32 @@
# -*- coding: utf-8 -*-
"""HTML Output Class for HTML formatted results."""

# pylint: disable=R0917; too-many positional args.

import logging
import os
import re

try:
from src.demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN
from src.demystify.libs import DemystifyAnalysisClass
except ModuleNotFoundError:
# Needed to run from root dir.
from demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN
from demystify.libs import DemystifyAnalysisClass
if os.name != "nt":
try:
from i18n.internationalstrings import AnalysisStringsEN as IN_EN
from libs import DemystifyAnalysisClass
except ModuleNotFoundError:
try:
from src.demystify.i18n.internationalstrings import (
AnalysisStringsEN as IN_EN,
)
from src.demystify.libs import DemystifyAnalysisClass
except ModuleNotFoundError:
from demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN
from demystify.libs import DemystifyAnalysisClass
else:
try:
from src.demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN
from src.demystify.libs import DemystifyAnalysisClass
except ModuleNotFoundError:
# Needed to run from root dir.
from demystify.i18n.internationalstrings import AnalysisStringsEN as IN_EN
from demystify.libs import DemystifyAnalysisClass

# NONE_REPLACE_DEBUG is a logging prompt to help us to understand what
# needs changing around 'None'/null values from the database. These
Expand Down
22 changes: 16 additions & 6 deletions src/demystify/libs/outputhandlers/roguesgalleryoutputclass.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,23 @@
# -*- coding: utf-8 -*-
"""Rogue output."""

import configparser as ConfigParser
import logging
import os

try:
from src.demystify.libs.HandleDenylistClass import HandleDenylist
except ModuleNotFoundError:
# Needed for PyPi import.
from demystify.libs.HandleDenylistClass import HandleDenylist
if os.name != "nt":
try:
from libs.HandleDenylistClass import HandleDenylist
except ModuleNotFoundError:
try:
from src.demystify.libs.HandleDenylistClass import HandleDenylist
except ModuleNotFoundError:
from demystify.libs.HandleDenylistClass import HandleDenylist
else:
try:
from src.demystify.libs.HandleDenylistClass import HandleDenylist
except ModuleNotFoundError:
# Needed for PyPi import.
from demystify.libs.HandleDenylistClass import HandleDenylist


class rogueoutputclass:
Expand Down
Loading
Loading