Review round 2 jk (#101)

* remove reference ListOfTestCases

* add folder artifacts to gitignore

* add generator for the list of test environments

* generate list of test environments

* add validator for list of test environments

* document generator

* implement evidence

* remove obsolete tests

* add tests for ListOfTestsGenerator

* repair pytest execution

---------

Signed-off-by: Erik Hu <135733975+Erikhu1@users.noreply.github.com>
Signed-off-by: Jonas-Kirchhoff <jonas.kirchhoff@d-fine.com>
Co-authored-by: Erik Hu <erik.hu@d-fine.com>
Co-authored-by: Erik Hu <135733975+Erikhu1@users.noreply.github.com>

Author: Jonas-Kirchhoff-d-fine

.dotstop_extensions/references.py

@@ -545,213 +545,6 @@ def __str__(self) -> str:
         # this is used as a title in the trudag report
         return f"function: [{self._name}]\n({str(self.path)})"
 
-class ListOfTestCases(BaseReference):
-
-    def __init__(self, test_files: list[str], recent_result_database: str = "artifacts/MemoryEfficientTestResults.db", recent_result_table: str = "test_results") -> None:
-        self._test_files = test_files
-        self._database = recent_result_database
-        self._table =  recent_result_table
-
-    @staticmethod    
-    def compile_string(items: list[str]) -> str:
-        # input: list of strings representing the structure of TEST_CASE, SECTION etc.,
-        # e.g. items = ["lexer class", "scan", "literal names"]
-        # output: the last item of the list, representing the most recent SECTION,
-        # indented as in the source code 
-        # throws error if input is empty
-        if len(items) == 0:
-            raise RuntimeError("Received empty structural list; nonempty list expected.")
-        result = ""
-        for _ in range(1, len(items)):
-            result += "    "
-        if items:
-            result += "* " + items[-1]
-        return result
-
-    @staticmethod    
-    def extract_quotation(s: str) -> str:
-        # input: string containing at least one quoted substring, e.g. s = "my \"input\""
-        # output: the first quoted substring of the input
-        # throws error if no quoted substring can be found.
-        first = s.find('"')
-        if first == -1:
-            raise RuntimeError("Expected quotation mark; none were detected.")
-        second = s.find('"', first + 1)
-        if second == -1:
-            raise RuntimeError("Expected quotation marks; only one was detected.")
-        return s[first + 1 : second]
-    
-    @staticmethod
-    def remove_and_count_indent(s: str) -> tuple[int, str]:
-        # input: string with possibly leading whitespace (space of horizontal tab)
-        # output: the number of leading spaces and the string with leading whitespace removed;
-        # tab counted as four spaces
-        cnt = 0
-        i = 0
-        n = len(s)
-        while i < n and (s[i] == " " or s[i] == "\t"):
-            if s[i] == " ":
-                cnt += 1
-            elif s[i] == "\t":
-                cnt += 4
-            i += 1
-        return (cnt, s[i:])
-
-    @staticmethod
-    def head_of_list() -> str:
-        return """## List of all unit-tests with test environments
-
-This list contains all unit-tests possibly running in this project.
-These tests are compiled from the source-code, where the individual unit-tests are arranged in TEST_CASEs containing possibly nested SECTIONs.
-To reflect the structure of the nested sections, nested lists are utilised, where the top-level list represents the list of TEST_CASEs. 
-
-It should be noted that not all unit-tests in a test-file are executed with every compiler-configuration.
-"""
-    
-    @staticmethod
-    def transform_test_file_to_test_name(test_file: str) -> str:
-        return "test-"+"-".join((test_file.split('.')[0]).split('-')[1:])
-
-    @classmethod
-    def type(cls) -> str:
-        return "list_of_test_cases"
-
-    def extract_test_structure(self, file_path: Path) -> str:
-        # input: path to a file potentially containing unit-tests
-        # output: the extracted arrangement of TEST_CASE and SECTION
-        # in the form of nested markdown lists
-
-        indent = 0 # the indent of the currently read line
-        current_indent = 0 # the indent of the last TEST_CASE or SECTION
-        current_path = [] # the current path
-        lines_out = [] # the collection of lines to be outputted
-
-        # open file_path as read-only, and process line by line
-        with file_path.open("r", encoding="utf-8", errors="replace") as source:
-            for line in source:
-                # count and remove leading whitespace
-                indent, trimmed = self.remove_and_count_indent(str(line))
-                
-                # check whether we have found a TEST_CASE
-                if trimmed.startswith("TEST_CASE(") or trimmed.startswith("TEST_CASE_TEMPLATE(") or trimmed.startswith("TEST_CASE_TEMPLATE_DEFINE("):
-                    # remember the current indent
-                    current_indent = indent
-                    # TEST_CASE is always the head of a new arrangement-structure
-                    # remove stored structure
-                    current_path.clear()
-                    # extract name of TEST_CASE and append path
-                    current_path.append(self.extract_quotation(trimmed))
-                    lines_out.append(self.compile_string(current_path))
-                
-                # check whether we have found a SECTION
-                if trimmed.startswith("SECTION("):
-                    # update path to reflect arrangement of current section
-                    while indent <= current_indent and current_path:
-                        current_path.pop()
-                        current_indent -= 4
-                    # remember the current indent
-                    current_indent = indent
-                    # extract name of SECTION and append path
-                    current_path.append(self.extract_quotation(trimmed))
-                    lines_out.append(self.compile_string(current_path))
-
-        # process extracted lines
-        return ("\n".join(lines_out) + "\n") if lines_out else ""
-
-    def extract_recent_test_environments(self) -> dict:
-        fetched_data = dict()
-        try:    
-            # initialise connection to test result database
-            connector = sqlite3.connect(self._database)
-            cursor = connector.cursor()
-            # verify that the expected table does exist
-            cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name = ?;",(self._table,))
-            if cursor.fetchone() is None: 
-                raise RuntimeError(f"Fatal Error: Could not find table {self._table} in database {self._database}.")
-        except sqlite3.Error as e:
-            raise RuntimeError(f"Fatal Error accessing database {self._database}: {e}")
-        # get all test-files from recent test executions
-        command = f"SELECT name FROM {self._table};"
-        cursor.execute(command)
-        raw_cases = cursor.fetchall()
-        cases = set([raw_case[0] for raw_case in raw_cases])
-        # for each test-file
-        for case in cases:
-            case_data = dict()
-            # get the test-environments
-            command = f"SELECT compiler, cpp_standard FROM {self._table} WHERE name = ? and skipped_cases == 0"
-            cursor.execute(command,(case,))
-            results = cursor.fetchall()
-            case_data["noskip"] = [{"compiler":result[0], "standard":result[1]} for result in results]
-            # some test-cases are skipped with certain environments
-            # It is unclear from the log, which cases are skipped;
-            # we leave this to the interested reader
-            command = f"SELECT compiler, cpp_standard, skipped_cases FROM {self._table} WHERE name = ? and skipped_cases != 0"
-            cursor.execute(command, (case,))
-            results = cursor.fetchall()
-            case_data["skip"] = [{"compiler": result[0], "standard": result[1], "skipped": result[2]} for result in results]
-            fetched_data[case] = case_data
-        return fetched_data
-
-    def fetch_all_test_data(self, input: list[str]):
-        # inputs: path(s) to directory potentially containing some test-data
-        extracted_test_data = []
-        recent_test_data = self.extract_recent_test_environments()
-        for arg in input:
-            p = Path(arg)
-            if p.is_file() and p.suffix == ".cpp" and p.name.startswith("unit-"):
-                extracted_test_data.append((p.name,self.extract_test_structure(p)))
-            elif p.is_dir():
-                for entry in p.rglob("*"):
-                    if entry.is_file() and entry.suffix == ".cpp" and entry.name.startswith("unit-"):
-                        extracted_test_data.append((entry.name,self.extract_test_structure(entry)))
-        extracted_test_data.sort(key= lambda x: x[0])
-        result = self.head_of_list()
-        for test_file, list_of_tests in extracted_test_data:
-            result += f"\n\n### List of tests in file {test_file}\n\n"
-            result += list_of_tests
-            result += "\n\n"
-            if recent_test_data.get(self.transform_test_file_to_test_name(test_file), None) is None:
-                result += "Unfortunately, none of the following tests seems to have been executed. Very strange indeed!\n\n"
-            else:
-                if recent_test_data.get(self.transform_test_file_to_test_name(test_file)).get("noskip",None) is not None:
-                    if len(recent_test_data.get(self.transform_test_file_to_test_name(test_file)).get("noskip")) != 0:
-                        result  += "\nAll tests in this file were run in the following configurations:\n\n"
-                        for datum in recent_test_data.get(self.transform_test_file_to_test_name(test_file)).get("noskip"):
-                            result += "* "
-                            result += datum.get("compiler",None)
-                            result += " with standard "
-                            result += datum.get("standard",None)
-                            result += "\n"
-                if recent_test_data.get(self.transform_test_file_to_test_name(test_file)).get("skip",None) is not None:
-                    if len(recent_test_data.get(self.transform_test_file_to_test_name(test_file)).get("skip")) != 0:
-                        result += "\nIn the following configuration, however, some test-cases were skipped:\n\n"
-                        for datum in recent_test_data.get(self.transform_test_file_to_test_name(test_file)).get("skip"):
-                            result += "* "
-                            how_many = datum.get("skipped",None)
-                            result += str(how_many)
-                            if how_many == 1:
-                                result += " test case was skipped when using "
-                            else:
-                                result += " test cases were skipped when using "
-                            result += datum.get("compiler",None)
-                            result += " with standard "
-                            result += datum.get("standard",None)
-                            result += "\n"
-        return result
-    
-    @property
-    def content(self) -> bytes:
-        # encoding is necessary since content will be hashed
-        return self.fetch_all_test_data(self._test_files).encode('utf-8')
-    
-    def as_markdown(self, filepath: None | str = None) -> str:
-        return self.content.decode('utf-8')
-
-    def __str__(self) -> str:
-        # this is used as a title in the trudag report
-        return "List of all unit-tests"
-    
 from trudag.dotstop.core.reference.references import LocalFileReference as LFR
 
 class VerboseFileReference(LFR):

.dotstop_extensions/test_references.py

@@ -2,8 +2,8 @@
 import tempfile
 from pathlib import Path
 from unittest.mock import patch
+from references import CPPTestReference, JSONTestsuiteReference, FunctionReference, ItemReference
 from validators import file_exists
-from references import CPPTestReference, JSONTestsuiteReference, FunctionReference, ListOfTestCases, ItemReference
 
 
 @pytest.fixture
@@ -673,41 +673,13 @@ def test_init_function_reference(temp_hpp_file):
     assert ref.path == temp_hpp_file
     assert ref._overload == 1
 
-def test_default_init_ListOfTestCases():
-    ref = ListOfTestCases(["file_1","file_2"])
-    assert ref._test_files == ["file_1","file_2"]
-    assert ref._database == "artifacts/MemoryEfficientTestResults.db"
-    assert ref._table == "test_results"
-
-def test_non_default_init_ListOfTestCases():
-    ref = ListOfTestCases(["file_1","file_2"],"my_database.db","my_fancy_table")
-    assert ref._test_files == ["file_1","file_2"]
-    assert ref._database == "my_database.db"
-    assert ref._table == "my_fancy_table"
-
-def test_compile_string():
-    with pytest.raises(RuntimeError):
-        ListOfTestCases.compile_string([])
-
-def test_remove_and_count_indent():
-    assert ListOfTestCases.remove_and_count_indent("Hallo")== (0,"Hallo")
-    assert ListOfTestCases.remove_and_count_indent(" Hallo") == (1,"Hallo")
-    assert ListOfTestCases.remove_and_count_indent("\t Hallo Welt \t\t") == (5,"Hallo Welt \t\t")
-
-def test_extract_quotation():
-    assert ListOfTestCases.extract_quotation("\"Hallo\" Welt") == "Hallo"
-    assert ListOfTestCases.extract_quotation("This is quite \"exciting\", isn't it.") == "exciting"
-    assert ListOfTestCases.extract_quotation("\"Hallo\" \"Welt\"") == "Hallo"
-
-def test_extract_faulty_quotation():
-    with pytest.raises(RuntimeError, match=r"Expected quotation mark; none were detected."):
-        ListOfTestCases.extract_quotation("Hallo Welt")
-    with pytest.raises(RuntimeError, match=r"Expected quotation marks; only one was detected."):
-        ListOfTestCases.extract_quotation("Hallo \"Welt")
-
-def test_transform_test_file_to_test_name():
-    assert ListOfTestCases.transform_test_file_to_test_name("unit-dummy-test.cpp") == "test-dummy-test"
-    assert ListOfTestCases.transform_test_file_to_test_name("unit-dummy_test.cpp") == "test-dummy_test"
+def test_faulty_init_ItemReference():
+    with pytest.raises(RuntimeError, match = r"Error: Can't initialise empty ItemReference."):
+        item_reference = ItemReference([])
+
+def test_init_ItemReference():
+    item_reference = ItemReference(["Hallo","Welt"])
+    assert item_reference._items == ["Hallo","Welt"]
 
 def test_file_exists(tmp_path):
     root = tmp_path / "direx"
@@ -725,10 +697,3 @@ def test_file_exists(tmp_path):
     assert score == 2/4
     assert any(isinstance(exception,Warning) for exception in exceptions)
     assert any(isinstance(exception,RuntimeError) for exception in exceptions)
-def test_faulty_init_ItemReference():
-    with pytest.raises(RuntimeError, match = r"Error: Can't initialise empty ItemReference."):
-        item_reference = ItemReference([])
-
-def test_init_ItemReference():
-    item_reference = ItemReference(["Hallo","Welt"])
-    assert item_reference._items == ["Hallo","Welt"]

.dotstop_extensions/validators.py

@@ -2,6 +2,7 @@
 import os
 import requests
 import sqlite3
+from TSF.scripts.generate_list_of_tests import ListOfTestsGenerator
 
 yaml: TypeAlias = str | int | float | list["yaml"] | dict[str, "yaml"]
 
@@ -232,3 +233,27 @@ def file_exists(configuration: dict[str, yaml]) -> tuple[float, list[Exception |
         else:
             found_files += 1 if os.path.isfile(file) else 0
     return (found_files/expected_files, exceptions)
+    
+def check_list_of_tests(configuration: dict[str, yaml]) -> tuple[float, list[Exception | Warning]]:
+    # initialise the generator
+    generator = ListOfTestsGenerator()
+    db = configuration.get("database",None)
+    if db is not None:
+        generator.set_database(db)
+    table = configuration.get("table",None)
+    if table is not None:
+        generator.set_table(table)
+    sources = configuration.get("sources",None)
+    if sources is not None:
+        generator.set_sources(sources)
+    
+    # fetch the expected result
+    try:
+        with open("./TSF/docs/list_of_test_environments.md", 'r') as f:
+            expected = f.read()
+            if expected == generator.fetch_all_test_data():
+                return(1.0,[])
+            else:
+                return(0.0,[Exception("The expected list of test-cases does not coincide with the fetched list.")])
+    except:
+        return(0.0,[Exception("An exception occurred when trying to compare the expected and the fetched list of tests.")])

.github/workflows/test_trudag_extensions.yml

@@ -31,7 +31,7 @@ jobs:
     - name: Run tests
       run: |
         cd .dotstop_extensions
-        pytest -v
+        PYTHONPATH=.. pytest -v
 
     - name: Generate test_trudag_extensions artifact
       run: |

.gitignore

@@ -55,6 +55,8 @@ user.bazelrc
 /TSF/temp
 
 /TSF/docs/generated
+# temporary folder used for generation of list of test-results
+/artifacts
 
 /.venv
 

TSF/__init__.py

@@ -0,0 +1 @@
+# This file makes the directory a Python package