diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index df1456117..0d60c134d 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@v10 + - uses: DeterminateSystems/nix-installer-action@v21 - name: Build using nix run: nix build - name: Run built Diffkemp @@ -76,7 +76,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@v10 + - uses: DeterminateSystems/nix-installer-action@v21 - name: Build DiffKemp run: > nix develop . --command bash -c " diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78ee6aedb..ee8a20288 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@v10 + - uses: DeterminateSystems/nix-installer-action@v21 - name: Delete unused software # This is necessary for running CScope database build as it takes diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml index 758f0c7d7..d143b209c 100644 --- a/.github/workflows/code-style.yml +++ b/.github/workflows/code-style.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@v10 + - uses: DeterminateSystems/nix-installer-action@v21 - name: Check Coding Style working-directory: ${{ github.workspace }} diff --git a/diffkemp/compare.py b/diffkemp/compare.py index 0a6d4f4a6..791d3eac5 100644 --- a/diffkemp/compare.py +++ b/diffkemp/compare.py @@ -43,6 +43,7 @@ def __init__(self, group, group_name, enable_module_cache, config, self.result_graph = None self.cache = SimpLLCache(mkdtemp()) self.module_cache = {} + self.group_result = None def _get_group_dir(self, output_dir): if output_dir is not None and self.group_name is not None: @@ -90,7 +91,8 @@ def __init__(self, cmd_args): self.config = Config.from_args(cmd_args) self.result = Result(Result.Kind.NONE, cmd_args.snapshot_dir_old, cmd_args.snapshot_dir_new, - start_time=default_timer()) + start_time=default_timer(), + hierarchy_level=Result.HierarchyLevel.OVERALL) self.regex_pattern = re.compile(cmd_args.regex_filter) \ if cmd_args.regex_filter else None self.output_dir = None @@ -139,9 +141,26 @@ def _compare_snapshots(self): return 0 def _compare_groups(self, group_info): + group_name = group_info.group_name + # Checks if there are groups (e.g. for sysctl parameters), + # creates group_result to be able to connect results in the group + if group_name is not None: + group_info.group_result = Result( + Result.Kind.NONE, + group_name, + group_name, + hierarchy_level=Result.HierarchyLevel.GROUP) + else: + group_info.group_result = self.result + for fun, old_fun_desc in sorted(group_info.group.functions.items()): self._compare_function(fun, old_fun_desc, group_info) + # add_inner method modifies result.graph so it has to be called when + # group_result.graph is already updated + if group_name is not None: + self.result.add_inner(group_info.group_result) + def _compare_function(self, fun, old_fun_desc, group_info): # Check if the function exists in the other snapshot new_fun_desc = \ @@ -179,16 +198,15 @@ def _modules_exist(old_fun_desc, new_fun_desc): def _handle_missing_module(self, fun, old_fun_desc, group_info): fun_result = Result(Result.Kind.UNKNOWN, fun, fun) - self.result.add_inner(fun_result) + group_info.group_result.add_inner(fun_result) self._print_fun_result(fun_result, fun, old_fun_desc, group_info) def _handle_fun_result(self, fun_result, fun, old_fun_desc, group_info): - if self.args.regex_filter is not None: # Filter results by regex self._filter_result_by_regex(fun_result) - self.result.add_inner(fun_result) + group_info.group_result.add_inner(fun_result) # Printing information about failures and non-equal functions. if fun_result.kind in [Result.Kind.NOT_EQUAL, diff --git a/diffkemp/output.py b/diffkemp/output.py index eff97f325..b99b4ee22 100644 --- a/diffkemp/output.py +++ b/diffkemp/output.py @@ -30,49 +30,66 @@ def _create_output(self): """Creates yaml representation of compare result.""" self.output["old-snapshot"] = self.old_dir self.output["new-snapshot"] = self.new_dir - - self._create_results() + self.output["results"] = self._create_results(self.result) self._create_definitions() - def _create_results(self): - """Creates call stacks of not-equal functions.""" - # list of all not-equal functions callstacks - results = [] - for fun_name, fun_result in self.result.inner.items(): - if fun_result.kind != Result.Kind.NOT_EQUAL: + def _create_results(self, current_result, name=None): + """:param current_result: node in the result class tree""" + if current_result.hierarchy_level == Result.HierarchyLevel.FUNCTION: + return self._create_function_result(current_result, name) + + inner_results = [] + for inner_name, inner_result in current_result.inner.items(): + result = self._create_results(inner_result, inner_name) + if result is not None: + inner_results.append(result) + + if current_result.hierarchy_level == Result.HierarchyLevel.OVERALL: + return inner_results + # hierarchy_level == GROUP + return { + "sysctl": name, # sysctl is the only usecase of groups + "results": inner_results + } + + def _create_function_result(self, fun_result, fun_name): + """ + Creates call stacks of not-equal functions. + :param fun_result: output of snapshot comparison + """ + if fun_result.kind != Result.Kind.NOT_EQUAL: + return + self.function_names.add(fun_name) + # list of not-equal functions callstacks for a function + diffs = [] + for called_res in sorted(fun_result.inner.values(), + key=lambda r: r.first.name): + if called_res.diff == "" and called_res.first.covered: + # Do not add empty diffs continue - self.function_names.add(fun_name) - # list of not-equal functions callstacks for a function - diffs = [] - for called_res in sorted(fun_result.inner.values(), - key=lambda r: r.first.name): - if called_res.diff == "" and called_res.first.covered: - # Do not add empty diffs - continue - # information about a function which is different - diff = {} - diff["function"] = called_res.first.name - # getting callstacks - diff["old-callstack"] = called_res.first \ - .callstack.to_output_yaml_with_rel_path(self.old_dir) \ - if called_res.first.callstack else [] - diff["new-callstack"] = called_res.second \ - .callstack.to_output_yaml_with_rel_path(self.new_dir) \ - if called_res.second.callstack else [] - # updating symbol names - if called_res.first.callstack: - function_names, macro_names, type_names = called_res \ - .first.callstack.get_symbol_names(fun_name) - self.function_names.update(function_names) - self.macro_names.update(macro_names) - self.type_names.update(type_names) - - diffs.append(diff) - results.append({ - "function": fun_name, - "diffs": diffs - }) - self.output["results"] = results + # information about a function which is different + diff = {} + diff["function"] = called_res.first.name + # getting callstacks + diff["old-callstack"] = called_res.first \ + .callstack.to_output_yaml_with_rel_path(self.old_dir) \ + if called_res.first.callstack else [] + diff["new-callstack"] = called_res.second \ + .callstack.to_output_yaml_with_rel_path(self.new_dir) \ + if called_res.second.callstack else [] + # updating symbol names + if called_res.first.callstack: + function_names, macro_names, type_names = called_res \ + .first.callstack.get_symbol_names(fun_name) + self.function_names.update(function_names) + self.macro_names.update(macro_names) + self.type_names.update(type_names) + + diffs.append(diff) + return { + "function": fun_name, + "diffs": diffs + } def _create_definitions(self): self.output["definitions"] = {} diff --git a/diffkemp/semdiff/result.py b/diffkemp/semdiff/result.py index 6240a7ebd..a02751461 100644 --- a/diffkemp/semdiff/result.py +++ b/diffkemp/semdiff/result.py @@ -183,8 +183,23 @@ def __init__(self, name, filename=None, line=None, callstack=None, self.diff_kind = diff_kind self.covered = covered + class HierarchyLevel(IntEnum): + """Describes position of result instance in result comparison tree""" + OVERALL = 0 + GROUP = 1 + FUNCTION = 2 + + class SymbolStat: + def __init__(self): + self.total = 0 + self.eq = 0 + self.neq = 0 + self.unkwn = 0 + self.errs = 0 + self.empty_diff = 0 + def __init__(self, kind, first_name, second_name, start_time=None, - stop_time=None): + stop_time=None, hierarchy_level=HierarchyLevel.FUNCTION): self.kind = kind self.first = Result.Entity(first_name) self.second = Result.Entity(second_name) @@ -194,6 +209,7 @@ def __init__(self, kind, first_name, second_name, start_time=None, self.inner = dict() self.start_time = start_time self.stop_time = stop_time + self.hierarchy_level = hierarchy_level def __str__(self): return str(self.kind) @@ -220,43 +236,56 @@ def report_symbol_stat(self, show_errors=False): Print numbers of equal, non-equal, unknown, and error results with percentage that each has from the total results. """ - total = len(self.inner) - eq = len([r for r in iter(self.inner.values()) - if r.kind == Result.Kind.EQUAL]) - neq = len([r for r in iter(self.inner.values()) - if r.kind == Result.Kind.NOT_EQUAL]) - unkwn = len([r for r in iter(self.inner.values()) - if r.kind == Result.Kind.UNKNOWN]) - errs = len([r for r in iter(self.inner.values()) - if r.kind in [Result.Kind.ERROR, Result.Kind.TIMEOUT]]) - empty_diff = len([r for r in iter(self.inner.values()) if all(map( - lambda x: x.diff == "", r.inner.values())) and - r.kind == Result.Kind.NOT_EQUAL]) - if total > 0: - print("Total symbols: {}".format(total)) - print("Equal: {0} ({1:.0f}%)".format(eq, eq / total * 100)) + stats = Result.SymbolStat() + self._populate_symbol_stat(stats) + + if stats.total > 0: + print("Total symbols: {}".format(stats.total)) + print("Equal: {0} ({1:.0f}%)".format( + stats.eq, stats.eq / stats.total * 100)) print("Not equal: {0} ({1:.0f}%)".format( - neq, neq / total * 100)) + stats.neq, stats.neq / stats.total * 100)) print("(empty diff): {0} ({1:.0f}%)".format( - empty_diff, empty_diff / total * 100)) - print("Unknown: {0} ({1:.0f}%)".format(unkwn, - unkwn / total * 100)) - print("Errors: {0} ({1:.0f}%)".format(errs, - errs / total * 100)) + stats.empty_diff, stats.empty_diff / stats.total * 100)) + print("Unknown: {0} ({1:.0f}%)".format( + stats.unkwn, stats.unkwn / stats.total * 100)) + print("Errors: {0} ({1:.0f}%)".format( + stats.errs, stats.errs / stats.total * 100)) + if show_errors: - if unkwn > 0: + if stats.unkwn > 0: print("\nFunctions that are unknown: ") - for f, r in sorted(self.inner.items()): - if r.kind == Result.Kind.UNKNOWN: - print(f) + self.print_function_names(Result.Kind.UNKNOWN) print() - if errs > 0: + if stats.errs > 0: print("\nFunctions whose comparison ended with an error: ") - for f, r in sorted(self.inner.items()): - if r.kind == Result.Kind.ERROR: - print(f) + self.print_function_names(Result.Kind.ERROR) print() + def print_function_names(self, kind): + for name, result in self.inner.items(): + if result.hierarchy_level == Result.HierarchyLevel.FUNCTION: + if result.kind == kind: + print(name) + else: + result.print_function_names(kind) + + def _populate_symbol_stat(self, stats): + """Aggregate statistics about analysed symbols.""" + for result in self.inner.values(): + if result.hierarchy_level == Result.HierarchyLevel.FUNCTION: + stats.total += 1 + stats.eq += result.kind == Result.Kind.EQUAL + stats.neq += result.kind == Result.Kind.NOT_EQUAL + stats.unkwn += result.kind == Result.Kind.UNKNOWN + stats.errs += result.kind in [Result.Kind.ERROR, + Result.Kind.TIMEOUT] + stats.empty_diff += (all(map(lambda x: x.diff == "", + result.inner.values())) and + result.kind == Result.Kind.NOT_EQUAL) + else: + result._populate_symbol_stat(stats) + def report_object_stat(self): """ Report detailed statistics about compared objects. @@ -276,14 +305,20 @@ def __eq__(self, other): def __hash__(self): return hash(self.res.first.name) + def _populate_unique_diffs(result, unique_diffs): + if result.hierarchy_level == Result.HierarchyLevel.FUNCTION: + for _, inner_res in result.inner.items(): + if (inner_res.diff == "" and + inner_res.first.covered): + continue + unique_diffs.add(UniqueDiff(inner_res)) + else: + for _, inner_result in result.inner.items(): + _populate_unique_diffs(inner_result, unique_diffs) + # Convert inner result to set of unique diffs unique_diffs = set() - for _, inner_res_out in self.inner.items(): - for _, inner_res in inner_res_out.inner.items(): - if (inner_res.diff == "" and - inner_res.first.covered): - continue - unique_diffs.add(UniqueDiff(inner_res)) + _populate_unique_diffs(self, unique_diffs) # Generate counts compared = len(self.graph.vertices) diff --git a/tests/unit_tests/result_test.py b/tests/unit_tests/result_test.py index c109d14ca..07749a17b 100644 --- a/tests/unit_tests/result_test.py +++ b/tests/unit_tests/result_test.py @@ -108,9 +108,7 @@ def test_callstack_get_symbol_names(callstack): @pytest.fixture -def result(graph): - result = Result(Result.Kind.NONE, "old-snapshot-path", "new-snapshot-path") - +def single_function_result(graph): comp1_result = Result(Result.Kind.NONE, "main_function", "main_function") objects_to_compare, *_ = \ graph.graph_to_fun_pair_list("main_function", "main_function", False) @@ -119,13 +117,60 @@ def result(graph): fun_result.first = fun_pair[0] fun_result.second = fun_pair[1] comp1_result.add_inner(fun_result) + yield comp1_result + + +@pytest.fixture +def multi_functions_result(graph, single_function_result): + result = Result(Result.Kind.NONE, "old-snapshot-path", "new-snapshot-path", + hierarchy_level=Result.HierarchyLevel.OVERALL) + result.add_inner(single_function_result) + result.graph = graph + yield result - result.add_inner(comp1_result) + +@pytest.fixture +def sysctl_group_result(graph, single_function_result): + result = Result(Result.Kind.NONE, "old-snapshot-path", "new-snapshot-path", + hierarchy_level=Result.HierarchyLevel.OVERALL) + group_result = Result(Result.Kind.NONE, "group_name", "group_name", + hierarchy_level=Result.HierarchyLevel.GROUP) + group_result.add_inner(single_function_result) + result.add_inner(group_result) result.graph = graph yield result -def test_yaml_output(result, mocker): +@pytest.fixture +def multi_groups_result(graph, single_function_result): + result = Result(Result.Kind.NONE, "old-snapshot-path", "new-snapshot-path", + hierarchy_level=Result.HierarchyLevel.OVERALL) + outer_group_result = Result(Result.Kind.NONE, "outer_name", "outer_name", + hierarchy_level=Result.HierarchyLevel.GROUP) + inner_group_result = Result(Result.Kind.NONE, "inner_name", "inner_name", + hierarchy_level=Result.HierarchyLevel.GROUP) + inner_group_result.add_inner(single_function_result) + outer_group_result.add_inner(inner_group_result) + result.add_inner(outer_group_result) + result.graph = graph + yield result + + +@pytest.fixture +def result(request): + return request.getfixturevalue(request.param) + + +@pytest.mark.parametrize( + "result,tree_kind", + [ + ("multi_functions_result",) * 2, + ("sysctl_group_result",) * 2, + ("multi_groups_result",) * 2 + ], + indirect=["result"] +) +def test_yaml_output(result, tree_kind, mocker): """Tests YAML representation of compare result made by YamlOutput class.""" mocker.patch("diffkemp.output.get_end_line", return_value=2958) @@ -141,7 +186,22 @@ def mock_rel_path(path, start): assert yaml["new-snapshot"] == "/abs/path/to/new-snapshot" assert len(yaml["results"]) == 1 - main_function_result = yaml["results"][0] + if tree_kind == "multi_functions_result": + main_function_result = yaml["results"][0] + elif tree_kind == "sysctl_group_result": + group_result = yaml["results"][0] + assert len(group_result["results"]) == 1 + assert group_result["sysctl"] == "group_name" + main_function_result = group_result["results"][0] + else: + outer_group_result = yaml["results"][0] + assert len(outer_group_result["results"]) == 1 + assert outer_group_result["sysctl"] == "outer_name" + inner_group_result = outer_group_result["results"][0] + assert len(inner_group_result["results"]) == 1 + assert inner_group_result["sysctl"] == "inner_name" + main_function_result = inner_group_result["results"][0] + assert main_function_result["function"] == "main_function" assert len(main_function_result["diffs"]) == 3 for diff in main_function_result["diffs"]: diff --git a/view/src/Result.js b/view/src/Result.js index 35c8e1def..1305b7bb3 100644 --- a/view/src/Result.js +++ b/view/src/Result.js @@ -36,6 +36,16 @@ export default class Result { /** Absolute path to new snapshot. */ newSnapshot; + /** + * Comparison results with the following structure: + * -results: + * -sysctl: name (this field exists only in case of sysctl comparison) + * results: + * -function: name + * -diffs + */ + results; + /** * Create Result from YAML file. * @param {Object} yaml YAML file describing result of diffkemp compare. @@ -44,42 +54,53 @@ export default class Result { this.definitions = yaml.definitions; this.oldSnapshot = yaml['old-snapshot']; this.newSnapshot = yaml['new-snapshot']; - // Sort results by compared function name - yaml.results.sort((resultA, resultB) => ( - compareFunctionNames(resultA.function, resultB.function) - )); + if (yaml.results.length !== 0 && !('function' in yaml.results[0])) { + this.results = yaml.results; + } else { + this.results = [{ results: yaml.results }]; + } + // Sort differing functions of each compared function - yaml.results.forEach((result) => { - result.diffs.sort((diffA, diffB) => compareFunctionNames(diffA.function, diffB.function)); + this.results.forEach((groupResult) => { + groupResult.results.forEach((result) => { + result.diffs.sort((diffA, diffB) => compareFunctionNames(diffA.function, diffB.function)); + }); }); - this.#createAllCompAndDiffFuns(yaml.results); + this.#createAllCompAndDiffFuns(); } /** * Creates allCompFuns and allDiffFuns fields. - * Expects that names of compared and differing functions are sorted in the `results`. - * @param results - Differences found by DiffKemp compare phase. + * Expects that names of differing functions are sorted in the `results`. */ - #createAllCompAndDiffFuns(results) { + #createAllCompAndDiffFuns() { // Map(compFunName, Map(diffFunName, diff)) this.#allCompFuns = new Map(); // Map(diffFunName, [...compFunNames]) this.#allDiffFuns = new Map(); - // foreach result (compared function) - results.forEach((result) => { - // Map(diffFunName, diff) - const diffFunsForComp = new Map(); - // for each diff (differing function) - result.diffs.forEach((diff) => { - diffFunsForComp.set(diff.function, diff); - if (this.#allDiffFuns.has(diff.function)) { - this.#allDiffFuns.get(diff.function).push(result.function); - } else { - this.#allDiffFuns.set(diff.function, [result.function]); + // foreach group (sysctl or all functions) + this.results.forEach((groupResult) => { + // foreach result (compared function) + groupResult.results.forEach((result) => { + if (!this.#allCompFuns.has(result.function)) { + // Map(diffFunName, diff) + const diffFunsForComp = new Map(); + // for each diff (differing function) + result.diffs.forEach((diff) => { + diffFunsForComp.set(diff.function, diff); + if (this.#allDiffFuns.has(diff.function)) { + this.#allDiffFuns.get(diff.function).push(result.function); + } else { + this.#allDiffFuns.set(diff.function, [result.function]); + } + }); + this.#allCompFuns.set(result.function, diffFunsForComp); } }); - this.#allCompFuns.set(result.function, diffFunsForComp); }); + // sort all compared functions + this.#allCompFuns = new Map([...this.#allCompFuns] + .sort((a, b) => compareFunctionNames(a[0], b[0]))); // sort keys (differing function names) in the map this.#allDiffFuns = new Map([...this.#allDiffFuns] .sort((a, b) => compareFunctionNames(a[0], b[0])));