diff --git a/tools/hw-oracle/compare_results.py b/tools/hw-oracle/compare_results.py index 9171bfe5d5..5626fb1f75 100644 --- a/tools/hw-oracle/compare_results.py +++ b/tools/hw-oracle/compare_results.py @@ -61,7 +61,7 @@ def read_report(path): return report -def compare(before, after): +def compare_limited(before, after, max_differences): for name in SETTINGS: if before[name] != after[name]: raise ValueError(f"incompatible reports: {name} differs") @@ -71,11 +71,18 @@ def compare(before, after): if left["input"] != right["input"]: raise ValueError(f"incompatible reports: row {index} input differs") differences = [] + difference_count = 0 for index, (left, right) in enumerate(zip(before["rows"], after["rows"]), 1): for register, old, new in zip(before["output_registers"], left["output"], right["output"]): if old != new: - differences.append((index, register, old, new)) - return differences + difference_count += 1 + if max_differences is None or len(differences) < max_differences: + differences.append((index, register, old, new)) + return difference_count, differences + + +def compare(before, after): + return compare_limited(before, after, None)[1] def nonnegative_integer(value): @@ -98,18 +105,21 @@ def main(): args = parser.parse_args() try: before, after = read_report(args.before), read_report(args.after) - differences = compare(before, after) + if args.max_differences is None: + differences = compare(before, after) + difference_count = len(differences) + else: + difference_count, differences = compare_limited(before, after, args.max_differences) except (OSError, UnicodeError, ValueError) as error: parser.error(str(error)) print(f"Assembly targets: {before['assembly_target']} -> {after['assembly_target']}") print(f"Compared {len(before['rows'])} rows x {len(before['output_registers'])} registers: " - f"{len(differences)} differing words") - visible = differences if args.max_differences is None else differences[:args.max_differences] - for row, register, old, new in visible: + f"{difference_count} differing words") + for row, register, old, new in differences: print(f"row {row}, {register}: 0x{old:08x} -> 0x{new:08x}") - if len(visible) < len(differences): - print(f"{len(differences) - len(visible)} additional differing words omitted") - return int(bool(differences)) + if len(differences) < difference_count: + print(f"{difference_count - len(differences)} additional differing words omitted") + return int(bool(difference_count)) if __name__ == "__main__": diff --git a/tools/hw-oracle/test_compare_results.py b/tools/hw-oracle/test_compare_results.py index 03c1816db5..23847f97a2 100644 --- a/tools/hw-oracle/test_compare_results.py +++ b/tools/hw-oracle/test_compare_results.py @@ -242,6 +242,23 @@ class CompareResultsTests(unittest.TestCase): self.assertEqual(result.returncode, 2) self.assertEqual(result.stdout, "") + def test_diagnostic_limit_retains_only_visible_differences_while_counting_all(self): + before = copy.deepcopy(self.report) + before["rows"] = [ + {"input": [index, 0, 0, 0], "output": [0, 0]} + for index in range(1000) + ] + after = copy.deepcopy(before) + for row in after["rows"]: + row["output"] = [1, 2] + count, differences = compare_results.compare_limited(before, after, 3) + self.assertEqual(count, 2000) + self.assertEqual(differences, [ + (1, "v10", 0, 1), + (1, "v11", 0, 2), + (2, "v10", 0, 1), + ]) + if __name__ == "__main__":