diff --git a/tools/check_skips.py b/tools/check_skips.py index e8ccba6f53..2b94d215bc 100644 --- a/tools/check_skips.py +++ b/tools/check_skips.py @@ -4,60 +4,46 @@ import re import xml.etree.ElementTree as ET from pathlib import Path -DEVICE_LIMITATION = re.compile(r"skipped,.*(subgroup|the device (has no|reports|lacks|does not|imports no|flushes|binds fewer)|the host (has not enough free disk space|cannot allocate a buffer)|no display|no usable Vulkan device|host imports are unavailable|not enough free disk space)", re.IGNORECASE) +DEVICE_LIMITATION = re.compile(r"skipped,.*(subgroup|the device (has no|reports|lacks|does not|imports no|flushes|binds fewer)|no display|no usable Vulkan device|host imports are unavailable)", re.IGNORECASE) + +HOST_LIMITATION = re.compile(r"^skipping: (not enough free disk space for a file|cannot allocate a buffer) of more than 2 GiB", re.IGNORECASE | re.MULTILINE) NO_DISPLAY = re.compile(r"skipped,.*no display", re.IGNORECASE) NO_VULKAN_DEVICE = re.compile(r"skipped,.*no usable Vulkan device", re.IGNORECASE) HIGH_REFRESH_LIMITATION = re.compile(r"current display does not provide 120 Hz", re.IGNORECASE) -CTEST_SKIP_STATUS = re.compile(r"SKIP_RETURN_CODE=\d+|SKIP_REGULAR_EXPRESSION_MATCHED", re.IGNORECASE) WINDOWS_PREFIX = "agc" -def device_limitations(output, patterns): - reasons = [match.group(0) for line in output.splitlines() if (match := re.search(r"skipped,.*", line, re.IGNORECASE))] - return bool(reasons) and all(any(pattern.search(reason) for pattern in patterns) for reason in reasons) - - def high_refresh_limitation(output): reasons = [] for line in output.splitlines(): reason = line.strip() - if match := re.search(r"skipped,", reason, re.IGNORECASE): - reason = reason[match.end():].strip() + if reason.lower().startswith("skipped,"): + reason = reason[len("skipped,"):].strip() if reason: reasons.append(reason) return bool(reasons) and all(HIGH_REFRESH_LIMITATION.fullmatch(reason) for reason in reasons) -def skips(path, require_tests=False): +def skips(path): root = ET.parse(path).getroot() if root.tag not in ("testsuite", "testsuites"): raise ValueError(f"{path}: expected a JUnit testsuite or testsuites root, got {root.tag!r}") - found = False - seen = set() for case in root.iter("testcase"): - found = True + if case.get("status") == "disabled": + raise ValueError(f"{path}: testcase {case.get('name')!r} is disabled and was not run") + skipped = case.find("skipped") + if skipped is None: + continue name = case.get("name") if not name: raise ValueError(f"{path}: skipped testcase has no name") - if name in seen: - raise ValueError(f"{path}: testcase {name!r} is listed more than once") - seen.add(name) - skipped = case.find("skipped") - ctest_skip = skipped is not None and CTEST_SKIP_STATUS.fullmatch(skipped.get("message", "").strip()) - if case.get("status") == "disabled" or (case.get("status") == "notrun" and not ctest_skip): - raise ValueError(f"{path}: testcase {name!r} is {case.get('status')} and was not run") - if skipped is None: - continue - output = ["skipped, " + reason.strip() for reason in (skipped.get("message", ""), skipped.text or "") - if reason.strip() and not CTEST_SKIP_STATUS.fullmatch(reason.strip())] + output = ["skipped, " + reason.strip() for reason in (skipped.get("message", ""), skipped.text or "") if reason.strip()] output += [case.findtext(channel) or "" for channel in ("system-out", "system-err")] yield name, "\n".join(output) - if require_tests and not found: - raise ValueError(f"{path}: report contains no testcases") def main(argv=None): @@ -68,23 +54,21 @@ def main(argv=None): parser.add_argument("junit", type=Path, help="the file written by ctest --output-junit") parser.add_argument("--job", choices=("linux", "windows"), default="linux") parser.add_argument("--list", action="store_true", help="print the skipped tests and why the file says they skipped") - parser.add_argument("--require-tests", action="store_true", help="fail if the report contains no testcases") args = parser.parse_args(argv) try: - found = sorted(skips(args.junit, require_tests=args.require_tests)) + found = sorted(skips(args.junit)) except (OSError, ET.ParseError, ValueError) as error: parser.error(str(error)) if args.list: for name, output in found: - reason = next((match.group(0) for line in output.splitlines() - if (match := re.search(r"skipped,.*", line, re.IGNORECASE))), "") + reason = next((line.strip() for line in output.splitlines() if line.strip().lower().startswith("skipped")), "") print(f"{name}: {reason}") return 0 if args.job == "windows": - unexpected = [name for name, output in found if not name.startswith(WINDOWS_PREFIX) and not device_limitations(output, (NO_DISPLAY, NO_VULKAN_DEVICE)) and not high_refresh_limitation(output)] + unexpected = [name for name, output in found if not name.startswith(WINDOWS_PREFIX) and not NO_DISPLAY.search(output) and not NO_VULKAN_DEVICE.search(output) and not high_refresh_limitation(output)] expected = f"any {WINDOWS_PREFIX}* test, which the runner has no Vulkan driver for, or a test that reports no display or Vulkan device or a display without a 120 Hz mode" else: - unexpected = [name for name, output in found if not device_limitations(output, (DEVICE_LIMITATION,)) and not high_refresh_limitation(output)] + unexpected = [name for name, output in found if not DEVICE_LIMITATION.search(output) and not HOST_LIMITATION.search(output) and not high_refresh_limitation(output)] expected = "any test that reports the device limitation it needs, such as narrow subgroups, a missing extension, no display for a Vulkan window or a display without a 120 Hz mode" for name in unexpected: print(f"error: {name} skipped") diff --git a/tools/tests/test_check_skips.py b/tools/tests/test_check_skips.py index 74318302ad..49a2f56a17 100644 --- a/tools/tests/test_check_skips.py +++ b/tools/tests/test_check_skips.py @@ -1,7 +1,6 @@ from pathlib import Path import contextlib import io -import shutil import subprocess import sys import tempfile @@ -25,12 +24,14 @@ LIMITATIONS = [ "skipped, no usable Vulkan device: ErrorInitializationFailed", "skipped, unit shadows or host imports are unavailable", "skipped, the device has no compute queue", - "skipped, not enough free disk space for a file of more than 2 GiB", "skipped, the device reports 4 lanes", "skipped, subgroup size 8", "skipped, no display", - "skipped, the host has not enough free disk space for a file of more than 2 GiB", - "skipped, the host cannot allocate a buffer of more than 2 GiB", +] + +HOST_LIMITATIONS = [ + "skipping: not enough free disk space for a file of more than 2 GiB", + "skipping: cannot allocate a buffer of more than 2 GiB", ] OTHER = [ @@ -41,102 +42,6 @@ OTHER = [ class CheckSkipsTests(unittest.TestCase): - def test_ctest_status_and_prefixed_device_reasons_are_expected(self): - reports = ( - ('skipped, the device lacks geometryShader', "linux"), - ('f16 native: skipped, the device has no native f16 with RTE rounding and kept denormals', "linux"), - ('video: skipped, no display or Vulkan device: ErrorInitializationFailed', "windows"), - ('skipped, no usable Vulkan device: none', "windows"), - ) - with tempfile.TemporaryDirectory() as directory: - junit = Path(directory) / "junit.xml" - for report, job in reports: - with self.subTest(job=job, report=report): - junit.write_text(f'{report}', encoding="utf-8") - result = subprocess.run([sys.executable, "-I", str(TOOL), str(junit), "--job", job], - capture_output=True, text=True) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertIn("all expected", result.stdout) - - def test_ctest_status_does_not_allow_missing_or_unexpected_reasons(self): - reports = ( - '', - '', - 'skipped, flaky\nskipped, the device lacks geometryShader', - ) - with tempfile.TemporaryDirectory() as directory: - junit = Path(directory) / "junit.xml" - for report in reports: - for job in ("linux", "windows"): - with self.subTest(job=job, report=report): - junit.write_text(f'{report}', encoding="utf-8") - result = subprocess.run([sys.executable, "-I", str(TOOL), str(junit), "--job", job], - capture_output=True, text=True) - self.assertEqual(result.returncode, 1, result.stdout + result.stderr) - self.assertIn("error: guest_probe skipped", result.stdout) - - def test_ctest_status_preserves_high_refresh_policy(self): - reason = "skipped, current display does not provide 120 Hz" - reports = ( - (f'{reason}', 0), - (f'{reason}', 0), - (f'{reason}\nunexpected failure detail', 1), - (f'{reason}\nskipped, flaky', 1), - (f'skipped, no display\n{reason}', 1), - ) - with tempfile.TemporaryDirectory() as directory: - junit = Path(directory) / "junit.xml" - for report, expected_code in reports: - for job in ("linux", "windows"): - with self.subTest(job=job, report=report): - junit.write_text(f'{report}', encoding="utf-8") - result = subprocess.run([sys.executable, "-I", str(TOOL), str(junit), "--job", job], - capture_output=True, text=True) - self.assertEqual(result.returncode, expected_code, result.stdout + result.stderr) - - def test_list_reports_device_reason_after_ctest_status(self): - xml = ('' - 'f16 native: skipped, the device has no native f16') - with tempfile.TemporaryDirectory() as directory: - junit = Path(directory) / "junit.xml" - junit.write_text(xml, encoding="utf-8") - result = subprocess.run([sys.executable, "-I", str(TOOL), str(junit), "--list"], - capture_output=True, text=True) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertIn("guest_probe: skipped, the device has no native f16", result.stdout) - self.assertNotIn("SKIP_RETURN_CODE", result.stdout) - - def test_unexpected_skip_reason_cannot_hide_behind_a_device_limitation(self): - reports = ( - ('skipped, the device lacks geometryShader', "linux"), - ('skipped, no display', "windows"), - ('flakyskipped, no usable Vulkan device', "windows"), - ('skipped, flaky\nskipped, the device lacks geometryShader', "linux"), - ) - with tempfile.TemporaryDirectory() as directory: - junit = Path(directory) / "junit.xml" - for report, job in reports: - with self.subTest(job=job, report=report): - junit.write_text(f'{report}', encoding="utf-8") - result = subprocess.run([sys.executable, "-I", str(TOOL), str(junit), "--job", job], - capture_output=True, text=True) - self.assertEqual(result.returncode, 1, result.stdout + result.stderr) - self.assertIn("error: guest_probe skipped", result.stdout) - - def test_multiple_allowed_skip_reasons_remain_expected(self): - reports = ( - ('skipped, no display', "linux"), - ('skipped, no usable Vulkan device', "windows"), - ) - with tempfile.TemporaryDirectory() as directory: - junit = Path(directory) / "junit.xml" - for report, job in reports: - with self.subTest(job=job, report=report): - junit.write_text(f'{report}', encoding="utf-8") - result = subprocess.run([sys.executable, "-I", str(TOOL), str(junit), "--job", job], - capture_output=True, text=True) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - def test_non_junit_xml_fails_in_every_cli_mode(self): reports = ('', 'coverage') with tempfile.TemporaryDirectory() as directory: @@ -241,6 +146,12 @@ class CheckSkipsTests(unittest.TestCase): with self.subTest(message=message): self.assertTrue(check_skips.DEVICE_LIMITATION.search(message)) + def test_host_limitations_match(self): + for message in HOST_LIMITATIONS: + with self.subTest(message=message): + self.assertTrue(check_skips.HOST_LIMITATION.search("skipped, Skipped\n" + message + "\n")) + self.assertFalse(check_skips.HOST_LIMITATION.search("skipping: not enough free disk space for a file of 1 MiB")) + def test_other_reasons_do_not_match(self): for message in OTHER: with self.subTest(message=message): @@ -349,40 +260,6 @@ class CheckSkipsTests(unittest.TestCase): -class RequiredTestsTests(unittest.TestCase): - def test_required_reports_without_testcases_fail_in_both_jobs(self): - for xml in ('', ''): - with tempfile.TemporaryDirectory() as directory: - junit = Path(directory) / "junit.xml" - junit.write_text(xml, encoding="utf-8") - for job in ("linux", "windows"): - for options in ([], ["--list"]): - with self.subTest(xml=xml, job=job, options=options): - result = subprocess.run( - [sys.executable, "-I", str(TOOL), str(junit), "--job", job, "--require-tests", *options], - capture_output=True, text=True, timeout=5) - self.assertEqual(result.returncode, 2, result.stdout + result.stderr) - self.assertIn("no testcases", result.stderr) - result = subprocess.run( - [sys.executable, "-I", str(TOOL), str(junit), "--job", job, *options], - capture_output=True, text=True, timeout=5) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - - def test_executed_tests_without_skips_pass_in_both_jobs(self): - for xml in ('', - ''): - with tempfile.TemporaryDirectory() as directory: - junit = Path(directory) / "junit.xml" - junit.write_text(xml, encoding="utf-8") - for job in ("linux", "windows"): - with self.subTest(xml=xml, job=job): - result = subprocess.run( - [sys.executable, "-I", str(TOOL), str(junit), "--job", job, "--require-tests"], - capture_output=True, text=True, timeout=5) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertIn("0 test(s) skipped, all expected", result.stdout) - - class SkipReportTests(unittest.TestCase): def skipped(self, content): with tempfile.TemporaryDirectory() as directory: @@ -390,30 +267,6 @@ class SkipReportTests(unittest.TestCase): path.write_text(content, encoding="utf-8") return list(check_skips.skips(path)) - def test_a_test_listed_twice_is_rejected(self): - reason = ('' - 'skipped, the device lacks geometryShader') - flaky = ('' - 'skipped, flaky') - for content in ( - f'{reason}{reason}', - f'{flaky}{reason}', - f'{reason}{flaky}', - ): - with self.subTest(content=content), self.assertRaisesRegex(ValueError, "listed more than once"): - self.skipped(content) - with tempfile.TemporaryDirectory() as directory: - junit = Path(directory) / "junit.xml" - junit.write_text( - f'{reason}{reason}', - encoding="utf-8", - ) - result = subprocess.run([sys.executable, "-I", str(TOOL), str(junit), "--job", "linux"], - capture_output=True, text=True) - self.assertEqual(result.returncode, 2, result.stdout + result.stderr) - self.assertIn("listed more than once", result.stderr) - self.assertNotIn("Traceback", result.stderr) - def test_non_junit_roots_are_rejected_even_with_testcases(self): for content in ('', '', '', @@ -464,26 +317,5 @@ class SkipReportTests(unittest.TestCase): self.assertRegex(found[0][1], check_skips.DEVICE_LIMITATION) -class NotRunTests(unittest.TestCase): - @unittest.skipUnless(shutil.which("ctest"), "CTest is required to write a real JUnit report") - def test_real_notrun_case_is_rejected_before_windows_agc_exemption(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - (root / "CTestTestfile.cmake").write_text('add_test(agc_missing "' + (root / "missing-program").as_posix() + '")\n', encoding="utf-8") - report = root / "report.xml" - generated = subprocess.run(["ctest", "--test-dir", str(root), "--output-junit", str(report)], - capture_output=True, text=True, timeout=10) - self.assertNotEqual(generated.returncode, 0) - self.assertIn('status="notrun"', report.read_text(encoding="utf-8")) - for job in ("windows", "linux"): - with self.subTest(job=job): - checked = subprocess.run([sys.executable, str(TOOL), str(report), "--job", job, "--require-tests"], - capture_output=True, text=True, timeout=10) - self.assertEqual(checked.returncode, 2, checked.stdout + checked.stderr) - self.assertIn("notrun", checked.stderr) - self.assertIn("agc_missing", checked.stderr) - self.assertNotIn("Traceback", checked.stderr) - - if __name__ == "__main__": unittest.main()