import argparse import re import subprocess import sys from pathlib import Path TEST_OUTPUT = r"tests[\\/]" COMMAND = re.compile(r"^\s*(add_test|set_tests_properties|subdirs)\s*\(", re.M | re.I) ARGUMENT = re.compile(r'"(?:\\.|[^"\\])*"|\[(?P=*)\[.*?\](?P=equals)\]|[^\s()]+', re.S) LEXICAL = re.compile(r'\\.|"(?:\\.|[^"\\])*"|\[(?P=*)\[.*?\](?P=literal_equals)\]' r'|(?P\#\[(?P=*)\[.*?\](?P=comment_equals)\]|\#[^\n]*)', re.S) KNOWN_UNRUN = set() SCRIPT = re.compile(r"(? block_indent: yield line continue block_indent = None sequence = re.match(r"-\s+", text) while parents and (parents[-1][0] > indent or (parents[-1][0] == indent and not (sequence and parents[-1][1] == "steps"))): parents.pop() path = [key for _, key in parents] step = len(path) == 3 and path[0] == "jobs" and path[2] == "steps" key_indent = indent if sequence: key_indent += sequence.end() text = text[sequence.end():] if step: step_key_indent = key_indent match = re.match(r"(['\"]?)([A-Za-z_][\w.-]*)\1\s*:(?:\s+(.*)|$)", text) if match is None: continue key, value = match[2], match[3] or "" if key == "steps" and len(path) == 2 and path[0] == "jobs": step_key_indent = None if step and key_indent == step_key_indent and key == "run": if value.startswith(("|", ">")): block_indent = key_indent elif value: yield value parents.append((key_indent, key)) def run_scripts(root, files): run = set() for name in files: if name.endswith("CMakeLists.txt") or name.endswith(".cmake"): content = (Path(root) / name).read_text(encoding="utf-8", errors="replace") for command, arguments in commands(content): if command == "add_test": run.update(argument.rpartition("}")[2].lstrip("/") for argument in arguments) elif name.startswith(".github/workflows/") and name.endswith((".yml", ".yaml")): for line in workflow_run_lines((Path(root) / name).read_text(encoding="utf-8", errors="replace")): if line.lstrip().startswith("#"): continue line = re.split(r"\s#", line)[0] if "python" in line: run.update(token[2:] if token.startswith("./") else token for token in SCRIPT.findall(line)) return {name for name in run if SCRIPT.fullmatch(name)} def unrun_scripts(root): files = listed(root) run = run_scripts(root, files) scripts = [name for name in files if re.match(r"^test_\w+\.py$", name.rpartition("/")[2])] return scripts, [name for name in scripts if not any(name == path or name.endswith("/" + path) for path in run)] def main(argv=None): for stream in (sys.stdout, sys.stderr): if hasattr(stream, "reconfigure"): stream.reconfigure(encoding="utf-8", errors="backslashreplace") parser = argparse.ArgumentParser(description="fail when a test executable is built that no ctest test runs") parser.add_argument("build", type=Path, nargs="?", default=Path("build"), help="configured build directory") parser.add_argument("--list", action="store_true", help="print every test executable and whether ctest runs it") parser.add_argument("--scripts", action="store_true", help="instead check that an add_test or a workflow runs every test_*.py of the source tree") args = parser.parse_args(argv) if args.scripts: scripts, unrun = unrun_scripts(Path(__file__).resolve().parents[1]) for name in unrun: print(f"error: {name} is a test script that no add_test or workflow runs") if unrun: print(f"{len(unrun)} test script(s) would never run: register each with add_test, or run it from a workflow") return 1 print(f"{len(scripts)} test script(s), each run by an add_test or a workflow") return 0 try: every = sorted(built(args.build)) run = registered(args.build) except (OSError, ValueError, subprocess.CalledProcessError) as error: parser.error(str(error)) if args.list: for name in every: print(f"{'run' if name in run else 'NOT RUN'} {name}") return 0 unrun = sorted(set(every) - run) if not every: print(f"no test executables found under {args.build}; is it configured with -DBUILD_TESTING=ON?") return 1 for name in sorted(set(unrun) & KNOWN_UNRUN): print(f"note: {name} is built and not run, as recorded") stale = sorted(KNOWN_UNRUN & run) for name in stale: print(f"error: {name} is run by ctest now; remove it from KNOWN_UNRUN") fresh = sorted(set(unrun) - KNOWN_UNRUN) for name in fresh: print(f"error: {name} is built but no ctest test runs it") if fresh: print(f"{len(fresh)} test executable(s) would never run: ctest reports a full pass over what it knows, so a test nobody registered is invisible in CI") if stale or fresh: return 1 print(f"{len(every)} test executable(s), {len(unrun)} built and not run as recorded, none new") return 0 if __name__ == "__main__": sys.exit(main())