fix(tools): skip unselected executables in fresh builds

tools/select_tests.py

@@ -1,5 +1,6 @@
 import argparse
 import collections
+import importlib.util
 import json
 import os
 import re
@@ -7,6 +8,10 @@ import subprocess
 import sys
 from pathlib import Path
 
+REGISTRATION_SPEC = importlib.util.spec_from_file_location("check_test_registration", Path(__file__).with_name("check_test_registration.py"))
+REGISTRATION = importlib.util.module_from_spec(REGISTRATION_SPEC)
+REGISTRATION_SPEC.loader.exec_module(REGISTRATION)
+
 FULL = re.compile(r"^CMakeLists\.txt$|^core/CMakeLists\.txt$|^core/libs/CMakeLists\.txt$|^\.github/workflows/build\.yml$"
                   r"|^3rdparty/|^cmake/|^\.gitmodules$|^tools/select_tests\.py$")
 LOCAL = re.compile(r"(^|/)CMakeLists\.txt$|\.cmake$")
@@ -87,8 +92,36 @@ class Build:
     def tests(self, *arguments):
         listing = json.loads(subprocess.run(["ctest", "--test-dir", self.path, "--show-only=json-v1", *arguments],
                                             capture_output=True, text=True, check=True).stdout)
+        missing = {test["name"] for test in listing["tests"] if not test.get("command")}
+        if missing:
+            commands = self.test_commands(missing)
+            for test in listing["tests"]:
+                candidates = commands.get(test["name"], set())
+                if test["name"] in missing and len(candidates) == 1:
+                    command = next(iter(candidates))
+                    if command and os.path.isabs(command[0]):
+                        test["command"] = list(command)
         return listing, listing["tests"]
 
+    def test_commands(self, names):
+        found, visited, pending = collections.defaultdict(set), set(), [Path(self.path)]
+        while pending:
+            directory = pending.pop().resolve()
+            file = directory / "CTestTestfile.cmake"
+            if directory in visited or not file.is_file():
+                continue
+            visited.add(directory)
+            try:
+                commands = list(REGISTRATION.commands(file.read_text(encoding="utf-8-sig")))
+            except (OSError, UnicodeError, ValueError):
+                continue
+            for command, arguments in commands:
+                if command == "subdirs":
+                    pending.extend(directory / argument for argument in arguments)
+                elif command == "add_test" and arguments and arguments[0] in names:
+                    found[arguments[0]].add(tuple(arguments[1:]))
+        return found
+
     def output(self, path):
         absolute = self.absolute(path)
         if not inside(absolute, self.path):
@@ -216,6 +249,8 @@ def plan(build, graph, picked):
     _, selected = build.tests("-I", "0,0,0," + ",".join(map(str, run)))
     names = {t["name"] for t in selected}
     run = [i for i, t in enumerate(listed, 1) if t["name"] in names]
+    if any(not t.get("command") for t in listed if t["name"] in names):
+        return run, build.leaves(), len(listed)
     keep = {build.output(a) for t in listed if t["name"] in names for a in t.get("command") or [] if os.path.isabs(a)}
     skip = {build.target(t) for t in listed if t["name"] not in names} - keep - {None}
     return run, [t for t in build.leaves() if t not in skip], len(listed)

tools/tests/test_select_tests.py

@@ -119,13 +119,61 @@ class SelectTests(unittest.TestCase):
         self.assertEqual(self.select("alpha/alpha.h", graph=older)[1], {"alpha", "alpha_data", "beta", "beta_setup", "script"})
 
     def test_selected_target_skips_unselected_executables(self):
-        self.select("alpha/main.c")
         subprocess.run(["ninja", "-C", str(self.build), "-t", "clean"], check=True, capture_output=True)
+        self.select("alpha/main.c")
         subprocess.run(["ninja", "-C", str(self.build), "-f", "select.ninja", "selected"], check=True, capture_output=True)
         built = {p.stem for p in self.build.rglob("*_tests*") if p.is_file() and os.access(p, os.X_OK)}
         subprocess.run(["cmake", "--build", str(self.build)], check=True, capture_output=True)
         self.assertEqual(built, {"alpha_tests"})
 
+    def test_fresh_build_keeps_a_shared_selected_executable(self):
+        with tempfile.TemporaryDirectory() as directory:
+            build = Path(directory) / "fresh"
+            compiler = shutil.which("gcc") or shutil.which("cc")
+            subprocess.run(["cmake", "-S", str(self.source), "-B", str(build), "-G", "Ninja", f"-DCMAKE_C_COMPILER={compiler}"],
+                           check=True, capture_output=True)
+            changed = build / "changed.txt"
+            changed.write_text("data/input.txt\n", encoding="utf-8")
+            self.tool("select", build, self.graph, changed)
+            subprocess.run(["ninja", "-C", str(build), "-f", "select.ninja", "selected"], check=True, capture_output=True)
+            built = {path.stem for path in build.rglob("*_tests*") if path.is_file() and os.access(path, os.X_OK)}
+            self.assertEqual(built, {"alpha_tests"})
+            output = self.tool("ctest", build, "--output-on-failure")
+            self.assertEqual(set(re.findall(r"Test +#\d+: (\S+)", output)), {"alpha_data", "script"})
+
+    def test_empty_selection_does_not_build_unselected_executables(self):
+        cmake = self.source / "CMakeLists.txt"
+        original = cmake.read_text(encoding="utf-8")
+        try:
+            cmake.write_text(original.replace('add_test(NAME script COMMAND ${CMAKE_COMMAND} -E true)\n', ''), encoding="utf-8")
+            subprocess.run(["cmake", "-S", str(self.source), "-B", str(self.build)], check=True, capture_output=True)
+            subprocess.run(["ninja", "-C", str(self.build), "-t", "clean"], check=True, capture_output=True)
+            self.assertEqual(self.select("README.md")[1], set())
+            subprocess.run(["ninja", "-C", str(self.build), "-f", "select.ninja", "selected"], check=True, capture_output=True)
+            self.assertFalse(any(path.is_file() and os.access(path, os.X_OK) for path in self.build.rglob("*_tests*")))
+        finally:
+            cmake.write_text(original, encoding="utf-8")
+            subprocess.run(["cmake", "-S", str(self.source), "-B", str(self.build)], check=True, capture_output=True)
+            subprocess.run(["cmake", "--build", str(self.build)], check=True, capture_output=True)
+
+    def test_retargeted_test_does_not_prune_its_previous_executable(self):
+        cmake = self.source / "alpha/CMakeLists.txt"
+        original = cmake.read_text(encoding="utf-8")
+        try:
+            cmake.write_text(original.replace("add_executable(alpha_tests main.c)",
+                                             "add_executable(alpha_tests main.c)\nadd_executable(replacement_tests main.c)")
+                             .replace("COMMAND alpha_tests", "COMMAND replacement_tests"), encoding="utf-8")
+            subprocess.run(["cmake", "-S", str(self.source), "-B", str(self.build)], check=True, capture_output=True)
+            subprocess.run(["ninja", "-C", str(self.build), "-t", "clean"], check=True, capture_output=True)
+            self.select("beta/main.c")
+            subprocess.run(["ninja", "-C", str(self.build), "-f", "select.ninja", "selected"], check=True, capture_output=True)
+            built = {path.stem for path in self.build.rglob("*_tests*") if path.is_file() and os.access(path, os.X_OK)}
+            self.assertEqual(built, {"alpha_tests", "beta_tests", "setup_tests"})
+        finally:
+            cmake.write_text(original, encoding="utf-8")
+            subprocess.run(["cmake", "-S", str(self.source), "-B", str(self.build)], check=True, capture_output=True)
+            subprocess.run(["cmake", "--build", str(self.build)], check=True, capture_output=True)
+
     def test_explain_reports_input_and_fixture_reasons_without_a_build_cache(self):
         changed = self.build / "explain-changed.txt"
         cache = self.build / "CMakeCache.txt"
@@ -182,8 +230,8 @@ class FixtureBuildTests(unittest.TestCase):
         marker.unlink(missing_ok=True)
         listed = self.build / "changed.txt"
         listed.write_text(changed + "\n", encoding="utf-8")
-        SelectTests.tool("select", self.build, graph, listed)
         subprocess.run(["ninja", "-C", str(self.build), "-t", "clean"], check=True, capture_output=True)
+        SelectTests.tool("select", self.build, graph, listed)
         subprocess.run(["ninja", "-C", str(self.build), "-f", "select.ninja", "selected"], check=True, capture_output=True)
         result = subprocess.run([sys.executable, str(TOOL), "ctest", str(self.build), "--output-on-failure"],
                                 capture_output=True, text=True)
@@ -292,6 +340,15 @@ class CaseInsensitivePathTests(unittest.TestCase):
 
 
 class PlanTests(unittest.TestCase):
+    def test_unresolved_selected_command_keeps_unselected_build_targets(self):
+        build = mock.Mock()
+        listed = [{"name": "selected"}, {"name": "other", "command": ["/build/tests/shared_tests"]}]
+        build.tests.side_effect = [({}, listed), ({}, [listed[0]])]
+        build.leaves.return_value = ["tests/shared_tests", "helper"]
+        build.target.side_effect = lambda test: "tests/shared_tests" if test.get("command") else None
+        graph = {"tests": {"selected": {}, "other": {}}}
+        self.assertEqual(select_tests.plan(build, graph, {"selected"}), ([1], ["tests/shared_tests", "helper"], 2))
+
     def test_plan_empty_run_avoids_invoking_ctest(self):
         build = mock.Mock()
         build.tests.return_value = ({}, [{"name": "alpha", "command": ["/build/alpha_tests"]}])
@@ -305,5 +362,35 @@ class PlanTests(unittest.TestCase):
         build.tests.assert_called_once_with()
 
 
+class MissingCommandTests(unittest.TestCase):
+    def test_only_unambiguous_absolute_commands_in_reachable_files_are_recovered(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            active, stale = root / "active tests", root / "stale"
+            active.mkdir()
+            stale.mkdir()
+            executable = (root / "tests/unit_tests").as_posix()
+            helper = (root / "tests/helper_tests").as_posix()
+            (root / "CTestTestfile.cmake").write_text('subdirs("active tests")\n', encoding="utf-8")
+            (active / "CTestTestfile.cmake").write_text(
+                f'add_test([=[absolute]=] "{executable}" "{helper}")\n'
+                'add_test(relative tests/relative_tests)\n'
+                'add_test(path unit_tests)\n'
+                f'if(CONFIG)\nadd_test(ambiguous "{executable}")\nelse()\nadd_test(ambiguous "{helper}")\nendif()\n'
+                f'add_test(existing "{executable}")\n'
+                'subdirs("..")\n', encoding="utf-8")
+            (stale / "CTestTestfile.cmake").write_text(f'add_test(stale "{executable}")\n', encoding="utf-8")
+            names = ("absolute", "relative", "path", "ambiguous", "existing", "stale")
+            listed = [{"name": name} for name in names]
+            listed[4]["command"] = [helper]
+            build = object.__new__(select_tests.Build)
+            build.path = str(root)
+            with mock.patch.object(select_tests.subprocess, "run", return_value=mock.Mock(stdout=json.dumps({"tests": listed}))):
+                _, recovered = build.tests()
+            commands = {test["name"]: test.get("command") for test in recovered}
+            self.assertEqual(commands, {"absolute": [executable, helper], "relative": None, "path": None,
+                                        "ambiguous": None, "existing": [helper], "stale": None})
+
+
 if __name__ == "__main__":
     unittest.main()