tests: Move required tool checks from rule generation to execution

This avoids regenerating the test makefile rules when the set of
installed tools changes and is a bit simpler overall.

tests/Makefile

@@ -31,12 +31,6 @@ make/rules/collect.mk: make/collect_tests.py
 make/rules/test/%.mk:
 	python3 make/test_rules.py $<
 
-ifdef NOSKIP
-SKIP_COMMAND := echo "NOSKIP was set, treating this as an error"; echo; false
-else
-SKIP_COMMAND := echo
-endif
-
 ifneq (help,$(MAKECMDGOALS))
 
 # This should run every time but only trigger anything depending on it whenever

tests/make/collect_tests.py

@@ -37,7 +37,7 @@ with out_file.open("w") as output:
         print(f"{out_file}: {checked_dir}", file=output)
 
     for test in tests:
-        print(f"make/rules/test/{test}.mk: {test} make/rules/found_tools", file=output)
+        print(f"make/rules/test/{test}.mk: {test}", file=output)
         for ext in [".sh", ".py"]:
             script_file = test.parent / (test.stem + ext)
             if script_file.exists():

tests/make/required_tools.py

@@ -17,11 +17,40 @@ REQUIRED_TOOLS = {
 }
 
 
+def found_tools():
+    with open("make/rules/found_tools") as found_tools_file:
+        return [tool.strip() for tool in found_tools_file.readlines()]
+
+
 if __name__ == "__main__":
     import subprocess
     import sys
+    import os
     from pathlib import Path
 
+    args = sys.argv[1:]
+
+    if args and args[0] == "run":
+        target, command, *required_tools = args[1:]
+
+        with open("make/rules/found_tools") as found_tools_file:
+            found_tools = set(tool.strip() for tool in found_tools_file.readlines())
+
+        missing_tools = sorted(
+            f"`{tool}`" for tool in required_tools if tool not in found_tools
+        )
+        if missing_tools:
+            noskip = "NOSKIP" in os.environ.get("MAKEFLAGS", "")
+            print()
+            print(f"SKIPPING {target}: {', '.join(missing_tools)} not found")
+            if noskip:
+                print("NOSKIP was set, treating this as an error")
+            print()
+            exit(noskip)
+
+        print(command, flush=True)
+        exit(subprocess.call(command, shell=True))
+
     found_tools = []
     check_tools = set()
     for tools in REQUIRED_TOOLS.values():
@@ -59,6 +88,3 @@ if __name__ == "__main__":
 
     with open("make/rules/found_tools", "w") as found_tools_file:
         found_tools_file.write(found_tools)
-else:
-    with open("make/rules/found_tools") as found_tools_file:
-        found_tools = [tool.strip() for tool in found_tools_file.readlines()]

tests/make/test_rules.py

@@ -2,9 +2,10 @@ import sys
 import os
 import subprocess
 import json
+import shlex
 from pathlib import Path
 
-from required_tools import REQUIRED_TOOLS, found_tools
+from required_tools import REQUIRED_TOOLS
 
 sby_file = Path(sys.argv[1])
 sby_dir = sby_file.parent
@@ -55,31 +56,19 @@ with rules_file.open("w") as rules:
                 solvers.add(solver)
                 engine_solvers.add((engine, solver))
 
+        required_tools = sorted(required_tools)
+
         print(f".PHONY: {target}", file=rules)
         print(f"{target}:", file=rules)
 
         shell_script = sby_dir / f"{sby_file.stem}.sh"
 
-        missing_tools = sorted(
-            f"`{tool}`" for tool in required_tools if tool not in found_tools
-        )
-
-        if missing_tools:
-            print(
-                f"\t@echo; echo 'SKIPPING {target}: {', '.join(missing_tools)} not found'; $(SKIP_COMMAND)",
-                file=rules,
-            )
-
-        elif shell_script.exists():
-            print(
-                f"\tcd {sby_dir} && SBY_FILE={sby_file.name} WORKDIR={workdirname} TASK={task} bash {shell_script.name}",
-                file=rules,
-            )
+        if shell_script.exists():
+            command = f"cd {sby_dir} && SBY_FILE={sby_file.name} WORKDIR={workdirname} TASK={task} bash {shell_script.name}"
         else:
-            print(
-                f"\tcd {sby_dir} && python3 $(SBY_MAIN) -f {sby_file.name} {task}",
-                file=rules,
-            )
+            command = f"cd {sby_dir} && python3 $(SBY_MAIN) -f {sby_file.name} {task}"
+
+        print(f"\t@python3 make/required_tools.py run {target} {shlex.quote(command)} {shlex.join(required_tools)}", file=rules)
 
         print(f".PHONY: clean-{target}", file=rules)
         print(f"clean-{target}:", file=rules)