Merge pull request #322 from jix/test_external_examples

allow running SBY tests with an external examples directory

docs/examples/Makefile

@@ -1,3 +1,3 @@
-SUBDIR=../docs/examples
+SUBDIR=examples
 TESTDIR=../../tests
 include $(TESTDIR)/make/subdir.mk

docs/examples/abstract/Makefile

@@ -1,3 +1,3 @@
-SUBDIR=../docs/examples/abstract
+SUBDIR=examples/abstract
 TESTDIR=../../../tests
 include $(TESTDIR)/make/subdir.mk

docs/examples/demos/Makefile

@@ -1,3 +1,3 @@
-SUBDIR=../docs/examples/demos
+SUBDIR=examples/demos
 TESTDIR=../../../tests
 include $(TESTDIR)/make/subdir.mk

docs/examples/fifo/Makefile

@@ -1,3 +1,3 @@
-SUBDIR=../docs/examples/fifo
+SUBDIR=examples/fifo
 TESTDIR=../../../tests
 include $(TESTDIR)/make/subdir.mk

docs/examples/indinv/Makefile

@@ -1,3 +1,3 @@
-SUBDIR=../docs/examples/indinv
+SUBDIR=examples/indinv
 TESTDIR=../../../tests
 include $(TESTDIR)/make/subdir.mk

docs/examples/multiclk/Makefile

@@ -1,3 +1,3 @@
-SUBDIR=../docs/examples/multiclk
+SUBDIR=examples/multiclk
 TESTDIR=../../../tests
 include $(TESTDIR)/make/subdir.mk

docs/examples/puzzles/Makefile

@@ -1,3 +1,3 @@
-SUBDIR=../docs/examples/puzzles
+SUBDIR=examples/puzzles
 TESTDIR=../../../tests
 include $(TESTDIR)/make/subdir.mk

docs/examples/quickstart/Makefile

@@ -1,3 +1,3 @@
-SUBDIR=../docs/examples/quickstart
+SUBDIR=examples/quickstart
 TESTDIR=../../../tests
 include $(TESTDIR)/make/subdir.mk

docs/examples/tristate/Makefile

@@ -1,3 +1,3 @@
-SUBDIR=../docs/examples/tristate
+SUBDIR=examples/tristate
 TESTDIR=../../../tests
 include $(TESTDIR)/make/subdir.mk

tests/Makefile

@@ -24,16 +24,28 @@ else
 SBY_MAIN := $(realpath $(dir $(firstword $(MAKEFILE_LIST)))/make/run_sby.py)
 endif
 
+ifeq ($(SBY_EXAMPLES),)
+EXAMPLE_DIR := ../docs/examples
+else
+EXAMPLE_DIR := $(SBY_EXAMPLES)
+endif
+
+CHECK_COLLECT := $(shell python3 make/collect_tests.py --check --output make/rules/collect.mk --examples $(EXAMPLE_DIR))
+ifneq (,$(CHECK_COLLECT))
+$(warning $(CHECK_COLLECT))
+endif
+
+
 ifeq (nt-unix-like,$(OS_NAME))
 SBY_MAIN := $(shell cygpath -w $(SBY_MAIN))
 endif
 export SBY_MAIN
 
 make/rules/collect.mk: make/collect_tests.py
-	python3 make/collect_tests.py
+	python3 make/collect_tests.py --output $@ --examples $(EXAMPLE_DIR)
 
 make/rules/test/%.mk:
-	python3 make/test_rules.py $<
+	python3 make/test_rules.py --rule $@ --source $<
 
 ifneq (help,$(MAKECMDGOALS))
 

tests/make/collect_tests.py

@@ -1,11 +1,32 @@
 from pathlib import Path
 import re
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--output", required=True)
+parser.add_argument("--examples")
+parser.add_argument("--check", action='store_true')
+args = parser.parse_args()
 
 tests = []
 checked_dirs = []
 
 SAFE_PATH = re.compile(r"^[a-zA-Z0-9_./\\]*$")
 
+out_file = Path(args.output)
+header_line = f"# example dir = {args.examples}"
+
+if args.check:
+    try:
+        with out_file.open("r") as f:
+            found_header = f.readline().strip()
+        if found_header != header_line:
+            out_file.unlink()
+    except FileNotFoundError:
+        pass
+    exit()
+
+out_file.parent.mkdir(exist_ok=True)
 
 def collect(path):
     # don't pick up any paths that need escaping nor any sby workdirs
@@ -13,6 +34,9 @@ def collect(path):
         not SAFE_PATH.match(str(path))
         or (path / "config.sby").exists()
         or (path / "status.sqlite").exists()
+        or (path / "config.mcy").exists()
+        or path.name == "__pycache__"
+        or path == out_file.parent
     ):
         return
 
@@ -24,6 +48,8 @@ def collect(path):
             continue
         if entry.name.startswith("skip_"):
             continue
+        if entry.with_suffix(".ivy").exists():
+            continue
         tests.append(entry)
     for entry in path.glob("*"):
         if entry.is_dir():
@@ -31,29 +57,41 @@ def collect(path):
 
 
 def unix_path(path):
-    return "/".join(path.parts)
+    source_path = "/".join(path.parts)
+    if source_path.startswith("//"):
+        source_path = source_path[1:]
+    if args.examples:
+        try:
+            relative = path.relative_to(args.examples)
+        except ValueError:
+            pass
+        else:
+            if '..' not in relative.parts:
+                return source_path, "examples/" + "/".join(relative.parts)
+    if '..' in path.parts:
+        raise RuntimeError("path escapes collect directories")
+    return source_path, source_path
 
 
-collect(Path("."))
-collect(Path("../docs/examples"))
-
-out_file = Path("make/rules/collect.mk")
-out_file.parent.mkdir(exist_ok=True)
+collect(Path('.'))
+if args.examples:
+    collect(Path(args.examples))
 
 with out_file.open("w") as output:
+    print(header_line, file=output)
 
     for checked_dir in checked_dirs:
         print(f"{out_file}: {checked_dir}", file=output)
 
     for test in tests:
-        test_unix = unix_path(test)
-        print(f"make/rules/test/{test_unix}.mk: {test_unix}", file=output)
+        test_unix_source, test_unix_rule = unix_path(test)
+        print(f"make/rules/test/{test_unix_rule}.mk: {test_unix_source}", file=output)
         for ext in [".sh", ".py"]:
             script_file = test.parent / (test.stem + ext)
             if script_file.exists():
-                script_file_unix = unix_path(script_file)
-                print(f"make/rules/test/{test_unix}.mk: {script_file_unix}", file=output)
-        print(f"make/rules/test/{test_unix}.mk: make/test_rules.py", file=output)
+                script_file_unix_source, script_file_unix_rule = unix_path(script_file)
+                print(f"make/rules/test/{test_unix_rule}.mk: {script_file_unix_source}", file=output)
+        print(f"make/rules/test/{test_unix_rule}.mk: make/test_rules.py", file=output)
     for test in tests:
-        test_unix = unix_path(test)
-        print(f"-include make/rules/test/{test_unix}.mk", file=output)
+        test_unix_source, test_unix_rule = unix_path(test)
+        print(f"-include make/rules/test/{test_unix_rule}.mk", file=output)

tests/make/subdir.mk

@@ -5,6 +5,9 @@ test:
 
 .PHONY: test refresh IMPLICIT_PHONY
 
+Makefile: IMPLICIT_PHONY
+%.mk: IMPLICIT_PHONY
+
 IMPLICIT_PHONY:
 
 refresh:

tests/make/test_rules.py

@@ -3,16 +3,27 @@ import os
 import subprocess
 import json
 import shlex
+import argparse
 from pathlib import Path
 
 from required_tools import REQUIRED_TOOLS
 
+parser = argparse.ArgumentParser()
+parser.add_argument("--rule", required=True)
+parser.add_argument("--source", required=True)
+args = parser.parse_args()
+
 
 def unix_path(path):
-    return "/".join(path.parts)
+    source_path = "/".join(path.parts)
+    if source_path.startswith("//"):
+        source_path = source_path[1:]
+    return source_path
 
 
-sby_file = Path(sys.argv[1])
+sby_file = Path(args.source)
+rules_file = Path(args.rule)
+rules_base = rules_file.parent.parts[3:]
 sby_dir = sby_file.parent
 
 
@@ -33,17 +44,19 @@ def parse_engine(engine):
     return engine, default_solvers.get(engine)
 
 
-rules_file = Path("make/rules/test") / sby_dir / (sby_file.name + ".mk")
 rules_file.parent.mkdir(exist_ok=True, parents=True)
 
 with rules_file.open("w") as rules:
-    name = str(sby_dir / sby_file.stem)
+    name = unix_path(sby_dir / sby_file.stem)
+    rule_name = "/".join(rules_base) + f"/{sby_file.stem}"
 
     for task, info in taskinfo.items():
         target = name
+        rule_target = rule_name
         workdirname = sby_file.stem
         if task:
             target += f"_{task}"
+            rule_target += f"_{task}"
             workdirname += f"_{task}"
 
         engines = set()
@@ -71,8 +84,8 @@ with rules_file.open("w") as rules:
 
         required_tools = sorted(required_tools)
 
-        print(f".PHONY: {target}", file=rules)
-        print(f"{target}:", file=rules)
+        print(f".PHONY: {rule_target}", file=rules)
+        print(f"{rule_target}:", file=rules)
 
         shell_script = sby_dir / f"{sby_file.stem}.sh"
 
@@ -81,15 +94,17 @@ with rules_file.open("w") as rules:
         if shell_script.exists():
             command = f"cd {sby_dir_unix} && env SBY_FILE={sby_file.name} WORKDIR={workdirname} TASK={task} bash {shell_script.name}"
         else:
-            command = f"cd {sby_dir_unix} && python3 $(SBY_MAIN) -f {sby_file.name} {task}"
+            command = (
+                f"cd {sby_dir_unix} && python3 $(SBY_MAIN) -f {sby_file.name} {task}"
+            )
 
         print(
             f"\t+@python3 make/required_tools.py run {target} {shlex.quote(command)} {shlex.join(required_tools)}",
             file=rules,
         )
 
-        print(f".PHONY: clean-{target}", file=rules)
-        print(f"clean-{target}:", file=rules)
+        print(f".PHONY: clean-{rule_target}", file=rules)
+        print(f"clean-{rule_target}:", file=rules)
         print(f"\trm -rf {target}", file=rules)
 
         test_groups = []
@@ -112,18 +127,18 @@ with rules_file.open("w") as rules:
 
         prefix = ""
 
-        for part in [*sby_dir.parts, ""]:
+        for part in [*rules_base, ""]:
             print(f".PHONY: {prefix}clean {prefix}test", file=rules)
-            print(f"{prefix}clean: clean-{target}", file=rules)
-            print(f"{prefix}test: {target}", file=rules)
+            print(f"{prefix}clean: clean-{rule_target}", file=rules)
+            print(f"{prefix}test: {rule_target}", file=rules)
 
             for test_group in test_groups:
                 print(f".PHONY: {prefix}{test_group}", file=rules)
-                print(f"{prefix}{test_group}: {target}", file=rules)
+                print(f"{prefix}{test_group}: {rule_target}", file=rules)
             prefix += f"{part}/"
 
     tasks = [task for task in taskinfo.keys() if task]
 
     if tasks:
-        print(f".PHONY: {name}", file=rules)
-        print(f"{name}:", *(f"{name}_{task}" for task in tasks), file=rules)
+        print(f".PHONY: {rule_name}", file=rules)
+        print(f"{rule_name}:", *(f"{rule_name}_{task}" for task in tasks), file=rules)