Save results, and create summary and report

tests/.gitignore

@@ -3,3 +3,4 @@
 *.err
 run-test.mk
 **/Makefile
+*.result

tests/Makefile

@@ -34,37 +34,37 @@ MK_TEST_DIRS += ./verilog
 
 # Tests that don't generate .mk
 SH_TEST_DIRS =
-SH_TEST_DIRS += ./simple
+#SH_TEST_DIRS += ./simple
-SH_TEST_DIRS += ./simple_abc9
+#SH_TEST_DIRS += ./simple_abc9
-SH_TEST_DIRS += ./hana
+#SH_TEST_DIRS += ./hana
-SH_TEST_DIRS += ./asicworld
+#SH_TEST_DIRS += ./asicworld
-SH_TEST_DIRS += ./realmath
+#SH_TEST_DIRS += ./realmath
-SH_TEST_DIRS += ./share
+#SH_TEST_DIRS += ./share
-SH_TEST_DIRS += ./opt_share
+#SH_TEST_DIRS += ./opt_share
-SH_TEST_DIRS += ./fsm
+#SH_TEST_DIRS += ./fsm
-SH_TEST_DIRS += ./memlib
+#SH_TEST_DIRS += ./memlib
-SH_TEST_DIRS += ./bram
+#SH_TEST_DIRS += ./bram
-SH_TEST_DIRS += ./svinterfaces
+#SH_TEST_DIRS += ./svinterfaces
-SH_TEST_DIRS += ./xprop
+#SH_TEST_DIRS += ./xprop
-SH_TEST_DIRS += ./select
+#SH_TEST_DIRS += ./select
-SH_TEST_DIRS += ./peepopt
+#SH_TEST_DIRS += ./peepopt
-SH_TEST_DIRS += ./proc
+#SH_TEST_DIRS += ./proc
-SH_TEST_DIRS += ./blif
+#SH_TEST_DIRS += ./blif
-SH_TEST_DIRS += ./arch
+#SH_TEST_DIRS += ./arch
-SH_TEST_DIRS += ./rpc
+#SH_TEST_DIRS += ./rpc
-SH_TEST_DIRS += ./memfile
+#SH_TEST_DIRS += ./memfile
-SH_TEST_DIRS += ./fmt
+#SH_TEST_DIRS += ./fmt
-SH_TEST_DIRS += ./cxxrtl
+#SH_TEST_DIRS += ./cxxrtl
-SH_TEST_DIRS += ./liberty
+#SH_TEST_DIRS += ./liberty
-ifeq ($(ENABLE_FUNCTIONAL_TESTS),1)
+#ifeq ($(ENABLE_FUNCTIONAL_TESTS),1)
-SH_TEST_DIRS += ./functional
+#SH_TEST_DIRS += ./functional
-endif
+#endif
 
 # Tests that don't generate .mk and need special args
 SH_ABC_TEST_DIRS =
-SH_ABC_TEST_DIRS += ./memories
+#SH_ABC_TEST_DIRS += ./memories
-SH_ABC_TEST_DIRS += ./aiger
+#SH_ABC_TEST_DIRS += ./aiger
-SH_ABC_TEST_DIRS += ./alumacc
+#SH_ABC_TEST_DIRS += ./alumacc
 
 all: vanilla-test
 
@@ -104,6 +104,15 @@ ifeq ($(YOSYS_NOVERIFIC),1)
 endif
 endif
 	@echo ""
+	@pass=$$(grep -h PASS **/*.result 2>/dev/null | wc -l); \
+	fail=$$(grep -h FAIL **/*.result 2>/dev/null | wc -l); \
+	total=$$((pass + fail)); \
+	echo "=========================="; \
+	echo "Tests: $$total"; \
+	echo "Passed: $$pass"; \
+	echo "Failed: $$fail"; \
+	echo "=========================="; \
+	test $$fail -eq 0
 
 clean:
 	@rm -rf ./asicworld/*.out ./asicworld/*.log
@@ -115,3 +124,14 @@ clean:
 	@rm -f ./svinterfaces/*.log_stdout ./svinterfaces/*.log_stderr ./svinterfaces/dut_result.txt ./svinterfaces/reference_result.txt ./svinterfaces/a.out ./svinterfaces/*_syn.v ./svinterfaces/*.diff
 	@rm -f  ./tools/cmp_tbdata
 	@rm -f $(addsuffix /run-test.mk,$(MK_TEST_DIRS))
+	@find . -name '*.result' -type f -exec rm -f {} +
+	@find . -mindepth 2 \( -path './sva*' -o -path './unit*' \) -prune -o -name 'Makefile' -type f -exec rm -f {} +
+
+.PHONY: report
+report:
+	@echo "=========================="
+	@echo "Failing tests:"
+	@find . -name '*.result' -type f -exec grep -H '^FAIL$$' {} + \
+	  | cut -d: -f1 \
+	  | sed 's|^\./||; s|\.result$$||'
+	@echo "=========================="

tests/gen_tests_makefile.py

@@ -15,8 +15,8 @@ def generate_target(name, command):
     print(f"all: {target}")
     print(f".PHONY: {target}")
     print(f"{target}:")
-    print(f"\t@YOSYS_MAX_THREADS=4 {command}")
+    print(f"\t@@$(call run_test,{target}, \\")
-    print(f"\t@echo 'Passed {target}'")
+    print(f"\tYOSYS_MAX_THREADS=4 {command})")
 
 def generate_ys_test(ys_file, yosys_args=""):
     cmd = f'$(YOSYS) -ql {ys_file}.err {yosys_args} {ys_file} >/dev/null 2>&1 && mv {ys_file}.err {ys_file}.log'
@@ -52,7 +52,18 @@ def generate_tests(argv):
 
     print(f"YOSYS ?= {yosys_basedir}/yosys")
     print()
-
+    print("define run_test")
+    print("	rc=0; \\")
+    print("	$(2) || rc=$$?; \\")
+    print("	if [ $$rc -eq 0 ]; then \\")
+    print("		echo \"PASS $1\"; \\")
+    print("		echo PASS > $1.result; \\")
+    print("	else \\")
+    print("		echo \"FAIL $1\"; \\")
+    print("		echo FAIL > $1.result; \\")
+    print("	fi")
+    print("endef")
+    print()
     print(".PHONY: all")
     print("all:")