Refactor tests

Organize tests into subdirectories and use a new makefile that scans
.sby files and allows selecting tests by mode, engine, solver and/or
subdirectory. Automatically skips tests that use engines/solvers that
are not found in the PATH.

See `cd tests; make help` for a description of supported make targets.

tests/keepgoing/Makefile

@@ -0,0 +1,2 @@
+SUBDIR=keepgoing
+include ../make/subdir.mk

tests/keepgoing/check_output.py

@@ -0,0 +1,17 @@
+import re
+
+
+def line_ref(dir, filename, pattern):
+    with open(dir + "/src/" + filename) as file:
+        if isinstance(pattern, str):
+            pattern_re = re.compile(re.escape(pattern))
+        else:
+            pattern_re = pattern
+            pattern = pattern.pattern
+
+        for number, line in enumerate(file, 1):
+            if pattern_re.search(line):
+                # Needs to match source locations for both verilog frontends
+                return fr"{filename}:(?:{number}|\d+.\d+-{number}.\d+)"
+
+        raise RuntimeError("%s: pattern `%s` not found" % (filename, pattern))

tests/keepgoing/keepgoing_multi_step.py

@@ -0,0 +1,31 @@
+import sys
+from check_output import *
+
+src = "keepgoing_multi_step.sv"
+
+workdir = sys.argv[1]
+
+assert_0 = line_ref(workdir, src, "assert(0)")
+step_3_7 = line_ref(workdir, src, "step 3,7")
+step_5 = line_ref(workdir, src, "step 5")
+step_7 = line_ref(workdir, src, "step 7")
+
+log = open(workdir + "/logfile.txt").read()
+log_per_trace = log.split("Writing trace to VCD file")[:-1]
+
+assert len(log_per_trace) == 4
+
+
+assert re.search(r"Assert failed in test: %s \(.*\)$" % assert_0, log_per_trace[0], re.M)
+
+for i in range(1, 4):
+    assert re.search(r"Assert failed in test: %s \(.*\) \[failed before\]$" % assert_0, log_per_trace[i], re.M)
+
+
+assert re.search(r"Assert failed in test: %s \(.*\)$" % step_3_7, log_per_trace[1], re.M)
+assert re.search(r"Assert failed in test: %s \(.*\)$" % step_5, log_per_trace[2], re.M)
+assert re.search(r"Assert failed in test: %s \(.*\) \[failed before\]$" % step_3_7, log_per_trace[3], re.M)
+assert re.search(r"Assert failed in test: %s \(.*\)$" % step_7, log_per_trace[3], re.M)
+
+pattern = f"Property ASSERT in test at {assert_0} failed. Trace file: engine_0/trace0.vcd"
+assert re.search(pattern, open(f"{workdir}/{workdir}.xml").read())

tests/keepgoing/keepgoing_multi_step.sby

@@ -0,0 +1,18 @@
+[tasks]
+bmc
+prove
+
+[options]
+bmc: mode bmc
+prove: mode prove
+expect fail
+
+[engines]
+smtbmc --keep-going boolector
+
+[script]
+read -sv keepgoing_multi_step.sv
+prep -top test
+
+[files]
+keepgoing_multi_step.sv

tests/keepgoing/keepgoing_multi_step.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+python3 $SBY_MAIN -f $SBY_FILE $TASK
+python3 ${SBY_FILE%.sby}.py $WORKDIR

tests/keepgoing/keepgoing_multi_step.sv

@@ -0,0 +1,22 @@
+module test (
+	input clk, a
+);
+	reg [7:0] counter = 0;
+
+	always @(posedge clk) begin
+		counter <= counter + 1;
+	end
+
+	always @(posedge clk) begin
+		assert(0);
+		if (counter == 3 || counter == 7) begin
+			assert(a); // step 3,7
+		end
+		if (counter == 5) begin
+			assert(a); // step 5
+		end
+		if (counter == 7) begin
+			assert(a); // step 7
+		end
+	end
+endmodule

tests/keepgoing/keepgoing_same_step.py

@@ -0,0 +1,19 @@
+import sys
+from check_output import *
+
+workdir = sys.argv[1]
+src = "keepgoing_same_step.sv"
+
+assert_a = line_ref(workdir, src, "assert(a)")
+assert_not_a = line_ref(workdir, src, "assert(!a)")
+assert_0 = line_ref(workdir, src, "assert(0)")
+
+log = open(workdir + "/logfile.txt").read()
+log_per_trace = log.split("Writing trace to VCD file")[:-1]
+
+assert len(log_per_trace) == 2
+
+assert re.search(r"Assert failed in test: %s \(.*\)$" % assert_a, log, re.M)
+assert re.search(r"Assert failed in test: %s \(.*\)$" % assert_not_a, log, re.M)
+assert re.search(r"Assert failed in test: %s \(.*\)$" % assert_0, log_per_trace[0], re.M)
+assert re.search(r"Assert failed in test: %s \(.*\) \[failed before\]$" % assert_0, log_per_trace[1], re.M)

tests/keepgoing/keepgoing_same_step.sby

@@ -0,0 +1,13 @@
+[options]
+mode bmc
+expect fail
+
+[engines]
+smtbmc --keep-going boolector
+
+[script]
+read -sv keepgoing_same_step.sv
+prep -top test
+
+[files]
+keepgoing_same_step.sv

tests/keepgoing/keepgoing_same_step.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+python3 $SBY_MAIN -f $SBY_FILE $TASK
+python3 ${SBY_FILE%.sby}.py $WORKDIR

tests/keepgoing/keepgoing_same_step.sv

@@ -0,0 +1,17 @@
+module test (
+	input clk, a
+);
+	reg [7:0] counter = 0;
+
+	always @(posedge clk) begin
+		counter <= counter + 1;
+	end
+
+	always @(posedge clk) begin
+		if (counter == 3) begin
+			assert(a);
+			assert(!a);
+			assert(0);
+		end
+	end
+endmodule

tests/keepgoing/keepgoing_smtc.py

@@ -0,0 +1,25 @@
+import sys
+from check_output import *
+
+workdir = sys.argv[1]
+src = "keepgoing_same_step.sv"
+
+assert_a = line_ref(workdir, src, "assert(a)")
+assert_not_a = line_ref(workdir, src, "assert(!a)")
+assert_0 = line_ref(workdir, src, "assert(0)")
+
+assert_false = line_ref(workdir, "extra.smtc", "assert false")
+assert_distinct = line_ref(workdir, "extra.smtc", "assert (distinct")
+
+log = open(workdir + "/logfile.txt").read()
+log_per_trace = log.split("Writing trace to VCD file")[:-1]
+
+assert len(log_per_trace) == 4
+
+assert re.search(r"Assert failed in test: %s \(.*\)$" % assert_a, log, re.M)
+assert re.search(r"Assert failed in test: %s \(.*\)$" % assert_not_a, log, re.M)
+
+assert re.search(r"Assert src/%s failed: false" % assert_false, log_per_trace[0], re.M)
+assert re.search(r"Assert failed in test: %s \(.*\)$" % assert_0, log_per_trace[1], re.M)
+assert re.search(r"Assert failed in test: %s \(.*\) \[failed before\]$" % assert_0, log_per_trace[2], re.M)
+assert re.search(r"Assert src/%s failed: \(distinct" % assert_distinct, log_per_trace[3], re.M)

tests/keepgoing/keepgoing_smtc.sby

@@ -0,0 +1,19 @@
+[options]
+mode bmc
+expect fail
+
+[engines]
+smtbmc --keep-going boolector -- --smtc src/extra.smtc
+
+[script]
+read -sv keepgoing_same_step.sv
+prep -top test
+
+[files]
+keepgoing_same_step.sv
+
+[file extra.smtc]
+state 2
+assert false
+always
+assert (distinct [counter] #b00000111)

tests/keepgoing/keepgoing_smtc.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+python3 $SBY_MAIN -f $SBY_FILE $TASK
+python3 ${SBY_FILE%.sby}.py $WORKDIR