mirror of
https://github.com/YosysHQ/sby.git
synced 2025-09-04 10:37:42 +00:00
Refactor tests
Organize tests into subdirectories and use a new makefile that scans .sby files and allows selecting tests by mode, engine, solver and/or subdirectory. Automatically skips tests that use engines/solvers that are not found in the PATH. See `cd tests; make help` for a description of supported make targets.
This commit is contained in:
Jannis Harder
parent
6daa434d85
commit
8da6f07cb3
60 changed files with 328 additions and 101 deletions
232
tests/junit/JUnit.xsd
Normal file
232
tests/junit/JUnit.xsd
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<xs:schema
|
||||
xmlns:xs="http://www.w3.org/2001/XMLSchema"
|
||||
elementFormDefault="qualified"
|
||||
attributeFormDefault="unqualified">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">JUnit test result schema for the Apache Ant JUnit and JUnitReport tasks
|
||||
Copyright © 2011, Windy Road Technology Pty. Limited
|
||||
The Apache Ant JUnit XML Schema is distributed under the terms of the Apache License Version 2.0 http://www.apache.org/licenses/
|
||||
Permission to waive conditions of this license may be requested from Windy Road Support (http://windyroad.org/support).</xs:documentation>
|
||||
</xs:annotation>
|
||||
<xs:element name="testsuite" type="testsuite"/>
|
||||
<xs:simpleType name="ISO8601_DATETIME_PATTERN">
|
||||
<xs:restriction base="xs:dateTime">
|
||||
<xs:pattern value="[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
<xs:element name="testsuites">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Contains an aggregation of testsuite results</xs:documentation>
|
||||
</xs:annotation>
|
||||
<xs:complexType>
|
||||
<xs:sequence>
|
||||
<xs:element name="testsuite" minOccurs="0" maxOccurs="unbounded">
|
||||
<xs:complexType>
|
||||
<xs:complexContent>
|
||||
<xs:extension base="testsuite">
|
||||
<xs:attribute name="package" type="xs:token" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Derived from testsuite/@name in the non-aggregated documents</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="id" type="xs:int" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Starts at '0' for the first testsuite and is incremented by 1 for each following testsuite</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
</xs:extension>
|
||||
</xs:complexContent>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
</xs:sequence>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:complexType name="testsuite">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Contains the results of exexuting a testsuite</xs:documentation>
|
||||
</xs:annotation>
|
||||
<xs:sequence>
|
||||
<xs:element name="properties">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Properties (e.g., environment settings) set during test execution</xs:documentation>
|
||||
</xs:annotation>
|
||||
<xs:complexType>
|
||||
<xs:sequence>
|
||||
<xs:element name="property" minOccurs="0" maxOccurs="unbounded">
|
||||
<xs:complexType>
|
||||
<xs:attribute name="name" use="required">
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:token">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="value" type="xs:string" use="required"/>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
</xs:sequence>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:element name="testcase" minOccurs="0" maxOccurs="unbounded">
|
||||
<xs:complexType>
|
||||
<xs:choice minOccurs="0">
|
||||
<xs:element name="skipped" />
|
||||
<xs:element name="error" minOccurs="0" maxOccurs="1">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Indicates that the test errored. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. Contains as a text node relevant data for the error, e.g., a stack trace</xs:documentation>
|
||||
</xs:annotation>
|
||||
<xs:complexType>
|
||||
<xs:simpleContent>
|
||||
<xs:extension base="pre-string">
|
||||
<xs:attribute name="message" type="xs:string">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">The error message. e.g., if a java exception is thrown, the return value of getMessage()</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="type" type="xs:string" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">The type of error that occured. e.g., if a java execption is thrown the full class name of the exception.</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
</xs:extension>
|
||||
</xs:simpleContent>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:element name="failure">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Indicates that the test failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals. Contains as a text node relevant data for the failure, e.g., a stack trace</xs:documentation>
|
||||
</xs:annotation>
|
||||
<xs:complexType>
|
||||
<xs:simpleContent>
|
||||
<xs:extension base="pre-string">
|
||||
<xs:attribute name="message" type="xs:string">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">The message specified in the assert</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="type" type="xs:string" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">The type of the assert.</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
</xs:extension>
|
||||
</xs:simpleContent>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
</xs:choice>
|
||||
<xs:attribute name="name" type="xs:token" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Name of the test method</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="classname" type="xs:token" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Full class name for the class the test method is in.</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="time" type="xs:decimal" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Time taken (in seconds) to execute the test</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="id" type="xs:string" use="optional">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Cell ID of the property</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="type" type="xs:token" use="optional">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Kind of property (assert, cover, live)</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="location" type="xs:token" use="optional">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Source location of the property</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="tracefile" type="xs:token" use="optional">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Tracefile for the property</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:element name="system-out">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Data that was written to standard out while the test was executed</xs:documentation>
|
||||
</xs:annotation>
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="pre-string">
|
||||
<xs:whiteSpace value="preserve"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:element>
|
||||
<xs:element name="system-err">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Data that was written to standard error while the test was executed</xs:documentation>
|
||||
</xs:annotation>
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="pre-string">
|
||||
<xs:whiteSpace value="preserve"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:element>
|
||||
</xs:sequence>
|
||||
<xs:attribute name="name" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Full class name of the test for non-aggregated testsuite documents. Class name without the package for aggregated testsuites documents</xs:documentation>
|
||||
</xs:annotation>
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:token">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="timestamp" type="ISO8601_DATETIME_PATTERN" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">when the test was executed. Timezone may not be specified.</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="hostname" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Host on which the tests were executed. 'localhost' should be used if the hostname cannot be determined.</xs:documentation>
|
||||
</xs:annotation>
|
||||
<xs:simpleType>
|
||||
<xs:restriction base="xs:token">
|
||||
<xs:minLength value="1"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="tests" type="xs:int" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">The total number of tests in the suite</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="failures" type="xs:int" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">The total number of tests in the suite that failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="errors" type="xs:int" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">The total number of tests in the suite that errored. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test.</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="skipped" type="xs:int" use="optional">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">The total number of ignored or skipped tests in the suite.</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="time" type="xs:decimal" use="required">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Time taken (in seconds) to execute the tests in the suite</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
</xs:complexType>
|
||||
<xs:simpleType name="pre-string">
|
||||
<xs:restriction base="xs:string">
|
||||
<xs:whiteSpace value="preserve"/>
|
||||
</xs:restriction>
|
||||
</xs:simpleType>
|
||||
</xs:schema>
|
||||
2
tests/junit/Makefile
Normal file
2
tests/junit/Makefile
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
SUBDIR=junit
|
||||
include ../make/subdir.mk
|
||||
38
tests/junit/junit_assert.sby
Normal file
38
tests/junit/junit_assert.sby
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
[tasks]
|
||||
pass
|
||||
fail
|
||||
preunsat
|
||||
|
||||
[options]
|
||||
mode bmc
|
||||
depth 1
|
||||
|
||||
pass: expect pass
|
||||
fail: expect fail
|
||||
preunsat: expect error
|
||||
|
||||
[engines]
|
||||
smtbmc boolector
|
||||
|
||||
[script]
|
||||
fail: read -define FAIL
|
||||
preunsat: read -define PREUNSAT
|
||||
read -sv test.sv
|
||||
prep -top top
|
||||
|
||||
[file test.sv]
|
||||
module test(input foo);
|
||||
always @* assert(foo);
|
||||
`ifdef FAIL
|
||||
always @* assert(!foo);
|
||||
`endif
|
||||
`ifdef PREUNSAT
|
||||
always @* assume(!foo);
|
||||
`endif
|
||||
endmodule
|
||||
|
||||
module top();
|
||||
test test_i (
|
||||
.foo(1'b1)
|
||||
);
|
||||
endmodule
|
||||
4
tests/junit/junit_assert.sh
Normal file
4
tests/junit/junit_assert.sh
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
python3 $SBY_MAIN -f $SBY_FILE $TASK
|
||||
python3 validate_junit.py $WORKDIR/$WORKDIR.xml
|
||||
43
tests/junit/junit_cover.sby
Normal file
43
tests/junit/junit_cover.sby
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
[tasks]
|
||||
pass
|
||||
uncovered fail
|
||||
assert fail
|
||||
preunsat
|
||||
|
||||
[options]
|
||||
mode cover
|
||||
depth 1
|
||||
|
||||
pass: expect pass
|
||||
fail: expect fail
|
||||
preunsat: expect fail
|
||||
|
||||
[engines]
|
||||
smtbmc boolector
|
||||
|
||||
[script]
|
||||
uncovered: read -define FAIL
|
||||
assert: read -define FAIL_ASSERT
|
||||
preunsat: read -define PREUNSAT
|
||||
read -sv test.sv
|
||||
prep -top top
|
||||
|
||||
[file test.sv]
|
||||
module test(input foo);
|
||||
`ifdef PREUNSAT
|
||||
always @* assume(!foo);
|
||||
`endif
|
||||
always @* cover(foo);
|
||||
`ifdef FAIL
|
||||
always @* cover(!foo);
|
||||
`endif
|
||||
`ifdef FAIL_ASSERT
|
||||
always @* assert(!foo);
|
||||
`endif
|
||||
endmodule
|
||||
|
||||
module top();
|
||||
test test_i (
|
||||
.foo(1'b1)
|
||||
);
|
||||
endmodule
|
||||
4
tests/junit/junit_cover.sh
Normal file
4
tests/junit/junit_cover.sh
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
python3 $SBY_MAIN -f $SBY_FILE $TASK
|
||||
python3 validate_junit.py $WORKDIR/$WORKDIR.xml
|
||||
16
tests/junit/junit_expect.sby
Normal file
16
tests/junit/junit_expect.sby
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
[options]
|
||||
mode bmc
|
||||
depth 1
|
||||
expect fail,timeout
|
||||
|
||||
[engines]
|
||||
smtbmc
|
||||
|
||||
[script]
|
||||
read -formal foo.v
|
||||
prep -top foo
|
||||
|
||||
[file foo.v]
|
||||
module foo;
|
||||
always_comb assert(1);
|
||||
endmodule
|
||||
4
tests/junit/junit_expect.sh
Normal file
4
tests/junit/junit_expect.sh
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
! python3 $SBY_MAIN -f $SBY_FILE $TASK
|
||||
grep '<failure type="EXPECT" message="Task returned status PASS. Expected values were: FAIL TIMEOUT" />' $WORKDIR/$WORKDIR.xml
|
||||
20
tests/junit/junit_nocodeloc.sby
Normal file
20
tests/junit/junit_nocodeloc.sby
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
[options]
|
||||
mode bmc
|
||||
|
||||
expect fail
|
||||
|
||||
[engines]
|
||||
smtbmc boolector
|
||||
|
||||
[script]
|
||||
read -sv multi_assert.v
|
||||
prep -top test
|
||||
setattr -unset src
|
||||
|
||||
[file multi_assert.v]
|
||||
module test();
|
||||
always @* begin
|
||||
assert (1);
|
||||
assert (0);
|
||||
end
|
||||
endmodule
|
||||
4
tests/junit/junit_nocodeloc.sh
Normal file
4
tests/junit/junit_nocodeloc.sh
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
python3 $SBY_MAIN -f $SBY_FILE $TASK
|
||||
python3 validate_junit.py $WORKDIR/$WORKDIR.xml
|
||||
42
tests/junit/junit_timeout_error.sby
Normal file
42
tests/junit/junit_timeout_error.sby
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
[tasks]
|
||||
syntax error
|
||||
solver error
|
||||
timeout
|
||||
|
||||
[options]
|
||||
mode cover
|
||||
depth 1
|
||||
timeout: timeout 1
|
||||
error: expect error
|
||||
timeout: expect timeout
|
||||
|
||||
[engines]
|
||||
~solver: smtbmc --dumpsmt2 --progress --stbv z3
|
||||
solver: smtbmc foo
|
||||
|
||||
[script]
|
||||
read -noverific
|
||||
syntax: read -define SYNTAX_ERROR
|
||||
read -sv primes.sv
|
||||
prep -top primes
|
||||
|
||||
[file primes.sv]
|
||||
module primes;
|
||||
parameter [8:0] offset = 7;
|
||||
(* anyconst *) reg [8:0] prime1;
|
||||
wire [9:0] prime2 = prime1 + offset;
|
||||
(* allconst *) reg [4:0] factor;
|
||||
|
||||
`ifdef SYNTAX_ERROR
|
||||
foo
|
||||
`endif
|
||||
|
||||
always @* begin
|
||||
if (1 < factor && factor < prime1)
|
||||
assume ((prime1 % factor) != 0);
|
||||
if (1 < factor && factor < prime2)
|
||||
assume ((prime2 % factor) != 0);
|
||||
assume (1 < prime1);
|
||||
cover (1);
|
||||
end
|
||||
endmodule
|
||||
4
tests/junit/junit_timeout_error.sh
Normal file
4
tests/junit/junit_timeout_error.sh
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
python3 $SBY_MAIN -f $SBY_FILE $TASK
|
||||
python3 validate_junit.py $WORKDIR/$WORKDIR.xml
|
||||
19
tests/junit/validate_junit.py
Normal file
19
tests/junit/validate_junit.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
from xmlschema import XMLSchema, XMLSchemaValidationError
|
||||
import argparse
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Validate JUnit output")
|
||||
parser.add_argument('xml')
|
||||
parser.add_argument('--xsd', default="JUnit.xsd")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
schema = XMLSchema(args.xsd)
|
||||
try:
|
||||
schema.validate(args.xml)
|
||||
except XMLSchemaValidationError as e:
|
||||
print(e)
|
||||
exit(1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue