diff --git a/Makefile b/Makefile index 36b4d0491..bcb855940 100644 --- a/Makefile +++ b/Makefile @@ -868,6 +868,7 @@ MK_TEST_DIRS += tests/arch/nexus MK_TEST_DIRS += tests/arch/quicklogic/pp3 MK_TEST_DIRS += tests/arch/quicklogic/qlf_k6n10f MK_TEST_DIRS += tests/arch/xilinx +MK_TEST_DIRS += tests/bugpoint MK_TEST_DIRS += tests/opt MK_TEST_DIRS += tests/sat MK_TEST_DIRS += tests/sim diff --git a/kernel/constids.inc b/kernel/constids.inc index c77a25c01..29872d45e 100644 --- a/kernel/constids.inc +++ b/kernel/constids.inc @@ -286,3 +286,4 @@ X(A_WIDTHS) X(B_WIDTHS) X(C_WIDTHS) X(C_SIGNED) +X(raise_error) diff --git a/passes/cmds/bugpoint.cc b/passes/cmds/bugpoint.cc index da06acedf..08dc76dda 100644 --- a/passes/cmds/bugpoint.cc +++ b/passes/cmds/bugpoint.cc @@ -20,6 +20,16 @@ #include "kernel/yosys.h" #include "backends/rtlil/rtlil_backend.h" +#if defined(_WIN32) +# include +# define WIFEXITED(x) 1 +# define WIFSIGNALED(x) 0 +# define WIFSTOPPED(x) 0 +# define WEXITSTATUS(x) ((x) & 0xff) +# define WTERMSIG(x) SIGTERM +# define WSTOPSIG(x) 0 +#endif + USING_YOSYS_NAMESPACE using namespace RTLIL_BACKEND; PRIVATE_NAMESPACE_BEGIN @@ -50,6 +60,10 @@ struct BugpointPass : public Pass { log(" -grep \"\"\n"); log(" only consider crashes that place this string in the log file.\n"); log("\n"); + log(" -expect-return \n"); + log(" only consider crashes that return the specified value. e.g. SEGFAULT\n"); + log(" returns a value of 139.\n"); + log("\n"); log(" -fast\n"); log(" run `proc_clean; clean -purge` after each minimization step. converges\n"); log(" faster, but produces larger testcases, and may fail to produce any\n"); @@ -60,6 +74,17 @@ struct BugpointPass : public Pass { log(" finishing. produces smaller and more useful testcases, but may fail to\n"); log(" produce any testcase at all if the crash is related to dangling wires.\n"); log("\n"); + log(" -runner \"\"\n"); + log(" child process wrapping command, e.g., \"timeout 30\", or valgrind.\n"); + log("\n"); + log(" -err-grep \"\"\n"); + log(" only consider crashes that print this string on stderr. useful for\n"); + log(" errors outside of yosys.\n"); + log("\n"); + log(" -suffix \"\"\n"); + log(" add suffix to generated file names. useful when running more than one\n"); + log(" instance of bugpoint in the same directory. limited to 8 characters.\n"); + log("\n"); log("It is possible to constrain which parts of the design will be considered for\n"); log("removal. Unless one or more of the following options are specified, all parts\n"); log("will be considered.\n"); @@ -89,24 +114,39 @@ struct BugpointPass : public Pass { log(" -updates\n"); log(" try to remove process updates from syncs.\n"); log("\n"); - log(" -runner \"\"\n"); - log(" child process wrapping command, e.g., \"timeout 30\", or valgrind.\n"); + log(" -wires\n"); + log(" try to remove wires. wires with a (* bugpoint_keep *) attribute will be\n"); + log(" skipped.\n"); log("\n"); } - bool run_yosys(RTLIL::Design *design, string runner, string yosys_cmd, string yosys_arg) + int run_yosys(RTLIL::Design *design, string runner, string yosys_cmd, string yosys_arg, string suffix, bool catch_err) { design->sort(); - std::ofstream f("bugpoint-case.il"); + string bugpoint_file = "bugpoint-case"; + if (suffix.size()) + bugpoint_file += stringf(".%.8s", suffix.c_str()); + + std::ofstream f(bugpoint_file + ".il"); RTLIL_BACKEND::dump_design(f, design, /*only_selected=*/false, /*flag_m=*/true, /*flag_n=*/false); f.close(); - string yosys_cmdline = stringf("%s %s -qq -L bugpoint-case.log %s bugpoint-case.il", runner.c_str(), yosys_cmd.c_str(), yosys_arg.c_str()); - return run_command(yosys_cmdline) == 0; + string yosys_cmdline = stringf("%s %s -qq -L %s.log %s %s.il", runner.c_str(), yosys_cmd.c_str(), bugpoint_file.c_str(), yosys_arg.c_str(), bugpoint_file.c_str()); + if (catch_err) yosys_cmdline += stringf(" 2>%s.err", bugpoint_file.c_str()); + auto status = run_command(yosys_cmdline); + // we're not processing lines, which means we're getting raw system() returns + if(WIFEXITED(status)) + return WEXITSTATUS(status); + else if(WIFSIGNALED(status)) + return WTERMSIG(status); + else if(WIFSTOPPED(status)) + return WSTOPSIG(status); + else + return 0; } - bool check_logfile(string grep) + bool check_logfile(string grep, string suffix, bool err=false) { if (grep.empty()) return true; @@ -114,7 +154,13 @@ struct BugpointPass : public Pass { if (grep.size() > 2 && grep.front() == '"' && grep.back() == '"') grep = grep.substr(1, grep.size() - 2); - std::ifstream f("bugpoint-case.log"); + string bugpoint_file = "bugpoint-case"; + if (suffix.size()) + bugpoint_file += stringf(".%.8s", suffix.c_str()); + bugpoint_file += err ? ".err" : ".log"; + + std::ifstream f(bugpoint_file); + while (!f.eof()) { string line; @@ -125,6 +171,11 @@ struct BugpointPass : public Pass { return false; } + bool check_logfiles(string grep, string err_grep, string suffix) + { + return check_logfile(grep, suffix) && check_logfile(err_grep, suffix, true); + } + RTLIL::Design *clean_design(RTLIL::Design *design, bool do_clean = true, bool do_delete = false) { if (!do_clean) @@ -399,7 +450,9 @@ struct BugpointPass : public Pass { void execute(std::vector args, RTLIL::Design *design) override { - string yosys_cmd = "yosys", yosys_arg, grep, runner; + string yosys_cmd = "yosys", yosys_arg, grep, err_grep, runner, suffix; + bool flag_expect_return = false, has_check = false, check_err = false; + int expect_return_value = 0; bool fast = false, clean = false; bool modules = false, ports = false, cells = false, connections = false, processes = false, assigns = false, updates = false, wires = false, has_part = false; @@ -426,9 +479,25 @@ struct BugpointPass : public Pass { continue; } if (args[argidx] == "-grep" && argidx + 1 < args.size()) { + has_check = true; grep = args[++argidx]; continue; } + if (args[argidx] == "-err-grep" && argidx + 1 < args.size()) { + has_check = true; + check_err = true; + err_grep = args[++argidx]; + continue; + } + if (args[argidx] == "-expect-return") { + flag_expect_return = true; + ++argidx; + if (argidx >= args.size()) + log_cmd_error("No expected return value specified.\n"); + + expect_return_value = atoi(args[argidx].c_str()); + continue; + } if (args[argidx] == "-fast") { fast = true; continue; @@ -485,6 +554,14 @@ struct BugpointPass : public Pass { } continue; } + if (args[argidx] == "-suffix" && argidx + 1 < args.size()) { + suffix = args[++argidx]; + if (suffix.size() && suffix.at(0) == '"') { + log_assert(suffix.back() == '"'); + suffix = suffix.substr(1, suffix.size() - 2); + } + continue; + } break; } extra_args(args, argidx, design); @@ -492,6 +569,9 @@ struct BugpointPass : public Pass { if (yosys_arg.empty()) log_cmd_error("Missing -script or -command option.\n"); + if (flag_expect_return && expect_return_value == 0 && !has_check) + log_cmd_error("Nothing to match on for -expect-return 0; change value or use -grep.\n"); + if (!has_part) { modules = true; @@ -508,10 +588,15 @@ struct BugpointPass : public Pass { log_cmd_error("This command only operates on fully selected designs!\n"); RTLIL::Design *crashing_design = clean_design(design, clean); - if (run_yosys(crashing_design, runner, yosys_cmd, yosys_arg)) + int retval = run_yosys(crashing_design, runner, yosys_cmd, yosys_arg, suffix, check_err); + if (flag_expect_return && retval != expect_return_value) + log_cmd_error("The provided script file or command and Yosys binary returned value %d instead of expected %d on this design!\n", retval, expect_return_value); + if (!flag_expect_return && retval == 0) log_cmd_error("The provided script file or command and Yosys binary do not crash on this design!\n"); - if (!check_logfile(grep)) + if (!check_logfile(grep, suffix)) log_cmd_error("The provided grep string is not found in the log file!\n"); + if (!check_logfile(err_grep, suffix, true)) + log_cmd_error("The provided grep string is not found in stderr log!\n"); int seed = 0; bool found_something = false, stage2 = false; @@ -521,21 +606,37 @@ struct BugpointPass : public Pass { { simplified = clean_design(simplified, fast, /*do_delete=*/true); - bool crashes; if (clean) { RTLIL::Design *testcase = clean_design(simplified); - crashes = !run_yosys(testcase, runner, yosys_cmd, yosys_arg); + retval = run_yosys(testcase, runner, yosys_cmd, yosys_arg, suffix, check_err); delete testcase; } else { - crashes = !run_yosys(simplified, runner, yosys_cmd, yosys_arg); + retval = run_yosys(simplified, runner, yosys_cmd, yosys_arg, suffix, check_err); } - if (crashes && check_logfile(grep)) + bool crashes = false; + if (flag_expect_return && retval == expect_return_value && check_logfiles(grep, err_grep, suffix)) + { + log("Testcase matches expected crash.\n"); + crashes = true; + } + else if (!flag_expect_return && retval == 0) + log("Testcase does not crash.\n"); + else if (!flag_expect_return && check_logfiles(grep, err_grep, suffix)) { log("Testcase crashes.\n"); + crashes = true; + } + else + // flag_expect_return && !(retval == expect_return_value && check_logfiles(grep, err_grep, suffix)) + // !flag_expect_return && !(retval == 0 && check_logfiles(grep, err_grep, suffix)) + log("Testcase does not match expected crash.\n"); + + if (crashes) + { if (crashing_design != design) delete crashing_design; crashing_design = simplified; @@ -543,7 +644,6 @@ struct BugpointPass : public Pass { } else { - log("Testcase does not crash.\n"); delete simplified; seed++; } diff --git a/passes/tests/Makefile.inc b/passes/tests/Makefile.inc index 531943d78..25e11967c 100644 --- a/passes/tests/Makefile.inc +++ b/passes/tests/Makefile.inc @@ -2,4 +2,5 @@ OBJS += passes/tests/test_autotb.o OBJS += passes/tests/test_cell.o OBJS += passes/tests/test_abcloop.o +OBJS += passes/tests/raise_error.o diff --git a/passes/tests/raise_error.cc b/passes/tests/raise_error.cc new file mode 100644 index 000000000..f9055e6d2 --- /dev/null +++ b/passes/tests/raise_error.cc @@ -0,0 +1,83 @@ +#include "kernel/yosys.h" + +USING_YOSYS_NAMESPACE +PRIVATE_NAMESPACE_BEGIN + +struct RaiseErrorPass : public Pass { + RaiseErrorPass() : Pass("raise_error", "raise errors from attributes") { } + void help() override + { + // |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---| + log("\n"); + log(" raise_error [options] [selection]\n"); + log("\n"); + log("Test error handling by raising arbitrary errors. This pass iterates over the\n"); + log("design (or selection of it) checking for objects with the 'raise_error'\n"); + log("attribute set. Assigning 'raise_error' to a string more than one character long\n"); + log("will log that string as an error message before exiting. Assigning 'raise_error'\n"); + log("to an integer (less than 256) will exit with that value as the exit code.\n"); + log("\n"); + log(" -stderr\n"); + log(" Log error messages directly to stderr instead of using 'log_error'.\n"); + log("\n"); + } + void execute(vector args, RTLIL::Design *design) override + { + log_header(design, "Executing RAISE_ERROR pass.\n"); + + bool use_stderr = false; + + int argidx; + for (argidx = 1; argidx < GetSize(args); argidx++) + { + if (args[argidx] == "-stderr") { + use_stderr = true; + continue; + } + break; + } + + extra_args(args, argidx, design, true); + + RTLIL::NamedObject *err_obj = nullptr; + + for (auto mod : design->all_selected_modules()) { + if (mod->has_attribute(ID::raise_error)) { + err_obj = mod->clone(); + break; + } + for (auto memb : mod->selected_members()) { + if (memb->has_attribute(ID::raise_error)) { + err_obj = memb; + break; + } + } + if (err_obj != nullptr) break; + } + + if (err_obj != nullptr) { + log("Raising error from '%s'.\n", log_id(err_obj)); + auto err_no = err_obj->attributes[ID::raise_error].as_int(); + if (err_no < 256) { + log_flush(); + } else { + auto err_msg = err_obj->get_string_attribute(ID::raise_error); + if (use_stderr) { + std::cerr << err_msg << std::endl; + err_no = 1; + } else { + log_error("%s\n", err_msg.c_str()); + } + } + #if defined(_MSC_VER) + _exit(err_no); + #else + _Exit(err_no); + #endif + } else { + log("'raise_error' attribute not found\n"); + } + } +} RaiseErrorPass; + +PRIVATE_NAMESPACE_END diff --git a/tests/bugpoint/.gitignore b/tests/bugpoint/.gitignore new file mode 100644 index 000000000..1012c66cc --- /dev/null +++ b/tests/bugpoint/.gitignore @@ -0,0 +1,5 @@ +bugpoint-case.* +*.log +*.err +*.temp +run-test.mk diff --git a/tests/bugpoint/failures.ys b/tests/bugpoint/failures.ys new file mode 100644 index 000000000..ce8daa8cc --- /dev/null +++ b/tests/bugpoint/failures.ys @@ -0,0 +1,29 @@ +write_file fail.temp << EOF +logger -expect error "Missing -script or -command option." 1 +bugpoint -suffix fail -yosys ../../yosys +EOF +exec -expect-return 0 -- ../../yosys -qq mods.il -s fail.temp + +write_file fail.temp << EOF +logger -expect error "do not crash on this design" 1 +bugpoint -suffix fail -yosys ../../yosys -command "dump" +EOF +exec -expect-return 0 -- ../../yosys -qq mods.il -s fail.temp + +write_file fail.temp << EOF +logger -expect error "returned value 3 instead of expected 7" 1 +bugpoint -suffix fail -yosys ../../yosys -command raise_error -expect-return 7 +EOF +exec -expect-return 0 -- ../../yosys -qq mods.il -s fail.temp + +write_file fail.temp << EOF +logger -expect error "not found in the log file!" 1 +bugpoint -suffix fail -yosys ../../yosys -command raise_error -grep "nope" +EOF +exec -expect-return 0 -- ../../yosys -qq mods.il -s fail.temp + +write_file fail.temp << EOF +logger -expect error "not found in stderr log!" 1 +bugpoint -suffix fail -yosys ../../yosys -command raise_error -err-grep "nope" +EOF +exec -expect-return 0 -- ../../yosys -qq mods.il -s fail.temp diff --git a/tests/bugpoint/mod_constraints.ys b/tests/bugpoint/mod_constraints.ys new file mode 100644 index 000000000..f35095510 --- /dev/null +++ b/tests/bugpoint/mod_constraints.ys @@ -0,0 +1,83 @@ +read_rtlil mods.il +select -assert-count 7 w:* +select -assert-mod-count 3 =* +select -assert-count 4 c:* +design -stash base + +# everything is removed by default +design -load base +bugpoint -suffix mods -yosys ../../yosys -command raise_error -expect-return 3 +select -assert-count 1 w:* +select -assert-mod-count 1 =* +select -assert-none c:* + +# don't remove wires +design -load base +bugpoint -suffix mods -yosys ../../yosys -command raise_error -expect-return 3 -modules -cells +select -assert-count 3 w:* +select -assert-mod-count 1 =* +select -assert-none c:* + +# don't remove cells or their connections +design -load base +bugpoint -suffix mods -yosys ../../yosys -command raise_error -expect-return 3 -wires -modules +select -assert-count 5 w:* +select -assert-mod-count 1 =* +select -assert-count 4 c:* + +# don't remove cells but do remove their connections +design -load base +bugpoint -suffix mods -yosys ../../yosys -command raise_error -expect-return 3 -wires -modules -connections +select -assert-count 1 w:* +select -assert-mod-count 1 =* +select -assert-count 4 c:* + +# don't remove modules +design -load base +bugpoint -suffix mods -yosys ../../yosys -command raise_error -expect-return 3 -wires -cells +select -assert-count 1 w:* +select -assert-mod-count 3 =* +select -assert-none c:* + +# can keep wires +design -load base +setattr -set bugpoint_keep 1 w:w_b +bugpoint -suffix mods -yosys ../../yosys -command raise_error -expect-return 3 +select -assert-count 2 w:* +select -assert-mod-count 1 =* +select -assert-none c:* + +# a wire with keep won't keep the cell/module containing it +design -load base +setattr -set bugpoint_keep 1 w:w_o +bugpoint -suffix mods -yosys ../../yosys -command raise_error -expect-return 3 +select -assert-count 1 w:* +select -assert-mod-count 1 =* +select -assert-none c:* + +# can keep cells (and do it without the associated module) +design -load base +setattr -set bugpoint_keep 1 c:c_a +bugpoint -suffix mods -yosys ../../yosys -command raise_error -expect-return 3 +select -assert-count 1 w:* +select -assert-mod-count 1 =* +select -assert-count 1 c:* + +# can keep modules +design -load base +setattr -mod -set bugpoint_keep 1 m_a +bugpoint -suffix mods -yosys ../../yosys -command raise_error -expect-return 3 +select -assert-count 1 w:* +select -assert-mod-count 2 =* +select -assert-none c:* + +# minimize to just the path connecting w_a and w_c +# which happens via w_b, w_i, w_o, m_a, c_a and c_b +write_file script.temp << EOF +select -assert-none w:w_a %co* w:w_c %ci* %i +EOF +design -load base +bugpoint -suffix mods -yosys ../../yosys -script script.temp -grep "Assertion failed" +select -assert-count 5 w:* +select -assert-mod-count 2 =* +select -assert-count 2 c:* diff --git a/tests/bugpoint/mods.il b/tests/bugpoint/mods.il new file mode 100644 index 000000000..fe3ce6522 --- /dev/null +++ b/tests/bugpoint/mods.il @@ -0,0 +1,38 @@ +module \m_a + wire input 1 \w_i + wire output 2 \w_o + connect \w_o \w_i +end + +module \m_b + wire input 1 \w_i + wire output 2 \w_o +end + +attribute \top 1 +module \top + attribute \raise_error 3 + wire \w_a + wire \w_b + wire \w_c + + cell \m_a \c_a + connect \w_i \w_a + connect \w_o \w_b + end + + cell \m_a \c_b + connect \w_i \w_b + connect \w_o \w_c + end + + cell \m_b \c_c + connect \w_i \w_c + connect \w_o \w_a + end + + cell \m_b \c_d + connect \w_i 1'0 + connect \w_o 1'1 + end +end diff --git a/tests/bugpoint/proc_constraints.ys b/tests/bugpoint/proc_constraints.ys new file mode 100644 index 000000000..22b8b3c60 --- /dev/null +++ b/tests/bugpoint/proc_constraints.ys @@ -0,0 +1,49 @@ +read_rtlil procs.il +select -assert-count 2 p:* +design -stash err_q + +# processes get removed by default +design -load err_q +bugpoint -suffix procs -yosys ../../yosys -command raise_error -expect-return 4 +select -assert-none p:* + +# individual processes can be kept +design -load err_q +setattr -set bugpoint_keep 1 p:proc_a +bugpoint -suffix procs -yosys ../../yosys -command raise_error -expect-return 4 +select -assert-count 1 p:* + +# all processes can be kept +design -load err_q +bugpoint -suffix procs -yosys ../../yosys -command raise_error -expect-return 4 -wires +select -assert-count 2 p:* + +# d and clock are connected after proc +design -load err_q +proc +select -assert-count 3 w:d %co +select -assert-count 3 w:clock %co + +# no assigns means no d +design -load err_q +bugpoint -suffix procs -yosys ../../yosys -command raise_error -expect-return 4 -assigns +proc +select -assert-count 1 w:d %co + +# no updates means no clock +design -load err_q +bugpoint -suffix procs -yosys ../../yosys -command raise_error -expect-return 4 -updates +proc +select -assert-count 1 w:clock %co + +# can remove ports +design -load err_q +select -assert-count 5 x:* +bugpoint -suffix procs -yosys ../../yosys -command raise_error -expect-return 4 -ports +select -assert-none x:* + +# can keep ports +design -load err_q +setattr -set bugpoint_keep 1 i:d o:q +bugpoint -suffix procs -yosys ../../yosys -command raise_error -expect-return 4 -ports +select -assert-count 2 x:* diff --git a/tests/bugpoint/procs.il b/tests/bugpoint/procs.il new file mode 100644 index 000000000..cb9f7c8dd --- /dev/null +++ b/tests/bugpoint/procs.il @@ -0,0 +1,42 @@ +module \ff_with_en_and_sync_reset + wire $0\q[1:1] + wire $0\q[0:0] + attribute \raise_error 4 + wire width 2 output 5 \q + wire width 2 input 4 \d + wire input 3 \enable + wire input 2 \reset + wire input 1 \clock + + process \proc_a + assign $0\q[0:0] \q [0] + switch \reset + case 1'1 + assign $0\q[0:0] 1'0 + case + switch \enable + case 1'1 + assign $0\q[0:0] \d [0] + case + end + end + sync posedge \clock + update \q [0] $0\q[0:0] + end + + process \proc_b + assign $0\q[1:1] \q [1] + switch \reset + case 1'1 + assign $0\q[1:1] 1'0 + case + switch \enable + case 1'1 + assign $0\q[1:1] \d [1] + case + end + end + sync posedge \clock + update \q [1] $0\q[1:1] + end +end diff --git a/tests/bugpoint/raise_error.ys b/tests/bugpoint/raise_error.ys new file mode 100644 index 000000000..a0a03f447 --- /dev/null +++ b/tests/bugpoint/raise_error.ys @@ -0,0 +1,43 @@ +read_verilog -noblackbox << EOF +(* raise_error=7 *) +module top(); +endmodule + +(* raise_error="help me" *) +module other(); +endmodule + +(* raise_error *) +module def(); +endmodule +EOF +select -assert-mod-count 3 =* +design -stash read + +# raise_error with int exits with status +design -load read +bugpoint -suffix error -yosys ../../yosys -command raise_error -expect-return 7 +select -assert-mod-count 1 =* +select -assert-mod-count 1 top + +# raise_error with string prints message and exits with 1 +design -load read +rename top abc +bugpoint -suffix error -yosys ../../yosys -command raise_error -grep "help me" -expect-return 1 +select -assert-mod-count 1 =* +select -assert-mod-count 1 other + +# raise_error with no value exits with 1 +design -load read +rename def zzy +delete other +bugpoint -suffix error -yosys ../../yosys -command raise_error -expect-return 1 +select -assert-mod-count 1 =* +select -assert-mod-count 1 zzy + +# raise_error -stderr prints to stderr and exits with 1 +design -load read +rename top abc +bugpoint -suffix error -yosys ../../yosys -command "raise_error -stderr" -err-grep "help me" -expect-return 1 +select -assert-mod-count 1 =* +select -assert-mod-count 1 other diff --git a/tests/bugpoint/run-test.sh b/tests/bugpoint/run-test.sh new file mode 100755 index 000000000..006c731e3 --- /dev/null +++ b/tests/bugpoint/run-test.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -eu +source ../gen-tests-makefile.sh +generate_mk --yosys-scripts