bugpoint: Add -greperr option

`-greperr <string>` redirects stderr to 'bugpoint-case.err', and then searches that file for `<string>`.
Move `-runner` option up with the other options to reduce ambiguity (i.e. so it doesn't look like it's another design parts constraint).
Also some shuffling of `err.ys`.

passes/cmds/bugpoint.cc

@@ -73,6 +73,13 @@ struct BugpointPass : public Pass {
 		log("        finishing. produces smaller and more useful testcases, but may fail to\n");
 		log("        produce any testcase at all if the crash is related to dangling wires.\n");
 		log("\n");
+		log("    -runner \"<prefix>\"\n");
+		log("        child process wrapping command, e.g., \"timeout 30\", or valgrind.\n");
+		log("\n");
+		log("    -greperr \"<string>\"\n");
+		log("        only consider crashes that print this string on stderr. useful for\n");
+		log("        errors outside of yosys.\n");
+		log("\n");
 		log("It is possible to constrain which parts of the design will be considered for\n");
 		log("removal. Unless one or more of the following options are specified, all parts\n");
 		log("will be considered.\n");
@@ -106,12 +113,9 @@ struct BugpointPass : public Pass {
 		log("        try to remove wires. wires with a (* bugpoint_keep *) attribute will be\n");
 		log("        skipped.\n");
 		log("\n");
-		log("    -runner \"<prefix>\"\n");
-		log("        child process wrapping command, e.g., \"timeout 30\", or valgrind.\n");
-		log("\n");
 	}
 
-	int run_yosys(RTLIL::Design *design, string runner, string yosys_cmd, string yosys_arg)
+	int run_yosys(RTLIL::Design *design, string runner, string yosys_cmd, string yosys_arg, bool catch_err)
 	{
 		design->sort();
 
@@ -120,6 +124,7 @@ struct BugpointPass : public Pass {
 		f.close();
 
 		string yosys_cmdline = stringf("%s %s -qq -L bugpoint-case.log %s bugpoint-case.il", runner.c_str(), yosys_cmd.c_str(), yosys_arg.c_str());
+		if (catch_err) yosys_cmdline += " 2>bugpoint-case.err";
 		auto status = run_command(yosys_cmdline);
 		// we're not processing lines, which means we're getting raw system() returns
 		if(WIFEXITED(status))
@@ -132,7 +137,7 @@ struct BugpointPass : public Pass {
 			return 0;
 	}
 
-	bool check_logfile(string grep)
+	bool check_logfile(string grep, bool err=false)
 	{
 		if (grep.empty())
 			return true;
@@ -140,7 +145,12 @@ struct BugpointPass : public Pass {
 		if (grep.size() > 2 && grep.front() == '"' && grep.back() == '"')
 			grep = grep.substr(1, grep.size() - 2);
 
-		std::ifstream f("bugpoint-case.log");
+		std::ifstream f;
+		if (err)
+			f = std::ifstream("bugpoint-case.err");
+		else
+			f = std::ifstream("bugpoint-case.log");
+
 		while (!f.eof())
 		{
 			string line;
@@ -151,6 +161,11 @@ struct BugpointPass : public Pass {
 		return false;
 	}
 
+	bool check_logfiles(string grep, string greperr)
+	{
+		return check_logfile(grep) && check_logfile(greperr, true);
+	}
+
 	RTLIL::Design *clean_design(RTLIL::Design *design, bool do_clean = true, bool do_delete = false)
 	{
 		if (!do_clean)
@@ -425,8 +440,8 @@ struct BugpointPass : public Pass {
 
 	void execute(std::vector<std::string> args, RTLIL::Design *design) override
 	{
-		string yosys_cmd = "yosys", yosys_arg, grep, runner;
+		string yosys_cmd = "yosys", yosys_arg, grep, greperr, runner;
-		bool flag_expect_return = false, has_check = false;
+		bool flag_expect_return = false, has_check = false, check_err = false;
 		int expect_return_value = 0;
 		bool fast = false, clean = false;
 		bool modules = false, ports = false, cells = false, connections = false, processes = false, assigns = false, updates = false, wires = false, has_part = false;
@@ -458,6 +473,12 @@ struct BugpointPass : public Pass {
 				grep = args[++argidx];
 				continue;
 			}
+			if (args[argidx] == "-greperr" && argidx + 1 < args.size()) {
+				has_check = true;
+				check_err = true;
+				greperr = args[++argidx];
+				continue;
+			}
 			if (args[argidx] == "-expect-return") {
 				flag_expect_return = true;
 				++argidx;
@@ -549,13 +570,15 @@ struct BugpointPass : public Pass {
 			log_cmd_error("This command only operates on fully selected designs!\n");
 
 		RTLIL::Design *crashing_design = clean_design(design, clean);
-		int retval = run_yosys(crashing_design, runner, yosys_cmd, yosys_arg);
+		int retval = run_yosys(crashing_design, runner, yosys_cmd, yosys_arg, check_err);
 		if (flag_expect_return && retval != expect_return_value)
 			log_cmd_error("The provided script file or command and Yosys binary returned value %d instead of expected %d on this design!\n", retval, expect_return_value);
 		if (!flag_expect_return && retval == 0)
 			log_cmd_error("The provided script file or command and Yosys binary do not crash on this design!\n");
 		if (!check_logfile(grep))
 			log_cmd_error("The provided grep string is not found in the log file!\n");
+		if (!check_logfile(greperr, true))
+			log_cmd_error("The provided grep string is not found in stderr log!\n");
 
 		int seed = 0;
 		bool found_something = false, stage2 = false;
@@ -568,30 +591,30 @@ struct BugpointPass : public Pass {
 				if (clean)
 				{
 					RTLIL::Design *testcase = clean_design(simplified);
-					retval = run_yosys(testcase, runner, yosys_cmd, yosys_arg);
+					retval = run_yosys(testcase, runner, yosys_cmd, yosys_arg, check_err);
 					delete testcase;
 				}
 				else
 				{
-					retval = run_yosys(simplified, runner, yosys_cmd, yosys_arg);
+					retval = run_yosys(simplified, runner, yosys_cmd, yosys_arg, check_err);
 				}
 
 				bool crashes = false;
-				if (flag_expect_return && retval == expect_return_value && check_logfile(grep))
+				if (flag_expect_return && retval == expect_return_value && check_logfiles(grep, greperr))
 				{
 					log("Testcase matches expected crash.\n");
 					crashes = true;
 				}
 				else if (!flag_expect_return && retval == 0)
 					log("Testcase does not crash.\n");
-				else if (!flag_expect_return && check_logfile(grep))
+				else if (!flag_expect_return && check_logfiles(grep, greperr))
 				{
 					log("Testcase crashes.\n");
 					crashes = true;
 				}
 				else
-					// flag_expect_return && !(retval == expect_return_value && check_logfile(grep))
+					// flag_expect_return && !(retval == expect_return_value && check_logfiles(grep, greperr))
-					// !flag_expect_return && !(retval == 0 && check_logfile(grep))
+					// !flag_expect_return && !(retval == 0 && check_logfiles(grep, greperr))
 					log("Testcase does not match expected crash.\n");
 
 				if (crashes)

tests/bugpoint/.gitignore

@@ -1,2 +1,3 @@
 *.il
-*.log
+*.log
+*.err

tests/bugpoint/err.ys

@@ -20,33 +20,24 @@ bugpoint -yosys ../../yosys -command raise_error -expect-return 7
 select -assert-mod-count 1 =*
 select -assert-mod-count 1 top
 
-# raise_error with string prints message
+# raise_error with string prints message and exits with 1
 design -load read
 rename top abc
-bugpoint -yosys ../../yosys -command raise_error -grep "help me"
+bugpoint -yosys ../../yosys -command raise_error -grep "help me" -expect-return 1
 select -assert-mod-count 1 =*
 select -assert-mod-count 1 other
 
 # raise_error with no value exits with 1
 design -load read
 rename def zzy
+delete other
 bugpoint -yosys ../../yosys -command raise_error -expect-return 1
 select -assert-mod-count 1 =*
 select -assert-mod-count 1 zzy
 
-# raise_error -stderr exits with 1
+# raise_error -stderr prints to stderr and exits with 1
 design -load read
 rename top abc
-delete def
+bugpoint -yosys ../../yosys -command "raise_error -stderr" -greperr "help me" -expect-return 1
-bugpoint -yosys ../../yosys -command "raise_error -stderr" -expect-return 1
 select -assert-mod-count 1 =*
 select -assert-mod-count 1 other
-
-#TODO
-# raise_error -stderr prints to stderr
-design -load read
-rename top abc
-delete def
-# bugpoint -yosys ../../yosys -command "raise_error -stderr" -grep "help me"
-# select -assert-mod-count 1 =*
-# select -assert-mod-count 1 other