From bd4094f216083c9c1cc9b65aeda3cbfb30114a16 Mon Sep 17 00:00:00 2001 From: Clifford Wolf Date: Sat, 9 Mar 2019 12:42:54 -0800 Subject: [PATCH 1/3] Add support for (re-)running in existing workdir Signed-off-by: Clifford Wolf --- sbysrc/sby.py | 49 ++++++++++++++++++++++++++++++++-------- sbysrc/sby_core.py | 25 +++++++++++++++----- sbysrc/sby_mode_bmc.py | 2 +- sbysrc/sby_mode_cover.py | 2 +- sbysrc/sby_mode_live.py | 2 +- sbysrc/sby_mode_prove.py | 2 +- 6 files changed, 62 insertions(+), 20 deletions(-) diff --git a/sbysrc/sby.py b/sbysrc/sby.py index f08f072..20a200f 100644 --- a/sbysrc/sby.py +++ b/sbysrc/sby.py @@ -32,10 +32,12 @@ exe_paths = dict() throw_err = False dump_cfg = False dump_tasks = False +reusedir = False +setupmode = False def usage(): print(""" -sby [options] [.sby [tasknames]] +sby [options] [.sby [tasknames] | ] -d set workdir name. default: (without .sby) @@ -69,13 +71,16 @@ sby [options] [.sby [tasknames]] --dumptasks print the list of tasks + + --setup + set up the working directory and exit """) sys.exit(1) try: opts, args = getopt.getopt(sys.argv[1:], "d:btfT:E", ["yosys=", "abc=", "smtbmc=", "suprove=", "aigbmc=", "avy=", "btormc=", - "dumpcfg", "dumptasks"]) + "dumpcfg", "dumptasks", "setup"]) except: usage() @@ -110,12 +115,27 @@ for o, a in opts: dump_cfg = True elif o == "--dumptasks": dump_tasks = True + elif o == "--setup": + setupmode = True else: usage() if len(args) > 0: sbyfile = args[0] - if not sbyfile.endswith(".sby"): + if os.path.isdir(sbyfile): + workdir = sbyfile + sbyfile += "/config.sby" + reusedir = True + if not opt_force and os.path.exists(workdir + "/model"): + print("ERROR: Use -f to re-run in existing directory.", file=sys.stderr) + sys.exit(1) + if len(args) > 1: + print("ERROR: Can't use tasks when running in existing directory.", file=sys.stderr) + sys.exit(1) + if setupmode: + print("ERROR: Can't use --setup with existing directory.", file=sys.stderr) + sys.exit(1) + elif not sbyfile.endswith(".sby"): print("ERROR: Sby file does not have .sby file extension.", file=sys.stderr) sys.exit(1) @@ -278,12 +298,18 @@ def run_job(taskname): early_log(my_workdir, "Moving direcory '%s' to '%s'." % (my_workdir, "%s.bak%03d" % (my_workdir, backup_idx))) shutil.move(my_workdir, "%s.bak%03d" % (my_workdir, backup_idx)) - if opt_force: + if opt_force and not reusedir: early_log(my_workdir, "Removing direcory '%s'." % (my_workdir)) if sbyfile: shutil.rmtree(my_workdir, ignore_errors=True) - os.makedirs(my_workdir) + if reusedir: + pass + elif os.path.isdir(my_workdir): + print("ERROR: Direcory '%s' already exists." % (my_workdir)) + sys.exit(1) + else: + os.makedirs(my_workdir) else: my_opt_tmpdir = True @@ -302,16 +328,16 @@ def run_job(taskname): junit_filename = "junit" sbyconfig, _ = read_sbyconfig(sbydata, taskname) - job = SbyJob(sbyconfig, my_workdir, early_logmsgs) + job = SbyJob(sbyconfig, my_workdir, early_logmsgs, reusedir) for k, v in exe_paths.items(): job.exe_paths[k] = v if throw_err: - job.run() + job.run(setupmode) else: try: - job.run() + job.run(setupmode) except SbyAbort: pass @@ -319,10 +345,13 @@ def run_job(taskname): job.log("Removing direcory '%s'." % (my_workdir)) shutil.rmtree(my_workdir, ignore_errors=True) - job.log("DONE (%s, rc=%d)" % (job.status, job.retcode)) + if setupmode: + job.log("SETUP COMPLETE (rc=%d)" % (job.retcode)) + else: + job.log("DONE (%s, rc=%d)" % (job.status, job.retcode)) job.logfile.close() - if not my_opt_tmpdir: + if not my_opt_tmpdir and not setupmode: with open("%s/%s.xml" % (job.workdir, junit_filename), "w") as f: junit_errors = 1 if job.retcode == 16 else 0 junit_failures = 1 if job.retcode != 0 and junit_errors == 0 else 0 diff --git a/sbysrc/sby_core.py b/sbysrc/sby_core.py index e572f5c..e41fb54 100644 --- a/sbysrc/sby_core.py +++ b/sbysrc/sby_core.py @@ -18,7 +18,7 @@ import os, re, resource, sys import subprocess, fcntl -from shutil import copyfile +from shutil import copyfile, rmtree from select import select from time import time, localtime @@ -133,7 +133,7 @@ class SbyAbort(BaseException): class SbyJob: - def __init__(self, sbyconfig, workdir, early_logs): + def __init__(self, sbyconfig, workdir, early_logs, reusedir): self.options = dict() self.used_options = set() self.engines = list() @@ -142,6 +142,7 @@ class SbyJob: self.verbatim_files = dict() self.models = dict() self.workdir = workdir + self.reusedir = reusedir self.status = "UNKNOWN" self.total_time = 0 self.expect = [] @@ -166,7 +167,7 @@ class SbyJob: self.summary = list() - self.logfile = open("%s/logfile.txt" % workdir, "w") + self.logfile = open("%s/logfile.txt" % workdir, "a") for line in early_logs: print(line, file=self.logfile) @@ -223,6 +224,11 @@ class SbyJob: print("ERROR: %s" % logmessage, file=f) raise SbyAbort(logmessage) + def makedirs(self, path): + if self.reusedir and os.path.isdir(path): + rmtree(path, ignore_errors=True) + os.makedirs(path) + def copy_src(self): os.makedirs(self.workdir + "/src") @@ -273,7 +279,7 @@ class SbyJob: self.__dict__["opt_" + option_name] = default_value def make_model(self, model_name): - if not os.path.exists("%s/model" % self.workdir): + if not os.path.isdir("%s/model" % self.workdir): os.makedirs("%s/model" % self.workdir) if model_name in ["base", "nomem"]: @@ -411,7 +417,7 @@ class SbyJob: else: assert 0 - def run(self): + def run(self, setupmode): mode = None key = None @@ -536,7 +542,14 @@ class SbyJob: if engine[0] not in ["smtbmc", "btor"]: self.error("Option skip is only valid for smtbmc and btor engines.") - self.copy_src() + if self.reusedir: + rmtree("%s/model" % self.workdir, ignore_errors=True) + else: + self.copy_src() + + if setupmode: + self.retcode = 0 + return if self.opt_mode == "bmc": import sby_mode_bmc diff --git a/sbysrc/sby_mode_bmc.py b/sbysrc/sby_mode_bmc.py index c3c55ea..edb9b99 100644 --- a/sbysrc/sby_mode_bmc.py +++ b/sbysrc/sby_mode_bmc.py @@ -29,7 +29,7 @@ def run(job): assert len(engine) > 0 job.log("engine_%d: %s" % (engine_idx, " ".join(engine))) - os.makedirs("%s/engine_%d" % (job.workdir, engine_idx)) + job.makedirs("%s/engine_%d" % (job.workdir, engine_idx)) if engine[0] == "smtbmc": import sby_engine_smtbmc diff --git a/sbysrc/sby_mode_cover.py b/sbysrc/sby_mode_cover.py index a4f3597..28c2f4d 100644 --- a/sbysrc/sby_mode_cover.py +++ b/sbysrc/sby_mode_cover.py @@ -28,7 +28,7 @@ def run(job): assert len(engine) > 0 job.log("engine_%d: %s" % (engine_idx, " ".join(engine))) - os.makedirs("%s/engine_%d" % (job.workdir, engine_idx)) + job.makedirs("%s/engine_%d" % (job.workdir, engine_idx)) if engine[0] == "smtbmc": import sby_engine_smtbmc diff --git a/sbysrc/sby_mode_live.py b/sbysrc/sby_mode_live.py index bee065c..3a64ff0 100644 --- a/sbysrc/sby_mode_live.py +++ b/sbysrc/sby_mode_live.py @@ -29,7 +29,7 @@ def run(job): assert len(engine) > 0 job.log("engine_%d: %s" % (engine_idx, " ".join(engine))) - os.makedirs("%s/engine_%d" % (job.workdir, engine_idx)) + job.makedirs("%s/engine_%d" % (job.workdir, engine_idx)) if engine[0] == "aiger": import sby_engine_aiger diff --git a/sbysrc/sby_mode_prove.py b/sbysrc/sby_mode_prove.py index d7742e7..daabe8f 100644 --- a/sbysrc/sby_mode_prove.py +++ b/sbysrc/sby_mode_prove.py @@ -36,7 +36,7 @@ def run(job): assert len(engine) > 0 job.log("engine_%d: %s" % (engine_idx, " ".join(engine))) - os.makedirs("%s/engine_%d" % (job.workdir, engine_idx)) + job.makedirs("%s/engine_%d" % (job.workdir, engine_idx)) if engine[0] == "smtbmc": import sby_engine_smtbmc From 577b5bcbc720e558107581c70d54d9a55ae4ebb8 Mon Sep 17 00:00:00 2001 From: Clifford Wolf Date: Sat, 9 Mar 2019 12:52:51 -0800 Subject: [PATCH 2/3] Improve rerun-in-existing-dir functionality Signed-off-by: Clifford Wolf --- sbysrc/sby.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sbysrc/sby.py b/sbysrc/sby.py index 20a200f..13d1cd1 100644 --- a/sbysrc/sby.py +++ b/sbysrc/sby.py @@ -135,6 +135,10 @@ if len(args) > 0: if setupmode: print("ERROR: Can't use --setup with existing directory.", file=sys.stderr) sys.exit(1) + if opt_force: + for f in "PASS FAIL UNKNOWN ERROR TIMEOUT".split(): + if os.path.exists(workdir + "/" + f): + os.remove(workdir + "/" + f) elif not sbyfile.endswith(".sby"): print("ERROR: Sby file does not have .sby file extension.", file=sys.stderr) sys.exit(1) @@ -318,7 +322,9 @@ def run_job(taskname): junit_ts_name = os.path.basename(sbyfile[:-4]) if sbyfile is not None else workdir if workdir is not None else "stdin" junit_tc_name = taskname if taskname is not None else "default" - if sbyfile is not None: + if reusedir: + junit_filename = os.path.basename(my_workdir) + elif sbyfile is not None: junit_filename = os.path.basename(sbyfile[:-4]) if taskname is not None: junit_filename += "_" + taskname From a14f0ac5afce778e45344d2547002247191a13e6 Mon Sep 17 00:00:00 2001 From: "William D. Jones" Date: Tue, 12 Mar 2019 00:50:52 -0400 Subject: [PATCH 3/3] Install launcher executable when running on Windows. Signed-off-by: William D. Jones --- Makefile | 13 ++ extern/launcher.c | 358 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 371 insertions(+) create mode 100644 extern/launcher.c diff --git a/Makefile b/Makefile index 689db67..62f452a 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,13 @@ DESTDIR = PREFIX = /usr/local +# On Windows, manually setting absolute path to Python binary may be required +# for launcher executable to work. From MSYS2, this can be done using the +# following command: "which python3 | cygpath -w -m -f -". +ifeq ($(OS), Windows_NT) +PYTHON = $(shell cygpath -w -m $(PREFIX)/bin/python3) +endif + help: @echo "" @echo "sudo make install" @@ -18,8 +25,14 @@ install: mkdir -p $(DESTDIR)$(PREFIX)/bin mkdir -p $(DESTDIR)$(PREFIX)/share/yosys/python3 cp sbysrc/sby_*.py $(DESTDIR)$(PREFIX)/share/yosys/python3/ +ifeq ($(OS), Windows_NT) + sed -e 's|##yosys-sys-path##|sys.path += [os.path.dirname(__file__) + p for p in ["/share/python3", "/../share/yosys/python3"]]|;' \ + -e "s|#!/usr/bin/env python3|#!$(PYTHON)|" < sbysrc/sby.py > $(DESTDIR)$(PREFIX)/bin/sby-script.py + gcc -DGUI=0 -O -s -o $(DESTDIR)$(PREFIX)/bin/sby.exe extern/launcher.c +else sed 's|##yosys-sys-path##|sys.path += [os.path.dirname(__file__) + p for p in ["/share/python3", "/../share/yosys/python3"]]|;' < sbysrc/sby.py > $(DESTDIR)$(PREFIX)/bin/sby chmod +x $(DESTDIR)$(PREFIX)/bin/sby +endif html: make -C docs html diff --git a/extern/launcher.c b/extern/launcher.c new file mode 100644 index 0000000..157d68c --- /dev/null +++ b/extern/launcher.c @@ -0,0 +1,358 @@ +/* This file comes from the PyPA Setuptools repository, commit 16e452a: +https://github.com/pypa/setuptools +Modifications include this comment and inline inclusion of the LICENSE text. */ + +/* Copyright (C) 2016 Jason R Coombs + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. */ + +/* Setuptools Script Launcher for Windows + + This is a stub executable for Windows that functions somewhat like + Effbot's "exemaker", in that it runs a script with the same name but + a .py extension, using information from a #! line. It differs in that + it spawns the actual Python executable, rather than attempting to + hook into the Python DLL. This means that the script will run with + sys.executable set to the Python executable, where exemaker ends up with + sys.executable pointing to itself. (Which means it won't work if you try + to run another Python process using sys.executable.) + + To build/rebuild with mingw32, do this in the setuptools project directory: + + gcc -DGUI=0 -mno-cygwin -O -s -o setuptools/cli.exe launcher.c + gcc -DGUI=1 -mwindows -mno-cygwin -O -s -o setuptools/gui.exe launcher.c + + To build for Windows RT, install both Visual Studio Express for Windows 8 + and for Windows Desktop (both freeware), create "win32" application using + "Windows Desktop" version, create new "ARM" target via + "Configuration Manager" menu and modify ".vcxproj" file by adding + "true" tag + as child of "PropertyGroup" tags that has "Debug|ARM" and "Release|ARM" + properties. + + It links to msvcrt.dll, but this shouldn't be a problem since it doesn't + actually run Python in the same process. Note that using 'exec' instead + of 'spawn' doesn't work, because on Windows this leads to the Python + executable running in the *background*, attached to the same console + window, meaning you get a command prompt back *before* Python even finishes + starting. So, we have to use spawnv() and wait for Python to exit before + continuing. :( +*/ + +#include +#include +#include +#include +#include +#include + +int child_pid=0; + +int fail(char *format, char *data) { + /* Print error message to stderr and return 2 */ + fprintf(stderr, format, data); + return 2; +} + +char *quoted(char *data) { + int i, ln = strlen(data), nb; + + /* We allocate twice as much space as needed to deal with worse-case + of having to escape everything. */ + char *result = calloc(ln*2+3, sizeof(char)); + char *presult = result; + + *presult++ = '"'; + for (nb=0, i=0; i < ln; i++) + { + if (data[i] == '\\') + nb += 1; + else if (data[i] == '"') + { + for (; nb > 0; nb--) + *presult++ = '\\'; + *presult++ = '\\'; + } + else + nb = 0; + *presult++ = data[i]; + } + + for (; nb > 0; nb--) /* Deal w trailing slashes */ + *presult++ = '\\'; + + *presult++ = '"'; + *presult++ = 0; + return result; +} + + + + + + + + + + +char *loadable_exe(char *exename) { + /* HINSTANCE hPython; DLL handle for python executable */ + char *result; + + /* hPython = LoadLibraryEx(exename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); + if (!hPython) return NULL; */ + + /* Return the absolute filename for spawnv */ + result = calloc(MAX_PATH, sizeof(char)); + strncpy(result, exename, MAX_PATH); + /*if (result) GetModuleFileNameA(hPython, result, MAX_PATH); + + FreeLibrary(hPython); */ + return result; +} + + +char *find_exe(char *exename, char *script) { + char drive[_MAX_DRIVE], dir[_MAX_DIR], fname[_MAX_FNAME], ext[_MAX_EXT]; + char path[_MAX_PATH], c, *result; + + /* convert slashes to backslashes for uniform search below */ + result = exename; + while (c = *result++) if (c=='/') result[-1] = '\\'; + + _splitpath(exename, drive, dir, fname, ext); + if (drive[0] || dir[0]=='\\') { + return loadable_exe(exename); /* absolute path, use directly */ + } + /* Use the script's parent directory, which should be the Python home + (This should only be used for bdist_wininst-installed scripts, because + easy_install-ed scripts use the absolute path to python[w].exe + */ + _splitpath(script, drive, dir, fname, ext); + result = dir + strlen(dir) -1; + if (*result == '\\') result--; + while (*result != '\\' && result>=dir) *result-- = 0; + _makepath(path, drive, dir, exename, NULL); + return loadable_exe(path); +} + + +char **parse_argv(char *cmdline, int *argc) +{ + /* Parse a command line in-place using MS C rules */ + + char **result = calloc(strlen(cmdline), sizeof(char *)); + char *output = cmdline; + char c; + int nb = 0; + int iq = 0; + *argc = 0; + + result[0] = output; + while (isspace(*cmdline)) cmdline++; /* skip leading spaces */ + + do { + c = *cmdline++; + if (!c || (isspace(c) && !iq)) { + while (nb) {*output++ = '\\'; nb--; } + *output++ = 0; + result[++*argc] = output; + if (!c) return result; + while (isspace(*cmdline)) cmdline++; /* skip leading spaces */ + if (!*cmdline) return result; /* avoid empty arg if trailing ws */ + continue; + } + if (c == '\\') + ++nb; /* count \'s */ + else { + if (c == '"') { + if (!(nb & 1)) { iq = !iq; c = 0; } /* skip " unless odd # of \ */ + nb = nb >> 1; /* cut \'s in half */ + } + while (nb) {*output++ = '\\'; nb--; } + if (c) *output++ = c; + } + } while (1); +} + +void pass_control_to_child(DWORD control_type) { + /* + * distribute-issue207 + * passes the control event to child process (Python) + */ + if (!child_pid) { + return; + } + GenerateConsoleCtrlEvent(child_pid,0); +} + +BOOL control_handler(DWORD control_type) { + /* + * distribute-issue207 + * control event handler callback function + */ + switch (control_type) { + case CTRL_C_EVENT: + pass_control_to_child(0); + break; + } + return TRUE; +} + +int create_and_wait_for_subprocess(char* command) { + /* + * distribute-issue207 + * launches child process (Python) + */ + DWORD return_value = 0; + LPSTR commandline = command; + STARTUPINFOA s_info; + PROCESS_INFORMATION p_info; + ZeroMemory(&p_info, sizeof(p_info)); + ZeroMemory(&s_info, sizeof(s_info)); + s_info.cb = sizeof(STARTUPINFO); + // set-up control handler callback funciotn + SetConsoleCtrlHandler((PHANDLER_ROUTINE) control_handler, TRUE); + if (!CreateProcessA(NULL, commandline, NULL, NULL, TRUE, 0, NULL, NULL, &s_info, &p_info)) { + fprintf(stderr, "failed to create process.\n"); + return 0; + } + child_pid = p_info.dwProcessId; + // wait for Python to exit + WaitForSingleObject(p_info.hProcess, INFINITE); + if (!GetExitCodeProcess(p_info.hProcess, &return_value)) { + fprintf(stderr, "failed to get exit code from process.\n"); + return 0; + } + return return_value; +} + +char* join_executable_and_args(char *executable, char **args, int argc) +{ + /* + * distribute-issue207 + * CreateProcess needs a long string of the executable and command-line arguments, + * so we need to convert it from the args that was built + */ + int len,counter; + char* cmdline; + + len=strlen(executable)+2; + for (counter=1; counterscript && *end != '.') + *end-- = '\0'; + *end-- = '\0'; + strcat(script, (GUI ? "-script.pyw" : "-script.py")); + + /* figure out the target python executable */ + + scriptf = open(script, O_RDONLY); + if (scriptf == -1) { + return fail("Cannot open %s\n", script); + } + end = python + read(scriptf, python, sizeof(python)); + close(scriptf); + + ptr = python-1; + while(++ptr < end && *ptr && *ptr!='\n' && *ptr!='\r') {;} + + *ptr-- = '\0'; + + if (strncmp(python, "#!", 2)) { + /* default to python.exe if no #! header */ + strcpy(python, "#!python.exe"); + } + + parsedargs = parse_argv(python+2, &parsedargc); + + /* Using spawnv() can fail strangely if you e.g. find the Cygwin + Python, so we'll make sure Windows can find and load it */ + + ptr = find_exe(parsedargs[0], script); + if (!ptr) { + return fail("Cannot find Python executable %s\n", parsedargs[0]); + } + + /* printf("Python executable: %s\n", ptr); */ + + /* Argument array needs to be + parsedargc + argc, plus 1 for null sentinel */ + + newargs = (char **)calloc(parsedargc + argc + 1, sizeof(char *)); + newargsp = newargs; + + *newargsp++ = quoted(ptr); + for (i = 1; i