From 3bf5be06376d4c9b44efa0eeb5abe52937403d31 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Tue, 29 Jul 2025 10:00:52 +1200 Subject: [PATCH 01/11] Add jsonl status format Replace `--statuscsv` and `--livecsv` with `--statusfmt ` and `--live str: +def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: if row is None: - csv_header = [ + data = [ "time", "task_name", "mode", @@ -520,7 +520,6 @@ def format_status_data_csvline(row: dict|None) -> str: "trace", "depth", ] - return ','.join(csv_header) else: engine = row['data'].get('engine', row['data'].get('source')) try: @@ -546,7 +545,11 @@ def format_status_data_csvline(row: dict|None) -> str: trace_path, depth, ] - return ','.join("" if v is None else str(v) for v in csv_line) + data = ["" if v is None else str(v) for v in csv_line] + if fmt == "csv": + return ','.join(data) + elif fmt == "jsonl": + return json.dumps(data) def filter_latest_task_ids(all_tasks: dict[int, dict[str]]): latest: dict[str, int] = {} diff --git a/tests/statusdb/timeout.sh b/tests/statusdb/timeout.sh index 2d70133..9896c3b 100644 --- a/tests/statusdb/timeout.sh +++ b/tests/statusdb/timeout.sh @@ -3,7 +3,7 @@ set -e python3 $SBY_MAIN -f $SBY_FILE $TASK STATUS_CSV=${WORKDIR}/status.csv -python3 $SBY_MAIN -f $SBY_FILE $TASK --statuscsv --latest | tee $STATUS_CSV +python3 $SBY_MAIN -f $SBY_FILE $TASK --statusfmt csv --latest | tee $STATUS_CSV if [[ $TASK =~ "_cover" ]]; then wc -l $STATUS_CSV | grep -q '6' From f05979a528de70c78125437b0b1fa119e4b4ad05 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Tue, 29 Jul 2025 10:04:40 +1200 Subject: [PATCH 02/11] Fix for statusfmt not going into status block --- sbysrc/sby.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbysrc/sby.py b/sbysrc/sby.py index f82f420..2d5a1da 100644 --- a/sbysrc/sby.py +++ b/sbysrc/sby.py @@ -72,7 +72,7 @@ if autotune and linkmode: print("ERROR: --link flag currently not available with --autotune") sys.exit(1) -if status_show or status_reset or task_status: +if status_show or status_reset or task_status or status_format: target = workdir_prefix or workdir or sbyfile if target is None: print("ERROR: Specify a .sby config file or working directory to use --status.") From 344236af41c777a96c303c9c24edbf45b49a13ed Mon Sep 17 00:00:00 2001 From: Jannis Harder Date: Tue, 29 Jul 2025 17:10:57 +0200 Subject: [PATCH 03/11] statusfmt: Make JSONL self-contained and escape CSV values --- sbysrc/sby_status.py | 69 +++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/sbysrc/sby_status.py b/sbysrc/sby_status.py index 35024f2..e914154 100644 --- a/sbysrc/sby_status.py +++ b/sbysrc/sby_status.py @@ -448,7 +448,8 @@ class SbyStatusDb: # print header header = format_status_data_fmtline(None, status_format) - print(header) + if header: + print(header) # find summary for each task/property combo prop_map: dict[(str, str), dict[str, (int, int)]] = {} @@ -506,20 +507,22 @@ def parse_status_data_row(raw: sqlite3.Row): row_dict["data"] = json.loads(row_dict.get("data") or "{}") return row_dict +fmtline_columns = [ + "time", + "task_name", + "mode", + "engine", + "name", + "location", + "kind", + "status", + "trace", + "depth", +] + def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: if row is None: - data = [ - "time", - "task_name", - "mode", - "engine", - "name", - "location", - "kind", - "status", - "trace", - "depth", - ] + data = None else: engine = row['data'].get('engine', row['data'].get('source')) try: @@ -533,22 +536,34 @@ def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: except TypeError: trace_path = None - csv_line = [ - round(time, 2), - row['task_name'], - row['mode'], - engine, - name or pretty_path(row['name']), - row['location'], - row['kind'], - row['status'] or "UNKNOWN", - trace_path, - depth, - ] - data = ["" if v is None else str(v) for v in csv_line] + data = { + "time": round(time, 2), + "task_name": row['task_name'], + "mode": row['mode'], + "engine": engine, + "name": name or pretty_path(row['name']), + "location": row['location'], + "kind": row['kind'], + "status": row['status'] or "UNKNOWN", + "trace": trace_path, + "depth": depth, + } if fmt == "csv": - return ','.join(data) + if data is None: + csv_line = fmtline_columns + else: + csv_line = [data[column] for column in fmtline_columns] + def csv_field(value): + if value is None: + return "" + value = str(value).replace('"', '""') + if any(c in value for c in '",\n'): + value = f'"{value}"' + return value + return ','.join(map(csv_field, csv_line)) elif fmt == "jsonl": + if data is None: + return "" return json.dumps(data) def filter_latest_task_ids(all_tasks: dict[int, dict[str]]): From 190ef869162b1c559e5b30313e410717d5510bd2 Mon Sep 17 00:00:00 2001 From: Jannis Harder Date: Tue, 29 Jul 2025 17:30:10 +0200 Subject: [PATCH 04/11] statusfmt: Skip missing fields in jsonl output --- sbysrc/sby_status.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sbysrc/sby_status.py b/sbysrc/sby_status.py index e914154..d59b1db 100644 --- a/sbysrc/sby_status.py +++ b/sbysrc/sby_status.py @@ -525,19 +525,10 @@ def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: data = None else: engine = row['data'].get('engine', row['data'].get('source')) - try: - time = row['status_created'] - row['created'] - except TypeError: - time = 0 name = row['hdlname'] depth = row['data'].get('step') - try: - trace_path = Path(row['workdir']) / row['path'] - except TypeError: - trace_path = None data = { - "time": round(time, 2), "task_name": row['task_name'], "mode": row['mode'], "engine": engine, @@ -545,14 +536,21 @@ def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: "location": row['location'], "kind": row['kind'], "status": row['status'] or "UNKNOWN", - "trace": trace_path, "depth": depth, } + try: + data["trace"] = str(Path(row['workdir']) / row['path']) + except TypeError: + pass + try: + data['time'] = round(row['status_created'] - row['created'], 2) + except TypeError: + pass if fmt == "csv": if data is None: csv_line = fmtline_columns else: - csv_line = [data[column] for column in fmtline_columns] + csv_line = [data.get(column) for column in fmtline_columns] def csv_field(value): if value is None: return "" @@ -564,6 +562,8 @@ def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: elif fmt == "jsonl": if data is None: return "" + # field order + data = {column: data[column] for column in fmtline_columns if column in data} return json.dumps(data) def filter_latest_task_ids(all_tasks: dict[int, dict[str]]): From d4864994ca3148a8c779b4eb301835e42d1d1dcf Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Fri, 1 Aug 2025 10:34:08 +1200 Subject: [PATCH 05/11] statusfmt: Skip null fields in jsonl output --- sbysrc/sby_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbysrc/sby_status.py b/sbysrc/sby_status.py index d59b1db..5c61463 100644 --- a/sbysrc/sby_status.py +++ b/sbysrc/sby_status.py @@ -563,7 +563,7 @@ def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: if data is None: return "" # field order - data = {column: data[column] for column in fmtline_columns if column in data} + data = {column: data[column] for column in fmtline_columns if data.get(column)} return json.dumps(data) def filter_latest_task_ids(all_tasks: dict[int, dict[str]]): From a906714c951c8b5ee010a30cb2d3399dc56566cf Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 2 Aug 2025 09:17:21 +1200 Subject: [PATCH 06/11] Add test for copying directories As per https://stackoverflow.com/a/54950959, `os.path.basename()` returns an empty string if the string ends with a trailing slash. This means that the target implied by `dir/` differs from an explicit target of `dir/`, and changes the behaviour to copy files to the root `src` directory instead. --- tests/links/symlink.py | 2 +- tests/links/symlink.sby | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/links/symlink.py b/tests/links/symlink.py index a6a06d5..bf5bd6b 100644 --- a/tests/links/symlink.py +++ b/tests/links/symlink.py @@ -1,4 +1,3 @@ -import os from pathlib import Path import sys @@ -13,6 +12,7 @@ def main(): assert(local_contents.strip() == 'log foo') else: assert(srcfile.is_symlink() == (task == "link")) + assert(srcfile.name != "script.ys") if __name__ == "__main__": main() diff --git a/tests/links/symlink.sby b/tests/links/symlink.sby index 52fa881..c3841ce 100644 --- a/tests/links/symlink.sby +++ b/tests/links/symlink.sby @@ -1,6 +1,8 @@ [tasks] link copy +dir_implicit: dir +dir_explicit: dir [options] mode prep @@ -15,7 +17,9 @@ script dir/script.ys [files] ../../docs/examples/demos/picorv32.v prv32fmcmp.v -dir +~dir: dir +dir_implicit: dir/ +dir_explicit: dir/ dir/ [file heredoc] log foo From b06781e19ecba2669a339f012e58e2a26c35f0d2 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 2 Aug 2025 09:17:55 +1200 Subject: [PATCH 07/11] Fix directory mismatch --- sbysrc/sby_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbysrc/sby_core.py b/sbysrc/sby_core.py index 2d0d1c2..ef0ed71 100644 --- a/sbysrc/sby_core.py +++ b/sbysrc/sby_core.py @@ -579,7 +579,7 @@ class SbyConfig: self.error(f"sby file syntax error: '[files]' section entry expects up to 2 arguments, {len(entries)} specified") if len(entries) == 1: - self.files[os.path.basename(entries[0])] = entries[0] + self.files[Path(entries[0]).name] = entries[0] elif len(entries) == 2: self.files[entries[0]] = entries[1] From ac419190d2e14b18b3e664a22e73232ef38e6b47 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 2 Aug 2025 10:06:30 +1200 Subject: [PATCH 08/11] Use more pathlib.Path --- sbysrc/sby_core.py | 61 ++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/sbysrc/sby_core.py b/sbysrc/sby_core.py index ef0ed71..52ea443 100644 --- a/sbysrc/sby_core.py +++ b/sbysrc/sby_core.py @@ -45,12 +45,8 @@ signal.signal(signal.SIGINT, force_shutdown) signal.signal(signal.SIGTERM, force_shutdown) def process_filename(filename): - if filename.startswith("~/"): - filename = os.environ['HOME'] + filename[1:] - filename = os.path.expandvars(filename) - - return filename + return Path(filename).expanduser() def dress_message(workdir, logmessage): tm = localtime() @@ -1033,13 +1029,14 @@ class SbyTask(SbyConfig): raise SbyAbort(logmessage) def makedirs(self, path): - if self.reusedir and os.path.isdir(path): + path = Path(path) + if self.reusedir and path.is_dir(): rmtree(path, ignore_errors=True) - if not os.path.isdir(path): - os.makedirs(path) + path.mkdir(parents=True, exist_ok=True) def copy_src(self, linkmode=False): - self.makedirs(self.workdir + "/src") + outdir = Path(self.workdir) / "src" + self.makedirs(outdir) for dstfile, lines in self.verbatim_files.items(): dstfile = self.workdir + "/src/" + dstfile @@ -1050,25 +1047,25 @@ class SbyTask(SbyConfig): f.write(line) for dstfile, srcfile in self.files.items(): - if dstfile.startswith("/") or dstfile.startswith("../") or ("/../" in dstfile): + dstfile = Path(dstfile) + if dstfile.is_absolute() or ".." in dstfile.parts: self.error(f"destination filename must be a relative path without /../: {dstfile}") - dstfile = self.workdir + "/src/" + dstfile + dstfile = outdir / dstfile srcfile = process_filename(srcfile) - basedir = os.path.dirname(dstfile) - if basedir != "" and not os.path.exists(basedir): - os.makedirs(basedir) + basedir = dstfile.parent + basedir.mkdir(parents=True, exist_ok=True) if linkmode: verb = "Link" else: verb = "Copy" - self.log(f"{verb} '{os.path.abspath(srcfile)}' to '{os.path.abspath(dstfile)}'.") + self.log(f"{verb} '{srcfile.absolute()}' to '{dstfile.absolute()}'.") if linkmode: - os.symlink(os.path.relpath(srcfile, basedir), dstfile) - elif os.path.isdir(srcfile): + os.symlink(srcfile.resolve(), dstfile) + elif srcfile.is_dir(): copytree(srcfile, dstfile, dirs_exist_ok=True) else: copyfile(srcfile, dstfile) @@ -1097,12 +1094,12 @@ class SbyTask(SbyConfig): self.__dict__["opt_" + option_name] = default_value def make_model(self, model_name): - if not os.path.isdir(f"{self.workdir}/model"): - os.makedirs(f"{self.workdir}/model") + modeldir = Path(self.workdir) / "model" + modeldir.mkdir(exist_ok=True) if model_name == "prep": - with open(f"""{self.workdir}/model/design_prep.ys""", "w") as f: - print(f"# running in {self.workdir}/model/", file=f) + with open(modeldir / "design_prep.ys", "w") as f: + print(f"# running in {modeldir}/", file=f) print(f"""read_rtlil design.il""", file=f) if not self.opt_skip_prep: print("scc -select; simplemap; select -clear", file=f) @@ -1146,7 +1143,7 @@ class SbyTask(SbyConfig): return [proc] if model_name == "base": - with open(f"""{self.workdir}/model/design.ys""", "w") as f: + with open(modeldir / "design.ys", "w") as f: print(f"# running in {self.workdir}/src/", file=f) for cmd in self.script: print(cmd, file=f) @@ -1168,7 +1165,7 @@ class SbyTask(SbyConfig): def instance_hierarchy_callback(retcode): if self.design == None: - with open(f"{self.workdir}/model/design.json") as f: + with open(modeldir / "design.json") as f: self.design = design_hierarchy(f) self.status_db.create_task_properties([ prop for prop in self.design.properties_by_path.values() @@ -1184,8 +1181,8 @@ class SbyTask(SbyConfig): return [proc] if re.match(r"^smt2(_syn)?(_nomem)?(_stbv|_stdt)?$", model_name): - with open(f"{self.workdir}/model/design_{model_name}.ys", "w") as f: - print(f"# running in {self.workdir}/model/", file=f) + with open(modeldir / f"design_{model_name}.ys", "w") as f: + print(f"# running in {modeldir}/", file=f) print(f"""read_rtlil design_prep.il""", file=f) print("hierarchy -smtcheck", file=f) print("delete */t:$print", file=f) @@ -1218,8 +1215,8 @@ class SbyTask(SbyConfig): return [proc] if re.match(r"^btor(_syn)?(_nomem)?$", model_name): - with open(f"{self.workdir}/model/design_{model_name}.ys", "w") as f: - print(f"# running in {self.workdir}/model/", file=f) + with open(modeldir / f"design_{model_name}.ys", "w") as f: + print(f"# running in {modeldir}/", file=f) print(f"""read_rtlil design_prep.il""", file=f) print("hierarchy -simcheck", file=f) print("delete */t:$print", file=f) @@ -1254,8 +1251,8 @@ class SbyTask(SbyConfig): return [proc] if model_name == "aig": - with open(f"{self.workdir}/model/design_aiger.ys", "w") as f: - print(f"# running in {self.workdir}/model/", file=f) + with open(modeldir / "design_aiger.ys", "w") as f: + print(f"# running in {modeldir}/", file=f) print("read_rtlil design_prep.il", file=f) print("delete */t:$print", file=f) print("hierarchy -simcheck", file=f) @@ -1281,7 +1278,7 @@ class SbyTask(SbyConfig): self, "aig", self.model("prep"), - f"""cd {self.workdir}/model; {self.exe_paths["yosys"]} -ql design_aiger.log design_aiger.ys""" + f"""cd {modeldir}; {self.exe_paths["yosys"]} -ql design_aiger.log design_aiger.ys""" ) proc.checkretcode = True @@ -1292,8 +1289,8 @@ class SbyTask(SbyConfig): self, model_name, self.model("aig"), - f"""cd {self.workdir}/model; {self.exe_paths["abc"]} -c 'read_aiger design_aiger.aig; fold{" -s" if self.opt_aigfolds else ""}; strash; write_aiger design_aiger_fold.aig'""", - logfile=open(f"{self.workdir}/model/design_aiger_fold.log", "w") + f"""cd {modeldir}; {self.exe_paths["abc"]} -c 'read_aiger design_aiger.aig; fold{" -s" if self.opt_aigfolds else ""}; strash; write_aiger design_aiger_fold.aig'""", + logfile=open(f"{modeldir}/design_aiger_fold.log", "w") ) proc.checkretcode = True From a215e3260a336c3b72e7b0f7f2db3c0ce6855ad3 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 2 Aug 2025 10:07:06 +1200 Subject: [PATCH 09/11] tests/links/symlink: Check file count And also that `src/dir/script.ys` exists --- tests/links/symlink.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/links/symlink.py b/tests/links/symlink.py index bf5bd6b..f274d61 100644 --- a/tests/links/symlink.py +++ b/tests/links/symlink.py @@ -4,6 +4,7 @@ import sys def main(): workdir, task = sys.argv[1:] src = Path(workdir) / "src" + count = 0 for srcfile in src.iterdir(): if srcfile.name == "heredoc": assert(not srcfile.is_symlink()) @@ -13,6 +14,10 @@ def main(): else: assert(srcfile.is_symlink() == (task == "link")) assert(srcfile.name != "script.ys") + count += 1 + assert(count == 4) + script_ys = src / "dir" / "script.ys" + assert(script_ys.exists()) if __name__ == "__main__": main() From 1d282943915f34e1e0ff9442ebc9389205df6fde Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 2 Aug 2025 10:38:35 +1200 Subject: [PATCH 10/11] More directory tests Confirming that the changes to path handling didn't break anything. `~/` works locally, but I'm not sure how to actually include that as a test. --- tests/links/more_dirs.sby | 17 +++++++++++++++++ tests/links/more_dirs.sh | 10 ++++++++++ 2 files changed, 27 insertions(+) create mode 100644 tests/links/more_dirs.sby create mode 100644 tests/links/more_dirs.sh diff --git a/tests/links/more_dirs.sby b/tests/links/more_dirs.sby new file mode 100644 index 0000000..b873726 --- /dev/null +++ b/tests/links/more_dirs.sby @@ -0,0 +1,17 @@ +[tasks] +link +copy + +[options] +mode prep + +[engines] +btor btormc + +[script] +read -noverific +script dir/script.ys + +[files] +here/dir ${WORKDIR}/../dir +a/b/c.v prv32fmcmp.v diff --git a/tests/links/more_dirs.sh b/tests/links/more_dirs.sh new file mode 100644 index 0000000..15d872e --- /dev/null +++ b/tests/links/more_dirs.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e +if [[ $TASK == link ]]; then + flags="--setup --link" +else + flags="--setup" +fi +python3 $SBY_MAIN -f $SBY_FILE $TASK $flags + +test -e ${WORKDIR}/src/here/dir -a -e ${WORKDIR}/src/a/b/c.v From 5fffe7eda6176d9a011ce5fa664b6a4ef2d2c4fa Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 2 Aug 2025 10:40:52 +1200 Subject: [PATCH 11/11] Fix heredoc in sub dir Also change log to use absolute path for consistency with the copy/link logs. --- sbysrc/sby_core.py | 5 +++-- tests/links/more_dirs.sby | 3 +++ tests/links/more_dirs.sh | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sbysrc/sby_core.py b/sbysrc/sby_core.py index 52ea443..17b1ae9 100644 --- a/sbysrc/sby_core.py +++ b/sbysrc/sby_core.py @@ -1039,8 +1039,9 @@ class SbyTask(SbyConfig): self.makedirs(outdir) for dstfile, lines in self.verbatim_files.items(): - dstfile = self.workdir + "/src/" + dstfile - self.log(f"Writing '{dstfile}'.") + dstfile = outdir / dstfile + self.log(f"Writing '{dstfile.absolute()}'.") + dstfile.parent.mkdir(parents=True, exist_ok=True) with open(dstfile, "w") as f: for line in lines: diff --git a/tests/links/more_dirs.sby b/tests/links/more_dirs.sby index b873726..9a926dd 100644 --- a/tests/links/more_dirs.sby +++ b/tests/links/more_dirs.sby @@ -15,3 +15,6 @@ script dir/script.ys [files] here/dir ${WORKDIR}/../dir a/b/c.v prv32fmcmp.v + +[file here/doc] +log foo diff --git a/tests/links/more_dirs.sh b/tests/links/more_dirs.sh index 15d872e..aa9ce11 100644 --- a/tests/links/more_dirs.sh +++ b/tests/links/more_dirs.sh @@ -7,4 +7,4 @@ else fi python3 $SBY_MAIN -f $SBY_FILE $TASK $flags -test -e ${WORKDIR}/src/here/dir -a -e ${WORKDIR}/src/a/b/c.v +test -e ${WORKDIR}/src/here/dir -a -e ${WORKDIR}/src/a/b/c.v -a -e ${WORKDIR}/src/here/doc