From 3bf5be06376d4c9b44efa0eeb5abe52937403d31 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Tue, 29 Jul 2025 10:00:52 +1200 Subject: [PATCH 1/5] Add jsonl status format Replace `--statuscsv` and `--livecsv` with `--statusfmt ` and `--live str: +def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: if row is None: - csv_header = [ + data = [ "time", "task_name", "mode", @@ -520,7 +520,6 @@ def format_status_data_csvline(row: dict|None) -> str: "trace", "depth", ] - return ','.join(csv_header) else: engine = row['data'].get('engine', row['data'].get('source')) try: @@ -546,7 +545,11 @@ def format_status_data_csvline(row: dict|None) -> str: trace_path, depth, ] - return ','.join("" if v is None else str(v) for v in csv_line) + data = ["" if v is None else str(v) for v in csv_line] + if fmt == "csv": + return ','.join(data) + elif fmt == "jsonl": + return json.dumps(data) def filter_latest_task_ids(all_tasks: dict[int, dict[str]]): latest: dict[str, int] = {} diff --git a/tests/statusdb/timeout.sh b/tests/statusdb/timeout.sh index 2d70133..9896c3b 100644 --- a/tests/statusdb/timeout.sh +++ b/tests/statusdb/timeout.sh @@ -3,7 +3,7 @@ set -e python3 $SBY_MAIN -f $SBY_FILE $TASK STATUS_CSV=${WORKDIR}/status.csv -python3 $SBY_MAIN -f $SBY_FILE $TASK --statuscsv --latest | tee $STATUS_CSV +python3 $SBY_MAIN -f $SBY_FILE $TASK --statusfmt csv --latest | tee $STATUS_CSV if [[ $TASK =~ "_cover" ]]; then wc -l $STATUS_CSV | grep -q '6' From f05979a528de70c78125437b0b1fa119e4b4ad05 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Tue, 29 Jul 2025 10:04:40 +1200 Subject: [PATCH 2/5] Fix for statusfmt not going into status block --- sbysrc/sby.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbysrc/sby.py b/sbysrc/sby.py index f82f420..2d5a1da 100644 --- a/sbysrc/sby.py +++ b/sbysrc/sby.py @@ -72,7 +72,7 @@ if autotune and linkmode: print("ERROR: --link flag currently not available with --autotune") sys.exit(1) -if status_show or status_reset or task_status: +if status_show or status_reset or task_status or status_format: target = workdir_prefix or workdir or sbyfile if target is None: print("ERROR: Specify a .sby config file or working directory to use --status.") From 344236af41c777a96c303c9c24edbf45b49a13ed Mon Sep 17 00:00:00 2001 From: Jannis Harder Date: Tue, 29 Jul 2025 17:10:57 +0200 Subject: [PATCH 3/5] statusfmt: Make JSONL self-contained and escape CSV values --- sbysrc/sby_status.py | 69 +++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/sbysrc/sby_status.py b/sbysrc/sby_status.py index 35024f2..e914154 100644 --- a/sbysrc/sby_status.py +++ b/sbysrc/sby_status.py @@ -448,7 +448,8 @@ class SbyStatusDb: # print header header = format_status_data_fmtline(None, status_format) - print(header) + if header: + print(header) # find summary for each task/property combo prop_map: dict[(str, str), dict[str, (int, int)]] = {} @@ -506,20 +507,22 @@ def parse_status_data_row(raw: sqlite3.Row): row_dict["data"] = json.loads(row_dict.get("data") or "{}") return row_dict +fmtline_columns = [ + "time", + "task_name", + "mode", + "engine", + "name", + "location", + "kind", + "status", + "trace", + "depth", +] + def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: if row is None: - data = [ - "time", - "task_name", - "mode", - "engine", - "name", - "location", - "kind", - "status", - "trace", - "depth", - ] + data = None else: engine = row['data'].get('engine', row['data'].get('source')) try: @@ -533,22 +536,34 @@ def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: except TypeError: trace_path = None - csv_line = [ - round(time, 2), - row['task_name'], - row['mode'], - engine, - name or pretty_path(row['name']), - row['location'], - row['kind'], - row['status'] or "UNKNOWN", - trace_path, - depth, - ] - data = ["" if v is None else str(v) for v in csv_line] + data = { + "time": round(time, 2), + "task_name": row['task_name'], + "mode": row['mode'], + "engine": engine, + "name": name or pretty_path(row['name']), + "location": row['location'], + "kind": row['kind'], + "status": row['status'] or "UNKNOWN", + "trace": trace_path, + "depth": depth, + } if fmt == "csv": - return ','.join(data) + if data is None: + csv_line = fmtline_columns + else: + csv_line = [data[column] for column in fmtline_columns] + def csv_field(value): + if value is None: + return "" + value = str(value).replace('"', '""') + if any(c in value for c in '",\n'): + value = f'"{value}"' + return value + return ','.join(map(csv_field, csv_line)) elif fmt == "jsonl": + if data is None: + return "" return json.dumps(data) def filter_latest_task_ids(all_tasks: dict[int, dict[str]]): From 190ef869162b1c559e5b30313e410717d5510bd2 Mon Sep 17 00:00:00 2001 From: Jannis Harder Date: Tue, 29 Jul 2025 17:30:10 +0200 Subject: [PATCH 4/5] statusfmt: Skip missing fields in jsonl output --- sbysrc/sby_status.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sbysrc/sby_status.py b/sbysrc/sby_status.py index e914154..d59b1db 100644 --- a/sbysrc/sby_status.py +++ b/sbysrc/sby_status.py @@ -525,19 +525,10 @@ def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: data = None else: engine = row['data'].get('engine', row['data'].get('source')) - try: - time = row['status_created'] - row['created'] - except TypeError: - time = 0 name = row['hdlname'] depth = row['data'].get('step') - try: - trace_path = Path(row['workdir']) / row['path'] - except TypeError: - trace_path = None data = { - "time": round(time, 2), "task_name": row['task_name'], "mode": row['mode'], "engine": engine, @@ -545,14 +536,21 @@ def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: "location": row['location'], "kind": row['kind'], "status": row['status'] or "UNKNOWN", - "trace": trace_path, "depth": depth, } + try: + data["trace"] = str(Path(row['workdir']) / row['path']) + except TypeError: + pass + try: + data['time'] = round(row['status_created'] - row['created'], 2) + except TypeError: + pass if fmt == "csv": if data is None: csv_line = fmtline_columns else: - csv_line = [data[column] for column in fmtline_columns] + csv_line = [data.get(column) for column in fmtline_columns] def csv_field(value): if value is None: return "" @@ -564,6 +562,8 @@ def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: elif fmt == "jsonl": if data is None: return "" + # field order + data = {column: data[column] for column in fmtline_columns if column in data} return json.dumps(data) def filter_latest_task_ids(all_tasks: dict[int, dict[str]]): From d4864994ca3148a8c779b4eb301835e42d1d1dcf Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Fri, 1 Aug 2025 10:34:08 +1200 Subject: [PATCH 5/5] statusfmt: Skip null fields in jsonl output --- sbysrc/sby_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbysrc/sby_status.py b/sbysrc/sby_status.py index d59b1db..5c61463 100644 --- a/sbysrc/sby_status.py +++ b/sbysrc/sby_status.py @@ -563,7 +563,7 @@ def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: if data is None: return "" # field order - data = {column: data[column] for column in fmtline_columns if column in data} + data = {column: data[column] for column in fmtline_columns if data.get(column)} return json.dumps(data) def filter_latest_task_ids(all_tasks: dict[int, dict[str]]):