From 3bf5be06376d4c9b44efa0eeb5abe52937403d31 Mon Sep 17 00:00:00 2001
From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com>
Date: Tue, 29 Jul 2025 10:00:52 +1200
Subject: [PATCH] Add jsonl status format

Replace `--statuscsv` and `--livecsv` with `--statusfmt <fmt>` and `--live <fmt` respectively.
Currently supports both csv and jsonl.
In the case of `--live`, updates can be printed in multiple formats, while `--statusfmt` only supports one at a time.
---
 sbysrc/sby.py             | 16 ++++++++--------
 sbysrc/sby_cmdline.py     |  8 ++++----
 sbysrc/sby_core.py        |  6 +++---
 sbysrc/sby_status.py      | 37 ++++++++++++++++++++-----------------
 tests/statusdb/timeout.sh |  2 +-
 5 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/sbysrc/sby.py b/sbysrc/sby.py
index e982fcc..f82f420 100644
--- a/sbysrc/sby.py
+++ b/sbysrc/sby.py
@@ -64,15 +64,15 @@ status_show = args.status
 status_reset = args.status_reset
 status_cancels = args.status_cancels
 task_status = args.task_status
-status_live_csv = args.livecsv
-status_show_csv = args.statuscsv
+status_live_formats = args.live_formats
+status_format = args.status_format
 status_latest = args.status_latest
 
 if autotune and linkmode:
     print("ERROR: --link flag currently not available with --autotune")
     sys.exit(1)
 
-if status_show or status_reset or task_status or status_show_csv:
+if status_show or status_reset or task_status:
     target = workdir_prefix or workdir or sbyfile
     if target is None:
         print("ERROR: Specify a .sby config file or working directory to use --status.")
@@ -104,16 +104,16 @@ if status_show or status_reset or task_status or status_show_csv:
     if status_show:
         status_db.print_status_summary(status_latest)
 
-    if status_show_csv:
-        status_db.print_status_summary_csv(tasknames, status_latest)
+    if status_format:
+        status_db.print_status_summary_fmt(tasknames, status_format, status_latest)
         
     if task_status:
         status_db.print_task_summary()
 
     status_db.db.close()
 
-    if status_live_csv:
-        print(f"WARNING: --livecsv flag found but not used.")
+    if status_live_formats:
+        print(f"WARNING: --live option found but not used.")
 
     sys.exit(0)
 elif status_latest:
@@ -496,7 +496,7 @@ def start_task(taskloop, taskname):
     else:
         junit_filename = "junit"
 
-    task = SbyTask(sbyconfig, my_workdir, early_logmsgs, reusedir, status_cancels, taskloop, name=taskname, live_csv=status_live_csv)
+    task = SbyTask(sbyconfig, my_workdir, early_logmsgs, reusedir, status_cancels, taskloop, name=taskname, live_formats=status_live_formats)
 
     for k, v in exe_paths.items():
         task.exe_paths[k] = v
diff --git a/sbysrc/sby_cmdline.py b/sbysrc/sby_cmdline.py
index cb2dedd..d7952ef 100644
--- a/sbysrc/sby_cmdline.py
+++ b/sbysrc/sby_cmdline.py
@@ -29,8 +29,8 @@ def parser_func(release_version='unknown SBY version'):
             help="maximum number of processes to run in parallel")
     parser.add_argument("--sequential", action="store_true", dest="sequential",
             help="run tasks in sequence, not in parallel")
-    parser.add_argument("--livecsv", action="store_true", dest="livecsv",
-            help="print live updates of property statuses during task execution in csv format")
+    parser.add_argument("--live", action="append", choices=["csv", "jsonl"], dest="live_formats",
+            help="print live updates of property statuses during task execution, may be specified multiple times")
 
     parser.add_argument("--autotune", action="store_true", dest="autotune",
             help="automatically find a well performing engine and engine configuration for each task")
@@ -77,8 +77,8 @@ def parser_func(release_version='unknown SBY version'):
 
     parser.add_argument("--status", action="store_true", dest="status",
             help="summarize the contents of the status database")
-    parser.add_argument("--statuscsv", action="store_true", dest="statuscsv",
-            help="print the most recent status for each property in csv format")
+    parser.add_argument("--statusfmt", action="store", default="", choices=["csv", "jsonl"], dest="status_format",
+            help="print the most recent status for each property in specified format")
     parser.add_argument("--latest", action="store_true", dest="status_latest",
             help="only check statuses from the most recent run of a task")
     parser.add_argument("--statusreset", action="store_true", dest="status_reset",
diff --git a/sbysrc/sby_core.py b/sbysrc/sby_core.py
index 93b7477..2d0d1c2 100644
--- a/sbysrc/sby_core.py
+++ b/sbysrc/sby_core.py
@@ -913,7 +913,7 @@ class SbySummary:
 
 
 class SbyTask(SbyConfig):
-    def __init__(self, sbyconfig, workdir, early_logs, reusedir, status_cancels=False, taskloop=None, logfile=None, name=None, live_csv=False):
+    def __init__(self, sbyconfig, workdir, early_logs, reusedir, status_cancels=False, taskloop=None, logfile=None, name=None, live_formats=[]):
         super().__init__()
         self.used_options = set()
         self.models = dict()
@@ -921,7 +921,7 @@ class SbyTask(SbyConfig):
         self.reusedir = reusedir
         self.status_cancels = status_cancels
         self.name = name
-        self.live_csv = live_csv
+        self.live_formats = live_formats
         self.status = "UNKNOWN"
         self.total_time = 0
         self.expect = list()
@@ -1430,7 +1430,7 @@ class SbyTask(SbyConfig):
             except FileNotFoundError:
                 status_path = f"{self.workdir}/status.sqlite"
 
-        self.status_db = SbyStatusDb(status_path, self, live_csv=self.live_csv)
+        self.status_db = SbyStatusDb(status_path, self, live_formats=self.live_formats)
 
     def setup_procs(self, setupmode, linkmode=False):
         self.handle_non_engine_options()
diff --git a/sbysrc/sby_status.py b/sbysrc/sby_status.py
index a8fc3a1..35024f2 100644
--- a/sbysrc/sby_status.py
+++ b/sbysrc/sby_status.py
@@ -106,10 +106,10 @@ class FileInUseError(Exception):
 
 
 class SbyStatusDb:
-    def __init__(self, path: Path, task, timeout: float = 5.0, live_csv = False):
+    def __init__(self, path: Path, task, timeout: float = 5.0, live_formats = []):
         self.debug = False
         self.task = task
-        self.live_csv = live_csv
+        self.live_formats = live_formats
 
         self.con = sqlite3.connect(path, isolation_level=None, timeout=timeout)
         self.db = self.con.cursor()
@@ -250,10 +250,11 @@ class SbyStatusDb:
             ),
         )
 
-        if self.live_csv:
+        if self.live_formats:
             row = self.get_status_data_joined(self.db.lastrowid)
-            csvline = format_status_data_csvline(row)
-            self.task.log(f"{click.style('csv', fg='yellow')}: {csvline}")
+            for fmt in self.live_formats:
+                fmtline = format_status_data_fmtline(row, fmt)
+                self.task.log(f"{click.style(fmt, fg='yellow')}: {fmtline}")
         
     @transaction
     def add_task_trace(
@@ -440,14 +441,14 @@ class SbyStatusDb:
 
         return {row["id"]: parse_status_data_row(row) for row in rows}
 
-    def print_status_summary_csv(self, tasknames: list[str], latest: bool):
+    def print_status_summary_fmt(self, tasknames: list[str], status_format: str, latest: bool):
         # get all statuses
         all_properties = self.all_status_data_joined()
         latest_task_ids = filter_latest_task_ids(self.all_tasks())
-        
-        # print csv header
-        csvheader = format_status_data_csvline(None)
-        print(csvheader)
+
+        # print header
+        header = format_status_data_fmtline(None, status_format)
+        print(header)
 
         # find summary for each task/property combo
         prop_map: dict[(str, str), dict[str, (int, int)]] = {}
@@ -488,9 +489,8 @@ class SbyStatusDb:
                 del prop["UNKNOWN"]
 
             for _, row in prop.values():
-                csvline = format_status_data_csvline(all_properties[row])
-                print(csvline)
-
+                line = format_status_data_fmtline(all_properties[row], status_format)
+                print(line)
 
 def combine_statuses(statuses):
     statuses = set(statuses)
@@ -506,9 +506,9 @@ def parse_status_data_row(raw: sqlite3.Row):
     row_dict["data"] = json.loads(row_dict.get("data") or "{}")
     return row_dict
 
-def format_status_data_csvline(row: dict|None) -> str:
+def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str:
     if row is None:
-        csv_header = [
+        data = [
             "time",
             "task_name",
             "mode",
@@ -520,7 +520,6 @@ def format_status_data_csvline(row: dict|None) -> str:
             "trace",
             "depth",
         ]
-        return ','.join(csv_header)
     else:
         engine = row['data'].get('engine', row['data'].get('source'))
         try:
@@ -546,7 +545,11 @@ def format_status_data_csvline(row: dict|None) -> str:
             trace_path,
             depth,
         ]
-        return ','.join("" if v is None else str(v) for v in csv_line)
+        data = ["" if v is None else str(v) for v in csv_line]
+    if fmt == "csv":
+        return ','.join(data)
+    elif fmt == "jsonl":
+        return json.dumps(data)
 
 def filter_latest_task_ids(all_tasks: dict[int, dict[str]]):
     latest: dict[str, int] = {}
diff --git a/tests/statusdb/timeout.sh b/tests/statusdb/timeout.sh
index 2d70133..9896c3b 100644
--- a/tests/statusdb/timeout.sh
+++ b/tests/statusdb/timeout.sh
@@ -3,7 +3,7 @@ set -e
 python3 $SBY_MAIN -f $SBY_FILE $TASK
 
 STATUS_CSV=${WORKDIR}/status.csv
-python3 $SBY_MAIN -f $SBY_FILE $TASK --statuscsv --latest | tee $STATUS_CSV
+python3 $SBY_MAIN -f $SBY_FILE $TASK --statusfmt csv --latest | tee $STATUS_CSV
 
 if [[ $TASK =~ "_cover" ]]; then
     wc -l $STATUS_CSV | grep -q '6'