diff --git a/sbysrc/sby.py b/sbysrc/sby.py index e982fcc..f82f420 100644 --- a/sbysrc/sby.py +++ b/sbysrc/sby.py @@ -64,15 +64,15 @@ status_show = args.status status_reset = args.status_reset status_cancels = args.status_cancels task_status = args.task_status -status_live_csv = args.livecsv -status_show_csv = args.statuscsv +status_live_formats = args.live_formats +status_format = args.status_format status_latest = args.status_latest if autotune and linkmode: print("ERROR: --link flag currently not available with --autotune") sys.exit(1) -if status_show or status_reset or task_status or status_show_csv: +if status_show or status_reset or task_status: target = workdir_prefix or workdir or sbyfile if target is None: print("ERROR: Specify a .sby config file or working directory to use --status.") @@ -104,16 +104,16 @@ if status_show or status_reset or task_status or status_show_csv: if status_show: status_db.print_status_summary(status_latest) - if status_show_csv: - status_db.print_status_summary_csv(tasknames, status_latest) + if status_format: + status_db.print_status_summary_fmt(tasknames, status_format, status_latest) if task_status: status_db.print_task_summary() status_db.db.close() - if status_live_csv: - print(f"WARNING: --livecsv flag found but not used.") + if status_live_formats: + print(f"WARNING: --live option found but not used.") sys.exit(0) elif status_latest: @@ -496,7 +496,7 @@ def start_task(taskloop, taskname): else: junit_filename = "junit" - task = SbyTask(sbyconfig, my_workdir, early_logmsgs, reusedir, status_cancels, taskloop, name=taskname, live_csv=status_live_csv) + task = SbyTask(sbyconfig, my_workdir, early_logmsgs, reusedir, status_cancels, taskloop, name=taskname, live_formats=status_live_formats) for k, v in exe_paths.items(): task.exe_paths[k] = v diff --git a/sbysrc/sby_cmdline.py b/sbysrc/sby_cmdline.py index cb2dedd..d7952ef 100644 --- a/sbysrc/sby_cmdline.py +++ b/sbysrc/sby_cmdline.py @@ -29,8 +29,8 @@ def parser_func(release_version='unknown SBY version'): help="maximum number of processes to run in parallel") parser.add_argument("--sequential", action="store_true", dest="sequential", help="run tasks in sequence, not in parallel") - parser.add_argument("--livecsv", action="store_true", dest="livecsv", - help="print live updates of property statuses during task execution in csv format") + parser.add_argument("--live", action="append", choices=["csv", "jsonl"], dest="live_formats", + help="print live updates of property statuses during task execution, may be specified multiple times") parser.add_argument("--autotune", action="store_true", dest="autotune", help="automatically find a well performing engine and engine configuration for each task") @@ -77,8 +77,8 @@ def parser_func(release_version='unknown SBY version'): parser.add_argument("--status", action="store_true", dest="status", help="summarize the contents of the status database") - parser.add_argument("--statuscsv", action="store_true", dest="statuscsv", - help="print the most recent status for each property in csv format") + parser.add_argument("--statusfmt", action="store", default="", choices=["csv", "jsonl"], dest="status_format", + help="print the most recent status for each property in specified format") parser.add_argument("--latest", action="store_true", dest="status_latest", help="only check statuses from the most recent run of a task") parser.add_argument("--statusreset", action="store_true", dest="status_reset", diff --git a/sbysrc/sby_core.py b/sbysrc/sby_core.py index 93b7477..2d0d1c2 100644 --- a/sbysrc/sby_core.py +++ b/sbysrc/sby_core.py @@ -913,7 +913,7 @@ class SbySummary: class SbyTask(SbyConfig): - def __init__(self, sbyconfig, workdir, early_logs, reusedir, status_cancels=False, taskloop=None, logfile=None, name=None, live_csv=False): + def __init__(self, sbyconfig, workdir, early_logs, reusedir, status_cancels=False, taskloop=None, logfile=None, name=None, live_formats=[]): super().__init__() self.used_options = set() self.models = dict() @@ -921,7 +921,7 @@ class SbyTask(SbyConfig): self.reusedir = reusedir self.status_cancels = status_cancels self.name = name - self.live_csv = live_csv + self.live_formats = live_formats self.status = "UNKNOWN" self.total_time = 0 self.expect = list() @@ -1430,7 +1430,7 @@ class SbyTask(SbyConfig): except FileNotFoundError: status_path = f"{self.workdir}/status.sqlite" - self.status_db = SbyStatusDb(status_path, self, live_csv=self.live_csv) + self.status_db = SbyStatusDb(status_path, self, live_formats=self.live_formats) def setup_procs(self, setupmode, linkmode=False): self.handle_non_engine_options() diff --git a/sbysrc/sby_status.py b/sbysrc/sby_status.py index a8fc3a1..35024f2 100644 --- a/sbysrc/sby_status.py +++ b/sbysrc/sby_status.py @@ -106,10 +106,10 @@ class FileInUseError(Exception): class SbyStatusDb: - def __init__(self, path: Path, task, timeout: float = 5.0, live_csv = False): + def __init__(self, path: Path, task, timeout: float = 5.0, live_formats = []): self.debug = False self.task = task - self.live_csv = live_csv + self.live_formats = live_formats self.con = sqlite3.connect(path, isolation_level=None, timeout=timeout) self.db = self.con.cursor() @@ -250,10 +250,11 @@ class SbyStatusDb: ), ) - if self.live_csv: + if self.live_formats: row = self.get_status_data_joined(self.db.lastrowid) - csvline = format_status_data_csvline(row) - self.task.log(f"{click.style('csv', fg='yellow')}: {csvline}") + for fmt in self.live_formats: + fmtline = format_status_data_fmtline(row, fmt) + self.task.log(f"{click.style(fmt, fg='yellow')}: {fmtline}") @transaction def add_task_trace( @@ -440,14 +441,14 @@ class SbyStatusDb: return {row["id"]: parse_status_data_row(row) for row in rows} - def print_status_summary_csv(self, tasknames: list[str], latest: bool): + def print_status_summary_fmt(self, tasknames: list[str], status_format: str, latest: bool): # get all statuses all_properties = self.all_status_data_joined() latest_task_ids = filter_latest_task_ids(self.all_tasks()) - - # print csv header - csvheader = format_status_data_csvline(None) - print(csvheader) + + # print header + header = format_status_data_fmtline(None, status_format) + print(header) # find summary for each task/property combo prop_map: dict[(str, str), dict[str, (int, int)]] = {} @@ -488,9 +489,8 @@ class SbyStatusDb: del prop["UNKNOWN"] for _, row in prop.values(): - csvline = format_status_data_csvline(all_properties[row]) - print(csvline) - + line = format_status_data_fmtline(all_properties[row], status_format) + print(line) def combine_statuses(statuses): statuses = set(statuses) @@ -506,9 +506,9 @@ def parse_status_data_row(raw: sqlite3.Row): row_dict["data"] = json.loads(row_dict.get("data") or "{}") return row_dict -def format_status_data_csvline(row: dict|None) -> str: +def format_status_data_fmtline(row: dict|None, fmt: str = "csv") -> str: if row is None: - csv_header = [ + data = [ "time", "task_name", "mode", @@ -520,7 +520,6 @@ def format_status_data_csvline(row: dict|None) -> str: "trace", "depth", ] - return ','.join(csv_header) else: engine = row['data'].get('engine', row['data'].get('source')) try: @@ -546,7 +545,11 @@ def format_status_data_csvline(row: dict|None) -> str: trace_path, depth, ] - return ','.join("" if v is None else str(v) for v in csv_line) + data = ["" if v is None else str(v) for v in csv_line] + if fmt == "csv": + return ','.join(data) + elif fmt == "jsonl": + return json.dumps(data) def filter_latest_task_ids(all_tasks: dict[int, dict[str]]): latest: dict[str, int] = {} diff --git a/tests/statusdb/timeout.sh b/tests/statusdb/timeout.sh index 2d70133..9896c3b 100644 --- a/tests/statusdb/timeout.sh +++ b/tests/statusdb/timeout.sh @@ -3,7 +3,7 @@ set -e python3 $SBY_MAIN -f $SBY_FILE $TASK STATUS_CSV=${WORKDIR}/status.csv -python3 $SBY_MAIN -f $SBY_FILE $TASK --statuscsv --latest | tee $STATUS_CSV +python3 $SBY_MAIN -f $SBY_FILE $TASK --statusfmt csv --latest | tee $STATUS_CSV if [[ $TASK =~ "_cover" ]]; then wc -l $STATUS_CSV | grep -q '6'