From 01ba32101471ce41805f427712b34d50532d7f8e Mon Sep 17 00:00:00 2001 From: Jacob Lifshay Date: Sun, 28 Sep 2025 23:05:24 -0700 Subject: [PATCH] switch to using new crate::build system --- Cargo.lock | 87 +- Cargo.toml | 3 +- crates/fayalite/Cargo.toml | 1 + crates/fayalite/examples/blinky.rs | 26 +- crates/fayalite/src/build.rs | 3352 +++++++++-------- crates/fayalite/src/build/external.rs | 1071 +++--- crates/fayalite/src/build/firrtl.rs | 120 +- crates/fayalite/src/build/formal.rs | 419 +++ crates/fayalite/src/build/graph.rs | 801 ++++ crates/fayalite/src/build/registry.rs | 341 ++ crates/fayalite/src/build/verilog.rs | 373 ++ crates/fayalite/src/cli.rs | 806 ---- crates/fayalite/src/firrtl.rs | 51 +- crates/fayalite/src/intern.rs | 53 + crates/fayalite/src/lib.rs | 1 - crates/fayalite/src/module.rs | 6 + .../src/module/transform/simplify_enums.rs | 6 +- crates/fayalite/src/prelude.rs | 3 +- crates/fayalite/src/testing.rs | 139 +- crates/fayalite/src/util.rs | 7 +- crates/fayalite/src/util/job_server.rs | 254 +- crates/fayalite/src/util/misc.rs | 321 ++ crates/fayalite/src/util/ready_valid.rs | 2 +- crates/fayalite/tests/formal.rs | 2 +- 24 files changed, 5202 insertions(+), 3043 deletions(-) create mode 100644 crates/fayalite/src/build/formal.rs create mode 100644 crates/fayalite/src/build/graph.rs create mode 100644 crates/fayalite/src/build/registry.rs create mode 100644 crates/fayalite/src/build/verilog.rs delete mode 100644 crates/fayalite/src/cli.rs diff --git a/Cargo.lock b/Cargo.lock index 0ff4521..f4b564a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,9 +25,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" @@ -155,9 +155,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.5.9" +version = "4.5.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" +checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" dependencies = [ "clap_builder", "clap_derive", @@ -165,9 +165,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.9" +version = "4.5.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" +checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" dependencies = [ "anstream", "anstyle", @@ -176,10 +176,19 @@ dependencies = [ ] [[package]] -name = "clap_derive" -version = "4.5.8" +name = "clap_complete" +version = "4.5.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" +checksum = "75bf0b32ad2e152de789bb635ea4d3078f6b838ad7974143e99b99f45a04af4a" +dependencies = [ + "clap", +] + +[[package]] +name = "clap_derive" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" dependencies = [ "heck", "proc-macro2", @@ -189,9 +198,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "colorchoice" @@ -301,6 +310,7 @@ dependencies = [ "bitvec", "blake3", "clap", + "clap_complete", "ctor", "eyre", "fayalite-proc-macros", @@ -383,12 +393,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "libc", + "r-efi", "wasi", ] @@ -455,23 +466,23 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobslot" -version = "0.2.19" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe10868679d7a24c2c67d862d0e64a342ce9aef7cdde9ce8019bd35d353d458d" +checksum = "58715c67c327da7f1558708348d68c207fd54900c4ae0529e29305d04d795b8c" dependencies = [ "cfg-if", "derive_destructure2", "getrandom", "libc", "scopeguard", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "libc" -version = "0.2.153" +version = "0.2.176" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" [[package]] name = "linux-raw-sys" @@ -553,6 +564,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "radium" version = "0.7.0" @@ -744,9 +761,21 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.14.7+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] [[package]] name = "which" @@ -791,6 +820,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-sys" version = "0.52.0" @@ -802,11 +837,11 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.59.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-targets", + "windows-link", ] [[package]] @@ -879,6 +914,12 @@ version = "0.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + [[package]] name = "wyz" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 46d749d..b905f73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,11 +22,12 @@ base64 = "0.22.1" bitvec = { version = "1.0.1", features = ["serde"] } blake3 = { version = "1.5.4", features = ["serde"] } clap = { version = "4.5.9", features = ["derive", "env", "string"] } +clap_complete = "4.5.58" ctor = "0.2.8" eyre = "0.6.12" hashbrown = "0.15.2" indexmap = { version = "2.5.0", features = ["serde"] } -jobslot = "0.2.19" +jobslot = "0.2.23" num-bigint = "0.4.6" num-traits = "0.2.16" petgraph = "0.8.1" diff --git a/crates/fayalite/Cargo.toml b/crates/fayalite/Cargo.toml index b3005fd..2403ff5 100644 --- a/crates/fayalite/Cargo.toml +++ b/crates/fayalite/Cargo.toml @@ -18,6 +18,7 @@ base64.workspace = true bitvec.workspace = true blake3.workspace = true clap.workspace = true +clap_complete.workspace = true ctor.workspace = true eyre.workspace = true fayalite-proc-macros.workspace = true diff --git a/crates/fayalite/examples/blinky.rs b/crates/fayalite/examples/blinky.rs index 87b77c1..40b22dc 100644 --- a/crates/fayalite/examples/blinky.rs +++ b/crates/fayalite/examples/blinky.rs @@ -1,7 +1,9 @@ // SPDX-License-Identifier: LGPL-3.0-or-later // See Notices.txt for copyright information -use clap::Parser; -use fayalite::{cli, prelude::*}; +use fayalite::{ + build::{ToArgs, WriteArgs}, + prelude::*, +}; #[hdl_module] fn blinky(clock_frequency: u64) { @@ -32,16 +34,22 @@ fn blinky(clock_frequency: u64) { connect(led, output_reg); } -#[derive(Parser)] -struct Cli { +#[derive(clap::Args, Clone, PartialEq, Eq, Hash, Debug)] +struct ExtraArgs { /// clock frequency in hertz #[arg(long, default_value = "1000000", value_parser = clap::value_parser!(u64).range(2..))] clock_frequency: u64, - #[command(subcommand)] - cli: cli::Cli, } -fn main() -> cli::Result { - let cli = Cli::parse(); - cli.cli.run(blinky(cli.clock_frequency)) +impl ToArgs for ExtraArgs { + fn to_args(&self, args: &mut (impl WriteArgs + ?Sized)) { + let Self { clock_frequency } = self; + args.write_arg(format_args!("--clock-frequency={clock_frequency}")); + } +} + +fn main() { + BuildCli::main(|_cli, ExtraArgs { clock_frequency }| { + Ok(JobParams::new(blinky(clock_frequency), "blinky")) + }); } diff --git a/crates/fayalite/src/build.rs b/crates/fayalite/src/build.rs index 9005f78..09ec3ee 100644 --- a/crates/fayalite/src/build.rs +++ b/crates/fayalite/src/build.rs @@ -2,77 +2,124 @@ // See Notices.txt for copyright information use crate::{ - bundle::Bundle, + build::graph::JobGraph, + bundle::{Bundle, BundleType}, intern::{Intern, Interned}, module::Module, - util::{HashMap, HashSet, job_server::AcquiredJob}, + util::job_server::AcquiredJob, }; -use hashbrown::hash_map::Entry; -use petgraph::{ - algo::{DfsSpace, kosaraju_scc, toposort}, - graph::DiGraph, - visit::{GraphBase, Visitable}, +use serde::{ + Deserialize, Deserializer, Serialize, Serializer, + de::{DeserializeOwned, Error as _}, + ser::Error as _, }; -use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error, ser::SerializeSeq}; use std::{ any::{Any, TypeId}, borrow::Cow, - cell::OnceCell, cmp::Ordering, - collections::{BTreeMap, BTreeSet, VecDeque}, - fmt::{self, Write}, + ffi::OsStr, + fmt, hash::{Hash, Hasher}, - marker::PhantomData, - panic, - rc::Rc, - sync::{Arc, OnceLock, RwLock, RwLockWriteGuard, mpsc}, - thread::{self, ScopedJoinHandle}, + path::{Path, PathBuf}, + sync::{Arc, OnceLock}, }; use tempfile::TempDir; pub mod external; pub mod firrtl; +pub mod formal; +pub mod graph; +pub mod registry; +pub mod verilog; -macro_rules! write_str { - ($s:expr, $($rest:tt)*) => { - String::write_fmt(&mut $s, format_args!($($rest)*)).expect("String::write_fmt can't fail") +pub(crate) const BUILT_IN_JOB_KINDS: &'static [fn() -> DynJobKind] = &[ + || DynJobKind::new(BaseJobKind), + || DynJobKind::new(CreateOutputDirJobKind), + || DynJobKind::new(firrtl::FirrtlJobKind), + || DynJobKind::new(external::ExternalCommandJobKind::::new()), + || DynJobKind::new(verilog::VerilogJobKind), + || DynJobKind::new(formal::WriteSbyFileJobKind), + || DynJobKind::new(external::ExternalCommandJobKind::::new()), +]; + +#[track_caller] +fn intern_known_utf8_path_buf(v: PathBuf) -> Interned { + let Ok(v) = v.into_os_string().into_string().map(str::intern_owned) else { + unreachable!("known to be valid UTF-8"); }; + v +} + +#[track_caller] +fn intern_known_utf8_str(v: impl AsRef) -> Interned { + let Some(v) = v.as_ref().to_str().map(str::intern) else { + unreachable!("known to be valid UTF-8"); + }; + v +} + +#[track_caller] +fn interned_known_utf8_method &OsStr>( + v: impl AsRef, + f: F, +) -> Interned { + intern_known_utf8_str(f(v.as_ref().as_ref())) +} + +#[track_caller] +fn interned_known_utf8_path_buf_method PathBuf>( + v: impl AsRef, + f: F, +) -> Interned { + intern_known_utf8_path_buf(f(v.as_ref().as_ref())) } #[derive(Clone, Hash, PartialEq, Eq, Debug)] +#[non_exhaustive] pub enum JobItem { - Module { value: Module }, - File { path: Interned }, + Path { + path: Interned, + }, + DynamicPaths { + paths: Vec>, + source_job_name: Interned, + }, } impl JobItem { pub fn name(&self) -> JobItemName { match self { - JobItem::Module { value } => JobItemName::Module { name: value.name() }, - &JobItem::File { path } => JobItemName::File { path }, + &JobItem::Path { path } => JobItemName::Path { path }, + &JobItem::DynamicPaths { + paths: _, + source_job_name, + } => JobItemName::DynamicPaths { source_job_name }, } } } #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] +#[non_exhaustive] pub enum JobItemName { - Module { name: Interned }, - File { path: Interned }, + Path { path: Interned }, + DynamicPaths { source_job_name: Interned }, } impl JobItemName { fn as_ref(&self) -> JobItemNameRef<'_> { match self { - JobItemName::Module { name } => JobItemNameRef::Module { name }, - JobItemName::File { path } => JobItemNameRef::File { path }, + JobItemName::Path { path } => JobItemNameRef::Path { path }, + JobItemName::DynamicPaths { source_job_name } => { + JobItemNameRef::DynamicPaths { source_job_name } + } } } } #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] enum JobItemNameRef<'a> { - Module { name: &'a str }, - File { path: &'a str }, + Path { path: &'a str }, + DynamicPaths { source_job_name: &'a str }, } /// ordered by string contents, not by `Interned` @@ -93,47 +140,507 @@ impl Ord for JobItemName { } } -pub trait JobKind: 'static + Send + Sync + Hash + Eq + fmt::Debug { - type Job: 'static + Send + Sync + Hash + Eq + fmt::Debug; - fn inputs_and_direct_dependencies<'a>( - &'a self, - job: &'a Self::Job, - ) -> Cow<'a, BTreeMap>>; - fn outputs(&self, job: &Self::Job) -> Interned<[JobItemName]>; - /// gets the part of the command line that is common for all members of this job kind -- usually the executable name/path and any global options and/or subcommands - fn command_line_prefix(&self) -> Interned<[Interned]>; - fn to_command_line(&self, job: &Self::Job) -> Interned<[Interned]>; - /// return the subcommand if this is an internal JobKind - fn subcommand(&self) -> Option; - /// Parse from [`ArgMatches`], this should only be called with the results of parsing [`subcommand()`]. - /// If [`subcommand()`] returned [`None`], you should not call this function since it will panic. - /// - /// [`ArgMatches`]: clap::ArgMatches - /// [`subcommand()`]: JobKind::subcommand - fn from_arg_matches(&self, matches: &mut clap::ArgMatches) -> clap::error::Result; - fn debug_name(&self, job: &Self::Job) -> String { - let name = self - .command_line_prefix() - .last() - .copied() - .or_else(|| self.to_command_line(job).first().copied()) - .unwrap_or_default(); - let name = match name.rsplit_once(['/', '\\']) { - Some((_, name)) if name.trim() != "" => name, - _ => &*name, - }; - format!("job:{name}") +pub trait WriteArgs: + for<'a> Extend<&'a str> + + Extend + + Extend> + + for<'a> Extend> +{ + fn write_args(&mut self, args: impl IntoIterator) { + self.extend(args.into_iter().map(|v| format!("{v}"))); } - fn parse_command_line( - &self, - command_line: Interned<[Interned]>, - ) -> clap::error::Result; + fn write_string_args(&mut self, args: impl IntoIterator) { + self.extend(args); + } + fn write_str_args<'a>(&mut self, args: impl IntoIterator) { + self.extend(args); + } + fn write_interned_args(&mut self, args: impl IntoIterator>) { + self.extend(args); + } + fn write_arg(&mut self, arg: impl fmt::Display) { + self.extend([format_args!("{arg}")]); + } + fn write_string_arg(&mut self, arg: String) { + self.extend([arg]); + } + fn write_str_arg(&mut self, arg: &str) { + self.extend([arg]); + } + fn write_interned_arg(&mut self, arg: Interned) { + self.extend([arg]); + } +} + +#[derive(Default)] +struct WriteArgsWrapper(W); + +impl<'a, W> Extend> for WriteArgsWrapper +where + Self: Extend, +{ + fn extend>>(&mut self, iter: T) { + self.extend(iter.into_iter().map(|v| v.to_string())) + } +} + +impl<'a> Extend<&'a str> for WriteArgsWrapper>> { + fn extend>(&mut self, iter: T) { + self.extend(iter.into_iter().map(str::intern)) + } +} + +impl Extend for WriteArgsWrapper>> { + fn extend>(&mut self, iter: T) { + self.extend(iter.into_iter().map(str::intern_owned)) + } +} + +impl Extend> for WriteArgsWrapper> { + fn extend>>(&mut self, iter: T) { + self.extend(iter.into_iter().map(String::from)) + } +} + +impl<'a> Extend<&'a str> for WriteArgsWrapper> { + fn extend>(&mut self, iter: T) { + self.extend(iter.into_iter().map(String::from)) + } +} + +impl Extend for WriteArgsWrapper> { + fn extend>(&mut self, iter: T) { + self.0.extend(iter); + } +} + +impl WriteArgs for WriteArgsWrapper> {} + +impl WriteArgs for WriteArgsWrapper>> {} + +pub trait ToArgs: clap::Args + 'static + Send + Sync + Hash + Eq + fmt::Debug + Clone { + fn to_args(&self, args: &mut (impl WriteArgs + ?Sized)); + fn to_interned_args(&self) -> Interned<[Interned]> { + Intern::intern_owned(self.to_interned_args_vec()) + } + fn to_interned_args_vec(&self) -> Vec> { + let mut retval = WriteArgsWrapper::default(); + self.to_args(&mut retval); + retval.0 + } + fn to_string_args(&self) -> Vec { + let mut retval = WriteArgsWrapper::default(); + self.to_args(&mut retval); + retval.0 + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct JobKindAndArgs { + pub kind: K, + pub args: K::Args, +} + +impl JobKindAndArgs { + pub fn args_to_jobs( + self, + dependencies: ::KindsAndArgs, + params: &JobParams, + ) -> eyre::Result> { + K::args_to_jobs( + JobArgsAndDependencies { + args: self, + dependencies, + }, + params, + ) + } +} + +impl> Copy for JobKindAndArgs {} + +impl From> for DynJobArgs { + fn from(value: JobKindAndArgs) -> Self { + let JobKindAndArgs { kind, args } = value; + DynJobArgs::new(kind, args) + } +} + +impl TryFrom for JobKindAndArgs { + type Error = DynJobArgs; + fn try_from(value: DynJobArgs) -> Result { + value.downcast() + } +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct JobAndKind { + pub kind: K, + pub job: K::Job, +} + +impl> Clone for JobAndKind { + fn clone(&self) -> Self { + Self { + kind: self.kind.clone(), + job: self.job.clone(), + } + } +} + +impl> Copy for JobAndKind {} + +impl From> for DynJob { + fn from(value: JobAndKind) -> Self { + let JobAndKind { kind, job } = value; + DynJob::new(kind, job) + } +} + +impl> TryFrom for JobAndKind { + type Error = DynJob; + fn try_from(value: DynJob) -> Result { + value.downcast() + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub struct JobKindAndDependencies { + pub kind: K, + pub dependencies: K::Dependencies, +} + +impl Default for JobKindAndDependencies { + fn default() -> Self { + Self::new(K::default()) + } +} + +impl JobKindAndDependencies { + pub fn new(kind: K) -> Self { + Self { + kind, + dependencies: kind.dependencies(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct JobAndDependencies { + pub job: JobAndKind, + pub dependencies: ::JobsAndKinds, +} + +impl Clone for JobAndDependencies +where + K::Job: Clone, + ::JobsAndKinds: Clone, +{ + fn clone(&self) -> Self { + Self { + job: self.job.clone(), + dependencies: self.dependencies.clone(), + } + } +} + +impl Copy for JobAndDependencies +where + K::Job: Copy, + ::JobsAndKinds: Copy, +{ +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct JobArgsAndDependencies { + pub args: JobKindAndArgs, + pub dependencies: ::KindsAndArgs, +} + +impl Copy for JobArgsAndDependencies +where + K::Args: Copy, + ::KindsAndArgs: Copy, +{ +} + +impl JobArgsAndDependencies { + pub fn args_to_jobs(self, params: &JobParams) -> eyre::Result> { + K::args_to_jobs(self, params) + } +} + +impl>, D: JobKind> JobArgsAndDependencies { + pub fn args_to_jobs_simple( + self, + params: &JobParams, + f: F, + ) -> eyre::Result> + where + F: FnOnce(K, K::Args, &mut JobAndDependencies) -> eyre::Result, + { + let Self { + args: JobKindAndArgs { kind, args }, + dependencies, + } = self; + let mut dependencies = dependencies.args_to_jobs(params)?; + let job = f(kind, args, &mut dependencies)?; + Ok(JobAndDependencies { + job: JobAndKind { kind, job }, + dependencies, + }) + } +} + +impl>, D: JobKind> + JobArgsAndDependencies> +{ + pub fn args_to_jobs_external_simple( + self, + params: &JobParams, + f: F, + ) -> eyre::Result<( + C::AdditionalJobData, + ::JobsAndKinds, + )> + where + F: FnOnce( + external::ExternalCommandArgs, + &mut JobAndDependencies, + ) -> eyre::Result, + { + let Self { + args: JobKindAndArgs { kind: _, args }, + dependencies, + } = self; + let mut dependencies = dependencies.args_to_jobs(params)?; + let additional_job_data = f(args, &mut dependencies)?; + Ok((additional_job_data, dependencies)) + } +} + +pub trait JobDependencies: 'static + Send + Sync + Hash + Eq + fmt::Debug + Copy { + type KindsAndArgs: 'static + Send + Sync + Hash + Eq + fmt::Debug + Clone; + type JobsAndKinds: 'static + Send + Sync + Hash + Eq + fmt::Debug; + fn kinds_dyn_extend>(self, dyn_kinds: &mut E); + fn kinds_dyn(self) -> Vec { + let mut retval = Vec::new(); + self.kinds_dyn_extend(&mut retval); + retval + } + fn into_dyn_jobs_extend>(jobs: Self::JobsAndKinds, dyn_jobs: &mut E); + fn into_dyn_jobs(jobs: Self::JobsAndKinds) -> Vec { + let mut retval = Vec::new(); + Self::into_dyn_jobs_extend(jobs, &mut retval); + retval + } + #[track_caller] + fn from_dyn_args_prefix>( + args: &mut I, + ) -> Self::KindsAndArgs; + #[track_caller] + fn from_dyn_args>(args: I) -> Self::KindsAndArgs { + let mut iter = args.into_iter(); + let retval = Self::from_dyn_args_prefix(&mut iter); + if iter.next().is_some() { + panic!("wrong number of dependencies"); + } + retval + } +} + +impl JobDependencies for JobKindAndDependencies { + type KindsAndArgs = JobArgsAndDependencies; + type JobsAndKinds = JobAndDependencies; + + fn kinds_dyn_extend>(self, dyn_kinds: &mut E) { + let Self { kind, dependencies } = self; + dependencies.kinds_dyn_extend(dyn_kinds); + dyn_kinds.extend([DynJobKind::new(kind)]); + } + + fn into_dyn_jobs_extend>( + jobs: Self::JobsAndKinds, + dyn_jobs: &mut E, + ) { + let JobAndDependencies { job, dependencies } = jobs; + K::Dependencies::into_dyn_jobs_extend(dependencies, dyn_jobs); + dyn_jobs.extend([job.into()]); + } + + #[track_caller] + fn from_dyn_args_prefix>( + args: &mut I, + ) -> Self::KindsAndArgs { + let dependencies = K::Dependencies::from_dyn_args_prefix(args); + let Some(args) = args.next() else { + panic!("wrong number of dependencies"); + }; + match args.downcast() { + Ok(args) => JobArgsAndDependencies { args, dependencies }, + Err(args) => { + panic!( + "wrong type of dependency, expected {} got:\n{args:?}", + std::any::type_name::() + ) + } + } + } +} + +macro_rules! impl_job_dependencies { + (@impl $(($v:ident: $T:ident),)*) => { + impl<$($T: JobDependencies),*> JobDependencies for ($($T,)*) { + type KindsAndArgs = ($($T::KindsAndArgs,)*); + type JobsAndKinds = ($($T::JobsAndKinds,)*); + + fn kinds_dyn_extend>(self, dyn_kinds: &mut E) { + #![allow(unused_variables)] + let ($($v,)*) = self; + $($T::kinds_dyn_extend($v, dyn_kinds);)* + } + + fn into_dyn_jobs_extend>( + jobs: Self::JobsAndKinds, + dyn_jobs: &mut E, + ) { + #![allow(unused_variables)] + let ($($v,)*) = jobs; + $($T::into_dyn_jobs_extend($v, dyn_jobs);)* + } + + #[track_caller] + fn from_dyn_args_prefix>( + args: &mut I, + ) -> Self::KindsAndArgs { + #![allow(unused_variables)] + $(let $v = $T::from_dyn_args_prefix(args);)* + ($($v,)*) + } + } + }; + ($($first:tt, $($rest:tt,)*)?) => { + impl_job_dependencies!(@impl $($first, $($rest,)*)?); + $(impl_job_dependencies!($($rest,)*);)? + }; +} + +impl_job_dependencies! { + (v0: T0), + (v1: T1), + (v2: T2), + (v3: T3), + (v4: T4), + (v5: T5), + (v6: T6), + (v7: T7), + (v8: T8), + (v9: T9), + (v10: T10), + (v11: T11), +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct JobParams { + main_module: Module, + application_name: Interned, +} + +impl AsRef for JobParams { + fn as_ref(&self) -> &Self { + self + } +} + +impl JobParams { + pub fn new_canonical(main_module: Module, application_name: Interned) -> Self { + Self { + main_module, + application_name, + } + } + pub fn new( + main_module: impl AsRef>, + application_name: impl AsRef, + ) -> Self { + Self::new_canonical( + main_module.as_ref().canonical(), + application_name.as_ref().intern(), + ) + } + pub fn main_module(&self) -> &Module { + &self.main_module + } + pub fn application_name(&self) -> Interned { + self.application_name + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +pub struct CommandParams { + pub command_line: Interned<[Interned]>, + pub current_dir: Option>, +} + +impl CommandParams { + fn to_unix_shell_line( + self, + output: &mut W, + mut escape_arg: impl FnMut(&str, &mut W) -> fmt::Result, + ) -> fmt::Result { + let Self { + command_line, + current_dir, + } = self; + let mut end = None; + let mut separator = if let Some(current_dir) = current_dir { + output.write_str("(cd ")?; + end = Some(")"); + if !current_dir.starts_with(|ch: char| { + ch.is_ascii_alphanumeric() || matches!(ch, '/' | '\\' | '.') + }) { + output.write_str("-- ")?; + } + escape_arg(¤t_dir, output)?; + "; exec -- " + } else { + "" + }; + for arg in command_line { + output.write_str(separator)?; + separator = " "; + escape_arg(&arg, output)?; + } + if let Some(end) = end { + output.write_str(end)?; + } + Ok(()) + } +} + +pub trait JobKind: 'static + Send + Sync + Hash + Eq + fmt::Debug + Sized + Copy { + type Args: ToArgs; + type Job: 'static + Send + Sync + Hash + Eq + fmt::Debug + Serialize + DeserializeOwned; + type Dependencies: JobDependencies; + fn dependencies(self) -> Self::Dependencies; + fn args_to_jobs( + args: JobArgsAndDependencies, + params: &JobParams, + ) -> eyre::Result>; + fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]>; + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]>; + fn name(self) -> Interned; + fn external_command_params(self, job: &Self::Job) -> Option; fn run( - &self, + self, job: &Self::Job, inputs: &[JobItem], + params: &JobParams, acquired_job: &mut AcquiredJob, ) -> eyre::Result>; + fn subcommand_hidden(self) -> bool { + false + } } trait DynJobKindTrait: 'static + Send + Sync + fmt::Debug { @@ -141,16 +648,21 @@ trait DynJobKindTrait: 'static + Send + Sync + fmt::Debug { fn as_arc_any(self: Arc) -> Arc; fn eq_dyn(&self, other: &dyn DynJobKindTrait) -> bool; fn hash_dyn(&self, state: &mut dyn Hasher); - fn command_line_prefix_dyn(&self) -> Interned<[Interned]>; - fn subcommand_dyn(&self) -> Option; + fn dependencies_kinds_dyn(&self) -> Vec; + fn args_group_id_dyn(&self) -> Option; + fn augment_args_dyn(&self, cmd: clap::Command) -> clap::Command; + fn augment_args_for_update_dyn(&self, cmd: clap::Command) -> clap::Command; fn from_arg_matches_dyn( - self: Arc, + &self, matches: &mut clap::ArgMatches, - ) -> clap::error::Result; - fn parse_command_line_dyn( + ) -> clap::error::Result; + fn name_dyn(&self) -> Interned; + fn subcommand_hidden_dyn(&self) -> bool; + fn deserialize_job_from_json_str(self: Arc, json: &str) -> serde_json::Result; + fn deserialize_job_from_json_value( self: Arc, - command_line: Interned<[Interned]>, - ) -> clap::error::Result; + json: &serde_json::Value, + ) -> serde_json::Result; } impl DynJobKindTrait for T { @@ -165,36 +677,57 @@ impl DynJobKindTrait for T { fn eq_dyn(&self, other: &dyn DynJobKindTrait) -> bool { other .as_any() - .downcast_ref::() + .downcast_ref::() .is_some_and(|other| self == other) } fn hash_dyn(&self, mut state: &mut dyn Hasher) { - self.hash(&mut state) + self.hash(&mut state); } - fn command_line_prefix_dyn(&self) -> Interned<[Interned]> { - self.command_line_prefix() + fn dependencies_kinds_dyn(&self) -> Vec { + self.dependencies().kinds_dyn() } - fn subcommand_dyn(&self) -> Option { - self.subcommand() + fn args_group_id_dyn(&self) -> Option { + ::group_id() + } + + fn augment_args_dyn(&self, cmd: clap::Command) -> clap::Command { + ::augment_args(cmd) + } + + fn augment_args_for_update_dyn(&self, cmd: clap::Command) -> clap::Command { + ::augment_args_for_update(cmd) } fn from_arg_matches_dyn( - self: Arc, + &self, matches: &mut clap::ArgMatches, - ) -> clap::error::Result { - let job = self.from_arg_matches(matches)?; - Ok(DynJob::from_arc(self, job)) + ) -> clap::error::Result { + Ok(DynJobArgs::new( + *self, + ::from_arg_matches_mut(matches)?, + )) } - fn parse_command_line_dyn( - self: Arc, - command_line: Interned<[Interned]>, - ) -> clap::error::Result { - let job = self.parse_command_line(command_line)?; - Ok(DynJob::from_arc(self, job)) + fn name_dyn(&self) -> Interned { + self.name() + } + + fn subcommand_hidden_dyn(&self) -> bool { + self.subcommand_hidden() + } + + fn deserialize_job_from_json_str(self: Arc, json: &str) -> serde_json::Result { + Ok(DynJob::from_arc(self, serde_json::from_str(json)?)) + } + + fn deserialize_job_from_json_value( + self: Arc, + json: &serde_json::Value, + ) -> serde_json::Result { + Ok(DynJob::from_arc(self, Deserialize::deserialize(json)?)) } } @@ -203,31 +736,19 @@ pub struct DynJobKind(Arc); impl DynJobKind { pub fn from_arc(job_kind: Arc) -> Self { - if TypeId::of::() == TypeId::of::() { - Self::clone( - &Arc::downcast::(job_kind.as_arc_any()) - .ok() - .expect("already checked type"), - ) - } else { - Self(job_kind) - } + Self(job_kind) } pub fn new(job_kind: T) -> Self { - if let Some(job_kind) = DynJobKindTrait::as_any(&job_kind).downcast_ref::() { - job_kind.clone() - } else { - Self(Arc::new(job_kind)) - } + Self(Arc::new(job_kind)) } pub fn type_id(&self) -> TypeId { DynJobKindTrait::as_any(&*self.0).type_id() } - pub fn downcast_ref(&self) -> Option<&T> { - DynJobKindTrait::as_any(&*self.0).downcast_ref() + pub fn downcast(&self) -> Option { + DynJobKindTrait::as_any(&*self.0).downcast_ref().copied() } pub fn downcast_arc(self) -> Result, Self> { - if self.downcast_ref::().is_some() { + if self.downcast::().is_some() { Ok(Arc::downcast::(self.0.as_arc_any()) .ok() .expect("already checked type")) @@ -235,18 +756,61 @@ impl DynJobKind { Err(self) } } - pub fn registry() -> JobKindRegistrySnapshot { - JobKindRegistrySnapshot(JobKindRegistry::get()) + pub fn dependencies_kinds(&self) -> Vec { + DynJobKindTrait::dependencies_kinds_dyn(&*self.0) } - #[track_caller] - pub fn register(self) { - JobKindRegistry::register(JobKindRegistry::lock(), self); + pub fn args_group_id(&self) -> Option { + DynJobKindTrait::args_group_id_dyn(&*self.0) + } + pub fn augment_args(&self, cmd: clap::Command) -> clap::Command { + DynJobKindTrait::augment_args_dyn(&*self.0, cmd) + } + pub fn augment_args_for_update(&self, cmd: clap::Command) -> clap::Command { + DynJobKindTrait::augment_args_for_update_dyn(&*self.0, cmd) + } + pub fn from_arg_matches( + &self, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result { + DynJobKindTrait::from_arg_matches_dyn(&*self.0, matches) + } + pub fn name(&self) -> Interned { + DynJobKindTrait::name_dyn(&*self.0) + } + pub fn subcommand_hidden(&self) -> bool { + DynJobKindTrait::subcommand_hidden_dyn(&*self.0) + } + pub fn deserialize_job_from_json_str(self, json: &str) -> serde_json::Result { + DynJobKindTrait::deserialize_job_from_json_str(self.0, json) + } + pub fn deserialize_job_from_json_value( + self, + json: &serde_json::Value, + ) -> serde_json::Result { + DynJobKindTrait::deserialize_job_from_json_value(self.0, json) + } + fn make_subcommand_without_args(&self) -> clap::Command { + clap::Command::new(Interned::into_inner(self.name())).hide(self.subcommand_hidden()) + } + pub fn make_subcommand(&self) -> clap::Command { + let mut subcommand = self.make_subcommand_without_args(); + for dependency in self.dependencies_kinds() { + subcommand = dependency.augment_args(subcommand); + } + self.augment_args(subcommand) + } + pub fn make_subcommand_for_update(&self) -> clap::Command { + let mut subcommand = self.make_subcommand_without_args(); + for dependency in self.dependencies_kinds() { + subcommand = dependency.augment_args_for_update(subcommand); + } + self.augment_args_for_update(subcommand) } } impl Hash for DynJobKind { fn hash(&self, state: &mut H) { - DynJobKindTrait::as_any(&*self.0).type_id().hash(state); + self.type_id().hash(state); DynJobKindTrait::hash_dyn(&*self.0, state); } } @@ -270,7 +834,7 @@ impl Serialize for DynJobKind { where S: Serializer, { - self.command_line_prefix().serialize(serializer) + self.name().serialize(serializer) } } @@ -279,303 +843,354 @@ impl<'de> Deserialize<'de> for DynJobKind { where D: Deserializer<'de>, { - let command_line_prefix: Cow<'_, [Interned]> = Cow::deserialize(deserializer)?; - match Self::registry().get_by_command_line_prefix(&command_line_prefix) { + let name = Cow::::deserialize(deserializer)?; + match Self::registry().get_by_name(&name) { Some(retval) => Ok(retval.clone()), None => Err(D::Error::custom(format_args!( - "unknown job kind: command line prefix not found in registry: {command_line_prefix:?}" + "unknown job kind: name not found in registry: {name:?}" ))), } } } -#[derive(Clone, Debug)] -struct JobKindRegistry { - command_line_prefix_to_job_kind_map: HashMap<&'static [Interned], DynJobKind>, - job_kinds: Vec, - subcommand_names: BTreeMap<&'static str, DynJobKind>, +#[derive(Copy, Clone, Debug, Default)] +pub struct DynJobKindValueParser; + +#[derive(Clone, PartialEq, Eq, Hash)] +struct DynJobKindValueEnum { + name: Interned, + job_kind: DynJobKind, } -enum JobKindRegisterError { - SameCommandLinePrefix { - command_line_prefix: &'static [Interned], - old_job_kind: DynJobKind, - new_job_kind: DynJobKind, - }, - CommandLinePrefixDoesNotMatchSubcommandName { - command_line_prefix: &'static [Interned], - expected: [Interned; 2], - job_kind: DynJobKind, - }, +impl clap::ValueEnum for DynJobKindValueEnum { + fn value_variants<'a>() -> &'a [Self] { + Interned::into_inner( + registry::JobKindRegistrySnapshot::get() + .iter_with_names() + .map(|(name, job_kind)| Self { + name, + job_kind: job_kind.clone(), + }) + .collect(), + ) + } + + fn to_possible_value(&self) -> Option { + Some(clap::builder::PossibleValue::new(Interned::into_inner( + self.name, + ))) + } } -impl fmt::Display for JobKindRegisterError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::SameCommandLinePrefix { - command_line_prefix, - old_job_kind, - new_job_kind, - } => write!( - f, - "two different `DynJobKind` can't share the same `command_line_prefix` of:\n\ - {command_line_prefix:?}\n\ - old job kind:\n\ - {old_job_kind:?}\n\ - new job kind:\n\ - {new_job_kind:?}", - ), - Self::CommandLinePrefixDoesNotMatchSubcommandName { - command_line_prefix, - expected, - job_kind, - } => write!( - f, - "`JobKind::subcommand()` returned `Some` but the `command_line_prefix` is not as expected\n\ - (it should be `[program_name_for_internal_jobs(), subcommand_name]`):\n\ - command_line_prefix:\n\ - {command_line_prefix:?}\n\ - expected:\n\ - {expected:?}\n\ - job kind:\n\ - {job_kind:?}", - ), +impl clap::builder::TypedValueParser for DynJobKindValueParser { + type Value = DynJobKind; + + fn parse_ref( + &self, + cmd: &clap::Command, + arg: Option<&clap::Arg>, + value: &std::ffi::OsStr, + ) -> clap::error::Result { + clap::builder::EnumValueParser::::new() + .parse_ref(cmd, arg, value) + .map(|v| v.job_kind) + } + + fn possible_values( + &self, + ) -> Option + '_>> { + static ENUM_VALUE_PARSER: OnceLock> = + OnceLock::new(); + ENUM_VALUE_PARSER + .get_or_init(clap::builder::EnumValueParser::::new) + .possible_values() + } +} + +impl clap::builder::ValueParserFactory for DynJobKind { + type Parser = DynJobKindValueParser; + + fn value_parser() -> Self::Parser { + DynJobKindValueParser::default() + } +} + +trait DynExtendInternedStr { + fn extend_from_slice(&mut self, items: &[Interned]); +} + +impl Extend> for dyn DynExtendInternedStr + '_ { + fn extend>>(&mut self, iter: T) { + let mut buf = [Interned::default(); 64]; + let mut buf_len = 0; + iter.into_iter().for_each(|item| { + buf[buf_len] = item; + buf_len += 1; + if buf_len == buf.len() { + ::extend_from_slice(self, &buf); + buf_len = 0; + } + }); + if buf_len > 0 { + ::extend_from_slice( + self, + &buf[..buf_len], + ); } } } -trait JobKindRegistryRegisterLock { - type Locked; - fn lock(self) -> Self::Locked; - fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry; -} - -impl JobKindRegistryRegisterLock for &'static RwLock> { - type Locked = RwLockWriteGuard<'static, Arc>; - fn lock(self) -> Self::Locked { - self.write().expect("shouldn't be poisoned") - } - fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry { - Arc::make_mut(locked) +impl>> DynExtendInternedStr for T { + fn extend_from_slice(&mut self, items: &[Interned]) { + self.extend(items.iter().copied()); } } -impl JobKindRegistryRegisterLock for &'_ mut JobKindRegistry { - type Locked = Self; +#[derive(PartialEq, Eq, Hash, Clone)] +struct DynJobArgsInner(JobKindAndArgs); - fn lock(self) -> Self::Locked { +impl fmt::Debug for DynJobArgsInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self(JobKindAndArgs { kind, args }) = self; + f.debug_struct("DynJobArgs") + .field("kind", kind) + .field("args", args) + .finish() + } +} + +trait DynJobArgsTrait: 'static + Send + Sync + fmt::Debug { + fn as_any(&self) -> &dyn Any; + fn as_arc_any(self: Arc) -> Arc; + fn kind_type_id(&self) -> TypeId; + fn eq_dyn(&self, other: &dyn DynJobArgsTrait) -> bool; + fn hash_dyn(&self, state: &mut dyn Hasher); + fn kind(&self) -> DynJobKind; + fn to_args_extend_vec(&self, args: Vec>) -> Vec>; + fn clone_into_arc(&self) -> Arc; + fn update_from_arg_matches( + &mut self, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result<()>; + #[track_caller] + fn args_to_jobs( + self: Arc, + dependencies_args: Vec, + params: &JobParams, + ) -> eyre::Result<(DynJob, Vec)>; +} + +impl DynJobArgsTrait for DynJobArgsInner { + fn as_any(&self) -> &dyn Any { self } - fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry { - locked - } -} - -impl JobKindRegistry { - fn lock() -> &'static RwLock> { - static REGISTRY: OnceLock>> = OnceLock::new(); - REGISTRY.get_or_init(Default::default) - } - fn try_register( - lock: L, - job_kind: DynJobKind, - ) -> Result<(), JobKindRegisterError> { - let command_line_prefix = Interned::into_inner(job_kind.command_line_prefix()); - let subcommand_name = job_kind - .subcommand() - .map(|subcommand| subcommand.get_name().intern()); - if let Some(subcommand_name) = subcommand_name { - let expected = [program_name_for_internal_jobs(), subcommand_name]; - if command_line_prefix != &expected { - return Err( - JobKindRegisterError::CommandLinePrefixDoesNotMatchSubcommandName { - command_line_prefix, - expected, - job_kind, - }, - ); - } - } - // run user code only outside of lock - let mut locked = lock.lock(); - let this = L::make_mut(&mut locked); - let result = match this - .command_line_prefix_to_job_kind_map - .entry(command_line_prefix) - { - Entry::Occupied(entry) => Err(JobKindRegisterError::SameCommandLinePrefix { - command_line_prefix, - old_job_kind: entry.get().clone(), - new_job_kind: job_kind, - }), - Entry::Vacant(entry) => { - this.job_kinds.push(job_kind.clone()); - if let Some(subcommand_name) = subcommand_name { - this.subcommand_names - .insert(Interned::into_inner(subcommand_name), job_kind.clone()); - } - entry.insert(job_kind); - Ok(()) - } - }; - drop(locked); - // outside of lock now, so we can test if it's the same DynJobKind - match result { - Err(JobKindRegisterError::SameCommandLinePrefix { - command_line_prefix: _, - old_job_kind, - new_job_kind, - }) if old_job_kind == new_job_kind => Ok(()), - result => result, - } - } - #[track_caller] - fn register(lock: L, job_kind: DynJobKind) { - match Self::try_register(lock, job_kind) { - Err(e) => panic!("{e}"), - Ok(()) => {} - } - } - fn get() -> Arc { - Self::lock().read().expect("shouldn't be poisoned").clone() - } -} - -impl Default for JobKindRegistry { - fn default() -> Self { - let mut retval = Self { - command_line_prefix_to_job_kind_map: HashMap::default(), - job_kinds: Vec::new(), - subcommand_names: BTreeMap::new(), - }; - for job_kind in [] { - Self::register(&mut retval, job_kind); - } - retval - } -} - -#[derive(Clone, Debug)] -pub struct JobKindRegistrySnapshot(Arc); - -impl JobKindRegistrySnapshot { - pub fn get() -> Self { - JobKindRegistrySnapshot(JobKindRegistry::get()) - } - pub fn get_by_command_line_prefix<'a>( - &'a self, - command_line_prefix: &[Interned], - ) -> Option<&'a DynJobKind> { - self.0 - .command_line_prefix_to_job_kind_map - .get(command_line_prefix) - } - pub fn job_kinds(&self) -> &[DynJobKind] { - &self.0.job_kinds - } -} - -#[derive(Clone, Debug, Hash, PartialEq, Eq)] -pub struct AnyInternalJob(pub DynJob); - -impl clap::Subcommand for AnyInternalJob { - fn augment_subcommands(mut cmd: clap::Command) -> clap::Command { - for job_kind in JobKindRegistrySnapshot::get().0.subcommand_names.values() { - let Some(subcommand) = job_kind.subcommand() else { - // shouldn't happen, ignore it - continue; - }; - cmd = cmd.subcommand(subcommand); - } - cmd + fn as_arc_any(self: Arc) -> Arc { + self } - fn augment_subcommands_for_update(cmd: clap::Command) -> clap::Command { - Self::augment_subcommands(cmd) + fn kind_type_id(&self) -> TypeId { + TypeId::of::() } - fn has_subcommand(name: &str) -> bool { - JobKindRegistrySnapshot::get() - .0 - .subcommand_names - .contains_key(name) - } -} - -impl clap::FromArgMatches for AnyInternalJob { - fn from_arg_matches(matches: &clap::ArgMatches) -> Result { - Self::from_arg_matches_mut(&mut matches.clone()) + fn eq_dyn(&self, other: &dyn DynJobArgsTrait) -> bool { + other + .as_any() + .downcast_ref::() + .is_some_and(|other| self == other) } - fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result { - if let Some((name, mut matches)) = matches.remove_subcommand() { - let job_kind_registry_snapshot = JobKindRegistrySnapshot::get(); - if let Some(job_kind) = job_kind_registry_snapshot.0.subcommand_names.get(&*name) { - Ok(Self(job_kind.from_arg_matches(&mut matches)?)) - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::InvalidSubcommand, - format!("the subcommand '{name}' wasn't recognized"), - )) - } - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::MissingSubcommand, - "a subcommand is required but one was not provided", - )) - } + fn hash_dyn(&self, mut state: &mut dyn Hasher) { + self.hash(&mut state); } - fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> { - *self = Self::from_arg_matches(matches)?; - Ok(()) + fn kind(&self) -> DynJobKind { + DynJobKind::new(self.0.kind) } - fn update_from_arg_matches_mut( + fn to_args_extend_vec(&self, args: Vec>) -> Vec> { + let mut writer = WriteArgsWrapper(args); + self.0.args.to_args(&mut writer); + writer.0 + } + + fn clone_into_arc(&self) -> Arc { + Arc::new(self.clone()) + } + + fn update_from_arg_matches( &mut self, matches: &mut clap::ArgMatches, - ) -> Result<(), clap::Error> { - *self = Self::from_arg_matches_mut(matches)?; - Ok(()) + ) -> clap::error::Result<()> { + clap::FromArgMatches::update_from_arg_matches_mut(&mut self.0.args, matches) + } + + #[track_caller] + fn args_to_jobs( + self: Arc, + dependencies_args: Vec, + params: &JobParams, + ) -> eyre::Result<(DynJob, Vec)> { + let JobAndDependencies { job, dependencies } = JobArgsAndDependencies { + args: Arc::unwrap_or_clone(self).0, + dependencies: K::Dependencies::from_dyn_args(dependencies_args), + } + .args_to_jobs(params)?; + Ok((job.into(), K::Dependencies::into_dyn_jobs(dependencies))) + } +} + +#[derive(Clone)] +pub struct DynJobArgs(Arc); + +impl DynJobArgs { + pub fn new(kind: K, args: K::Args) -> Self { + Self(Arc::new(DynJobArgsInner(JobKindAndArgs { kind, args }))) + } + pub fn kind_type_id(&self) -> TypeId { + DynJobArgsTrait::kind_type_id(&*self.0) + } + pub fn downcast_ref(&self) -> Option<(&K, &K::Args)> { + let DynJobArgsInner::(JobKindAndArgs { kind, args }) = + DynJobArgsTrait::as_any(&*self.0).downcast_ref()?; + Some((kind, args)) + } + pub fn downcast(self) -> Result, Self> { + if self.downcast_ref::().is_some() { + let this = Arc::downcast::>(self.0.as_arc_any()) + .ok() + .expect("already checked type"); + Ok(Arc::unwrap_or_clone(this).0) + } else { + Err(self) + } + } + pub fn kind(&self) -> DynJobKind { + DynJobArgsTrait::kind(&*self.0) + } + pub fn to_args_vec(&self) -> Vec> { + self.to_args_extend_vec(Vec::new()) + } + pub fn to_args_extend_vec(&self, args: Vec>) -> Vec> { + DynJobArgsTrait::to_args_extend_vec(&*self.0, args) + } + fn make_mut(&mut self) -> &mut dyn DynJobArgsTrait { + // can't just return the reference if the first get_mut returns Some since + // as of rustc 1.90.0 this causes a false-positive lifetime error. + if Arc::get_mut(&mut self.0).is_none() { + self.0 = DynJobArgsTrait::clone_into_arc(&*self.0); + } + Arc::get_mut(&mut self.0).expect("clone_into_arc returns a new arc with a ref-count of 1") + } + pub fn update_from_arg_matches( + &mut self, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result<()> { + DynJobArgsTrait::update_from_arg_matches(self.make_mut(), matches) + } + pub fn args_to_jobs( + self, + dependencies_args: Vec, + params: &JobParams, + ) -> eyre::Result<(DynJob, Vec)> { + DynJobArgsTrait::args_to_jobs(self.0, dependencies_args, params) + } +} + +impl Hash for DynJobArgs { + fn hash(&self, state: &mut H) { + self.kind_type_id().hash(state); + DynJobArgsTrait::hash_dyn(&*self.0, state); + } +} + +impl PartialEq for DynJobArgs { + fn eq(&self, other: &Self) -> bool { + DynJobArgsTrait::eq_dyn(&*self.0, &*other.0) + } +} + +impl Eq for DynJobArgs {} + +impl fmt::Debug for DynJobArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[derive(PartialEq, Eq, Hash)] +struct DynJobInner { + kind: Arc, + job: T::Job, + inputs: Interned<[JobItemName]>, + outputs: Interned<[JobItemName]>, + external_command_params: Option, +} + +impl> Clone for DynJobInner { + fn clone(&self) -> Self { + Self { + kind: self.kind.clone(), + job: self.job.clone(), + inputs: self.inputs, + outputs: self.outputs, + external_command_params: self.external_command_params, + } + } +} + +impl fmt::Debug for DynJobInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + kind, + job, + inputs, + outputs, + external_command_params, + } = self; + f.debug_struct("DynJob") + .field("kind", kind) + .field("job", job) + .field("inputs", inputs) + .field("outputs", outputs) + .field("external_command_params", external_command_params) + .finish() } } trait DynJobTrait: 'static + Send + Sync + fmt::Debug { fn as_any(&self) -> &dyn Any; + fn as_arc_any(self: Arc) -> Arc; fn eq_dyn(&self, other: &dyn DynJobTrait) -> bool; fn hash_dyn(&self, state: &mut dyn Hasher); fn kind_type_id(&self) -> TypeId; fn kind(&self) -> DynJobKind; - fn inputs_and_direct_dependencies<'a>(&'a self) -> &'a BTreeMap>; + fn inputs(&self) -> Interned<[JobItemName]>; fn outputs(&self) -> Interned<[JobItemName]>; - fn to_command_line(&self) -> Interned<[Interned]>; - fn debug_name(&self) -> String; - fn run(&self, inputs: &[JobItem], acquired_job: &mut AcquiredJob) - -> eyre::Result>; + fn external_command_params(&self) -> Option; + fn serialize_to_json_ascii(&self) -> serde_json::Result; + fn serialize_to_json_value(&self) -> serde_json::Result; + fn run( + &self, + inputs: &[JobItem], + params: &JobParams, + acquired_job: &mut AcquiredJob, + ) -> eyre::Result>; } -mod inner { - use super::*; - - #[derive(Debug, PartialEq, Eq, Hash)] - pub(crate) struct DynJob { - pub(crate) kind: Arc, - pub(crate) job: T::Job, - pub(crate) inputs_and_direct_dependencies: BTreeMap>, - pub(crate) outputs: Interned<[JobItemName]>, - } -} - -impl DynJobTrait for inner::DynJob { +impl DynJobTrait for DynJobInner { fn as_any(&self) -> &dyn Any { self } + fn as_arc_any(self: Arc) -> Arc { + self + } + fn eq_dyn(&self, other: &dyn DynJobTrait) -> bool { other .as_any() - .downcast_ref::>() + .downcast_ref::() .is_some_and(|other| self == other) } @@ -591,28 +1206,33 @@ impl DynJobTrait for inner::DynJob { DynJobKind(self.kind.clone()) } - fn inputs_and_direct_dependencies<'a>(&'a self) -> &'a BTreeMap> { - &self.inputs_and_direct_dependencies + fn inputs(&self) -> Interned<[JobItemName]> { + self.inputs } fn outputs(&self) -> Interned<[JobItemName]> { self.outputs } - fn to_command_line(&self) -> Interned<[Interned]> { - self.kind.to_command_line(&self.job) + fn external_command_params(&self) -> Option { + self.external_command_params } - fn debug_name(&self) -> String { - self.kind.debug_name(&self.job) + fn serialize_to_json_ascii(&self) -> serde_json::Result { + crate::util::serialize_to_json_ascii(&self.job) + } + + fn serialize_to_json_value(&self) -> serde_json::Result { + serde_json::to_value(&self.job) } fn run( &self, inputs: &[JobItem], + params: &JobParams, acquired_job: &mut AcquiredJob, ) -> eyre::Result> { - self.kind.run(&self.job, inputs, acquired_job) + self.kind.run(&self.job, inputs, params, acquired_job) } } @@ -620,65 +1240,111 @@ impl DynJobTrait for inner::DynJob { pub struct DynJob(Arc); impl DynJob { - fn new_unchecked(job_kind: Arc, job: T::Job) -> Self { - let inputs_and_direct_dependencies = - job_kind.inputs_and_direct_dependencies(&job).into_owned(); + pub fn from_arc(job_kind: Arc, job: T::Job) -> Self { + let inputs = job_kind.inputs(&job); let outputs = job_kind.outputs(&job); - Self(Arc::new(inner::DynJob { + let external_command_params = job_kind.external_command_params(&job); + Self(Arc::new(DynJobInner { kind: job_kind, job, - inputs_and_direct_dependencies, + inputs, outputs, + external_command_params, })) } - pub fn from_arc(job_kind: Arc, job: T::Job) -> Self { - if TypeId::of::() == TypeId::of::() { - ::downcast_ref::(&job) - .expect("already checked type") - .clone() - } else { - Self::new_unchecked(job_kind, job) - } - } pub fn new(job_kind: T, job: T::Job) -> Self { - if TypeId::of::() == TypeId::of::() { - ::downcast_ref::(&job) - .expect("already checked type") - .clone() - } else { - Self::new_unchecked(Arc::new(job_kind), job) - } + Self::from_arc(Arc::new(job_kind), job) } pub fn kind_type_id(&self) -> TypeId { self.0.kind_type_id() } - pub fn downcast(&self) -> Option<(&T, &T::Job)> { - let inner::DynJob { kind, job, .. } = self.0.as_any().downcast_ref()?; + pub fn downcast_ref(&self) -> Option<(&K, &K::Job)> { + let DynJobInner { kind, job, .. } = self.0.as_any().downcast_ref()?; Some((kind, job)) } + pub fn downcast>(self) -> Result, Self> { + if self.kind_type_id() == TypeId::of::() { + let DynJobInner { kind, job, .. } = Arc::unwrap_or_clone( + self.0 + .as_arc_any() + .downcast::>() + .expect("already checked type"), + ); + Ok(JobAndKind { kind: *kind, job }) + } else { + Err(self) + } + } pub fn kind(&self) -> DynJobKind { DynJobTrait::kind(&*self.0) } - pub fn inputs_and_direct_dependencies<'a>( - &'a self, - ) -> &'a BTreeMap> { - DynJobTrait::inputs_and_direct_dependencies(&*self.0) + pub fn inputs(&self) -> Interned<[JobItemName]> { + DynJobTrait::inputs(&*self.0) } pub fn outputs(&self) -> Interned<[JobItemName]> { DynJobTrait::outputs(&*self.0) } - pub fn to_command_line(&self) -> Interned<[Interned]> { - DynJobTrait::to_command_line(&*self.0) + pub fn serialize_to_json_ascii(&self) -> serde_json::Result { + DynJobTrait::serialize_to_json_ascii(&*self.0) } - pub fn debug_name(&self) -> String { - DynJobTrait::debug_name(&*self.0) + pub fn serialize_to_json_value(&self) -> serde_json::Result { + DynJobTrait::serialize_to_json_value(&*self.0) + } + pub fn external_command_params(&self) -> Option { + DynJobTrait::external_command_params(&*self.0) + } + #[track_caller] + pub fn internal_command_params_with_program_prefix( + &self, + internal_program_prefix: &[Interned], + extra_args: &[Interned], + ) -> CommandParams { + let mut command_line = internal_program_prefix.to_vec(); + let command_line = match RunSingleJob::try_add_subcommand(self, &mut command_line) { + Ok(()) => { + command_line.extend_from_slice(extra_args); + Intern::intern_owned(command_line) + } + Err(e) => panic!("Serializing job {:?} failed: {e}", self.kind().name()), + }; + CommandParams { + command_line, + current_dir: None, + } + } + #[track_caller] + pub fn internal_command_params(&self, extra_args: &[Interned]) -> CommandParams { + self.internal_command_params_with_program_prefix( + &[program_name_for_internal_jobs()], + extra_args, + ) + } + #[track_caller] + pub fn command_params_with_internal_program_prefix( + &self, + internal_program_prefix: &[Interned], + extra_args: &[Interned], + ) -> CommandParams { + match self.external_command_params() { + Some(v) => v, + None => self + .internal_command_params_with_program_prefix(internal_program_prefix, extra_args), + } + } + #[track_caller] + pub fn command_params(&self, extra_args: &[Interned]) -> CommandParams { + self.command_params_with_internal_program_prefix( + &[program_name_for_internal_jobs()], + extra_args, + ) } pub fn run( &self, inputs: &[JobItem], + params: &JobParams, acquired_job: &mut AcquiredJob, ) -> eyre::Result> { - DynJobTrait::run(&*self.0, inputs, acquired_job) + DynJobTrait::run(&*self.0, inputs, params, acquired_job) } } @@ -700,7 +1366,7 @@ impl Hash for DynJob { #[serde(rename = "DynJob")] struct DynJobSerde { kind: DynJobKind, - command_line: Interned<[Interned]>, + job: serde_json::Value, } impl Serialize for DynJob { @@ -710,7 +1376,7 @@ impl Serialize for DynJob { { DynJobSerde { kind: self.kind(), - command_line: self.to_command_line(), + job: self.serialize_to_json_value().map_err(S::Error::custom)?, } .serialize(serializer) } @@ -721,1082 +1387,149 @@ impl<'de> Deserialize<'de> for DynJob { where D: Deserializer<'de>, { - let DynJobSerde { kind, command_line } = Deserialize::deserialize(deserializer)?; - kind.parse_command_line(command_line) + let DynJobSerde { kind, job } = Deserialize::deserialize(deserializer)?; + kind.deserialize_job_from_json_value(&job) .map_err(D::Error::custom) } } -impl JobKind for DynJobKind { - type Job = DynJob; - - fn inputs_and_direct_dependencies<'a>( - &'a self, - job: &'a Self::Job, - ) -> Cow<'a, BTreeMap>> { - Cow::Borrowed(job.inputs_and_direct_dependencies()) - } - - fn outputs(&self, job: &Self::Job) -> Interned<[JobItemName]> { - job.outputs() - } - - fn command_line_prefix(&self) -> Interned<[Interned]> { - self.0.command_line_prefix_dyn() - } - - fn to_command_line(&self, job: &Self::Job) -> Interned<[Interned]> { - job.to_command_line() - } - - fn subcommand(&self) -> Option { - self.0.subcommand_dyn() - } - - fn from_arg_matches(&self, matches: &mut clap::ArgMatches) -> clap::error::Result { - self.0.clone().from_arg_matches_dyn(matches) - } - - fn parse_command_line( - &self, - command_line: Interned<[Interned]>, - ) -> clap::error::Result { - self.0.clone().parse_command_line_dyn(command_line) - } - - fn run( - &self, - job: &Self::Job, - inputs: &[JobItem], - acquired_job: &mut AcquiredJob, - ) -> eyre::Result> { - job.run(inputs, acquired_job) - } -} - -#[derive(Clone, Debug)] -enum JobGraphNode { - Job(DynJob), - Item { - #[allow(dead_code, reason = "name used for debugging")] - name: JobItemName, - source_job: Option, - }, -} - -type JobGraphInner = DiGraph; - -#[derive(Clone, Default)] -pub struct JobGraph { - jobs: HashMap::NodeId>, - items: HashMap::NodeId>, - graph: JobGraphInner, - topological_order: Vec<::NodeId>, - space: DfsSpace<::NodeId, ::Map>, -} - -impl fmt::Debug for JobGraph { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Self { - jobs: _, - items: _, - graph, - topological_order, - space: _, - } = self; - f.debug_struct("JobGraph") - .field("graph", graph) - .field("topological_order", topological_order) - .finish_non_exhaustive() - } -} - -#[derive(Clone, Debug)] -pub enum JobGraphError { - CycleError { - job: DynJob, - output: JobItemName, - }, - MultipleJobsCreateSameOutput { - output_item: JobItemName, - existing_job: DynJob, - new_job: DynJob, - }, -} - -impl std::error::Error for JobGraphError {} - -impl fmt::Display for JobGraphError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::CycleError { job, output } => write!( - f, - "job can't be added to job graph because it would introduce a cyclic dependency through this job output:\n\ - {output:?}\n\ - job:\n{job:?}", - ), - JobGraphError::MultipleJobsCreateSameOutput { - output_item, - existing_job, - new_job, - } => write!( - f, - "job can't be added to job graph because the new job has an output that is also produced by an existing job.\n\ - conflicting output:\n\ - {output_item:?}\n\ - existing job:\n\ - {existing_job:?}\n\ - new job:\n\ - {new_job:?}", - ), +pub trait RunBuild: Sized { + fn main(make_params: F) + where + Self: clap::Parser + Clone, + F: FnOnce(Self, Extra) -> eyre::Result, + { + match Self::try_main(make_params) { + Ok(()) => {} + Err(e) => { + eprintln!("{e:#}"); + std::process::exit(1); + } } } + fn try_main(make_params: F) -> eyre::Result<()> + where + Self: clap::Parser + Clone, + F: FnOnce(Self, Extra) -> eyre::Result, + { + let args = Self::parse(); + args.clone() + .run(|extra| make_params(args, extra), Self::command()) + } + fn run(self, make_params: F, cmd: clap::Command) -> eyre::Result<()> + where + F: FnOnce(Extra) -> eyre::Result; } -#[derive(Copy, Clone, Debug)] -enum EscapeForUnixShellState { - DollarSingleQuote, - SingleQuote, - Unquoted, -} - -#[derive(Clone)] -pub struct EscapeForUnixShell<'a> { - state: EscapeForUnixShellState, - prefix: [u8; 3], - bytes: &'a [u8], -} - -impl<'a> fmt::Debug for EscapeForUnixShell<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self, f) +impl RunBuild for JobArgsAndDependencies { + fn run(self, make_params: F, cmd: clap::Command) -> eyre::Result<()> + where + F: FnOnce(NoArgs) -> eyre::Result, + { + let params = make_params(NoArgs)?; + self.args_to_jobs(¶ms)?.run(|_| Ok(params), cmd) } } -impl<'a> fmt::Display for EscapeForUnixShell<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - for c in self.clone() { - f.write_char(c)?; - } +impl RunBuild for JobAndDependencies { + fn run(self, make_params: F, cmd: clap::Command) -> eyre::Result<()> + where + F: FnOnce(NoArgs) -> eyre::Result, + { + let _ = cmd; + let params = make_params(NoArgs)?; + let Self { job, dependencies } = self; + let mut jobs = vec![DynJob::from(job)]; + K::Dependencies::into_dyn_jobs_extend(dependencies, &mut jobs); + let mut job_graph = JobGraph::new(); + job_graph.add_jobs(jobs); // add all at once to avoid recomputing graph properties multiple times + job_graph.run(¶ms) + } +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct RunSingleJob { + pub job: DynJob, + pub extra: Extra, +} + +impl RunSingleJob { + pub const SUBCOMMAND_NAME: &'static str = "run-single-job"; + fn try_add_subcommand( + job: &DynJob, + subcommand_line: &mut Vec>, + ) -> serde_json::Result<()> { + let mut json = job.serialize_to_json_ascii()?; + json.insert_str(0, "--json="); + subcommand_line.extend([ + Self::SUBCOMMAND_NAME.intern(), + Intern::intern_owned(format!("--name={}", job.kind().name())), + Intern::intern_owned(json), + ]); Ok(()) } } -impl<'a> EscapeForUnixShell<'a> { - pub fn new(s: &'a str) -> Self { - Self::from_bytes(s.as_bytes()) - } - fn make_prefix(bytes: &[u8]) -> [u8; 3] { - let mut prefix = [0; 3]; - prefix[..bytes.len()].copy_from_slice(bytes); - prefix - } - pub fn from_bytes(bytes: &'a [u8]) -> Self { - let mut needs_single_quote = bytes.is_empty(); - for &b in bytes { - match b { - b'!' | b'\'' | b'\"' | b' ' => needs_single_quote = true, - 0..0x20 | 0x7F.. => { - return Self { - state: EscapeForUnixShellState::DollarSingleQuote, - prefix: Self::make_prefix(b"$'"), - bytes, - }; - } - _ => {} - } - } - if needs_single_quote { - Self { - state: EscapeForUnixShellState::SingleQuote, - prefix: Self::make_prefix(b"'"), - bytes, - } - } else { - Self { - state: EscapeForUnixShellState::Unquoted, - prefix: Self::make_prefix(b""), - bytes, - } - } +impl TryFrom> for RunSingleJob { + type Error = clap::Error; + + fn try_from(value: RunSingleJobClap) -> Result { + let RunSingleJobClap::RunSingleJob { + name: job_kind, + json, + extra, + } = value; + let name = job_kind.name(); + job_kind + .deserialize_job_from_json_str(&json) + .map_err(|e| { + clap::Error::raw( + clap::error::ErrorKind::ValueValidation, + format_args!("failed to parse job {name} from JSON: {e}"), + ) + }) + .map(|job| Self { job, extra }) } } -impl Iterator for EscapeForUnixShell<'_> { - type Item = char; - - fn next(&mut self) -> Option { - match &mut self.prefix { - [0, 0, 0] => {} - [0, 0, v] | // find first - [0, v, _] | // non-zero byte - [v, _, _] => { - let retval = *v as char; - *v = 0; - return Some(retval); - } - } - let Some(&next_byte) = self.bytes.split_off_first() else { - return match self.state { - EscapeForUnixShellState::DollarSingleQuote - | EscapeForUnixShellState::SingleQuote => { - self.state = EscapeForUnixShellState::Unquoted; - Some('\'') - } - EscapeForUnixShellState::Unquoted => None, - }; - }; - match self.state { - EscapeForUnixShellState::DollarSingleQuote => match next_byte { - b'\'' | b'\\' => { - self.prefix = Self::make_prefix(&[next_byte]); - Some('\\') - } - b'\t' => { - self.prefix = Self::make_prefix(b"t"); - Some('\\') - } - b'\n' => { - self.prefix = Self::make_prefix(b"n"); - Some('\\') - } - b'\r' => { - self.prefix = Self::make_prefix(b"r"); - Some('\\') - } - 0x20..=0x7E => Some(next_byte as char), - _ => { - self.prefix = [ - b'x', - char::from_digit(next_byte as u32 >> 4, 0x10).expect("known to be in range") - as u8, - char::from_digit(next_byte as u32 & 0xF, 0x10) - .expect("known to be in range") as u8, - ]; - Some('\\') - } - }, - EscapeForUnixShellState::SingleQuote => { - if next_byte == b'\'' { - self.prefix = Self::make_prefix(b"\\''"); - Some('\'') - } else { - Some(next_byte as char) - } - } - EscapeForUnixShellState::Unquoted => match next_byte { - b' ' | b'!' | b'"' | b'#' | b'$' | b'&' | b'\'' | b'(' | b')' | b'*' | b',' - | b';' | b'<' | b'>' | b'?' | b'[' | b'\\' | b']' | b'^' | b'`' | b'{' | b'|' - | b'}' | b'~' => { - self.prefix = Self::make_prefix(&[next_byte]); - Some('\\') - } - _ => Some(next_byte as char), - }, - } - } +#[derive(clap::Subcommand)] +enum RunSingleJobClap { + #[command(name = RunSingleJob::SUBCOMMAND_NAME, hide = true)] + RunSingleJob { + #[arg(long)] + name: DynJobKind, + #[arg(long)] + json: String, + #[command(flatten)] + extra: Extra, + }, } -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] -#[non_exhaustive] -pub enum UnixMakefileEscapeKind { - NonRecipe, - RecipeWithoutShellEscaping, - RecipeWithShellEscaping, -} - -#[derive(Copy, Clone)] -pub struct EscapeForUnixMakefile<'a> { - s: &'a str, - kind: UnixMakefileEscapeKind, -} - -impl<'a> fmt::Debug for EscapeForUnixMakefile<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self, f) - } -} - -impl<'a> fmt::Display for EscapeForUnixMakefile<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.do_write(f, fmt::Write::write_str, fmt::Write::write_char, |_, _| { - Ok(()) - }) - } -} - -impl<'a> EscapeForUnixMakefile<'a> { - fn do_write( - &self, - state: &mut S, - write_str: impl Fn(&mut S, &str) -> Result<(), E>, - write_char: impl Fn(&mut S, char) -> Result<(), E>, - add_variable: impl Fn(&mut S, &'static str) -> Result<(), E>, - ) -> Result<(), E> { - let escape_recipe_char = |c| match c { - '$' => write_str(state, "$$"), - '\0'..='\x1F' | '\x7F' => { - panic!("can't escape a control character for Unix Makefile: {c:?}"); - } - _ => write_char(state, c), - }; - match self.kind { - UnixMakefileEscapeKind::NonRecipe => self.s.chars().try_for_each(|c| match c { - '=' => { - add_variable(state, "EQUALS = =")?; - write_str(state, "$(EQUALS)") - } - ';' => panic!("can't escape a semicolon (;) for Unix Makefile"), - '$' => write_str(state, "$$"), - '\\' | ' ' | '#' | ':' | '%' | '*' | '?' | '[' | ']' | '~' => { - write_char(state, '\\')?; - write_char(state, c) - } - '\0'..='\x1F' | '\x7F' => { - panic!("can't escape a control character for Unix Makefile: {c:?}"); - } - _ => write_char(state, c), - }), - UnixMakefileEscapeKind::RecipeWithoutShellEscaping => { - self.s.chars().try_for_each(escape_recipe_char) - } - UnixMakefileEscapeKind::RecipeWithShellEscaping => { - EscapeForUnixShell::new(self.s).try_for_each(escape_recipe_char) - } - } - } - pub fn new( - s: &'a str, - kind: UnixMakefileEscapeKind, - needed_variables: &mut BTreeSet<&'static str>, - ) -> Self { - let retval = Self { s, kind }; - let Ok(()) = retval.do_write( - needed_variables, - |_, _| Ok(()), - |_, _| Ok(()), - |needed_variables, variable| -> Result<(), std::convert::Infallible> { - needed_variables.insert(variable); - Ok(()) - }, - ); - retval - } -} - -impl JobGraph { - pub fn new() -> Self { - Self::default() - } - fn try_add_item_node( - &mut self, - name: JobItemName, - new_source_job: Option, - new_nodes: &mut HashSet<::NodeId>, - ) -> Result<::NodeId, JobGraphError> { - match self.items.entry(name) { - Entry::Occupied(item_entry) => { - let node_id = *item_entry.get(); - let JobGraphNode::Item { - name: _, - source_job, - } = &mut self.graph[node_id] - else { - unreachable!("known to be an item"); - }; - if let Some(new_source_job) = new_source_job { - if let Some(source_job) = source_job { - return Err(JobGraphError::MultipleJobsCreateSameOutput { - output_item: item_entry.key().clone(), - existing_job: source_job.clone(), - new_job: new_source_job, - }); - } else { - *source_job = Some(new_source_job); - } - } - Ok(node_id) - } - Entry::Vacant(item_entry) => { - let node_id = self.graph.add_node(JobGraphNode::Item { - name, - source_job: new_source_job, - }); - new_nodes.insert(node_id); - item_entry.insert(node_id); - Ok(node_id) - } - } - } - pub fn try_add_jobs>( - &mut self, - jobs: I, - ) -> Result<(), JobGraphError> { - let jobs = jobs.into_iter(); - struct RemoveNewNodesOnError<'a> { - this: &'a mut JobGraph, - new_nodes: HashSet<::NodeId>, - } - impl Drop for RemoveNewNodesOnError<'_> { - fn drop(&mut self) { - for node in self.new_nodes.drain() { - self.this.graph.remove_node(node); - } - } - } - let mut remove_new_nodes_on_error = RemoveNewNodesOnError { - this: self, - new_nodes: HashSet::with_capacity_and_hasher(jobs.size_hint().0, Default::default()), - }; - let new_nodes = &mut remove_new_nodes_on_error.new_nodes; - let this = &mut *remove_new_nodes_on_error.this; - let mut worklist = Vec::from_iter(jobs); - while let Some(job) = worklist.pop() { - let Entry::Vacant(job_entry) = this.jobs.entry(job.clone()) else { - continue; - }; - let job_node_id = this - .graph - .add_node(JobGraphNode::Job(job_entry.key().clone())); - new_nodes.insert(job_node_id); - job_entry.insert(job_node_id); - for name in job.outputs() { - let item_node_id = this.try_add_item_node(name, Some(job.clone()), new_nodes)?; - this.graph.add_edge(job_node_id, item_node_id, ()); - } - for (&name, direct_dependency) in job.inputs_and_direct_dependencies() { - worklist.extend(direct_dependency.clone()); - let item_node_id = this.try_add_item_node(name, None, new_nodes)?; - this.graph.add_edge(item_node_id, job_node_id, ()); - } - } - match toposort(&this.graph, Some(&mut this.space)) { - Ok(v) => { - this.topological_order = v; - // no need to remove any of the new nodes on drop since we didn't encounter any errors - remove_new_nodes_on_error.new_nodes.clear(); - Ok(()) - } - Err(_) => { - // there's at least one cycle, find one! - let cycle = kosaraju_scc(&this.graph) - .into_iter() - .find_map(|scc| { - if scc.len() <= 1 { - // can't be a cycle since our graph is bipartite -- - // jobs only connect to items, never jobs to jobs or items to items - None - } else { - Some(scc) - } - }) - .expect("we know there's a cycle"); - let cycle_set = HashSet::from_iter(cycle.iter().copied()); - let job = cycle - .into_iter() - .find_map(|node_id| { - if let JobGraphNode::Job(job) = &this.graph[node_id] { - Some(job.clone()) - } else { - None - } - }) - .expect("a job must be part of the cycle"); - let output = job - .outputs() - .into_iter() - .find(|output| cycle_set.contains(&this.items[output])) - .expect("an output must be part of the cycle"); - Err(JobGraphError::CycleError { job, output }) - } - } - } - #[track_caller] - pub fn add_jobs>(&mut self, jobs: I) { - match self.try_add_jobs(jobs) { - Ok(()) => {} - Err(e) => panic!("error: {e}"), - } - } - pub fn to_unix_makefile(&self) -> String { - let mut retval = String::new(); - let mut needed_variables = BTreeSet::new(); - for &node_id in &self.topological_order { - let JobGraphNode::Job(job) = &self.graph[node_id] else { - continue; - }; - for (index, output) in job.outputs().into_iter().enumerate() { - match output { - JobItemName::Module { .. } => continue, - JobItemName::File { path } => { - if index != 0 { - retval.push_str(" "); - } - write_str!( - retval, - "{}", - EscapeForUnixMakefile::new( - &path, - UnixMakefileEscapeKind::NonRecipe, - &mut needed_variables - ) - ); - } - } - } - retval.push_str(":"); - for input in job.inputs_and_direct_dependencies().keys() { - match input { - JobItemName::Module { .. } => continue, - JobItemName::File { path } => { - write_str!( - retval, - " {}", - EscapeForUnixMakefile::new( - &path, - UnixMakefileEscapeKind::NonRecipe, - &mut needed_variables - ) - ); - } - } - } - retval.push_str("\n\t"); - for (index, arg) in job.to_command_line().into_iter().enumerate() { - if index != 0 { - retval.push_str(" "); - } - write_str!( - retval, - "{}", - EscapeForUnixMakefile::new( - &arg, - UnixMakefileEscapeKind::RecipeWithShellEscaping, - &mut needed_variables - ) - ); - } - retval.push_str("\n\n"); - } - if !needed_variables.is_empty() { - retval.insert_str( - 0, - &String::from_iter(needed_variables.into_iter().map(|v| format!("{v}\n"))), - ); - } - retval - } - pub fn to_unix_shell_script(&self) -> String { - let mut retval = String::from( - "#!/bin/sh\n\ - set -ex\n", - ); - for &node_id in &self.topological_order { - let JobGraphNode::Job(job) = &self.graph[node_id] else { - continue; - }; - for (index, arg) in job.to_command_line().into_iter().enumerate() { - if index != 0 { - retval.push_str(" "); - } - write_str!(retval, "{}", EscapeForUnixShell::new(&arg)); - } - retval.push_str("\n"); - } - retval - } - pub fn run(&self) -> eyre::Result<()> { - // use scope to auto-join threads on errors - thread::scope(|scope| { - struct WaitingJobState { - job_node_id: ::NodeId, - job: DynJob, - inputs: BTreeMap>, - } - let mut ready_jobs = VecDeque::new(); - let mut item_name_to_waiting_jobs_map = HashMap::<_, Vec<_>>::default(); - for &node_id in &self.topological_order { - let JobGraphNode::Job(job) = &self.graph[node_id] else { - continue; - }; - let waiting_job = WaitingJobState { - job_node_id: node_id, - job: job.clone(), - inputs: job - .inputs_and_direct_dependencies() - .keys() - .map(|&name| (name, OnceCell::new())) - .collect(), - }; - if waiting_job.inputs.is_empty() { - ready_jobs.push_back(waiting_job); - } else { - let waiting_job = Rc::new(waiting_job); - for &input_item in waiting_job.inputs.keys() { - item_name_to_waiting_jobs_map - .entry(input_item) - .or_default() - .push(waiting_job.clone()); - } - } - } - struct RunningJob<'scope> { - job: DynJob, - thread: ScopedJoinHandle<'scope, eyre::Result>>, - } - let mut running_jobs = HashMap::default(); - let (finished_jobs_sender, finished_jobs_receiver) = mpsc::channel(); - loop { - while let Some(finished_job) = finished_jobs_receiver.try_recv().ok() { - let Some(RunningJob { job, thread }) = running_jobs.remove(&finished_job) - else { - unreachable!(); - }; - let output_items = thread.join().map_err(panic::resume_unwind)??; - assert!( - output_items.iter().map(JobItem::name).eq(job.outputs()), - "job's run() method returned the wrong output items:\n\ - output items:\n\ - {output_items:?}\n\ - expected outputs:\n\ - {:?}\n\ - job:\n\ - {job:?}", - job.outputs(), - ); - for output_item in output_items { - for waiting_job in item_name_to_waiting_jobs_map - .remove(&output_item.name()) - .unwrap_or_default() - { - let Ok(()) = - waiting_job.inputs[&output_item.name()].set(output_item.clone()) - else { - unreachable!(); - }; - if let Some(waiting_job) = Rc::into_inner(waiting_job) { - ready_jobs.push_back(waiting_job); - } - } - } - } - if let Some(WaitingJobState { - job_node_id, - job, - inputs, - }) = ready_jobs.pop_front() - { - struct RunningJobInThread { - job_node_id: ::NodeId, - job: DynJob, - inputs: Vec, - acquired_job: AcquiredJob, - finished_jobs_sender: mpsc::Sender<::NodeId>, - } - impl RunningJobInThread { - fn run(mut self) -> eyre::Result> { - self.job.run(&self.inputs, &mut self.acquired_job) - } - } - impl Drop for RunningJobInThread { - fn drop(&mut self) { - let _ = self.finished_jobs_sender.send(self.job_node_id); - } - } - let name = job.debug_name(); - let running_job_in_thread = RunningJobInThread { - job_node_id, - job: job.clone(), - inputs: Vec::from_iter( - inputs - .into_values() - .map(|input| input.into_inner().expect("was set earlier")), - ), - acquired_job: AcquiredJob::acquire(), - finished_jobs_sender: finished_jobs_sender.clone(), - }; - running_jobs.insert( - job_node_id, - RunningJob { - job, - thread: thread::Builder::new() - .name(name) - .spawn_scoped(scope, move || running_job_in_thread.run()) - .expect("failed to spawn thread for job"), - }, - ); - } - if running_jobs.is_empty() { - assert!(item_name_to_waiting_jobs_map.is_empty()); - assert!(ready_jobs.is_empty()); - return Ok(()); - } - } - }) - } -} - -impl Extend for JobGraph { - #[track_caller] - fn extend>(&mut self, iter: T) { - self.add_jobs(iter); - } -} - -impl FromIterator for JobGraph { - #[track_caller] - fn from_iter>(iter: T) -> Self { - let mut retval = Self::new(); - retval.add_jobs(iter); - retval - } -} - -impl Serialize for JobGraph { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut serializer = serializer.serialize_seq(Some(self.jobs.len()))?; - for &node_id in &self.topological_order { - let JobGraphNode::Job(job) = &self.graph[node_id] else { - continue; - }; - serializer.serialize_element(job)?; - } - serializer.end() - } -} - -impl<'de> Deserialize<'de> for JobGraph { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let jobs = Vec::::deserialize(deserializer)?; - let mut retval = JobGraph::new(); - retval.try_add_jobs(jobs).map_err(D::Error::custom)?; - Ok(retval) - } -} - -pub fn program_name_for_internal_jobs() -> Interned { - static PROGRAM_NAME: OnceLock> = OnceLock::new(); - *PROGRAM_NAME - .get_or_init(|| str::intern_owned(std::env::args().next().expect("can't get program name"))) -} - -pub trait InternalJobTrait: clap::Args + 'static + fmt::Debug + Eq + Hash + Send + Sync { - fn subcommand_name() -> Interned; - fn to_args(&self) -> Vec>; - fn inputs_and_direct_dependencies<'a>( - &'a self, - ) -> Cow<'a, BTreeMap>>; - fn outputs(&self) -> Interned<[JobItemName]>; - fn run(&self, inputs: &[JobItem], acquired_job: &mut AcquiredJob) - -> eyre::Result>; -} - -#[derive(Hash, PartialEq, Eq)] -pub struct InternalJobKind(PhantomData Job>); - -impl Clone for InternalJobKind { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for InternalJobKind {} - -impl InternalJobKind { - pub const fn new() -> Self { - Self(PhantomData) - } -} - -impl Default for InternalJobKind { - fn default() -> Self { - Self::new() - } -} - -impl fmt::Debug for InternalJobKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "InternalJobKind<{}>", std::any::type_name::()) - } -} - -impl JobKind for InternalJobKind { - type Job = InternalJob; - - fn inputs_and_direct_dependencies<'a>( - &'a self, - job: &'a Self::Job, - ) -> Cow<'a, BTreeMap>> { - job.0.inputs_and_direct_dependencies() - } - - fn outputs(&self, job: &Self::Job) -> Interned<[JobItemName]> { - job.0.outputs() - } - - fn command_line_prefix(&self) -> Interned<[Interned]> { - [program_name_for_internal_jobs(), Job::subcommand_name()][..].intern() - } - - fn subcommand(&self) -> Option { - Some(Job::augment_args(clap::Command::new(Interned::into_inner( - Job::subcommand_name(), - )))) - } - - fn from_arg_matches(&self, matches: &mut clap::ArgMatches) -> clap::error::Result { - clap::FromArgMatches::from_arg_matches_mut(matches) - } - - fn to_command_line(&self, job: &Self::Job) -> Interned<[Interned]> { - Interned::from_iter( - self.command_line_prefix() - .iter() - .copied() - .chain(job.0.to_args()), - ) - } - - fn parse_command_line( - &self, - command_line: Interned<[Interned]>, - ) -> clap::error::Result { - let cmd = clap::Command::new(Interned::into_inner(program_name_for_internal_jobs())); - let mut matches = as clap::Subcommand>::augment_subcommands(cmd) - .subcommand_required(true) - .arg_required_else_help(true) - .try_get_matches_from(command_line.iter().map(|arg| &**arg))?; - as clap::FromArgMatches>::from_arg_matches_mut(&mut matches) - } - - fn run( - &self, - job: &Self::Job, - inputs: &[JobItem], - acquired_job: &mut AcquiredJob, - ) -> eyre::Result> { - job.0.run(inputs, acquired_job) - } -} - -#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Default, PartialOrd, Ord)] -pub struct InternalJob(pub Job); - -impl clap::FromArgMatches for InternalJob { - fn from_arg_matches(matches: &clap::ArgMatches) -> Result { - Self::from_arg_matches_mut(&mut matches.clone()) - } - - fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result { - if let Some((name, mut matches)) = matches.remove_subcommand() { - if *name == *Job::subcommand_name() { - Ok(Self(Job::from_arg_matches_mut(&mut matches)?)) - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::InvalidSubcommand, - format!("the subcommand '{name}' wasn't recognized"), - )) - } - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::MissingSubcommand, - "a subcommand is required but one was not provided", - )) - } - } - - fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> { - self.update_from_arg_matches_mut(&mut matches.clone()) - } - - fn update_from_arg_matches_mut( - &mut self, - matches: &mut clap::ArgMatches, - ) -> Result<(), clap::Error> { - if let Some((name, mut matches)) = matches.remove_subcommand() { - if *name == *Job::subcommand_name() { - self.0.update_from_arg_matches_mut(&mut matches)?; - Ok(()) - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::InvalidSubcommand, - format!("the subcommand '{name}' wasn't recognized"), - )) - } - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::MissingSubcommand, - "a subcommand is required but one was not provided", - )) - } - } -} - -impl clap::Subcommand for InternalJob { +impl clap::Subcommand for RunSingleJob { fn augment_subcommands(cmd: clap::Command) -> clap::Command { - cmd.subcommand( - InternalJobKind::::new() - .subcommand() - .expect("known to return Some"), - ) + RunSingleJobClap::::augment_subcommands(cmd) } fn augment_subcommands_for_update(cmd: clap::Command) -> clap::Command { - cmd.subcommand(Job::augment_args_for_update(clap::Command::new( - Interned::into_inner(Job::subcommand_name()), - ))) + RunSingleJobClap::::augment_subcommands(cmd) } fn has_subcommand(name: &str) -> bool { - *name == *Job::subcommand_name() + RunSingleJobClap::::has_subcommand(name) } } -#[derive(clap::Args)] -#[clap(id = "OutputDir")] -struct OutputDirArgs { - /// the directory to put the generated main output file and associated files in - #[arg(short, long, value_hint = clap::ValueHint::DirPath, required = true)] - output: Option, - #[arg(long, env = "FAYALITE_KEEP_TEMP_DIR")] - keep_temp_dir: bool, -} - -#[derive(Debug, Clone)] -pub struct OutputDir { - output: String, - temp_dir: Option>, - keep_temp_dir: bool, -} - -impl Eq for OutputDir {} - -impl AsRef for OutputDir { - fn as_ref(&self) -> &str { - self.path() - } -} - -impl AsRef for OutputDir { - fn as_ref(&self) -> &std::path::Path { - self.path().as_ref() - } -} - -impl OutputDir { - pub fn path(&self) -> &str { - &self.output - } - pub fn new(output: String) -> Self { - Self { - output, - temp_dir: None, - keep_temp_dir: false, - } - } - pub fn with_keep_temp_dir(output: String, keep_temp_dir: bool) -> Self { - Self { - output, - temp_dir: None, - keep_temp_dir, - } - } - pub fn temp(keep_temp_dir: bool) -> std::io::Result { - let temp_dir = TempDir::new()?; - let output = String::from(temp_dir.path().as_os_str().to_str().ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidFilename, - format!( - "temporary directory path is not valid UTF-8: {:?}", - temp_dir.path() - ), - ) - })?); - let temp_dir = if keep_temp_dir { - println!( - "created temporary directory: {}", - temp_dir.into_path().display() - ); - None - } else { - Some(Arc::new(temp_dir)) - }; - Ok(Self { - output, - temp_dir, - keep_temp_dir, - }) - } - pub fn to_args(&self) -> Vec> { - let Self { - output, - temp_dir: _, - keep_temp_dir, - } = self; - let mut retval = Vec::new(); - retval.push(str::intern_owned(format!("--output={output}"))); - if *keep_temp_dir { - retval.push("--keep-temp-dir".intern()); - } - retval - } - fn compare_key(&self) -> (&str, bool, bool) { - let Self { - output, - temp_dir, - keep_temp_dir, - } = self; - (output, temp_dir.is_some(), *keep_temp_dir) - } -} - -impl PartialEq for OutputDir { - fn eq(&self, other: &Self) -> bool { - self.compare_key() == other.compare_key() - } -} - -impl Hash for OutputDir { - fn hash(&self, state: &mut H) { - self.compare_key().hash(state); - } -} - -impl TryFrom for OutputDir { - type Error = clap::Error; - - fn try_from(value: OutputDirArgs) -> Result { - let OutputDirArgs { - output, - keep_temp_dir, - } = value; - match output { - Some(output) => Ok(Self::with_keep_temp_dir(output, keep_temp_dir)), - None => Ok(Self::temp(keep_temp_dir)?), - } - } -} - -impl clap::FromArgMatches for OutputDir { +impl clap::FromArgMatches for RunSingleJob { fn from_arg_matches(matches: &clap::ArgMatches) -> clap::error::Result { - OutputDirArgs::from_arg_matches(matches)?.try_into() + RunSingleJobClap::from_arg_matches(matches)?.try_into() } - fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> clap::error::Result { - OutputDirArgs::from_arg_matches_mut(matches)?.try_into() + RunSingleJobClap::from_arg_matches_mut(matches)?.try_into() } - fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> clap::error::Result<()> { *self = Self::from_arg_matches(matches)?; Ok(()) } - fn update_from_arg_matches_mut( &mut self, matches: &mut clap::ArgMatches, @@ -1806,53 +1539,730 @@ impl clap::FromArgMatches for OutputDir { } } -impl clap::Args for OutputDir { - fn group_id() -> Option { - OutputDirArgs::group_id() - } - - fn augment_args(cmd: clap::Command) -> clap::Command { - OutputDirArgs::augment_args(cmd) - } - - fn augment_args_for_update(cmd: clap::Command) -> clap::Command { - OutputDirArgs::augment_args_for_update(cmd) +impl RunBuild for RunSingleJob { + fn run(self, make_params: F, cmd: clap::Command) -> eyre::Result<()> + where + F: FnOnce(Extra) -> eyre::Result, + { + let _ = cmd; + let params = make_params(self.extra)?; + let mut job_graph = JobGraph::new(); + job_graph.add_jobs([self.job]); + job_graph.run(¶ms) } } -#[derive(clap::Parser, Debug, Clone, Hash, PartialEq, Eq)] -#[non_exhaustive] -pub struct BaseArgs { +#[derive(Clone, PartialEq, Eq, Hash, clap::Subcommand)] +pub enum Completions { + #[non_exhaustive] + Completions { + #[arg(default_value = Self::shell_str_from_env(), required = Self::shell_from_env().is_none())] + shell: clap_complete::aot::Shell, + }, +} + +impl Completions { + pub fn new(shell: clap_complete::aot::Shell) -> Self { + Self::Completions { shell } + } + pub fn from_env() -> Option { + Some(Self::Completions { + shell: Self::shell_from_env()?, + }) + } + fn shell_from_env() -> Option { + static SHELL: OnceLock> = OnceLock::new(); + *SHELL.get_or_init(clap_complete::aot::Shell::from_env) + } + fn shell_str_from_env() -> clap::builder::Resettable { + static SHELL_STR: OnceLock> = OnceLock::new(); + SHELL_STR + .get_or_init(|| Self::shell_from_env().map(|v| v.to_string())) + .as_deref() + .map(Into::into) + .into() + } +} + +impl RunBuild for Completions { + fn run(self, _make_params: F, mut cmd: clap::Command) -> eyre::Result<()> + where + F: FnOnce(NoArgs) -> eyre::Result, + { + let Self::Completions { shell } = self; + let bin_name = cmd + .get_bin_name() + .map(str::intern) + .unwrap_or_else(|| program_name_for_internal_jobs()); + clap_complete::aot::generate( + shell, + &mut cmd, + &*bin_name, + &mut std::io::BufWriter::new(std::io::stdout().lock()), + ); + Ok(()) + } +} + +#[derive( + clap::Args, + Copy, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Debug, + Default, + Serialize, + Deserialize, +)] +pub struct NoArgs; + +impl ToArgs for NoArgs { + fn to_args(&self, _args: &mut (impl WriteArgs + ?Sized)) { + let Self {} = self; + } +} + +#[derive(Clone, PartialEq, Eq, Hash, clap::Parser)] +pub enum BuildCli { + #[clap(flatten)] + Job(AnyJobSubcommand), + #[clap(flatten)] + RunSingleJob(RunSingleJob), + #[clap(flatten)] + Completions(Completions), + #[clap(flatten)] + CreateUnixShellScript(CreateUnixShellScript), +} + +impl RunBuild for BuildCli { + fn run(self, make_params: F, cmd: clap::Command) -> eyre::Result<()> + where + F: FnOnce(Extra) -> eyre::Result, + { + match self { + BuildCli::Job(v) => v.run(make_params, cmd), + BuildCli::RunSingleJob(v) => v.run(make_params, cmd), + BuildCli::Completions(v) => v.run(|NoArgs {}| unreachable!(), cmd), + BuildCli::CreateUnixShellScript(v) => v.run(make_params, cmd), + } + } +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug, clap::Subcommand)] +enum CreateUnixShellScriptInner { + CreateUnixShellScript { + #[command(subcommand)] + inner: AnyJobSubcommand, + }, +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct CreateUnixShellScript(CreateUnixShellScriptInner); + +impl RunBuild for CreateUnixShellScript { + fn run(self, make_params: F, cmd: clap::Command) -> eyre::Result<()> + where + F: FnOnce(Extra) -> eyre::Result, + { + let CreateUnixShellScriptInner::CreateUnixShellScript { + inner: + AnyJobSubcommand { + args, + dependencies_args, + extra, + }, + } = self.0; + let extra_args = extra.to_interned_args_vec(); + let params = make_params(extra)?; + let (job, dependencies) = args.args_to_jobs(dependencies_args, ¶ms)?; + let mut job_graph = JobGraph::new(); + job_graph.add_jobs([job].into_iter().chain(dependencies)); + println!( + "{}", + job_graph.to_unix_shell_script_with_internal_program_prefix( + &[cmd + .get_bin_name() + .map(str::intern) + .unwrap_or_else(|| program_name_for_internal_jobs())], + &extra_args, + ) + ); + Ok(()) + } +} + +impl clap::FromArgMatches for CreateUnixShellScript { + fn from_arg_matches(matches: &clap::ArgMatches) -> Result { + clap::FromArgMatches::from_arg_matches(matches).map(Self) + } + fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result { + clap::FromArgMatches::from_arg_matches_mut(matches).map(Self) + } + fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> { + self.0.update_from_arg_matches(matches) + } + fn update_from_arg_matches_mut( + &mut self, + matches: &mut clap::ArgMatches, + ) -> Result<(), clap::Error> { + self.0.update_from_arg_matches_mut(matches) + } +} + +impl clap::Subcommand for CreateUnixShellScript { + fn augment_subcommands(cmd: clap::Command) -> clap::Command { + CreateUnixShellScriptInner::::augment_subcommands(cmd) + } + + fn augment_subcommands_for_update(cmd: clap::Command) -> clap::Command { + CreateUnixShellScriptInner::::augment_subcommands_for_update(cmd) + } + + fn has_subcommand(name: &str) -> bool { + CreateUnixShellScriptInner::::has_subcommand(name) + } +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct AnyJobSubcommand { + pub args: DynJobArgs, + pub dependencies_args: Vec, + pub extra: Extra, +} + +impl AnyJobSubcommand { + pub fn from_subcommand_arg_matches( + job_kind: &DynJobKind, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result { + let dependencies = job_kind.dependencies_kinds(); + let dependencies_args = Result::from_iter( + dependencies + .into_iter() + .map(|dependency| dependency.from_arg_matches(matches)), + )?; + Ok(Self { + args: job_kind.clone().from_arg_matches(matches)?, + dependencies_args, + extra: Extra::from_arg_matches_mut(matches)?, + }) + } + pub fn update_from_subcommand_arg_matches( + &mut self, + job_kind: &DynJobKind, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result<()> { + let Self { + args, + dependencies_args, + extra, + } = self; + if *job_kind == args.kind() { + for dependency in dependencies_args { + dependency.update_from_arg_matches(matches)?; + } + args.update_from_arg_matches(matches)?; + } else { + let dependencies = job_kind.dependencies_kinds(); + let new_dependencies_args = Result::from_iter( + dependencies + .into_iter() + .map(|dependency| dependency.from_arg_matches(matches)), + )?; + *args = job_kind.clone().from_arg_matches(matches)?; + *dependencies_args = new_dependencies_args; + } + extra.update_from_arg_matches_mut(matches) + } +} + +impl clap::Subcommand for AnyJobSubcommand { + fn augment_subcommands(mut cmd: clap::Command) -> clap::Command { + let snapshot = registry::JobKindRegistrySnapshot::get(); + for job_kind in &snapshot { + cmd = cmd.subcommand(Extra::augment_args(job_kind.make_subcommand())); + } + cmd + } + + fn augment_subcommands_for_update(mut cmd: clap::Command) -> clap::Command { + let snapshot = registry::JobKindRegistrySnapshot::get(); + for job_kind in &snapshot { + cmd = cmd.subcommand(Extra::augment_args_for_update( + job_kind.make_subcommand_for_update(), + )); + } + cmd + } + + fn has_subcommand(name: &str) -> bool { + registry::JobKindRegistrySnapshot::get() + .get_by_name(name) + .is_some() + } +} + +impl clap::FromArgMatches for AnyJobSubcommand { + fn from_arg_matches(matches: &clap::ArgMatches) -> clap::error::Result { + Self::from_arg_matches_mut(&mut matches.clone()) + } + + fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> clap::error::Result { + if let Some((name, mut matches)) = matches.remove_subcommand() { + let job_kind_registry_snapshot = registry::JobKindRegistrySnapshot::get(); + if let Some(job_kind) = job_kind_registry_snapshot.get_by_name(&name) { + Self::from_subcommand_arg_matches(job_kind, &mut matches) + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::InvalidSubcommand, + format!("the subcommand '{name}' wasn't recognized"), + )) + } + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::MissingSubcommand, + "a subcommand is required but one was not provided", + )) + } + } + + fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> clap::error::Result<()> { + Self::update_from_arg_matches_mut(self, &mut matches.clone()) + } + + fn update_from_arg_matches_mut( + &mut self, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result<()> { + if let Some((name, mut matches)) = matches.remove_subcommand() { + let job_kind_registry_snapshot = registry::JobKindRegistrySnapshot::get(); + if let Some(job_kind) = job_kind_registry_snapshot.get_by_name(&name) { + self.update_from_subcommand_arg_matches(job_kind, &mut matches) + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::InvalidSubcommand, + format!("the subcommand '{name}' wasn't recognized"), + )) + } + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::MissingSubcommand, + "a subcommand is required but one was not provided", + )) + } + } +} + +impl RunBuild for AnyJobSubcommand { + fn run(self, make_params: F, cmd: clap::Command) -> eyre::Result<()> + where + F: FnOnce(Extra) -> eyre::Result, + { + let _ = cmd; + let Self { + args, + dependencies_args, + extra, + } = self; + let params = make_params(extra)?; + let (job, dependencies) = args.args_to_jobs(dependencies_args, ¶ms)?; + let mut job_graph = JobGraph::new(); + job_graph.add_jobs([job].into_iter().chain(dependencies)); // add all at once to avoid recomputing graph properties multiple times + job_graph.run(¶ms) + } +} + +pub fn program_name_for_internal_jobs() -> Interned { + static PROGRAM_NAME: OnceLock> = OnceLock::new(); + *PROGRAM_NAME + .get_or_init(|| str::intern_owned(std::env::args().next().expect("can't get program name"))) +} + +#[derive(clap::Args, PartialEq, Eq, Hash, Debug, Clone)] +#[group(id = "CreateOutputDir")] +pub struct CreateOutputDirArgs { /// the directory to put the generated main output file and associated files in + #[arg(short, long, value_hint = clap::ValueHint::DirPath)] + pub output: Option, + #[arg(long, env = "FAYALITE_KEEP_TEMP_DIR")] + pub keep_temp_dir: bool, +} + +impl ToArgs for CreateOutputDirArgs { + fn to_args(&self, args: &mut (impl WriteArgs + ?Sized)) { + let Self { + output, + keep_temp_dir, + } = self; + if let Some(output) = output { + args.write_arg(format_args!("--output={output}")); + } + if *keep_temp_dir { + args.write_str_arg("--keep-temp-dir"); + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CreateOutputDir { + output_dir: Interned, + #[serde(skip)] + temp_dir: Option>, +} + +impl Eq for CreateOutputDir {} + +impl PartialEq for CreateOutputDir { + fn eq(&self, other: &Self) -> bool { + self.compare_key() == other.compare_key() + } +} + +impl Hash for CreateOutputDir { + fn hash(&self, state: &mut H) { + self.compare_key().hash(state); + } +} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Debug)] +pub struct CreateOutputDirJobKind; + +impl JobKind for CreateOutputDirJobKind { + type Args = CreateOutputDirArgs; + type Job = CreateOutputDir; + type Dependencies = (); + + fn dependencies(self) -> Self::Dependencies { + () + } + + fn args_to_jobs( + args: JobArgsAndDependencies, + _params: &JobParams, + ) -> eyre::Result> { + let JobArgsAndDependencies { + args: + JobKindAndArgs { + kind, + args: + CreateOutputDirArgs { + output, + keep_temp_dir, + }, + }, + dependencies: (), + } = args; + let (output_dir, temp_dir) = if let Some(output) = output { + (Intern::intern_owned(output), None) + } else { + // we create the temp dir here rather than in run so other + // jobs can have their paths based on the chosen temp dir + let temp_dir = TempDir::new()?; + let output_dir = temp_dir + .path() + .as_os_str() + .to_str() + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidFilename, + format!( + "temporary directory path is not valid UTF-8: {:?}", + temp_dir.path() + ), + ) + })? + .intern(); + let temp_dir = if keep_temp_dir { + // use TempDir::into_path() to no longer automatically delete the temp dir + let temp_dir_path = temp_dir.into_path(); + println!("created temporary directory: {}", temp_dir_path.display()); + None + } else { + Some(Arc::new(temp_dir)) + }; + (output_dir, temp_dir) + }; + Ok(JobAndDependencies { + job: JobAndKind { + kind, + job: CreateOutputDir { + output_dir, + temp_dir, + }, + }, + dependencies: (), + }) + } + + fn inputs(self, _job: &Self::Job) -> Interned<[JobItemName]> { + Interned::default() + } + + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + [JobItemName::Path { + path: job.output_dir, + }][..] + .intern() + } + + fn name(self) -> Interned { + "create-output-dir".intern() + } + + fn external_command_params(self, job: &Self::Job) -> Option { + Some(CommandParams { + command_line: [ + "mkdir".intern(), + "-p".intern(), + "--".intern(), + job.output_dir, + ][..] + .intern(), + current_dir: None, + }) + } + + fn run( + self, + job: &Self::Job, + inputs: &[JobItem], + _params: &JobParams, + _acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + let [] = inputs else { + panic!("invalid inputs for CreateOutputDir"); + }; + std::fs::create_dir_all(&*job.output_dir)?; + Ok(vec![JobItem::Path { + path: job.output_dir, + }]) + } + + fn subcommand_hidden(self) -> bool { + true + } +} + +impl CreateOutputDir { + pub fn output_dir(&self) -> Interned { + self.output_dir + } + fn compare_key(&self) -> (&str, bool) { + let Self { + output_dir, + temp_dir, + } = self; + (output_dir, temp_dir.is_some()) + } +} + +#[derive(clap::Args, Debug, Clone, Hash, PartialEq, Eq)] +#[group(id = "BaseJob")] +#[non_exhaustive] +pub struct BaseJobArgs { + /// rather than having CreateOutputDir be a normal dependency, it's nested in BaseJob to avoid a cyclic dependency #[command(flatten)] - pub output: OutputDir, + pub create_output_dir_args: CreateOutputDirArgs, /// the stem of the generated main output file, e.g. to get foo.v, pass --file-stem=foo #[arg(long)] pub file_stem: Option, - pub module_name: String, + /// run commands even if their results are already cached + #[arg(long, env = Self::RUN_EVEN_IF_CACHED_ENV_NAME)] + pub run_even_if_cached: bool, } -impl BaseArgs { - pub fn to_args(&self) -> Vec> { - let Self { - output, - file_stem, - module_name, - } = self; - let mut retval = output.to_args(); - if let Some(file_stem) = file_stem { - retval.push(str::intern_owned(format!("--file-stem={file_stem}"))); +impl BaseJobArgs { + pub const RUN_EVEN_IF_CACHED_ENV_NAME: &'static str = "FAYALITE_RUN_EVEN_IF_CACHED"; + pub fn from_output_dir_and_env(output: String) -> Self { + Self { + create_output_dir_args: CreateOutputDirArgs { + output: Some(output), + keep_temp_dir: false, + }, + file_stem: None, + run_even_if_cached: std::env::var_os(Self::RUN_EVEN_IF_CACHED_ENV_NAME).is_some(), } - retval.push(str::intern(module_name)); - retval - } - pub fn file_with_ext(&self, ext: &str) -> String { - let mut retval = std::path::Path::new(self.output.path()) - .join(self.file_stem.as_ref().unwrap_or(&self.module_name)); - retval.set_extension(ext); - retval - .into_os_string() - .into_string() - .expect("known to be UTF-8") + } +} + +impl ToArgs for BaseJobArgs { + fn to_args(&self, args: &mut (impl WriteArgs + ?Sized)) { + let Self { + create_output_dir_args, + file_stem, + run_even_if_cached, + } = self; + create_output_dir_args.to_args(args); + if let Some(file_stem) = file_stem { + args.write_arg(format_args!("--file-stem={file_stem}")); + } + if *run_even_if_cached { + args.write_str_arg("--run-even-if-cached"); + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct BaseJob { + /// rather than having CreateOutputDir be a normal dependency, it's nested in BaseJob to avoid a cyclic dependency + #[serde(flatten)] + create_output_dir: CreateOutputDir, + file_stem: Interned, + run_even_if_cached: bool, +} + +impl BaseJob { + pub fn output_dir(&self) -> Interned { + self.create_output_dir.output_dir() + } + pub fn file_stem(&self) -> Interned { + self.file_stem + } + pub fn file_with_ext(&self, ext: &str) -> Interned { + let mut retval = std::path::Path::new(&self.output_dir()).join(self.file_stem()); + retval.set_extension(ext); + intern_known_utf8_path_buf(retval) + } + pub fn run_even_if_cached(&self) -> bool { + self.run_even_if_cached + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)] +pub struct BaseJobKind; + +impl JobKind for BaseJobKind { + type Args = BaseJobArgs; + type Job = BaseJob; + type Dependencies = (); + + fn dependencies(self) -> Self::Dependencies { + () + } + + fn args_to_jobs( + args: JobArgsAndDependencies, + params: &JobParams, + ) -> eyre::Result> { + let BaseJobArgs { + create_output_dir_args, + file_stem, + run_even_if_cached, + } = args.args.args; + let create_output_dir_args = JobKindAndArgs { + kind: CreateOutputDirJobKind, + args: create_output_dir_args, + }; + let create_output_dir = create_output_dir_args.args_to_jobs((), params)?.job.job; + let file_stem = file_stem + .map(Intern::intern_owned) + .unwrap_or(params.main_module().name()); + Ok(JobAndDependencies { + job: JobAndKind { + kind: BaseJobKind, + job: BaseJob { + create_output_dir, + file_stem, + run_even_if_cached, + }, + }, + dependencies: (), + }) + } + + fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + CreateOutputDirJobKind.inputs(&job.create_output_dir) + } + + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + CreateOutputDirJobKind.outputs(&job.create_output_dir) + } + + fn name(self) -> Interned { + "base-job".intern() + } + + fn external_command_params(self, job: &Self::Job) -> Option { + CreateOutputDirJobKind.external_command_params(&job.create_output_dir) + } + + fn run( + self, + job: &Self::Job, + inputs: &[JobItem], + params: &JobParams, + acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + CreateOutputDirJobKind.run(&job.create_output_dir, inputs, params, acquired_job) + } + + fn subcommand_hidden(self) -> bool { + true + } +} + +pub trait GetBaseJob { + fn base_job(&self) -> &BaseJob; +} + +impl GetBaseJob for &'_ T { + fn base_job(&self) -> &BaseJob { + T::base_job(self) + } +} + +impl GetBaseJob for &'_ mut T { + fn base_job(&self) -> &BaseJob { + T::base_job(self) + } +} + +impl GetBaseJob for Box { + fn base_job(&self) -> &BaseJob { + T::base_job(self) + } +} + +impl GetBaseJob for BaseJob { + fn base_job(&self) -> &BaseJob { + self + } +} + +impl GetBaseJob for JobAndKind { + fn base_job(&self) -> &BaseJob { + &self.job + } +} + +impl GetBaseJob for JobAndDependencies { + fn base_job(&self) -> &BaseJob { + &self.job.job + } +} + +impl GetBaseJob for JobAndDependencies +where + K::Dependencies: JobDependencies, + ::JobsAndKinds: GetBaseJob, +{ + fn base_job(&self) -> &BaseJob { + self.dependencies.base_job() + } +} + +impl GetBaseJob for (T, U) { + fn base_job(&self) -> &BaseJob { + self.0.base_job() + } +} + +impl GetBaseJob for (T, U, V) { + fn base_job(&self) -> &BaseJob { + self.0.base_job() } } diff --git a/crates/fayalite/src/build/external.rs b/crates/fayalite/src/build/external.rs index 360ad6e..2664d3f 100644 --- a/crates/fayalite/src/build/external.rs +++ b/crates/fayalite/src/build/external.rs @@ -2,39 +2,33 @@ // See Notices.txt for copyright information use crate::{ - build::{DynJob, EscapeForUnixShell, JobItem, JobItemName, JobKind}, + build::{ + CommandParams, GetBaseJob, JobAndDependencies, JobAndKind, JobArgsAndDependencies, + JobDependencies, JobItem, JobItemName, JobKind, JobKindAndArgs, JobParams, ToArgs, + WriteArgs, intern_known_utf8_path_buf, + }, intern::{Intern, Interned}, util::{job_server::AcquiredJob, streaming_read_utf8::streaming_read_utf8}, }; use base64::{Engine, prelude::BASE64_URL_SAFE_NO_PAD}; -use clap::builder::StyledStr; -use eyre::{Context, ensure, eyre}; -use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error}; +use clap::builder::OsStringValueParser; +use eyre::{Context, bail, ensure, eyre}; +use serde::{ + Deserialize, Deserializer, Serialize, Serializer, + de::{DeserializeOwned, Error}, +}; use std::{ borrow::Cow, collections::BTreeMap, - env, - fmt::{self, Write}, - mem, + ffi::{OsStr, OsString}, + fmt, + hash::{Hash, Hasher}, + io::Write, + marker::PhantomData, + path::{Path, PathBuf}, + sync::OnceLock, }; -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -enum TemplateArg { - Literal(String), - InputPath { before: String, after: String }, - OutputPath { before: String, after: String }, -} - -impl TemplateArg { - fn after_mut(&mut self) -> &mut String { - match self { - TemplateArg::Literal(after) - | TemplateArg::InputPath { after, .. } - | TemplateArg::OutputPath { after, .. } => after, - } - } -} - #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize, Deserialize)] #[non_exhaustive] pub enum ExternalJobCacheVersion { @@ -114,14 +108,15 @@ impl From for MaybeUtf8 { } #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] -pub struct ExternalJobCache { +#[serde(rename = "ExternalJobCache")] +pub struct ExternalJobCacheV2 { pub version: ExternalJobCacheVersion, pub inputs_hash: blake3::Hash, pub stdout_stderr: String, pub result: Result, String>, } -impl ExternalJobCache { +impl ExternalJobCacheV2 { fn read_from_file(cache_json_path: Interned) -> eyre::Result { let cache_str = std::fs::read_to_string(&*cache_json_path) .wrap_err_with(|| format!("can't read {cache_json_path}"))?; @@ -136,8 +131,8 @@ impl ExternalJobCache { #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] pub struct ExternalJobCaching { - pub cache_json_path: Interned, - pub run_even_if_cached: bool, + cache_json_path: Interned, + run_even_if_cached: bool, } #[derive(Default)] @@ -180,20 +175,64 @@ impl JobCacheHasher { } } -impl ExternalJobCaching { - pub fn new(cache_json_path: Interned) -> Self { - Self { - cache_json_path, - run_even_if_cached: false, - } +fn write_file_atomically_no_clobber C, C: AsRef<[u8]>>( + path: impl AsRef, + containing_dir: impl AsRef, + contents: F, +) -> std::io::Result<()> { + let path = path.as_ref(); + let containing_dir = containing_dir.as_ref(); + if !matches!(std::fs::exists(&path), Ok(true)) { + // use File::create_new rather than tempfile's code to get normal file permissions rather than mode 600 on Unix. + let mut file = tempfile::Builder::new() + .make_in(containing_dir, |path| std::fs::File::create_new(path))?; + file.write_all(contents().as_ref())?; // write all in one operation to avoid a bunch of tiny writes + file.into_temp_path().persist_noclobber(path)?; } - #[track_caller] - pub fn from_path(cache_json_path: impl AsRef) -> Self { - let cache_json_path = cache_json_path.as_ref(); - let Some(cache_json_path) = cache_json_path.as_os_str().to_str() else { - panic!("non-UTF-8 path to cache json: {cache_json_path:?}"); - }; - Self::new(cache_json_path.intern()) + Ok(()) +} + +impl ExternalJobCaching { + pub fn get_cache_dir_from_output_dir(output_dir: &str) -> PathBuf { + Path::join(output_dir.as_ref(), ".cache") + } + pub fn make_cache_dir( + cache_dir: impl AsRef, + application_name: &str, + ) -> std::io::Result<()> { + let cache_dir = cache_dir.as_ref(); + std::fs::create_dir_all(cache_dir)?; + write_file_atomically_no_clobber(cache_dir.join("CACHEDIR.TAG"), cache_dir, || { + format!( + "Signature: 8a477f597d28d172789f06886806bc55\n\ + # This file is a cache directory tag created by {application_name}.\n\ + # For information about cache directory tags see https://bford.info/cachedir/\n" + ) + })?; + write_file_atomically_no_clobber(cache_dir.join(".gitignore"), cache_dir, || { + format!( + "# This is a cache directory created by {application_name}.\n\ + # ignore all files\n\ + *\n" + ) + }) + } + pub fn new( + output_dir: &str, + application_name: &str, + json_file_stem: &str, + run_even_if_cached: bool, + ) -> std::io::Result { + let cache_dir = Self::get_cache_dir_from_output_dir(output_dir); + Self::make_cache_dir(&cache_dir, application_name)?; + let mut cache_json_path = cache_dir; + cache_json_path.push(json_file_stem); + cache_json_path.set_extension("json"); + let cache_json_path = intern_known_utf8_path_buf(cache_json_path); + Ok(Self { + cache_json_path, + run_even_if_cached, + }) } fn write_stdout_stderr(stdout_stderr: &str) { if stdout_stderr == "" { @@ -215,12 +254,12 @@ impl ExternalJobCaching { if self.run_even_if_cached { return Err(()); } - let Ok(ExternalJobCache { + let Ok(ExternalJobCacheV2 { version: ExternalJobCacheVersion::CURRENT, inputs_hash: cached_inputs_hash, stdout_stderr, result, - }) = ExternalJobCache::read_from_file(self.cache_json_path) + }) = ExternalJobCacheV2::read_from_file(self.cache_json_path) else { return Err(()); }; @@ -253,7 +292,7 @@ impl ExternalJobCaching { fn make_command( command_line: Interned<[Interned]>, ) -> eyre::Result { - ensure!(command_line.is_empty(), "command line must not be empty"); + ensure!(!command_line.is_empty(), "command line must not be empty"); let mut cmd = std::process::Command::new(&*command_line[0]); cmd.args(command_line[1..].iter().map(|arg| &**arg)) .stdin(std::process::Stdio::null()); @@ -314,7 +353,7 @@ impl ExternalJobCaching { .expect("spawn shouldn't fail"); run_fn(cmd) }); - ExternalJobCache { + ExternalJobCacheV2 { version: ExternalJobCacheVersion::CURRENT, inputs_hash, stdout_stderr, @@ -350,486 +389,528 @@ impl ExternalJobCaching { } } -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct TemplatedExternalJobKind { - template: Interned<[TemplateArg]>, - command_line_prefix: Interned<[Interned]>, - caching: Option, -} +#[derive(Clone, Eq, Hash)] +pub struct ExternalCommandJobKind(PhantomData); -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum Token { - Char(char), - ArgSeparator, -} - -impl Token { - fn as_ident_start(self) -> Option { - match self { - Self::Char(ch @ '_') => Some(ch), - Self::Char(ch) if ch.is_alphabetic() => Some(ch), - Self::Char(_) | Self::ArgSeparator => None, - } - } - fn as_ident_continue(self) -> Option { - match self { - Self::Char(ch @ '_') => Some(ch), - Self::Char(ch) if ch.is_alphanumeric() => Some(ch), - Self::Char(_) | Self::ArgSeparator => None, - } - } -} - -#[derive(Clone, Debug)] -struct Tokens<'a> { - current: std::str::Chars<'a>, - rest: std::slice::Iter<'a, &'a str>, -} - -impl<'a> Tokens<'a> { - fn new(args: &'a [&'a str]) -> Self { - Self { - current: "".chars(), - rest: args.iter(), - } - } -} - -impl Iterator for Tokens<'_> { - type Item = Token; - - fn next(&mut self) -> Option { - match self.current.next() { - Some(c) => Some(Token::Char(c)), - None => { - self.current = self.rest.next()?.chars(); - Some(Token::ArgSeparator) - } - } - } -} - -struct Parser<'a> { - tokens: std::iter::Peekable>, - template: Vec, -} - -impl<'a> Parser<'a> { - fn new(args_template: &'a [&'a str]) -> Self { - Self { - tokens: Tokens::new(args_template).peekable(), - template: vec![TemplateArg::Literal(String::new())], // placeholder for program path - } - } - fn parse_var(&mut self) -> Result<(), ParseErrorKind> { - let last_arg = self.template.last_mut().expect("known to be non-empty"); - let TemplateArg::Literal(before) = last_arg else { - return Err(ParseErrorKind::EachArgMustHaveAtMostOneVar); - }; - let before = mem::take(before); - self.tokens - .next_if_eq(&Token::Char('$')) - .ok_or(ParseErrorKind::ExpectedVar)?; - self.tokens - .next_if_eq(&Token::Char('{')) - .ok_or(ParseErrorKind::ExpectedVar)?; - let mut var_name = String::new(); - self.tokens - .next_if(|token| { - token.as_ident_start().is_some_and(|ch| { - var_name.push(ch); - true - }) - }) - .ok_or(ParseErrorKind::ExpectedVar)?; - while let Some(_) = self.tokens.next_if(|token| { - token.as_ident_continue().is_some_and(|ch| { - var_name.push(ch); - true - }) - }) {} - self.tokens - .next_if_eq(&Token::Char('}')) - .ok_or(ParseErrorKind::ExpectedVar)?; - let after = String::new(); - *last_arg = match &*var_name { - "input" => TemplateArg::InputPath { before, after }, - "output" => TemplateArg::OutputPath { before, after }, - "" => return Err(ParseErrorKind::ExpectedVar), - _ => { - return Err(ParseErrorKind::UnknownIdentifierExpectedInputOrOutput( - var_name, - )); - } - }; - Ok(()) - } - fn parse(&mut self) -> Result<(), ParseErrorKind> { - while let Some(&peek) = self.tokens.peek() { - match peek { - Token::ArgSeparator => { - self.template.push(TemplateArg::Literal(String::new())); - let _ = self.tokens.next(); - } - Token::Char('$') => self.parse_var()?, - Token::Char(ch) => { - self.template - .last_mut() - .expect("known to be non-empty") - .after_mut() - .push(ch); - let _ = self.tokens.next(); - } - } - } - Ok(()) - } - fn finish( - self, - program_path: String, - caching: Option, - ) -> TemplatedExternalJobKind { - let Self { - mut tokens, - mut template, - } = self; - assert!( - tokens.next().is_none(), - "parse() must be called before finish()" - ); - assert_eq!(template[0], TemplateArg::Literal(String::new())); - *template[0].after_mut() = program_path; - let template: Interned<[_]> = Intern::intern_owned(template); - let mut command_line_prefix = Vec::new(); - for arg in &template { - match arg { - TemplateArg::Literal(arg) => command_line_prefix.push(str::intern(arg)), - TemplateArg::InputPath { before, after: _ } - | TemplateArg::OutputPath { before, after: _ } => { - command_line_prefix.push(str::intern(before)); - break; - } - } - } - TemplatedExternalJobKind { - template, - command_line_prefix: Intern::intern_owned(command_line_prefix), - caching, - } - } -} - -pub fn find_program<'a>( - default_program_name: &'a str, - program_path_env_var: Option<&str>, -) -> eyre::Result { - let var = program_path_env_var - .and_then(env::var_os) - .filter(|v| !v.is_empty()); - let program_path = var.as_deref().unwrap_or(default_program_name.as_ref()); - let program_path = which::which(program_path) - .wrap_err_with(|| format!("can't find program {program_path:?}"))?; - program_path - .into_os_string() - .into_string() - .map_err(|program_path| eyre!("path to program is not valid UTF-8: {program_path:?}")) -} - -#[derive(Clone, Debug)] -enum ParseErrorKind { - ExpectedVar, - UnknownIdentifierExpectedInputOrOutput(String), - EachArgMustHaveAtMostOneVar, -} - -#[derive(Clone, Debug)] -pub struct TemplateParseError(ParseErrorKind); - -impl From for TemplateParseError { - fn from(value: ParseErrorKind) -> Self { - Self(value) - } -} - -impl fmt::Display for TemplateParseError { +impl fmt::Debug for ExternalCommandJobKind { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match &self.0 { - ParseErrorKind::ExpectedVar => { - f.write_str("expected `${{ident}}` for some identifier `ident`") - } - ParseErrorKind::UnknownIdentifierExpectedInputOrOutput(ident) => write!( - f, - "unknown identifier: expected `input` or `output`: {ident:?}", - ), - ParseErrorKind::EachArgMustHaveAtMostOneVar => { - f.write_str("each argument must have at most one variable") - } - } + write!(f, "ExternalCommandJobKind<{}>", std::any::type_name::()) } } -impl std::error::Error for TemplateParseError {} - -impl TemplatedExternalJobKind { - pub fn try_new( - default_program_name: &str, - program_path_env_var: Option<&str>, - args_template: &[&str], - caching: Option, - ) -> Result, TemplateParseError> { - let mut parser = Parser::new(args_template); - parser.parse()?; - Ok(find_program(default_program_name, program_path_env_var) - .map(|program_path| parser.finish(program_path, caching))) - } - #[track_caller] - pub fn new( - default_program_name: &str, - program_path_env_var: Option<&str>, - args_template: &[&str], - caching: Option, - ) -> eyre::Result { - match Self::try_new( - default_program_name, - program_path_env_var, - args_template, - caching, - ) { - Ok(retval) => retval, - Err(e) => panic!("{e}"), - } - } - fn usage(&self) -> StyledStr { - let mut retval = String::from("Usage:"); - let mut last_input_index = 0usize; - let mut last_output_index = 0usize; - for arg in &self.template { - let mut write_arg = |before: &str, middle: fmt::Arguments<'_>, after: &str| { - retval.push_str(" "); - let start_len = retval.len(); - if before != "" { - write!(retval, "{}", EscapeForUnixShell::new(before)).expect("won't error"); - } - retval.write_fmt(middle).expect("won't error"); - if after != "" { - write!(retval, "{}", EscapeForUnixShell::new(after)).expect("won't error"); - } - if retval.len() == start_len { - write!(retval, "{}", EscapeForUnixShell::new("")).expect("won't error"); - } - }; - match arg { - TemplateArg::Literal(s) => write_arg(s, format_args!(""), ""), - TemplateArg::InputPath { before, after } => { - last_input_index += 1; - write_arg(before, format_args!(""), after); - } - TemplateArg::OutputPath { before, after } => { - last_output_index += 1; - write_arg(before, format_args!(""), after); - } - } - } - retval.into() - } - fn with_usage(&self, mut e: clap::Error) -> clap::Error { - e.insert( - clap::error::ContextKind::Usage, - clap::error::ContextValue::StyledStr(self.usage()), - ); - e +impl PartialEq for ExternalCommandJobKind { + fn eq(&self, _other: &Self) -> bool { + true } } -impl JobKind for TemplatedExternalJobKind { - type Job = TemplatedExternalJob; - - fn inputs_and_direct_dependencies<'a>( - &'a self, - job: &'a Self::Job, - ) -> Cow<'a, BTreeMap>> { - Cow::Borrowed(&job.inputs) +impl Ord for ExternalCommandJobKind { + fn cmp(&self, _other: &Self) -> std::cmp::Ordering { + std::cmp::Ordering::Equal } +} - fn outputs(&self, job: &Self::Job) -> Interned<[JobItemName]> { - job.outputs +impl PartialOrd for ExternalCommandJobKind { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) } +} - fn command_line_prefix(&self) -> Interned<[Interned]> { - self.command_line_prefix +impl Default for ExternalCommandJobKind { + fn default() -> Self { + Self(PhantomData) } +} - fn to_command_line(&self, job: &Self::Job) -> Interned<[Interned]> { - job.command_line +impl Copy for ExternalCommandJobKind {} + +impl ExternalCommandJobKind { + pub const fn new() -> Self { + Self(PhantomData) } +} - fn subcommand(&self) -> Option { - None - } +#[derive(Copy, Clone)] +struct ExternalCommandProgramPathValueParser(PhantomData); - fn from_arg_matches(&self, _matches: &mut clap::ArgMatches) -> Result { - panic!( - "a TemplatedExternalJob is not a subcommand of this program -- TemplatedExternalJobKind::subcommand() always returns None" - ); - } +fn parse_which_result( + which_result: which::Result, + program_name: impl Into, + program_path_arg_name: impl FnOnce() -> String, +) -> Result, ResolveProgramPathError> { + let which_result = match which_result { + Ok(v) => v, + Err(e) => { + return Err(ResolveProgramPathError { + inner: ResolveProgramPathErrorInner::Which(e), + program_name: program_name.into(), + program_path_arg_name: program_path_arg_name(), + }); + } + }; + Ok(str::intern_owned( + which_result + .into_os_string() + .into_string() + .map_err(|_| ResolveProgramPathError { + inner: ResolveProgramPathErrorInner::NotValidUtf8, + program_name: program_name.into(), + program_path_arg_name: program_path_arg_name(), + })?, + )) +} - fn parse_command_line( +impl clap::builder::TypedValueParser + for ExternalCommandProgramPathValueParser +{ + type Value = Interned; + + fn parse_ref( &self, - command_line: Interned<[Interned]>, - ) -> clap::error::Result { - let mut inputs = BTreeMap::new(); - let mut outputs = Vec::new(); - let mut command_line_iter = command_line.iter(); - for template_arg in &self.template { - let Some(command_line_arg) = command_line_iter.next() else { - return Err(self.with_usage(clap::Error::new( - clap::error::ErrorKind::MissingRequiredArgument, - ))); - }; - let match_io = |before: &str, after: &str| -> clap::error::Result<_> { - Ok(JobItemName::File { - path: command_line_arg - .strip_prefix(before) - .and_then(|s| s.strip_suffix(after)) - .ok_or_else(|| { - self.with_usage(clap::Error::new( - clap::error::ErrorKind::MissingRequiredArgument, - )) - })? - .intern(), + cmd: &clap::Command, + arg: Option<&clap::Arg>, + value: &OsStr, + ) -> clap::error::Result { + OsStringValueParser::new() + .try_map(|program_name| { + parse_which_result(which::which(&program_name), program_name, || { + T::program_path_arg_name().into() }) - }; - match template_arg { - TemplateArg::Literal(template_arg) => { - if **command_line_arg != **template_arg { - return Err(self.with_usage(clap::Error::new( - clap::error::ErrorKind::MissingRequiredArgument, - ))); - } - } - TemplateArg::InputPath { before, after } => { - inputs.insert(match_io(before, after)?, None); - } - TemplateArg::OutputPath { before, after } => outputs.push(match_io(before, after)?), - } - } - if let Some(_) = command_line_iter.next() { - Err(self.with_usage(clap::Error::new(clap::error::ErrorKind::UnknownArgument))) - } else { - Ok(TemplatedExternalJob { - command_line, - inputs, - outputs: Intern::intern_owned(outputs), }) + .parse_ref(cmd, arg, value) + } +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug, clap::Args)] +#[group(id = T::args_group_id())] +#[non_exhaustive] +pub struct ExternalCommandArgs { + #[arg( + name = Interned::into_inner(T::program_path_arg_name()), + long = T::program_path_arg_name(), + value_name = T::program_path_arg_value_name(), + env = T::program_path_env_var_name().map(Interned::into_inner), + value_parser = ExternalCommandProgramPathValueParser::(PhantomData), + default_value = T::default_program_name(), + value_hint = clap::ValueHint::CommandName, + )] + pub program_path: Interned, + #[arg( + name = Interned::into_inner(T::run_even_if_cached_arg_name()), + long = T::run_even_if_cached_arg_name(), + )] + pub run_even_if_cached: bool, + #[command(flatten)] + pub additional_args: T::AdditionalArgs, +} + +#[derive(Clone)] +enum ResolveProgramPathErrorInner { + Which(which::Error), + NotValidUtf8, +} + +impl fmt::Debug for ResolveProgramPathErrorInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Which(v) => v.fmt(f), + Self::NotValidUtf8 => f.write_str("NotValidUtf8"), } } +} - fn run( - &self, - job: &Self::Job, - inputs: &[JobItem], - acquired_job: &mut AcquiredJob, - ) -> eyre::Result> { - assert!( - inputs - .iter() - .map(JobItem::name) - .eq(job.inputs.keys().copied()) - ); - let output_file_paths = job.outputs.iter().map(|&output| match output { - JobItemName::Module { .. } => unreachable!(), - JobItemName::File { path } => path, - }); - let input_file_paths = job.inputs.keys().map(|name| match name { - JobItemName::Module { .. } => unreachable!(), - JobItemName::File { path } => *path, - }); - ExternalJobCaching::run_maybe_cached( - self.caching, - job.command_line, - input_file_paths, - output_file_paths.clone(), - |cmd| { - acquired_job - .run_command(cmd, |cmd: &mut std::process::Command| { - let status = cmd.status()?; - if status.success() { - Ok(()) - } else { - Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("process exited with status: {status}"), - )) - } - }) - .wrap_err_with(|| format!("when trying to run: {:?}", job.command_line)) - }, - )?; - Ok(Vec::from_iter( - output_file_paths.map(|path| JobItem::File { path }), +impl fmt::Display for ResolveProgramPathErrorInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Which(v) => v.fmt(f), + Self::NotValidUtf8 => f.write_str("path is not valid UTF-8"), + } + } +} + +#[derive(Clone, Debug)] +pub struct ResolveProgramPathError { + inner: ResolveProgramPathErrorInner, + program_name: std::ffi::OsString, + program_path_arg_name: String, +} + +impl fmt::Display for ResolveProgramPathError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + inner, + program_name, + program_path_arg_name, + } = self; + write!( + f, + "{program_path_arg_name}: failed to resolve {program_name:?} to a valid program: {inner}", + ) + } +} + +impl std::error::Error for ResolveProgramPathError {} + +pub fn resolve_program_path( + program_name: Option<&OsStr>, + default_program_name: impl AsRef, + program_path_env_var_name: Option<&OsStr>, +) -> Result, ResolveProgramPathError> { + let default_program_name = default_program_name.as_ref(); + let owned_program_name; + let program_name = if let Some(program_name) = program_name { + program_name + } else if let Some(v) = program_path_env_var_name.and_then(std::env::var_os) { + owned_program_name = v; + &owned_program_name + } else { + default_program_name + }; + parse_which_result(which::which(program_name), program_name, || { + default_program_name.display().to_string() + }) +} + +impl ExternalCommandArgs { + pub fn with_resolved_program_path( + program_path: Interned, + additional_args: T::AdditionalArgs, + ) -> Self { + Self { + program_path, + run_even_if_cached: false, + additional_args, + } + } + pub fn new( + program_name: Option<&OsStr>, + additional_args: T::AdditionalArgs, + ) -> Result { + Ok(Self::with_resolved_program_path( + resolve_program_path( + program_name, + T::default_program_name(), + T::program_path_env_var_name().as_ref().map(AsRef::as_ref), + )?, + additional_args, )) } } -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct TemplatedExternalJob { - command_line: Interned<[Interned]>, - inputs: BTreeMap>, - outputs: Interned<[JobItemName]>, +impl ToArgs for ExternalCommandArgs { + fn to_args(&self, args: &mut (impl WriteArgs + ?Sized)) { + let Self { + program_path, + run_even_if_cached, + ref additional_args, + } = *self; + args.write_arg(format_args!( + "--{}={program_path}", + T::program_path_arg_name() + )); + if run_even_if_cached { + args.write_arg(format_args!("--{}", T::run_even_if_cached_arg_name())); + } + additional_args.to_args(args); + } } -impl TemplatedExternalJob { - pub fn try_add_direct_dependency(&mut self, new_dependency: DynJob) -> eyre::Result<()> { - let mut added = false; - for output in new_dependency.outputs() { - if let Some(existing_dependency) = self.inputs.get_mut(&output) { - eyre::ensure!( - existing_dependency - .as_ref() - .is_none_or(|v| *v == new_dependency), - "job can't have the same output as some other job:\n\ - output: {output:?}\n\ - existing job:\n\ - {existing_dependency:?}\n\ - new job:\n\ - {new_dependency:?}", - ); - *existing_dependency = Some(new_dependency.clone()); - added = true; - } +#[derive(Copy, Clone)] +struct ExternalCommandJobParams { + command_params: CommandParams, + inputs: Interned<[JobItemName]>, + outputs: Interned<[JobItemName]>, + output_paths: Interned<[Interned]>, +} + +impl ExternalCommandJobParams { + fn new(job: &ExternalCommandJob) -> Self { + let output_paths = T::output_paths(job); + Self { + command_params: CommandParams { + command_line: Interned::from_iter( + [job.program_path] + .into_iter() + .chain(T::command_line_args(job).iter().copied()), + ), + current_dir: T::current_dir(job), + }, + inputs: T::inputs(job), + outputs: output_paths + .iter() + .map(|&path| JobItemName::Path { path }) + .collect(), + output_paths, } - eyre::ensure!( - added, - "job (that we'll call `A`) can't be added as a direct dependency of another\n\ - job (that we'll call `B`) when none of job `A`'s outputs are an input of job `B`\n\ - job `A`:\n\ - {new_dependency:?}\n\ - job `B`:\n\ - {self:?}" - ); - Ok(()) - } - pub fn try_add_direct_dependencies>( - &mut self, - dependencies: I, - ) -> eyre::Result<()> { - dependencies - .into_iter() - .try_for_each(|new_dependency| self.try_add_direct_dependency(new_dependency)) - } - #[track_caller] - pub fn add_direct_dependencies>(&mut self, dependencies: I) { - match self.try_add_direct_dependencies(dependencies) { - Ok(()) => {} - Err(e) => panic!("error adding dependencies: {e}"), - } - } - #[track_caller] - pub fn with_direct_dependencies>( - mut self, - dependencies: I, - ) -> Self { - self.add_direct_dependencies(dependencies); - self + } +} + +#[derive(Deserialize, Serialize)] +pub struct ExternalCommandJob { + additional_job_data: T::AdditionalJobData, + program_path: Interned, + output_dir: Interned, + run_even_if_cached: bool, + #[serde(skip)] + params_cache: OnceLock, +} + +impl Eq for ExternalCommandJob {} + +impl> Clone for ExternalCommandJob { + fn clone(&self) -> Self { + let Self { + ref additional_job_data, + program_path, + output_dir, + run_even_if_cached, + ref params_cache, + } = *self; + Self { + additional_job_data: additional_job_data.clone(), + program_path, + output_dir, + run_even_if_cached, + params_cache: params_cache.clone(), + } + } +} + +impl fmt::Debug for ExternalCommandJob { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + additional_job_data, + program_path, + output_dir, + run_even_if_cached, + params_cache: _, + } = self; + write!(f, "ExternalCommandJob<{}>", std::any::type_name::())?; + f.debug_struct("") + .field("additional_job_data", additional_job_data) + .field("program_path", program_path) + .field("output_dir", output_dir) + .field("run_even_if_cached", run_even_if_cached) + .finish() + } +} + +impl PartialEq for ExternalCommandJob { + fn eq(&self, other: &Self) -> bool { + let Self { + additional_job_data, + program_path, + output_dir, + run_even_if_cached, + params_cache: _, + } = self; + *additional_job_data == other.additional_job_data + && *program_path == other.program_path + && *output_dir == other.output_dir + && *run_even_if_cached == other.run_even_if_cached + } +} + +impl Hash for ExternalCommandJob { + fn hash(&self, state: &mut H) { + let Self { + additional_job_data, + program_path, + output_dir, + run_even_if_cached, + params_cache: _, + } = self; + additional_job_data.hash(state); + program_path.hash(state); + output_dir.hash(state); + run_even_if_cached.hash(state); + } +} + +impl ExternalCommandJob { + pub fn additional_job_data(&self) -> &T::AdditionalJobData { + &self.additional_job_data + } + pub fn program_path(&self) -> Interned { + self.program_path + } + pub fn output_dir(&self) -> Interned { + self.output_dir + } + pub fn run_even_if_cached(&self) -> bool { + self.run_even_if_cached + } + fn params(&self) -> &ExternalCommandJobParams { + self.params_cache + .get_or_init(|| ExternalCommandJobParams::new(self)) + } + pub fn command_params(&self) -> CommandParams { + self.params().command_params + } + pub fn inputs(&self) -> Interned<[JobItemName]> { + self.params().inputs + } + pub fn output_paths(&self) -> Interned<[Interned]> { + self.params().output_paths + } + pub fn outputs(&self) -> Interned<[JobItemName]> { + self.params().outputs + } +} + +pub trait ExternalCommand: 'static + Send + Sync + Hash + Eq + fmt::Debug + Sized + Clone { + type AdditionalArgs: ToArgs; + type AdditionalJobData: 'static + + Send + + Sync + + Hash + + Eq + + fmt::Debug + + Serialize + + DeserializeOwned; + type Dependencies: JobDependencies; + fn dependencies() -> Self::Dependencies; + fn args_to_jobs( + args: JobArgsAndDependencies>, + params: &JobParams, + ) -> eyre::Result<( + Self::AdditionalJobData, + ::JobsAndKinds, + )>; + fn inputs(job: &ExternalCommandJob) -> Interned<[JobItemName]>; + fn output_paths(job: &ExternalCommandJob) -> Interned<[Interned]>; + fn command_line_args(job: &ExternalCommandJob) -> Interned<[Interned]>; + fn current_dir(job: &ExternalCommandJob) -> Option>; + fn job_kind_name() -> Interned; + fn args_group_id() -> clap::Id { + Interned::into_inner(Self::job_kind_name()).into() + } + fn program_path_arg_name() -> Interned { + Self::default_program_name() + } + fn program_path_arg_value_name() -> Interned { + Intern::intern_owned(Self::program_path_arg_name().to_uppercase()) + } + fn default_program_name() -> Interned; + fn program_path_env_var_name() -> Option> { + Some(Intern::intern_owned( + Self::program_path_arg_name() + .to_uppercase() + .replace('-', "_"), + )) + } + fn run_even_if_cached_arg_name() -> Interned { + Intern::intern_owned(format!("{}-run-even-if-cached", Self::job_kind_name())) + } + fn subcommand_hidden() -> bool { + false + } +} + +impl JobKind for ExternalCommandJobKind { + type Args = ExternalCommandArgs; + type Job = ExternalCommandJob; + type Dependencies = T::Dependencies; + + fn dependencies(self) -> Self::Dependencies { + T::dependencies() + } + + fn args_to_jobs( + args: JobArgsAndDependencies, + params: &JobParams, + ) -> eyre::Result> { + let JobKindAndArgs { + kind, + args: + ExternalCommandArgs { + program_path, + run_even_if_cached, + additional_args: _, + }, + } = args.args; + let (additional_job_data, dependencies) = T::args_to_jobs(args, params)?; + let base_job = dependencies.base_job(); + let job = ExternalCommandJob { + additional_job_data, + program_path, + output_dir: base_job.output_dir(), + run_even_if_cached: base_job.run_even_if_cached() | run_even_if_cached, + params_cache: OnceLock::new(), + }; + job.params(); // fill cache + Ok(JobAndDependencies { + job: JobAndKind { kind, job }, + dependencies, + }) + } + + fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + job.inputs() + } + + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + job.outputs() + } + + fn name(self) -> Interned { + T::job_kind_name() + } + + fn external_command_params(self, job: &Self::Job) -> Option { + Some(job.command_params()) + } + + fn run( + self, + job: &Self::Job, + inputs: &[JobItem], + params: &JobParams, + acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + assert!(inputs.iter().map(JobItem::name).eq(job.inputs())); + let CommandParams { + command_line, + current_dir, + } = job.command_params(); + ExternalJobCaching::new( + &job.output_dir, + ¶ms.application_name(), + &T::job_kind_name(), + job.run_even_if_cached, + )? + .run( + command_line, + inputs + .iter() + .flat_map(|item| match item { + JobItem::Path { path } => std::slice::from_ref(path), + JobItem::DynamicPaths { + paths, + source_job_name: _, + } => paths, + }) + .copied(), + job.output_paths(), + |mut cmd| { + if let Some(current_dir) = current_dir { + cmd.current_dir(current_dir); + } + let status = acquired_job.run_command(cmd, |cmd| cmd.status())?; + if !status.success() { + bail!("running {command_line:?} failed: {status}") + } + Ok(()) + }, + )?; + Ok(job + .output_paths() + .iter() + .map(|&path| JobItem::Path { path }) + .collect()) + } + + fn subcommand_hidden(self) -> bool { + T::subcommand_hidden() } } diff --git a/crates/fayalite/src/build/firrtl.rs b/crates/fayalite/src/build/firrtl.rs index 7c9998f..5dc4106 100644 --- a/crates/fayalite/src/build/firrtl.rs +++ b/crates/fayalite/src/build/firrtl.rs @@ -2,81 +2,119 @@ // See Notices.txt for copyright information use crate::{ - build::{BaseArgs, DynJob, InternalJobTrait, JobItem, JobItemName}, + build::{ + BaseJob, BaseJobKind, CommandParams, JobAndDependencies, JobArgsAndDependencies, JobItem, + JobItemName, JobKind, JobKindAndDependencies, JobParams, ToArgs, WriteArgs, + }, firrtl::{ExportOptions, FileBackend}, intern::{Intern, Interned}, util::job_server::AcquiredJob, }; -use clap::Parser; -use std::{borrow::Cow, collections::BTreeMap}; +use clap::Args; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; -#[derive(Parser, Debug, Clone, Hash, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Debug)] +pub struct FirrtlJobKind; + +#[derive(Args, Debug, Clone, Hash, PartialEq, Eq)] +#[group(id = "Firrtl")] #[non_exhaustive] pub struct FirrtlArgs { - #[command(flatten)] - pub base: BaseArgs, #[command(flatten)] pub export_options: ExportOptions, } -impl FirrtlArgs { +impl ToArgs for FirrtlArgs { + fn to_args(&self, args: &mut (impl WriteArgs + ?Sized)) { + let Self { export_options } = self; + export_options.to_args(args); + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Firrtl { + base: BaseJob, + export_options: ExportOptions, +} + +impl Firrtl { fn make_firrtl_file_backend(&self) -> FileBackend { FileBackend { - dir_path: self.base.output.path().into(), - top_fir_file_stem: self.base.file_stem.clone(), + dir_path: PathBuf::from(&*self.base.output_dir()), + top_fir_file_stem: Some(String::from(&*self.base.file_stem())), circuit_name: None, } } - pub fn firrtl_file(&self) -> String { + pub fn firrtl_file(&self) -> Interned { self.base.file_with_ext("fir") } } -impl InternalJobTrait for FirrtlArgs { - fn subcommand_name() -> Interned { - "firrtl".intern() +impl JobKind for FirrtlJobKind { + type Args = FirrtlArgs; + type Job = Firrtl; + type Dependencies = JobKindAndDependencies; + + fn dependencies(self) -> Self::Dependencies { + JobKindAndDependencies::new(BaseJobKind) } - fn to_args(&self) -> Vec> { - let Self { - base, - export_options, - } = self; - let mut retval = base.to_args(); - retval.extend(export_options.to_args()); - retval - } - - fn inputs_and_direct_dependencies<'a>( - &'a self, - ) -> Cow<'a, BTreeMap>> { - Cow::Owned(BTreeMap::from_iter([( - JobItemName::Module { - name: str::intern(&self.base.module_name), + fn args_to_jobs( + args: JobArgsAndDependencies, + params: &JobParams, + ) -> eyre::Result> { + args.args_to_jobs_simple( + params, + |_kind, FirrtlArgs { export_options }, dependencies| { + Ok(Firrtl { + base: dependencies.job.job.clone(), + export_options, + }) }, - None, - )])) + ) } - fn outputs(&self) -> Interned<[JobItemName]> { - [JobItemName::File { - path: str::intern_owned(self.firrtl_file()), + fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + [JobItemName::Path { + path: job.base.output_dir(), }][..] .intern() } + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + [JobItemName::Path { + path: job.firrtl_file(), + }][..] + .intern() + } + + fn name(self) -> Interned { + "firrtl".intern() + } + + fn external_command_params(self, _job: &Self::Job) -> Option { + None + } + fn run( - &self, + self, + job: &Self::Job, inputs: &[JobItem], + params: &JobParams, _acquired_job: &mut AcquiredJob, ) -> eyre::Result> { - let [JobItem::Module { value: module }] = inputs else { - panic!("wrong inputs, expected a single `Module`"); + let [JobItem::Path { path: input_path }] = *inputs else { + panic!("wrong inputs, expected a single `Path`"); }; - assert_eq!(*module.name(), *self.base.module_name); - crate::firrtl::export(self.make_firrtl_file_backend(), module, self.export_options)?; - Ok(vec![JobItem::File { - path: str::intern_owned(self.firrtl_file()), + assert_eq!(input_path, job.base.output_dir()); + crate::firrtl::export( + job.make_firrtl_file_backend(), + params.main_module(), + job.export_options, + )?; + Ok(vec![JobItem::Path { + path: job.firrtl_file(), }]) } } diff --git a/crates/fayalite/src/build/formal.rs b/crates/fayalite/src/build/formal.rs new file mode 100644 index 0000000..366ce83 --- /dev/null +++ b/crates/fayalite/src/build/formal.rs @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// See Notices.txt for copyright information + +use crate::{ + build::{ + CommandParams, GetBaseJob, JobAndDependencies, JobArgsAndDependencies, JobDependencies, + JobItem, JobItemName, JobKind, JobKindAndDependencies, JobParams, ToArgs, WriteArgs, + external::{ExternalCommand, ExternalCommandJob, ExternalCommandJobKind}, + interned_known_utf8_method, + verilog::{VerilogDialect, VerilogJobKind}, + }, + intern::{Intern, Interned}, + module::NameId, + util::job_server::AcquiredJob, +}; +use clap::{Args, ValueEnum}; +use eyre::{Context, eyre}; +use serde::{Deserialize, Serialize}; +use std::fmt; + +#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq, Hash, Default, Deserialize, Serialize)] +#[non_exhaustive] +pub enum FormalMode { + #[default] + BMC, + Prove, + Live, + Cover, +} + +impl FormalMode { + pub fn as_str(self) -> &'static str { + match self { + FormalMode::BMC => "bmc", + FormalMode::Prove => "prove", + FormalMode::Live => "live", + FormalMode::Cover => "cover", + } + } +} + +impl fmt::Display for FormalMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Args, Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub struct FormalArgs { + #[arg(long = "sby-extra-arg", value_name = "ARG")] + pub sby_extra_args: Vec, + #[arg(long, default_value_t)] + pub formal_mode: FormalMode, + #[arg(long, default_value_t = Self::DEFAULT_DEPTH)] + pub formal_depth: u64, + #[arg(long, default_value = Self::DEFAULT_SOLVER)] + pub formal_solver: String, + #[arg(long = "smtbmc-extra-arg", value_name = "ARG")] + pub smtbmc_extra_args: Vec, +} + +impl FormalArgs { + pub const DEFAULT_DEPTH: u64 = 20; + pub const DEFAULT_SOLVER: &'static str = "z3"; +} + +impl ToArgs for FormalArgs { + fn to_args(&self, args: &mut (impl WriteArgs + ?Sized)) { + let Self { + sby_extra_args, + formal_mode, + formal_depth, + formal_solver, + smtbmc_extra_args, + } = self; + args.extend( + sby_extra_args + .iter() + .map(|v| format!("--sby-extra-arg={v}")), + ); + args.extend([ + format_args!("--formal-mode={formal_mode}"), + format_args!("--formal-depth={formal_depth}"), + format_args!("--formal-solver={formal_solver}"), + ]); + args.extend( + smtbmc_extra_args + .iter() + .map(|v| format!("--smtbmc-extra-arg={v}")), + ); + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +pub struct WriteSbyFileJobKind; + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)] +pub struct WriteSbyFileJob { + sby_extra_args: Interned<[Interned]>, + formal_mode: FormalMode, + formal_depth: u64, + formal_solver: Interned, + smtbmc_extra_args: Interned<[Interned]>, + sby_file: Interned, + output_dir: Interned, + main_verilog_file: Interned, +} + +impl WriteSbyFileJob { + pub fn sby_extra_args(&self) -> Interned<[Interned]> { + self.sby_extra_args + } + pub fn formal_mode(&self) -> FormalMode { + self.formal_mode + } + pub fn formal_depth(&self) -> u64 { + self.formal_depth + } + pub fn formal_solver(&self) -> Interned { + self.formal_solver + } + pub fn smtbmc_extra_args(&self) -> Interned<[Interned]> { + self.smtbmc_extra_args + } + pub fn sby_file(&self) -> Interned { + self.sby_file + } + pub fn output_dir(&self) -> Interned { + self.output_dir + } + pub fn main_verilog_file(&self) -> Interned { + self.main_verilog_file + } + fn write_sby( + &self, + output: &mut W, + additional_files: &[Interned], + main_module_name_id: NameId, + ) -> Result, fmt::Error> { + let Self { + sby_extra_args: _, + formal_mode, + formal_depth, + formal_solver, + smtbmc_extra_args, + sby_file: _, + output_dir: _, + main_verilog_file, + } = self; + write!( + output, + "[options]\n\ + mode {formal_mode}\n\ + depth {formal_depth}\n\ + wait on\n\ + \n\ + [engines]\n\ + smtbmc {formal_solver} -- --" + )?; + for i in smtbmc_extra_args { + output.write_str(" ")?; + output.write_str(i)?; + } + output.write_str( + "\n\ + \n\ + [script]\n", + )?; + for verilog_file in [main_verilog_file].into_iter().chain(additional_files) { + if !(verilog_file.ends_with(".v") || verilog_file.ends_with(".sv")) { + continue; + } + let verilog_file = match std::path::absolute(verilog_file) + .and_then(|v| { + v.into_os_string().into_string().map_err(|_| { + std::io::Error::new(std::io::ErrorKind::Other, "path is not valid UTF-8") + }) + }) + .wrap_err_with(|| format!("converting {verilog_file:?} to an absolute path failed")) + { + Ok(v) => v, + Err(e) => return Ok(Err(e)), + }; + if verilog_file.contains(|ch: char| { + (ch != ' ' && ch != '\t' && ch.is_ascii_whitespace()) || ch == '"' + }) { + return Ok(Err(eyre!( + "verilog file path contains characters that aren't permitted" + ))); + } + writeln!(output, "read_verilog -sv -formal \"{verilog_file}\"")?; + } + let circuit_name = crate::firrtl::get_circuit_name(main_module_name_id); + // workaround for wires disappearing -- set `keep` on all wires + writeln!( + output, + "hierarchy -top {circuit_name}\n\ + proc\n\ + setattr -set keep 1 w:\\*\n\ + prep", + )?; + Ok(Ok(())) + } +} + +impl JobKind for WriteSbyFileJobKind { + type Args = FormalArgs; + type Job = WriteSbyFileJob; + type Dependencies = JobKindAndDependencies; + + fn dependencies(self) -> Self::Dependencies { + Default::default() + } + + fn args_to_jobs( + mut args: JobArgsAndDependencies, + params: &JobParams, + ) -> eyre::Result> { + args.dependencies + .dependencies + .args + .args + .additional_args + .verilog_dialect + .get_or_insert(VerilogDialect::Yosys); + args.args_to_jobs_simple(params, |_kind, args, dependencies| { + let FormalArgs { + sby_extra_args, + formal_mode, + formal_depth, + formal_solver, + smtbmc_extra_args, + } = args; + Ok(WriteSbyFileJob { + sby_extra_args: sby_extra_args.into_iter().map(str::intern_owned).collect(), + formal_mode, + formal_depth, + formal_solver: str::intern_owned(formal_solver), + smtbmc_extra_args: smtbmc_extra_args + .into_iter() + .map(str::intern_owned) + .collect(), + sby_file: dependencies.base_job().file_with_ext("sby"), + output_dir: dependencies.base_job().output_dir(), + main_verilog_file: dependencies.job.job.main_verilog_file(), + }) + }) + } + + fn inputs(self, _job: &Self::Job) -> Interned<[JobItemName]> { + [JobItemName::DynamicPaths { + source_job_name: VerilogJobKind.name(), + }][..] + .intern() + } + + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + [JobItemName::Path { path: job.sby_file }][..].intern() + } + + fn name(self) -> Interned { + "write-sby-file".intern() + } + + fn external_command_params(self, _job: &Self::Job) -> Option { + None + } + + fn run( + self, + job: &Self::Job, + inputs: &[JobItem], + params: &JobParams, + _acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + assert!(inputs.iter().map(JobItem::name).eq(self.inputs(job))); + let [ + JobItem::DynamicPaths { + paths: additional_files, + .. + }, + ] = inputs + else { + unreachable!(); + }; + let mut contents = String::new(); + match job.write_sby( + &mut contents, + additional_files, + params.main_module().name_id(), + ) { + Ok(result) => result?, + Err(fmt::Error) => unreachable!("writing to String can't fail"), + } + std::fs::write(job.sby_file, contents) + .wrap_err_with(|| format!("writing {} failed", job.sby_file))?; + Ok(vec![JobItem::Path { path: job.sby_file }]) + } + + fn subcommand_hidden(self) -> bool { + true + } +} + +#[derive(Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub struct Formal { + #[serde(flatten)] + write_sby_file: WriteSbyFileJob, + sby_file_name: Interned, +} + +impl fmt::Debug for Formal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + write_sby_file: + WriteSbyFileJob { + sby_extra_args, + formal_mode, + formal_depth, + formal_solver, + smtbmc_extra_args, + sby_file, + output_dir: _, + main_verilog_file, + }, + sby_file_name, + } = self; + f.debug_struct("Formal") + .field("sby_extra_args", sby_extra_args) + .field("formal_mode", formal_mode) + .field("formal_depth", formal_depth) + .field("formal_solver", formal_solver) + .field("smtbmc_extra_args", smtbmc_extra_args) + .field("sby_file", sby_file) + .field("sby_file_name", sby_file_name) + .field("main_verilog_file", main_verilog_file) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Hash, PartialEq, Eq, Debug, Args)] +pub struct FormalAdditionalArgs {} + +impl ToArgs for FormalAdditionalArgs { + fn to_args(&self, _args: &mut (impl WriteArgs + ?Sized)) { + let Self {} = self; + } +} + +impl ExternalCommand for Formal { + type AdditionalArgs = FormalAdditionalArgs; + type AdditionalJobData = Formal; + type Dependencies = JobKindAndDependencies; + + fn dependencies() -> Self::Dependencies { + Default::default() + } + + fn args_to_jobs( + args: JobArgsAndDependencies>, + params: &JobParams, + ) -> eyre::Result<( + Self::AdditionalJobData, + ::JobsAndKinds, + )> { + args.args_to_jobs_external_simple(params, |args, dependencies| { + let FormalAdditionalArgs {} = args.additional_args; + Ok(Formal { + write_sby_file: dependencies.job.job.clone(), + sby_file_name: interned_known_utf8_method(dependencies.job.job.sby_file(), |v| { + v.file_name().expect("known to have file name") + }), + }) + }) + } + + fn inputs(job: &ExternalCommandJob) -> Interned<[JobItemName]> { + [ + JobItemName::Path { + path: job.additional_job_data().write_sby_file.sby_file(), + }, + JobItemName::Path { + path: job.additional_job_data().write_sby_file.main_verilog_file(), + }, + JobItemName::DynamicPaths { + source_job_name: VerilogJobKind.name(), + }, + ][..] + .intern() + } + + fn output_paths(_job: &ExternalCommandJob) -> Interned<[Interned]> { + Interned::default() + } + + fn command_line_args(job: &ExternalCommandJob) -> Interned<[Interned]> { + [ + // "-j1".intern(), // sby seems not to respect job count in parallel mode + "-f".intern(), + job.additional_job_data().sby_file_name, + ] + .into_iter() + .chain(job.additional_job_data().write_sby_file.sby_extra_args()) + .collect() + } + + fn current_dir(job: &ExternalCommandJob) -> Option> { + Some(job.output_dir()) + } + + fn job_kind_name() -> Interned { + "formal".intern() + } + + fn default_program_name() -> Interned { + "sby".intern() + } +} diff --git a/crates/fayalite/src/build/graph.rs b/crates/fayalite/src/build/graph.rs new file mode 100644 index 0000000..a90d839 --- /dev/null +++ b/crates/fayalite/src/build/graph.rs @@ -0,0 +1,801 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// See Notices.txt for copyright information + +use crate::{ + build::{DynJob, JobItem, JobItemName, JobParams, program_name_for_internal_jobs}, + intern::Interned, + util::{HashMap, HashSet, job_server::AcquiredJob}, +}; +use eyre::{ContextCompat, eyre}; +use petgraph::{ + algo::{DfsSpace, kosaraju_scc, toposort}, + graph::DiGraph, + visit::{GraphBase, Visitable}, +}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error, ser::SerializeSeq}; +use std::{ + cell::OnceCell, + collections::{BTreeMap, BTreeSet, VecDeque}, + fmt::{self, Write}, + panic, + rc::Rc, + sync::mpsc, + thread::{self, ScopedJoinHandle}, +}; + +macro_rules! write_str { + ($s:expr, $($rest:tt)*) => { + write!($s, $($rest)*).expect("String::write_fmt can't fail") + }; +} + +#[derive(Clone, Debug)] +enum JobGraphNode { + Job(DynJob), + Item { + #[allow(dead_code, reason = "name used for debugging")] + name: JobItemName, + source_job: Option, + }, +} + +type JobGraphInner = DiGraph; + +#[derive(Clone, Default)] +pub struct JobGraph { + jobs: HashMap::NodeId>, + items: HashMap::NodeId>, + graph: JobGraphInner, + topological_order: Vec<::NodeId>, + space: DfsSpace<::NodeId, ::Map>, +} + +impl fmt::Debug for JobGraph { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + jobs: _, + items: _, + graph, + topological_order, + space: _, + } = self; + f.debug_struct("JobGraph") + .field("graph", graph) + .field("topological_order", topological_order) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug)] +pub enum JobGraphError { + CycleError { + job: DynJob, + output: JobItemName, + }, + MultipleJobsCreateSameOutput { + output_item: JobItemName, + existing_job: DynJob, + new_job: DynJob, + }, +} + +impl std::error::Error for JobGraphError {} + +impl fmt::Display for JobGraphError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CycleError { job, output } => write!( + f, + "job can't be added to job graph because it would introduce a cyclic dependency through this job output:\n\ + {output:?}\n\ + job:\n{job:?}", + ), + JobGraphError::MultipleJobsCreateSameOutput { + output_item, + existing_job, + new_job, + } => write!( + f, + "job can't be added to job graph because the new job has an output that is also produced by an existing job.\n\ + conflicting output:\n\ + {output_item:?}\n\ + existing job:\n\ + {existing_job:?}\n\ + new job:\n\ + {new_job:?}", + ), + } + } +} + +#[derive(Copy, Clone, Debug)] +enum EscapeForUnixShellState { + DollarSingleQuote, + SingleQuote, + Unquoted, +} + +#[derive(Clone)] +pub struct EscapeForUnixShell<'a> { + state: EscapeForUnixShellState, + prefix: [u8; 3], + bytes: &'a [u8], +} + +impl<'a> fmt::Debug for EscapeForUnixShell<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl<'a> fmt::Display for EscapeForUnixShell<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for c in self.clone() { + f.write_char(c)?; + } + Ok(()) + } +} + +impl<'a> EscapeForUnixShell<'a> { + pub fn new(s: &'a str) -> Self { + Self::from_bytes(s.as_bytes()) + } + fn make_prefix(bytes: &[u8]) -> [u8; 3] { + let mut prefix = [0; 3]; + prefix[..bytes.len()].copy_from_slice(bytes); + prefix + } + pub fn from_bytes(bytes: &'a [u8]) -> Self { + let mut needs_single_quote = bytes.is_empty(); + for &b in bytes { + match b { + b'!' | b'\'' | b'\"' | b' ' => needs_single_quote = true, + 0..0x20 | 0x7F.. => { + return Self { + state: EscapeForUnixShellState::DollarSingleQuote, + prefix: Self::make_prefix(b"$'"), + bytes, + }; + } + _ => {} + } + } + if needs_single_quote { + Self { + state: EscapeForUnixShellState::SingleQuote, + prefix: Self::make_prefix(b"'"), + bytes, + } + } else { + Self { + state: EscapeForUnixShellState::Unquoted, + prefix: Self::make_prefix(b""), + bytes, + } + } + } +} + +impl Iterator for EscapeForUnixShell<'_> { + type Item = char; + + fn next(&mut self) -> Option { + match &mut self.prefix { + [0, 0, 0] => {} + [0, 0, v] | // find first + [0, v, _] | // non-zero byte + [v, _, _] => { + let retval = *v as char; + *v = 0; + return Some(retval); + } + } + let Some(&next_byte) = self.bytes.split_off_first() else { + return match self.state { + EscapeForUnixShellState::DollarSingleQuote + | EscapeForUnixShellState::SingleQuote => { + self.state = EscapeForUnixShellState::Unquoted; + Some('\'') + } + EscapeForUnixShellState::Unquoted => None, + }; + }; + match self.state { + EscapeForUnixShellState::DollarSingleQuote => match next_byte { + b'\'' | b'\\' => { + self.prefix = Self::make_prefix(&[next_byte]); + Some('\\') + } + b'\t' => { + self.prefix = Self::make_prefix(b"t"); + Some('\\') + } + b'\n' => { + self.prefix = Self::make_prefix(b"n"); + Some('\\') + } + b'\r' => { + self.prefix = Self::make_prefix(b"r"); + Some('\\') + } + 0x20..=0x7E => Some(next_byte as char), + _ => { + self.prefix = [ + b'x', + char::from_digit(next_byte as u32 >> 4, 0x10).expect("known to be in range") + as u8, + char::from_digit(next_byte as u32 & 0xF, 0x10) + .expect("known to be in range") as u8, + ]; + Some('\\') + } + }, + EscapeForUnixShellState::SingleQuote => { + if next_byte == b'\'' { + self.prefix = Self::make_prefix(b"\\''"); + Some('\'') + } else { + Some(next_byte as char) + } + } + EscapeForUnixShellState::Unquoted => match next_byte { + b' ' | b'!' | b'"' | b'#' | b'$' | b'&' | b'\'' | b'(' | b')' | b'*' | b',' + | b';' | b'<' | b'>' | b'?' | b'[' | b'\\' | b']' | b'^' | b'`' | b'{' | b'|' + | b'}' | b'~' => { + self.prefix = Self::make_prefix(&[next_byte]); + Some('\\') + } + _ => Some(next_byte as char), + }, + } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +#[non_exhaustive] +pub enum UnixMakefileEscapeKind { + NonRecipe, + RecipeWithoutShellEscaping, + RecipeWithShellEscaping, +} + +#[derive(Copy, Clone)] +pub struct EscapeForUnixMakefile<'a> { + s: &'a str, + kind: UnixMakefileEscapeKind, +} + +impl<'a> fmt::Debug for EscapeForUnixMakefile<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl<'a> fmt::Display for EscapeForUnixMakefile<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.do_write(f, fmt::Write::write_str, fmt::Write::write_char, |_, _| { + Ok(()) + }) + } +} + +impl<'a> EscapeForUnixMakefile<'a> { + fn do_write( + &self, + state: &mut S, + write_str: impl Fn(&mut S, &str) -> Result<(), E>, + write_char: impl Fn(&mut S, char) -> Result<(), E>, + add_variable: impl Fn(&mut S, &'static str) -> Result<(), E>, + ) -> Result<(), E> { + let escape_recipe_char = |c| match c { + '$' => write_str(state, "$$"), + '\0'..='\x1F' | '\x7F' => { + panic!("can't escape a control character for Unix Makefile: {c:?}"); + } + _ => write_char(state, c), + }; + match self.kind { + UnixMakefileEscapeKind::NonRecipe => self.s.chars().try_for_each(|c| match c { + '=' => { + add_variable(state, "EQUALS = =")?; + write_str(state, "$(EQUALS)") + } + ';' => panic!("can't escape a semicolon (;) for Unix Makefile"), + '$' => write_str(state, "$$"), + '\\' | ' ' | '#' | ':' | '%' | '*' | '?' | '[' | ']' | '~' => { + write_char(state, '\\')?; + write_char(state, c) + } + '\0'..='\x1F' | '\x7F' => { + panic!("can't escape a control character for Unix Makefile: {c:?}"); + } + _ => write_char(state, c), + }), + UnixMakefileEscapeKind::RecipeWithoutShellEscaping => { + self.s.chars().try_for_each(escape_recipe_char) + } + UnixMakefileEscapeKind::RecipeWithShellEscaping => { + EscapeForUnixShell::new(self.s).try_for_each(escape_recipe_char) + } + } + } + pub fn new( + s: &'a str, + kind: UnixMakefileEscapeKind, + needed_variables: &mut BTreeSet<&'static str>, + ) -> Self { + let retval = Self { s, kind }; + let Ok(()) = retval.do_write( + needed_variables, + |_, _| Ok(()), + |_, _| Ok(()), + |needed_variables, variable| -> Result<(), std::convert::Infallible> { + needed_variables.insert(variable); + Ok(()) + }, + ); + retval + } +} + +impl JobGraph { + pub fn new() -> Self { + Self::default() + } + fn try_add_item_node( + &mut self, + name: JobItemName, + new_source_job: Option, + new_nodes: &mut HashSet<::NodeId>, + ) -> Result<::NodeId, JobGraphError> { + use hashbrown::hash_map::Entry; + match self.items.entry(name) { + Entry::Occupied(item_entry) => { + let node_id = *item_entry.get(); + let JobGraphNode::Item { + name: _, + source_job, + } = &mut self.graph[node_id] + else { + unreachable!("known to be an item"); + }; + if let Some(new_source_job) = new_source_job { + if let Some(source_job) = source_job { + return Err(JobGraphError::MultipleJobsCreateSameOutput { + output_item: item_entry.key().clone(), + existing_job: source_job.clone(), + new_job: new_source_job, + }); + } else { + *source_job = Some(new_source_job); + } + } + Ok(node_id) + } + Entry::Vacant(item_entry) => { + let node_id = self.graph.add_node(JobGraphNode::Item { + name, + source_job: new_source_job, + }); + new_nodes.insert(node_id); + item_entry.insert(node_id); + Ok(node_id) + } + } + } + pub fn try_add_jobs>( + &mut self, + jobs: I, + ) -> Result<(), JobGraphError> { + use hashbrown::hash_map::Entry; + let jobs = jobs.into_iter(); + struct RemoveNewNodesOnError<'a> { + this: &'a mut JobGraph, + new_nodes: HashSet<::NodeId>, + } + impl Drop for RemoveNewNodesOnError<'_> { + fn drop(&mut self) { + for node in self.new_nodes.drain() { + self.this.graph.remove_node(node); + } + } + } + let mut remove_new_nodes_on_error = RemoveNewNodesOnError { + this: self, + new_nodes: HashSet::with_capacity_and_hasher(jobs.size_hint().0, Default::default()), + }; + let new_nodes = &mut remove_new_nodes_on_error.new_nodes; + let this = &mut *remove_new_nodes_on_error.this; + for job in jobs { + let Entry::Vacant(job_entry) = this.jobs.entry(job.clone()) else { + continue; + }; + let job_node_id = this + .graph + .add_node(JobGraphNode::Job(job_entry.key().clone())); + new_nodes.insert(job_node_id); + job_entry.insert(job_node_id); + for name in job.outputs() { + let item_node_id = this.try_add_item_node(name, Some(job.clone()), new_nodes)?; + this.graph.add_edge(job_node_id, item_node_id, ()); + } + for name in job.inputs() { + let item_node_id = this.try_add_item_node(name, None, new_nodes)?; + this.graph.add_edge(item_node_id, job_node_id, ()); + } + } + match toposort(&this.graph, Some(&mut this.space)) { + Ok(v) => { + this.topological_order = v; + // no need to remove any of the new nodes on drop since we didn't encounter any errors + remove_new_nodes_on_error.new_nodes.clear(); + Ok(()) + } + Err(_) => { + // there's at least one cycle, find one! + let cycle = kosaraju_scc(&this.graph) + .into_iter() + .find_map(|scc| { + if scc.len() <= 1 { + // can't be a cycle since our graph is bipartite -- + // jobs only connect to items, never jobs to jobs or items to items + None + } else { + Some(scc) + } + }) + .expect("we know there's a cycle"); + let cycle_set = HashSet::from_iter(cycle.iter().copied()); + let job = cycle + .into_iter() + .find_map(|node_id| { + if let JobGraphNode::Job(job) = &this.graph[node_id] { + Some(job.clone()) + } else { + None + } + }) + .expect("a job must be part of the cycle"); + let output = job + .outputs() + .into_iter() + .find(|output| cycle_set.contains(&this.items[output])) + .expect("an output must be part of the cycle"); + Err(JobGraphError::CycleError { job, output }) + } + } + } + #[track_caller] + pub fn add_jobs>(&mut self, jobs: I) { + match self.try_add_jobs(jobs) { + Ok(()) => {} + Err(e) => panic!("error: {e}"), + } + } + pub fn to_unix_makefile(&self, extra_args: &[Interned]) -> String { + self.to_unix_makefile_with_internal_program_prefix( + &[program_name_for_internal_jobs()], + extra_args, + ) + } + pub fn to_unix_makefile_with_internal_program_prefix( + &self, + internal_program_prefix: &[Interned], + extra_args: &[Interned], + ) -> String { + let mut retval = String::new(); + let mut needed_variables = BTreeSet::new(); + let mut phony_targets = BTreeSet::new(); + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + let outputs = job.outputs(); + if outputs.is_empty() { + retval.push_str(":"); + } else { + for output in job.outputs() { + match output { + JobItemName::Path { path } => { + write_str!( + retval, + "{} ", + EscapeForUnixMakefile::new( + &path, + UnixMakefileEscapeKind::NonRecipe, + &mut needed_variables + ) + ); + } + JobItemName::DynamicPaths { source_job_name } => { + write_str!( + retval, + "{} ", + EscapeForUnixMakefile::new( + &source_job_name, + UnixMakefileEscapeKind::NonRecipe, + &mut needed_variables + ) + ); + phony_targets.insert(Interned::into_inner(source_job_name)); + } + } + } + if outputs.len() == 1 { + retval.push_str(":"); + } else { + retval.push_str("&:"); + } + } + for input in job.inputs() { + match input { + JobItemName::Path { path } => { + write_str!( + retval, + " {}", + EscapeForUnixMakefile::new( + &path, + UnixMakefileEscapeKind::NonRecipe, + &mut needed_variables + ) + ); + } + JobItemName::DynamicPaths { source_job_name } => { + write_str!( + retval, + " {}", + EscapeForUnixMakefile::new( + &source_job_name, + UnixMakefileEscapeKind::NonRecipe, + &mut needed_variables + ) + ); + phony_targets.insert(Interned::into_inner(source_job_name)); + } + } + } + retval.push_str("\n\t"); + job.command_params_with_internal_program_prefix(internal_program_prefix, extra_args) + .to_unix_shell_line(&mut retval, |arg, output| { + write!( + output, + "{}", + EscapeForUnixMakefile::new( + arg, + UnixMakefileEscapeKind::RecipeWithShellEscaping, + &mut needed_variables + ) + ) + }) + .expect("writing to String never fails"); + retval.push_str("\n\n"); + } + if !phony_targets.is_empty() { + retval.push_str("\n.PHONY:"); + for phony_target in phony_targets { + write_str!( + retval, + " {}", + EscapeForUnixMakefile::new( + phony_target, + UnixMakefileEscapeKind::NonRecipe, + &mut needed_variables + ) + ); + } + retval.push_str("\n"); + } + if !needed_variables.is_empty() { + retval.insert_str( + 0, + &String::from_iter(needed_variables.into_iter().map(|v| format!("{v}\n"))), + ); + } + retval + } + pub fn to_unix_shell_script(&self, extra_args: &[Interned]) -> String { + self.to_unix_shell_script_with_internal_program_prefix( + &[program_name_for_internal_jobs()], + extra_args, + ) + } + pub fn to_unix_shell_script_with_internal_program_prefix( + &self, + internal_program_prefix: &[Interned], + extra_args: &[Interned], + ) -> String { + let mut retval = String::from( + "#!/bin/sh\n\ + set -ex\n", + ); + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + job.command_params_with_internal_program_prefix(internal_program_prefix, extra_args) + .to_unix_shell_line(&mut retval, |arg, output| { + write!(output, "{}", EscapeForUnixShell::new(&arg)) + }) + .expect("writing to String never fails"); + retval.push_str("\n"); + } + retval + } + pub fn run(&self, params: &JobParams) -> eyre::Result<()> { + // use scope to auto-join threads on errors + thread::scope(|scope| { + struct WaitingJobState { + job_node_id: ::NodeId, + job: DynJob, + inputs: BTreeMap>, + } + let mut ready_jobs = VecDeque::new(); + let mut item_name_to_waiting_jobs_map = HashMap::<_, Vec<_>>::default(); + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + let waiting_job = WaitingJobState { + job_node_id: node_id, + job: job.clone(), + inputs: job + .inputs() + .iter() + .map(|&name| (name, OnceCell::new())) + .collect(), + }; + if waiting_job.inputs.is_empty() { + ready_jobs.push_back(waiting_job); + } else { + let waiting_job = Rc::new(waiting_job); + for &input_item in waiting_job.inputs.keys() { + item_name_to_waiting_jobs_map + .entry(input_item) + .or_default() + .push(waiting_job.clone()); + } + } + } + struct RunningJob<'scope> { + job: DynJob, + thread: ScopedJoinHandle<'scope, eyre::Result>>, + } + let mut running_jobs = HashMap::default(); + let (finished_jobs_sender, finished_jobs_receiver) = mpsc::channel(); + loop { + while let Some(finished_job) = finished_jobs_receiver.try_recv().ok() { + let Some(RunningJob { job, thread }) = running_jobs.remove(&finished_job) + else { + unreachable!(); + }; + let output_items = thread.join().map_err(panic::resume_unwind)??; + assert!( + output_items.iter().map(JobItem::name).eq(job.outputs()), + "job's run() method returned the wrong output items:\n\ + output items:\n\ + {output_items:?}\n\ + expected outputs:\n\ + {:?}\n\ + job:\n\ + {job:?}", + job.outputs(), + ); + for output_item in output_items { + for waiting_job in item_name_to_waiting_jobs_map + .remove(&output_item.name()) + .unwrap_or_default() + { + let Ok(()) = + waiting_job.inputs[&output_item.name()].set(output_item.clone()) + else { + unreachable!(); + }; + if let Some(waiting_job) = Rc::into_inner(waiting_job) { + ready_jobs.push_back(waiting_job); + } + } + } + } + if let Some(WaitingJobState { + job_node_id, + job, + inputs, + }) = ready_jobs.pop_front() + { + struct RunningJobInThread<'a> { + job_node_id: ::NodeId, + job: DynJob, + inputs: Vec, + params: &'a JobParams, + acquired_job: AcquiredJob, + finished_jobs_sender: mpsc::Sender<::NodeId>, + } + impl RunningJobInThread<'_> { + fn run(mut self) -> eyre::Result> { + self.job + .run(&self.inputs, self.params, &mut self.acquired_job) + } + } + impl Drop for RunningJobInThread<'_> { + fn drop(&mut self) { + let _ = self.finished_jobs_sender.send(self.job_node_id); + } + } + let name = job.kind().name(); + let running_job_in_thread = RunningJobInThread { + job_node_id, + job: job.clone(), + inputs: Result::from_iter(inputs.into_iter().map(|(input_name, input)| { + input.into_inner().wrap_err_with(|| { + eyre!("failed when trying to run job {name}: nothing provided the input item: {input_name:?}") + }) + }))?, + params, + acquired_job: AcquiredJob::acquire()?, + finished_jobs_sender: finished_jobs_sender.clone(), + }; + running_jobs.insert( + job_node_id, + RunningJob { + job, + thread: thread::Builder::new() + .name(format!("job:{name}")) + .spawn_scoped(scope, move || running_job_in_thread.run()) + .expect("failed to spawn thread for job"), + }, + ); + } + if running_jobs.is_empty() { + assert!(item_name_to_waiting_jobs_map.is_empty()); + assert!(ready_jobs.is_empty()); + return Ok(()); + } + } + }) + } +} + +impl Extend for JobGraph { + #[track_caller] + fn extend>(&mut self, iter: T) { + self.add_jobs(iter); + } +} + +impl FromIterator for JobGraph { + #[track_caller] + fn from_iter>(iter: T) -> Self { + let mut retval = Self::new(); + retval.add_jobs(iter); + retval + } +} + +impl Serialize for JobGraph { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut serializer = serializer.serialize_seq(Some(self.jobs.len()))?; + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + serializer.serialize_element(job)?; + } + serializer.end() + } +} + +impl<'de> Deserialize<'de> for JobGraph { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let jobs = Vec::::deserialize(deserializer)?; + let mut retval = JobGraph::new(); + retval.try_add_jobs(jobs).map_err(D::Error::custom)?; + Ok(retval) + } +} diff --git a/crates/fayalite/src/build/registry.rs b/crates/fayalite/src/build/registry.rs new file mode 100644 index 0000000..93d06f5 --- /dev/null +++ b/crates/fayalite/src/build/registry.rs @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// See Notices.txt for copyright information + +use crate::{ + build::{BUILT_IN_JOB_KINDS, DynJobKind, JobKind}, + intern::Interned, +}; +use std::{ + borrow::Borrow, + cmp::Ordering, + collections::BTreeMap, + fmt, + sync::{Arc, OnceLock, RwLock, RwLockWriteGuard}, +}; + +impl DynJobKind { + pub fn registry() -> JobKindRegistrySnapshot { + JobKindRegistrySnapshot(JobKindRegistry::get()) + } + #[track_caller] + pub fn register(self) { + JobKindRegistry::register(JobKindRegistry::lock(), self); + } +} + +#[derive(Copy, Clone, PartialEq, Eq)] +struct InternedStrCompareAsStr(Interned); + +impl fmt::Debug for InternedStrCompareAsStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl Ord for InternedStrCompareAsStr { + fn cmp(&self, other: &Self) -> Ordering { + str::cmp(&self.0, &other.0) + } +} + +impl PartialOrd for InternedStrCompareAsStr { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Borrow for InternedStrCompareAsStr { + fn borrow(&self) -> &str { + &self.0 + } +} + +#[derive(Clone, Debug)] +struct JobKindRegistry { + job_kinds: BTreeMap, +} + +enum JobKindRegisterError { + SameName { + name: InternedStrCompareAsStr, + old_job_kind: DynJobKind, + new_job_kind: DynJobKind, + }, +} + +impl fmt::Display for JobKindRegisterError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SameName { + name, + old_job_kind, + new_job_kind, + } => write!( + f, + "two different `JobKind` can't share the same name:\n\ + {name:?}\n\ + old job kind:\n\ + {old_job_kind:?}\n\ + new job kind:\n\ + {new_job_kind:?}", + ), + } + } +} + +trait JobKindRegistryRegisterLock { + type Locked; + fn lock(self) -> Self::Locked; + fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry; +} + +impl JobKindRegistryRegisterLock for &'static RwLock> { + type Locked = RwLockWriteGuard<'static, Arc>; + fn lock(self) -> Self::Locked { + self.write().expect("shouldn't be poisoned") + } + fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry { + Arc::make_mut(locked) + } +} + +impl JobKindRegistryRegisterLock for &'_ mut JobKindRegistry { + type Locked = Self; + + fn lock(self) -> Self::Locked { + self + } + + fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry { + locked + } +} + +impl JobKindRegistry { + fn lock() -> &'static RwLock> { + static REGISTRY: OnceLock>> = OnceLock::new(); + REGISTRY.get_or_init(Default::default) + } + fn try_register( + lock: L, + job_kind: DynJobKind, + ) -> Result<(), JobKindRegisterError> { + use std::collections::btree_map::Entry; + let name = InternedStrCompareAsStr(job_kind.name()); + // run user code only outside of lock + let mut locked = lock.lock(); + let this = L::make_mut(&mut locked); + let result = match this.job_kinds.entry(name) { + Entry::Occupied(entry) => Err(JobKindRegisterError::SameName { + name, + old_job_kind: entry.get().clone(), + new_job_kind: job_kind, + }), + Entry::Vacant(entry) => { + entry.insert(job_kind); + Ok(()) + } + }; + drop(locked); + // outside of lock now, so we can test if it's the same DynJobKind + match result { + Err(JobKindRegisterError::SameName { + name: _, + old_job_kind, + new_job_kind, + }) if old_job_kind == new_job_kind => Ok(()), + result => result, + } + } + #[track_caller] + fn register(lock: L, job_kind: DynJobKind) { + match Self::try_register(lock, job_kind) { + Err(e) => panic!("{e}"), + Ok(()) => {} + } + } + fn get() -> Arc { + Self::lock().read().expect("shouldn't be poisoned").clone() + } +} + +impl Default for JobKindRegistry { + fn default() -> Self { + let mut retval = Self { + job_kinds: BTreeMap::new(), + }; + for job_kind in BUILT_IN_JOB_KINDS { + Self::register(&mut retval, job_kind()); + } + retval + } +} + +#[derive(Clone, Debug)] +pub struct JobKindRegistrySnapshot(Arc); + +impl JobKindRegistrySnapshot { + pub fn get() -> Self { + JobKindRegistrySnapshot(JobKindRegistry::get()) + } + pub fn get_by_name<'a>(&'a self, name: &str) -> Option<&'a DynJobKind> { + self.0.job_kinds.get(name) + } + pub fn iter_with_names(&self) -> JobKindRegistryIterWithNames<'_> { + JobKindRegistryIterWithNames(self.0.job_kinds.iter()) + } + pub fn iter(&self) -> JobKindRegistryIter<'_> { + JobKindRegistryIter(self.0.job_kinds.values()) + } +} + +impl<'a> IntoIterator for &'a JobKindRegistrySnapshot { + type Item = &'a DynJobKind; + type IntoIter = JobKindRegistryIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a> IntoIterator for &'a mut JobKindRegistrySnapshot { + type Item = &'a DynJobKind; + type IntoIter = JobKindRegistryIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +#[derive(Clone, Debug)] +pub struct JobKindRegistryIter<'a>( + std::collections::btree_map::Values<'a, InternedStrCompareAsStr, DynJobKind>, +); + +impl<'a> Iterator for JobKindRegistryIter<'a> { + type Item = &'a DynJobKind; + + fn next(&mut self) -> Option { + self.0.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } + + fn count(self) -> usize + where + Self: Sized, + { + self.0.count() + } + + fn last(self) -> Option { + self.0.last() + } + + fn nth(&mut self, n: usize) -> Option { + self.0.nth(n) + } + + fn fold(self, init: B, f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + self.0.fold(init, f) + } +} + +impl<'a> std::iter::FusedIterator for JobKindRegistryIter<'a> {} + +impl<'a> ExactSizeIterator for JobKindRegistryIter<'a> {} + +impl<'a> DoubleEndedIterator for JobKindRegistryIter<'a> { + fn next_back(&mut self) -> Option { + self.0.next_back() + } + + fn nth_back(&mut self, n: usize) -> Option { + self.0.nth_back(n) + } + + fn rfold(self, init: B, f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + self.0.rfold(init, f) + } +} + +#[derive(Clone, Debug)] +pub struct JobKindRegistryIterWithNames<'a>( + std::collections::btree_map::Iter<'a, InternedStrCompareAsStr, DynJobKind>, +); + +impl<'a> Iterator for JobKindRegistryIterWithNames<'a> { + type Item = (Interned, &'a DynJobKind); + + fn next(&mut self) -> Option { + self.0.next().map(|(name, job_kind)| (name.0, job_kind)) + } + + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } + + fn count(self) -> usize + where + Self: Sized, + { + self.0.count() + } + + fn last(self) -> Option { + self.0.last().map(|(name, job_kind)| (name.0, job_kind)) + } + + fn nth(&mut self, n: usize) -> Option { + self.0.nth(n).map(|(name, job_kind)| (name.0, job_kind)) + } + + fn fold(self, init: B, f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + self.0 + .map(|(name, job_kind)| (name.0, job_kind)) + .fold(init, f) + } +} + +impl<'a> std::iter::FusedIterator for JobKindRegistryIterWithNames<'a> {} + +impl<'a> ExactSizeIterator for JobKindRegistryIterWithNames<'a> {} + +impl<'a> DoubleEndedIterator for JobKindRegistryIterWithNames<'a> { + fn next_back(&mut self) -> Option { + self.0 + .next_back() + .map(|(name, job_kind)| (name.0, job_kind)) + } + + fn nth_back(&mut self, n: usize) -> Option { + self.0 + .nth_back(n) + .map(|(name, job_kind)| (name.0, job_kind)) + } + + fn rfold(self, init: B, f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + self.0 + .map(|(name, job_kind)| (name.0, job_kind)) + .rfold(init, f) + } +} + +#[track_caller] +pub fn register_job_kind(kind: K) { + DynJobKind::new(kind).register(); +} diff --git a/crates/fayalite/src/build/verilog.rs b/crates/fayalite/src/build/verilog.rs new file mode 100644 index 0000000..219dda8 --- /dev/null +++ b/crates/fayalite/src/build/verilog.rs @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// See Notices.txt for copyright information + +use crate::{ + build::{ + CommandParams, GetBaseJob, JobAndDependencies, JobArgsAndDependencies, JobDependencies, + JobItem, JobItemName, JobKind, JobKindAndDependencies, JobParams, ToArgs, WriteArgs, + external::{ExternalCommand, ExternalCommandJob, ExternalCommandJobKind}, + firrtl::FirrtlJobKind, + interned_known_utf8_method, interned_known_utf8_path_buf_method, + }, + intern::{Intern, Interned}, + util::job_server::AcquiredJob, +}; +use clap::Args; +use eyre::bail; +use serde::{Deserialize, Serialize}; +use std::{fmt, mem}; + +/// based on [LLVM Circt's recommended lowering options][lowering-options] +/// +/// [lowering-options]: https://circt.llvm.org/docs/VerilogGeneration/#recommended-loweringoptions-by-target +#[derive(clap::ValueEnum, Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[non_exhaustive] +pub enum VerilogDialect { + Questa, + Spyglass, + Verilator, + Vivado, + Yosys, +} + +impl fmt::Display for VerilogDialect { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl VerilogDialect { + pub fn as_str(self) -> &'static str { + match self { + VerilogDialect::Questa => "questa", + VerilogDialect::Spyglass => "spyglass", + VerilogDialect::Verilator => "verilator", + VerilogDialect::Vivado => "vivado", + VerilogDialect::Yosys => "yosys", + } + } + pub fn firtool_extra_args(self) -> &'static [&'static str] { + match self { + VerilogDialect::Questa => &["--lowering-options=emitWireInPorts"], + VerilogDialect::Spyglass => { + &["--lowering-options=explicitBitcast,disallowExpressionInliningInPorts"] + } + VerilogDialect::Verilator => &[ + "--lowering-options=locationInfoStyle=wrapInAtSquareBracket,disallowLocalVariables", + ], + VerilogDialect::Vivado => &["--lowering-options=mitigateVivadoArrayIndexConstPropBug"], + VerilogDialect::Yosys => { + &["--lowering-options=disallowLocalVariables,disallowPackedArrays"] + } + } + } +} + +#[derive(Args, Debug, Clone, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub struct UnadjustedVerilogArgs { + #[arg(long = "firtool-extra-arg", value_name = "ARG")] + pub firtool_extra_args: Vec, + /// adapt the generated Verilog for a particular toolchain + #[arg(long)] + pub verilog_dialect: Option, + #[arg(long)] + pub verilog_debug: bool, +} + +impl ToArgs for UnadjustedVerilogArgs { + fn to_args(&self, args: &mut (impl WriteArgs + ?Sized)) { + let Self { + ref firtool_extra_args, + verilog_dialect, + verilog_debug, + } = *self; + args.extend( + firtool_extra_args + .iter() + .map(|arg| format!("--firtool-extra-arg={arg}")), + ); + if let Some(verilog_dialect) = verilog_dialect { + args.write_arg(format_args!("--verilog-dialect={verilog_dialect}")); + } + if verilog_debug { + args.write_str_arg("--verilog-debug"); + } + } +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug, Deserialize, Serialize)] +pub struct UnadjustedVerilog { + firrtl_file: Interned, + firrtl_file_name: Interned, + unadjusted_verilog_file: Interned, + unadjusted_verilog_file_name: Interned, + firtool_extra_args: Interned<[Interned]>, + verilog_dialect: Option, + verilog_debug: bool, +} + +impl UnadjustedVerilog { + pub fn firrtl_file(&self) -> Interned { + self.firrtl_file + } + pub fn unadjusted_verilog_file(&self) -> Interned { + self.unadjusted_verilog_file + } + pub fn firtool_extra_args(&self) -> Interned<[Interned]> { + self.firtool_extra_args + } + pub fn verilog_dialect(&self) -> Option { + self.verilog_dialect + } + pub fn verilog_debug(&self) -> bool { + self.verilog_debug + } +} + +impl ExternalCommand for UnadjustedVerilog { + type AdditionalArgs = UnadjustedVerilogArgs; + type AdditionalJobData = UnadjustedVerilog; + type Dependencies = JobKindAndDependencies; + + fn dependencies() -> Self::Dependencies { + Default::default() + } + + fn args_to_jobs( + args: JobArgsAndDependencies>, + params: &JobParams, + ) -> eyre::Result<( + Self::AdditionalJobData, + ::JobsAndKinds, + )> { + args.args_to_jobs_external_simple(params, |args, dependencies| { + let UnadjustedVerilogArgs { + firtool_extra_args, + verilog_dialect, + verilog_debug, + } = args.additional_args; + let unadjusted_verilog_file = dependencies + .dependencies + .job + .job + .file_with_ext("unadjusted.v"); + Ok(UnadjustedVerilog { + firrtl_file: dependencies.job.job.firrtl_file(), + firrtl_file_name: interned_known_utf8_method( + dependencies.job.job.firrtl_file(), + |v| v.file_name().expect("known to have file name"), + ), + unadjusted_verilog_file, + unadjusted_verilog_file_name: interned_known_utf8_method( + unadjusted_verilog_file, + |v| v.file_name().expect("known to have file name"), + ), + firtool_extra_args: firtool_extra_args + .into_iter() + .map(str::intern_owned) + .collect(), + verilog_dialect, + verilog_debug, + }) + }) + } + + fn inputs(job: &ExternalCommandJob) -> Interned<[JobItemName]> { + [JobItemName::Path { + path: job.additional_job_data().firrtl_file, + }][..] + .intern() + } + + fn output_paths(job: &ExternalCommandJob) -> Interned<[Interned]> { + [job.additional_job_data().unadjusted_verilog_file][..].intern() + } + + fn command_line_args(job: &ExternalCommandJob) -> Interned<[Interned]> { + let UnadjustedVerilog { + firrtl_file: _, + firrtl_file_name, + unadjusted_verilog_file: _, + unadjusted_verilog_file_name, + firtool_extra_args, + verilog_dialect, + verilog_debug, + } = *job.additional_job_data(); + let mut retval = vec![ + firrtl_file_name, + "-o".intern(), + unadjusted_verilog_file_name, + ]; + if verilog_debug { + retval.push("-g".intern()); + retval.push("--preserve-values=all".intern()); + } + if let Some(dialect) = verilog_dialect { + retval.extend( + dialect + .firtool_extra_args() + .iter() + .copied() + .map(str::intern), + ); + } + retval.extend_from_slice(&firtool_extra_args); + Intern::intern_owned(retval) + } + + fn current_dir(job: &ExternalCommandJob) -> Option> { + Some(job.output_dir()) + } + + fn job_kind_name() -> Interned { + "unadjusted-verilog".intern() + } + + fn default_program_name() -> Interned { + "firtool".intern() + } + + fn subcommand_hidden() -> bool { + true + } + + fn run_even_if_cached_arg_name() -> Interned { + "firtool-run-even-if-cached".intern() + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct VerilogJobKind; + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Args)] +#[non_exhaustive] +pub struct VerilogJobArgs {} + +impl ToArgs for VerilogJobArgs { + fn to_args(&self, _args: &mut (impl WriteArgs + ?Sized)) { + let Self {} = self; + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct VerilogJob { + output_dir: Interned, + unadjusted_verilog_file: Interned, + main_verilog_file: Interned, +} + +impl VerilogJob { + pub fn output_dir(&self) -> Interned { + self.output_dir + } + pub fn unadjusted_verilog_file(&self) -> Interned { + self.unadjusted_verilog_file + } + pub fn main_verilog_file(&self) -> Interned { + self.main_verilog_file + } +} + +impl JobKind for VerilogJobKind { + type Args = VerilogJobArgs; + type Job = VerilogJob; + type Dependencies = JobKindAndDependencies>; + + fn dependencies(self) -> Self::Dependencies { + Default::default() + } + + fn args_to_jobs( + args: JobArgsAndDependencies, + params: &JobParams, + ) -> eyre::Result> { + args.args_to_jobs_simple(params, |_kind, args, dependencies| { + let VerilogJobArgs {} = args; + Ok(VerilogJob { + output_dir: dependencies.base_job().output_dir(), + unadjusted_verilog_file: dependencies + .job + .job + .additional_job_data() + .unadjusted_verilog_file(), + main_verilog_file: dependencies.base_job().file_with_ext("v"), + }) + }) + } + + fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + [JobItemName::Path { + path: job.unadjusted_verilog_file, + }][..] + .intern() + } + + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + [ + JobItemName::Path { + path: job.main_verilog_file, + }, + JobItemName::DynamicPaths { + source_job_name: self.name(), + }, + ][..] + .intern() + } + + fn name(self) -> Interned { + "verilog".intern() + } + + fn external_command_params(self, _job: &Self::Job) -> Option { + None + } + + fn run( + self, + job: &Self::Job, + inputs: &[JobItem], + _params: &JobParams, + _acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + assert!(inputs.iter().map(JobItem::name).eq(self.inputs(job))); + let input = std::fs::read_to_string(job.unadjusted_verilog_file())?; + let file_separator_prefix = "\n// ----- 8< ----- FILE \""; + let file_separator_suffix = "\" ----- 8< -----\n\n"; + let mut input = &*input; + let main_verilog_file = job.main_verilog_file(); + let mut file_name = Some(main_verilog_file); + let mut additional_outputs = Vec::new(); + loop { + let (chunk, next_file_name) = if let Some((chunk, rest)) = + input.split_once(file_separator_prefix) + { + let Some((next_file_name, rest)) = rest.split_once(file_separator_suffix) else { + bail!( + "parsing firtool's output failed: found {file_separator_prefix:?} but no {file_separator_suffix:?}" + ); + }; + input = rest; + let next_file_name = + interned_known_utf8_path_buf_method(job.output_dir, |v| v.join(next_file_name)); + additional_outputs.push(next_file_name); + (chunk, Some(next_file_name)) + } else { + (mem::take(&mut input), None) + }; + let Some(file_name) = mem::replace(&mut file_name, next_file_name) else { + break; + }; + std::fs::write(&file_name, chunk)?; + } + Ok(vec![ + JobItem::Path { + path: main_verilog_file, + }, + JobItem::DynamicPaths { + paths: additional_outputs, + source_job_name: self.name(), + }, + ]) + } +} diff --git a/crates/fayalite/src/cli.rs b/crates/fayalite/src/cli.rs deleted file mode 100644 index 85095da..0000000 --- a/crates/fayalite/src/cli.rs +++ /dev/null @@ -1,806 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-or-later -// See Notices.txt for copyright information -use crate::{ - bundle::{Bundle, BundleType}, - firrtl::{self, ExportOptions}, - intern::Interned, - module::Module, - util::{job_server::AcquiredJob, streaming_read_utf8::streaming_read_utf8}, -}; -use clap::{ - Parser, Subcommand, ValueEnum, ValueHint, - builder::{OsStringValueParser, TypedValueParser}, -}; -use eyre::{Report, eyre}; -use serde::{Deserialize, Serialize}; -use std::{ - error, - ffi::OsString, - fmt::{self, Write}, - fs, io, mem, - path::{Path, PathBuf}, - process, -}; -use tempfile::TempDir; - -pub type Result = std::result::Result; - -pub struct CliError(Report); - -impl fmt::Debug for CliError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -impl fmt::Display for CliError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -impl error::Error for CliError {} - -impl From for CliError { - fn from(value: io::Error) -> Self { - CliError(Report::new(value)) - } -} - -pub trait RunPhase { - type Output; - fn run(&self, arg: Arg) -> Result { - self.run_with_job(arg, &mut AcquiredJob::acquire()) - } - fn run_with_job(&self, arg: Arg, acquired_job: &mut AcquiredJob) -> Result; -} - -#[derive(Parser, Debug, Clone)] -#[non_exhaustive] -pub struct BaseArgs { - /// the directory to put the generated main output file and associated files in - #[arg(short, long, value_hint = ValueHint::DirPath, required = true)] - pub output: Option, - /// the stem of the generated main output file, e.g. to get foo.v, pass --file-stem=foo - #[arg(long)] - pub file_stem: Option, - #[arg(long, env = "FAYALITE_KEEP_TEMP_DIR")] - pub keep_temp_dir: bool, - #[arg(skip = false)] - pub redirect_output_for_rust_test: bool, -} - -impl BaseArgs { - fn make_firrtl_file_backend(&self) -> Result<(firrtl::FileBackend, Option)> { - let (dir_path, temp_dir) = match &self.output { - Some(output) => (output.clone(), None), - None => { - let temp_dir = TempDir::new()?; - if self.keep_temp_dir { - let temp_dir = temp_dir.into_path(); - println!("created temporary directory: {}", temp_dir.display()); - (temp_dir, None) - } else { - (temp_dir.path().to_path_buf(), Some(temp_dir)) - } - } - }; - Ok(( - firrtl::FileBackend { - dir_path, - top_fir_file_stem: self.file_stem.clone(), - circuit_name: None, - }, - temp_dir, - )) - } - /// handles possibly redirecting the command's output for Rust tests - pub fn run_external_command( - &self, - _acquired_job: &mut AcquiredJob, - mut command: process::Command, - mut captured_output: Option<&mut String>, - ) -> io::Result { - if self.redirect_output_for_rust_test || captured_output.is_some() { - let (reader, writer) = io::pipe()?; - let mut reader = io::BufReader::new(reader); - command.stderr(writer.try_clone()?); - command.stdout(writer); // must not leave writer around after spawning child - command.stdin(process::Stdio::null()); - let mut child = command.spawn()?; - drop(command); // close writers - Ok(loop { - let status = child.try_wait()?; - streaming_read_utf8(&mut reader, |s| { - if let Some(captured_output) = captured_output.as_deref_mut() { - captured_output.push_str(s); - } - // use print! so output goes to Rust test output capture - print!("{s}"); - io::Result::Ok(()) - })?; - if let Some(status) = status { - break status; - } - }) - } else { - command.status() - } - } -} - -#[derive(Parser, Debug, Clone)] -#[non_exhaustive] -pub struct FirrtlArgs { - #[command(flatten)] - pub base: BaseArgs, - #[command(flatten)] - pub export_options: ExportOptions, -} - -#[derive(Debug)] -#[non_exhaustive] -pub struct FirrtlOutput { - pub file_stem: String, - pub top_module: String, - pub output_dir: PathBuf, - pub temp_dir: Option, -} - -impl FirrtlOutput { - pub fn file_with_ext(&self, ext: &str) -> PathBuf { - let mut retval = self.output_dir.join(&self.file_stem); - retval.set_extension(ext); - retval - } - pub fn firrtl_file(&self) -> PathBuf { - self.file_with_ext("fir") - } -} - -impl FirrtlArgs { - fn run_impl( - &self, - top_module: Module, - _acquired_job: &mut AcquiredJob, - ) -> Result { - let (file_backend, temp_dir) = self.base.make_firrtl_file_backend()?; - let firrtl::FileBackend { - top_fir_file_stem, - circuit_name, - dir_path, - } = firrtl::export(file_backend, &top_module, self.export_options)?; - Ok(FirrtlOutput { - file_stem: top_fir_file_stem.expect( - "export is known to set the file stem from the circuit name if not provided", - ), - top_module: circuit_name.expect("export is known to set the circuit name"), - output_dir: dir_path, - temp_dir, - }) - } -} - -impl RunPhase> for FirrtlArgs { - type Output = FirrtlOutput; - fn run_with_job( - &self, - top_module: Module, - acquired_job: &mut AcquiredJob, - ) -> Result { - self.run_impl(top_module.canonical(), acquired_job) - } -} - -impl RunPhase>> for FirrtlArgs { - type Output = FirrtlOutput; - fn run_with_job( - &self, - top_module: Interned>, - acquired_job: &mut AcquiredJob, - ) -> Result { - self.run_with_job(*top_module, acquired_job) - } -} - -/// based on [LLVM Circt's recommended lowering options -/// ](https://circt.llvm.org/docs/VerilogGeneration/#recommended-loweringoptions-by-target) -#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq, Hash)] -#[non_exhaustive] -pub enum VerilogDialect { - Questa, - Spyglass, - Verilator, - Vivado, - Yosys, -} - -impl fmt::Display for VerilogDialect { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -impl VerilogDialect { - pub fn as_str(self) -> &'static str { - match self { - VerilogDialect::Questa => "questa", - VerilogDialect::Spyglass => "spyglass", - VerilogDialect::Verilator => "verilator", - VerilogDialect::Vivado => "vivado", - VerilogDialect::Yosys => "yosys", - } - } - pub fn firtool_extra_args(self) -> &'static [&'static str] { - match self { - VerilogDialect::Questa => &["--lowering-options=emitWireInPorts"], - VerilogDialect::Spyglass => { - &["--lowering-options=explicitBitcast,disallowExpressionInliningInPorts"] - } - VerilogDialect::Verilator => &[ - "--lowering-options=locationInfoStyle=wrapInAtSquareBracket,disallowLocalVariables", - ], - VerilogDialect::Vivado => &["--lowering-options=mitigateVivadoArrayIndexConstPropBug"], - VerilogDialect::Yosys => { - &["--lowering-options=disallowLocalVariables,disallowPackedArrays"] - } - } - } -} - -#[derive(Parser, Debug, Clone)] -#[non_exhaustive] -pub struct VerilogArgs { - #[command(flatten)] - pub firrtl: FirrtlArgs, - #[arg( - long, - default_value = "firtool", - env = "FIRTOOL", - value_hint = ValueHint::CommandName, - value_parser = OsStringValueParser::new().try_map(which) - )] - pub firtool: PathBuf, - #[arg(long)] - pub firtool_extra_args: Vec, - /// adapt the generated Verilog for a particular toolchain - #[arg(long)] - pub verilog_dialect: Option, - #[arg(long, short = 'g')] - pub debug: bool, -} - -#[derive(Debug)] -#[non_exhaustive] -pub struct VerilogOutput { - pub firrtl: FirrtlOutput, - pub verilog_files: Vec, - pub contents_hash: Option, -} - -impl VerilogOutput { - pub fn main_verilog_file(&self) -> PathBuf { - self.firrtl.file_with_ext("v") - } - fn unadjusted_verilog_file(&self) -> PathBuf { - self.firrtl.file_with_ext("unadjusted.v") - } -} - -impl VerilogArgs { - fn process_unadjusted_verilog_file(&self, mut output: VerilogOutput) -> Result { - let input = fs::read_to_string(output.unadjusted_verilog_file())?; - let file_separator_prefix = "\n// ----- 8< ----- FILE \""; - let file_separator_suffix = "\" ----- 8< -----\n\n"; - let mut input = &*input; - output.contents_hash = Some(blake3::hash(input.as_bytes())); - let main_verilog_file = output.main_verilog_file(); - let mut file_name: Option<&Path> = Some(&main_verilog_file); - loop { - let (chunk, next_file_name) = if let Some((chunk, rest)) = - input.split_once(file_separator_prefix) - { - let Some((next_file_name, rest)) = rest.split_once(file_separator_suffix) else { - return Err(CliError(eyre!( - "parsing firtool's output failed: found {file_separator_prefix:?} but no {file_separator_suffix:?}" - ))); - }; - input = rest; - (chunk, Some(next_file_name.as_ref())) - } else { - (mem::take(&mut input), None) - }; - let Some(file_name) = mem::replace(&mut file_name, next_file_name) else { - break; - }; - let file_name = output.firrtl.output_dir.join(file_name); - fs::write(&file_name, chunk)?; - if let Some(extension) = file_name.extension() { - if extension == "v" || extension == "sv" { - output.verilog_files.push(file_name); - } - } - } - Ok(output) - } - fn run_impl( - &self, - firrtl_output: FirrtlOutput, - acquired_job: &mut AcquiredJob, - ) -> Result { - let Self { - firrtl, - firtool, - firtool_extra_args, - verilog_dialect, - debug, - } = self; - let output = VerilogOutput { - firrtl: firrtl_output, - verilog_files: vec![], - contents_hash: None, - }; - let mut cmd = process::Command::new(firtool); - cmd.arg(output.firrtl.firrtl_file()); - cmd.arg("-o"); - cmd.arg(output.unadjusted_verilog_file()); - if *debug { - cmd.arg("-g"); - cmd.arg("--preserve-values=all"); - } - if let Some(dialect) = verilog_dialect { - cmd.args(dialect.firtool_extra_args()); - } - cmd.args(firtool_extra_args); - cmd.current_dir(&output.firrtl.output_dir); - let status = firrtl.base.run_external_command(acquired_job, cmd, None)?; - if status.success() { - self.process_unadjusted_verilog_file(output) - } else { - Err(CliError(eyre!( - "running {} failed: {status}", - self.firtool.display() - ))) - } - } -} - -impl RunPhase for VerilogArgs -where - FirrtlArgs: RunPhase, -{ - type Output = VerilogOutput; - fn run_with_job(&self, arg: Arg, acquired_job: &mut AcquiredJob) -> Result { - let firrtl_output = self.firrtl.run_with_job(arg, acquired_job)?; - self.run_impl(firrtl_output, acquired_job) - } -} - -#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq, Hash, Default)] -#[non_exhaustive] -pub enum FormalMode { - #[default] - BMC, - Prove, - Live, - Cover, -} - -impl FormalMode { - pub fn as_str(self) -> &'static str { - match self { - FormalMode::BMC => "bmc", - FormalMode::Prove => "prove", - FormalMode::Live => "live", - FormalMode::Cover => "cover", - } - } -} - -impl fmt::Display for FormalMode { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -#[derive(Clone)] -struct FormalAdjustArgs; - -impl clap::FromArgMatches for FormalAdjustArgs { - fn from_arg_matches(_matches: &clap::ArgMatches) -> Result { - Ok(Self) - } - - fn update_from_arg_matches(&mut self, _matches: &clap::ArgMatches) -> Result<(), clap::Error> { - Ok(()) - } -} - -impl clap::Args for FormalAdjustArgs { - fn augment_args(cmd: clap::Command) -> clap::Command { - cmd.mut_arg("output", |arg| arg.required(false)) - .mut_arg("verilog_dialect", |arg| { - arg.default_value(VerilogDialect::Yosys.to_string()) - .hide(true) - }) - } - - fn augment_args_for_update(cmd: clap::Command) -> clap::Command { - Self::augment_args(cmd) - } -} - -fn which(v: std::ffi::OsString) -> which::Result { - #[cfg(not(miri))] - return which::which(v); - #[cfg(miri)] - return Ok(Path::new("/").join(v)); -} - -#[derive(Parser, Clone)] -#[non_exhaustive] -pub struct FormalArgs { - #[command(flatten)] - pub verilog: VerilogArgs, - #[arg( - long, - default_value = "sby", - env = "SBY", - value_hint = ValueHint::CommandName, - value_parser = OsStringValueParser::new().try_map(which) - )] - pub sby: PathBuf, - #[arg(long)] - pub sby_extra_args: Vec, - #[arg(long, default_value_t)] - pub mode: FormalMode, - #[arg(long, default_value_t = Self::DEFAULT_DEPTH)] - pub depth: u64, - #[arg(long, default_value = "z3")] - pub solver: String, - #[arg(long)] - pub smtbmc_extra_args: Vec, - #[arg(long, default_value_t = true, env = "FAYALITE_CACHE_RESULTS")] - pub cache_results: bool, - #[command(flatten)] - _formal_adjust_args: FormalAdjustArgs, -} - -impl fmt::Debug for FormalArgs { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Self { - verilog, - sby, - sby_extra_args, - mode, - depth, - solver, - smtbmc_extra_args, - cache_results, - _formal_adjust_args: _, - } = self; - f.debug_struct("FormalArgs") - .field("verilog", verilog) - .field("sby", sby) - .field("sby_extra_args", sby_extra_args) - .field("mode", mode) - .field("depth", depth) - .field("solver", solver) - .field("smtbmc_extra_args", smtbmc_extra_args) - .field("cache_results", cache_results) - .finish_non_exhaustive() - } -} - -impl FormalArgs { - pub const DEFAULT_DEPTH: u64 = 20; -} - -#[derive(Debug)] -#[non_exhaustive] -pub struct FormalOutput { - pub verilog: VerilogOutput, -} - -impl FormalOutput { - pub fn sby_file(&self) -> PathBuf { - self.verilog.firrtl.file_with_ext("sby") - } - pub fn cache_file(&self) -> PathBuf { - self.verilog.firrtl.file_with_ext("cache.json") - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[non_exhaustive] -pub struct FormalCacheOutput {} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[non_exhaustive] -pub enum FormalCacheVersion { - V1, -} - -impl FormalCacheVersion { - pub const CURRENT: Self = Self::V1; -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[non_exhaustive] -pub struct FormalCache { - pub version: FormalCacheVersion, - pub contents_hash: blake3::Hash, - pub stdout_stderr: String, - pub result: Result, -} - -impl FormalCache { - pub fn new( - version: FormalCacheVersion, - contents_hash: blake3::Hash, - stdout_stderr: String, - result: Result, - ) -> Self { - Self { - version, - contents_hash, - stdout_stderr, - result, - } - } -} - -impl FormalArgs { - fn sby_contents(&self, output: &FormalOutput) -> Result { - let Self { - verilog: _, - sby: _, - sby_extra_args: _, - mode, - depth, - smtbmc_extra_args, - solver, - cache_results: _, - _formal_adjust_args: _, - } = self; - let smtbmc_options = smtbmc_extra_args.join(" "); - let top_module = &output.verilog.firrtl.top_module; - let mut retval = format!( - "[options]\n\ - mode {mode}\n\ - depth {depth}\n\ - wait on\n\ - \n\ - [engines]\n\ - smtbmc {solver} -- -- {smtbmc_options}\n\ - \n\ - [script]\n" - ); - for verilog_file in &output.verilog.verilog_files { - let verilog_file = verilog_file - .to_str() - .ok_or_else(|| CliError(eyre!("verilog file path is not UTF-8")))?; - if verilog_file.contains(|ch: char| { - (ch != ' ' && ch != '\t' && ch.is_ascii_whitespace()) || ch == '"' - }) { - return Err(CliError(eyre!( - "verilog file path contains characters that aren't permitted" - ))); - } - writeln!(retval, "read_verilog -sv -formal \"{verilog_file}\"").unwrap(); - } - // workaround for wires disappearing -- set `keep` on all wires - writeln!(retval, "hierarchy -top {top_module}").unwrap(); - writeln!(retval, "proc").unwrap(); - writeln!(retval, "setattr -set keep 1 w:\\*").unwrap(); - writeln!(retval, "prep").unwrap(); - Ok(retval) - } - fn run_impl( - &self, - verilog_output: VerilogOutput, - acquired_job: &mut AcquiredJob, - ) -> Result { - let output = FormalOutput { - verilog: verilog_output, - }; - let sby_file = output.sby_file(); - let sby_contents = self.sby_contents(&output)?; - let contents_hash = output.verilog.contents_hash.map(|verilog_hash| { - let mut hasher = blake3::Hasher::new(); - hasher.update(verilog_hash.as_bytes()); - hasher.update(sby_contents.as_bytes()); - hasher.update(&(self.sby_extra_args.len() as u64).to_le_bytes()); - for sby_extra_arg in self.sby_extra_args.iter() { - hasher.update(&(sby_extra_arg.len() as u64).to_le_bytes()); - hasher.update(sby_extra_arg.as_bytes()); - } - hasher.finalize() - }); - std::fs::write(&sby_file, sby_contents)?; - let mut cmd = process::Command::new(&self.sby); - cmd.arg("-j1"); // sby seems not to respect job count in parallel mode - cmd.arg("-f"); - cmd.arg(sby_file.file_name().unwrap()); - cmd.args(&self.sby_extra_args); - cmd.current_dir(&output.verilog.firrtl.output_dir); - let mut captured_output = String::new(); - let cache_file = output.cache_file(); - let do_cache = if let Some(contents_hash) = contents_hash.filter(|_| self.cache_results) { - if let Some(FormalCache { - version: FormalCacheVersion::CURRENT, - contents_hash: cache_contents_hash, - stdout_stderr, - result, - }) = fs::read(&cache_file) - .ok() - .and_then(|v| serde_json::from_slice(&v).ok()) - { - if cache_contents_hash == contents_hash { - println!("Using cached formal result:\n{stdout_stderr}"); - return match result { - Ok(FormalCacheOutput {}) => Ok(output), - Err(error) => Err(CliError(eyre::Report::msg(error))), - }; - } - } - true - } else { - false - }; - let _ = fs::remove_file(&cache_file); - let status = self.verilog.firrtl.base.run_external_command( - acquired_job, - cmd, - do_cache.then_some(&mut captured_output), - )?; - let result = if status.success() { - Ok(output) - } else { - Err(CliError(eyre!( - "running {} failed: {status}", - self.sby.display() - ))) - }; - fs::write( - cache_file, - serde_json::to_string_pretty(&FormalCache { - version: FormalCacheVersion::CURRENT, - contents_hash: contents_hash.unwrap(), - stdout_stderr: captured_output, - result: match &result { - Ok(FormalOutput { verilog: _ }) => Ok(FormalCacheOutput {}), - Err(error) => Err(error.to_string()), - }, - }) - .expect("serialization shouldn't ever fail"), - )?; - result - } -} - -impl RunPhase for FormalArgs -where - VerilogArgs: RunPhase, -{ - type Output = FormalOutput; - fn run_with_job(&self, arg: Arg, acquired_job: &mut AcquiredJob) -> Result { - let verilog_output = self.verilog.run_with_job(arg, acquired_job)?; - self.run_impl(verilog_output, acquired_job) - } -} - -#[derive(Subcommand, Debug)] -enum CliCommand { - /// Generate FIRRTL - Firrtl(FirrtlArgs), - /// Generate Verilog - Verilog(VerilogArgs), - /// Run a formal proof - Formal(FormalArgs), -} - -/// a simple CLI -/// -/// Use like: -/// -/// ```no_run -/// # use fayalite::prelude::*; -/// # #[hdl_module] -/// # fn my_module() {} -/// use fayalite::cli; -/// -/// fn main() -> cli::Result { -/// cli::Cli::parse().run(my_module()) -/// } -/// ``` -/// -/// You can also use it with a larger [`clap`]-based CLI like so: -/// -/// ```no_run -/// # use fayalite::prelude::*; -/// # #[hdl_module] -/// # fn my_module() {} -/// use clap::{Subcommand, Parser}; -/// use fayalite::cli; -/// -/// #[derive(Subcommand)] -/// pub enum Cmd { -/// #[command(flatten)] -/// Fayalite(cli::Cli), -/// MySpecialCommand { -/// #[arg(long)] -/// foo: bool, -/// }, -/// } -/// -/// #[derive(Parser)] -/// pub struct Cli { -/// #[command(subcommand)] -/// cmd: Cmd, // or just use cli::Cli directly if you don't need more subcommands -/// } -/// -/// fn main() -> cli::Result { -/// match Cli::parse().cmd { -/// Cmd::Fayalite(v) => v.run(my_module())?, -/// Cmd::MySpecialCommand { foo } => println!("special: foo={foo}"), -/// } -/// Ok(()) -/// } -/// ``` -#[derive(Parser, Debug)] -// clear things that would be crate-specific -#[command(name = "Fayalite Simple CLI", about = None, long_about = None)] -pub struct Cli { - #[command(subcommand)] - subcommand: CliCommand, -} - -impl clap::Subcommand for Cli { - fn augment_subcommands(cmd: clap::Command) -> clap::Command { - CliCommand::augment_subcommands(cmd) - } - - fn augment_subcommands_for_update(cmd: clap::Command) -> clap::Command { - CliCommand::augment_subcommands_for_update(cmd) - } - - fn has_subcommand(name: &str) -> bool { - CliCommand::has_subcommand(name) - } -} - -impl RunPhase for Cli -where - FirrtlArgs: RunPhase, -{ - type Output = (); - fn run_with_job(&self, arg: T, acquired_job: &mut AcquiredJob) -> Result { - match &self.subcommand { - CliCommand::Firrtl(c) => { - c.run_with_job(arg, acquired_job)?; - } - CliCommand::Verilog(c) => { - c.run_with_job(arg, acquired_job)?; - } - CliCommand::Formal(c) => { - c.run_with_job(arg, acquired_job)?; - } - } - Ok(()) - } -} - -impl Cli { - /// forwards to [`clap::Parser::parse()`] so you don't have to import [`clap::Parser`] - pub fn parse() -> Self { - clap::Parser::parse() - } - /// forwards to [`RunPhase::run()`] so you don't have to import [`RunPhase`] - pub fn run(&self, top_module: T) -> Result<()> - where - Self: RunPhase, - { - RunPhase::run(self, top_module) - } -} diff --git a/crates/fayalite/src/firrtl.rs b/crates/fayalite/src/firrtl.rs index 5fa6644..d13ecc2 100644 --- a/crates/fayalite/src/firrtl.rs +++ b/crates/fayalite/src/firrtl.rs @@ -7,6 +7,7 @@ use crate::{ DocStringAnnotation, DontTouchAnnotation, SVAttributeAnnotation, }, array::Array, + build::{ToArgs, WriteArgs}, bundle::{Bundle, BundleField, BundleType}, clock::Clock, enum_::{Enum, EnumType, EnumVariant}, @@ -23,7 +24,7 @@ use crate::{ memory::{Mem, PortKind, PortName, ReadUnderWrite}, module::{ AnnotatedModuleIO, Block, ExternModuleBody, ExternModuleParameter, - ExternModuleParameterValue, Module, ModuleBody, NameOptId, NormalModuleBody, Stmt, + ExternModuleParameterValue, Module, ModuleBody, NameId, NameOptId, NormalModuleBody, Stmt, StmtConnect, StmtDeclaration, StmtFormal, StmtIf, StmtInstance, StmtMatch, StmtReg, StmtWire, transform::{ @@ -42,7 +43,7 @@ use crate::{ use bitvec::slice::BitSlice; use clap::value_parser; use num_traits::Signed; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::{ cell::{Cell, RefCell}, cmp::Ordering, @@ -2749,14 +2750,23 @@ impl clap::builder::TypedValueParser for OptionSimplifyEnumsKindValueParser { #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct ExportOptionsPrivate(()); -#[derive(clap::Parser, Copy, Clone, PartialEq, Eq, Hash)] +impl ExportOptionsPrivate { + fn private_new() -> Self { + Self(()) + } +} + +#[derive(clap::Parser, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct ExportOptions { #[clap(long = "no-simplify-memories", action = clap::ArgAction::SetFalse)] + #[serde(default = "ExportOptions::default_simplify_memories")] pub simplify_memories: bool, #[clap(long, value_parser = OptionSimplifyEnumsKindValueParser, default_value = "replace-with-bundle-of-uints")] + #[serde(default = "ExportOptions::default_simplify_enums")] pub simplify_enums: std::option::Option, // use std::option::Option instead of Option to avoid clap mis-parsing #[doc(hidden)] #[clap(skip = ExportOptionsPrivate(()))] + #[serde(skip, default = "ExportOptionsPrivate::private_new")] /// `#[non_exhaustive]` except allowing struct update syntax pub __private: ExportOptionsPrivate, } @@ -2767,16 +2777,15 @@ impl fmt::Debug for ExportOptions { } } -impl ExportOptions { - pub fn to_args(&self) -> Vec> { +impl ToArgs for ExportOptions { + fn to_args(&self, args: &mut (impl WriteArgs + ?Sized)) { let Self { simplify_memories, simplify_enums, __private: ExportOptionsPrivate(()), - } = self; - let mut retval = Vec::new(); - if !*simplify_memories { - retval.push("--no-simplify-memories".intern()); + } = *self; + if !simplify_memories { + args.write_str_arg("--no-simplify-memories"); } let simplify_enums = simplify_enums.map(|v| { clap::ValueEnum::to_possible_value(&v).expect("there are no skipped variants") @@ -2785,10 +2794,16 @@ impl ExportOptions { None => OptionSimplifyEnumsKindValueParser::NONE_NAME, Some(v) => v.get_name(), }; - retval.push(str::intern_owned(format!( - "--simplify-enums={simplify_enums}" - ))); - retval + args.write_arg(format_args!("--simplify-enums={simplify_enums}")); + } +} + +impl ExportOptions { + fn default_simplify_memories() -> bool { + true + } + fn default_simplify_enums() -> Option { + Some(SimplifyEnumsKind::ReplaceWithBundleOfUInts) } fn debug_fmt( &self, @@ -2846,13 +2861,19 @@ impl ExportOptions { impl Default for ExportOptions { fn default() -> Self { Self { - simplify_memories: true, - simplify_enums: Some(SimplifyEnumsKind::ReplaceWithBundleOfUInts), + simplify_memories: Self::default_simplify_memories(), + simplify_enums: Self::default_simplify_enums(), __private: ExportOptionsPrivate(()), } } } +pub fn get_circuit_name(top_module_name_id: NameId) -> Interned { + let mut global_ns = Namespace::default(); + let circuit_name = global_ns.get(top_module_name_id); + circuit_name.0 +} + pub fn export( file_backend: B, top_module: &Module, diff --git a/crates/fayalite/src/intern.rs b/crates/fayalite/src/intern.rs index af91f0a..9d57146 100644 --- a/crates/fayalite/src/intern.rs +++ b/crates/fayalite/src/intern.rs @@ -9,11 +9,13 @@ use std::{ any::{Any, TypeId}, borrow::{Borrow, Cow}, cmp::Ordering, + ffi::OsStr, fmt, hash::{BuildHasher, Hash, Hasher}, iter::FusedIterator, marker::PhantomData, ops::Deref, + path::Path, sync::{Mutex, RwLock}, }; @@ -416,6 +418,12 @@ forward_fmt_trait!(Pointer); forward_fmt_trait!(UpperExp); forward_fmt_trait!(UpperHex); +impl AsRef for Interned { + fn as_ref(&self) -> &T { + self + } +} + #[derive(Clone, Debug)] pub struct InternedSliceIter { slice: Interned<[T]>, @@ -485,6 +493,51 @@ where } } +impl FromIterator for Interned +where + String: FromIterator, +{ + fn from_iter>(iter: T) -> Self { + str::intern_owned(FromIterator::from_iter(iter)) + } +} + +impl AsRef for Interned { + fn as_ref(&self) -> &OsStr { + str::as_ref(self) + } +} + +impl AsRef for Interned { + fn as_ref(&self) -> &Path { + str::as_ref(self) + } +} + +impl From> for clap::builder::Str { + fn from(value: Interned) -> Self { + Interned::into_inner(value).into() + } +} + +impl From> for clap::builder::OsStr { + fn from(value: Interned) -> Self { + Interned::into_inner(value).into() + } +} + +impl From> for clap::builder::StyledStr { + fn from(value: Interned) -> Self { + Interned::into_inner(value).into() + } +} + +impl From> for clap::Id { + fn from(value: Interned) -> Self { + Interned::into_inner(value).into() + } +} + impl From> for Vec { fn from(value: Interned<[T]>) -> Self { Vec::from(&*value) diff --git a/crates/fayalite/src/lib.rs b/crates/fayalite/src/lib.rs index 3a67c5c..e5323f3 100644 --- a/crates/fayalite/src/lib.rs +++ b/crates/fayalite/src/lib.rs @@ -89,7 +89,6 @@ pub mod annotations; pub mod array; pub mod build; pub mod bundle; -pub mod cli; pub mod clock; pub mod enum_; pub mod expr; diff --git a/crates/fayalite/src/module.rs b/crates/fayalite/src/module.rs index a81893d..6757597 100644 --- a/crates/fayalite/src/module.rs +++ b/crates/fayalite/src/module.rs @@ -1212,6 +1212,12 @@ pub struct Module { module_annotations: Interned<[Annotation]>, } +impl AsRef for Module { + fn as_ref(&self) -> &Self { + self + } +} + #[derive(Default)] struct DebugFmtModulesState { seen: HashSet, diff --git a/crates/fayalite/src/module/transform/simplify_enums.rs b/crates/fayalite/src/module/transform/simplify_enums.rs index ccdecf6..0839881 100644 --- a/crates/fayalite/src/module/transform/simplify_enums.rs +++ b/crates/fayalite/src/module/transform/simplify_enums.rs @@ -22,6 +22,7 @@ use crate::{ wire::Wire, }; use core::fmt; +use serde::{Deserialize, Serialize}; #[derive(Debug)] pub enum SimplifyEnumsError { @@ -955,12 +956,15 @@ impl Folder for State { } } -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, clap::ValueEnum)] +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, clap::ValueEnum, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] pub enum SimplifyEnumsKind { SimplifyToEnumsWithNoBody, #[clap(name = "replace-with-bundle-of-uints")] + #[serde(rename = "replace-with-bundle-of-uints")] ReplaceWithBundleOfUInts, #[clap(name = "replace-with-uint")] + #[serde(rename = "replace-with-uint")] ReplaceWithUInt, } diff --git a/crates/fayalite/src/prelude.rs b/crates/fayalite/src/prelude.rs index 4fca2ad..5ff3a64 100644 --- a/crates/fayalite/src/prelude.rs +++ b/crates/fayalite/src/prelude.rs @@ -7,8 +7,8 @@ pub use crate::{ DocStringAnnotation, DontTouchAnnotation, SVAttributeAnnotation, }, array::{Array, ArrayType}, + build::{BuildCli, JobParams, RunBuild}, bundle::Bundle, - cli::Cli, clock::{Clock, ClockDomain, ToClock}, enum_::{Enum, HdlNone, HdlOption, HdlSome}, expr::{ @@ -36,6 +36,7 @@ pub use crate::{ value::{SimOnly, SimOnlyValue, SimValue, ToSimValue, ToSimValueWithType}, }, source_location::SourceLocation, + testing::assert_formal, ty::{AsMask, CanonicalType, Type}, util::{ConstUsize, GenericConstUsize}, wire::Wire, diff --git a/crates/fayalite/src/testing.rs b/crates/fayalite/src/testing.rs index b81bc3f..e7465bf 100644 --- a/crates/fayalite/src/testing.rs +++ b/crates/fayalite/src/testing.rs @@ -1,11 +1,20 @@ // SPDX-License-Identifier: LGPL-3.0-or-later // See Notices.txt for copyright information use crate::{ - cli::{FormalArgs, FormalMode, FormalOutput, RunPhase}, + build::{ + BaseJobArgs, BaseJobKind, JobArgsAndDependencies, JobKindAndArgs, JobParams, NoArgs, + RunBuild, + external::{ExternalCommandArgs, ExternalCommandJobKind}, + firrtl::{FirrtlArgs, FirrtlJobKind}, + formal::{Formal, FormalAdditionalArgs, FormalArgs, FormalMode, WriteSbyFileJobKind}, + verilog::{UnadjustedVerilogArgs, VerilogJobArgs, VerilogJobKind}, + }, + bundle::BundleType, firrtl::ExportOptions, + module::Module, util::HashMap, }; -use clap::Parser; +use eyre::eyre; use serde::Deserialize; use std::{ fmt::Write, @@ -14,14 +23,6 @@ use std::{ sync::{Mutex, OnceLock}, }; -fn assert_formal_helper() -> FormalArgs { - static FORMAL_ARGS: OnceLock = OnceLock::new(); - // ensure we only run parsing once, so errors from env vars don't produce overlapping output if we're called on multiple threads - FORMAL_ARGS - .get_or_init(|| FormalArgs::parse_from(["fayalite::testing::assert_formal"])) - .clone() -} - #[derive(Deserialize)] struct CargoMetadata { target_directory: String, @@ -97,26 +98,104 @@ fn get_assert_formal_target_path(test_name: &dyn std::fmt::Display) -> PathBuf { .join(dir) } -#[track_caller] -pub fn assert_formal( - test_name: impl std::fmt::Display, - module: M, - mode: FormalMode, - depth: u64, +fn make_assert_formal_args( + test_name: &dyn std::fmt::Display, + formal_mode: FormalMode, + formal_depth: u64, solver: Option<&str>, export_options: ExportOptions, -) where - FormalArgs: RunPhase, -{ - let mut args = assert_formal_helper(); - args.verilog.firrtl.base.redirect_output_for_rust_test = true; - args.verilog.firrtl.base.output = Some(get_assert_formal_target_path(&test_name)); - args.verilog.firrtl.export_options = export_options; - args.verilog.debug = true; - args.mode = mode; - args.depth = depth; - if let Some(solver) = solver { - args.solver = solver.into(); - } - args.run(module).expect("testing::assert_formal() failed"); +) -> eyre::Result>> { + let args = JobKindAndArgs { + kind: BaseJobKind, + args: BaseJobArgs::from_output_dir_and_env( + get_assert_formal_target_path(&test_name) + .into_os_string() + .into_string() + .map_err(|_| eyre!("path is not valid UTF-8"))?, + ), + }; + let dependencies = JobArgsAndDependencies { + args, + dependencies: (), + }; + let args = JobKindAndArgs { + kind: FirrtlJobKind, + args: FirrtlArgs { export_options }, + }; + let dependencies = JobArgsAndDependencies { args, dependencies }; + let args = JobKindAndArgs { + kind: ExternalCommandJobKind::new(), + args: ExternalCommandArgs::new( + None, + UnadjustedVerilogArgs { + firtool_extra_args: vec![], + verilog_dialect: None, + verilog_debug: true, + }, + )?, + }; + let dependencies = JobArgsAndDependencies { args, dependencies }; + let args = JobKindAndArgs { + kind: VerilogJobKind, + args: VerilogJobArgs {}, + }; + let dependencies = JobArgsAndDependencies { args, dependencies }; + let args = JobKindAndArgs { + kind: WriteSbyFileJobKind, + args: FormalArgs { + sby_extra_args: vec![], + formal_mode, + formal_depth, + formal_solver: solver.unwrap_or(FormalArgs::DEFAULT_SOLVER).into(), + smtbmc_extra_args: vec![], + }, + }; + let dependencies = JobArgsAndDependencies { args, dependencies }; + let args = JobKindAndArgs { + kind: ExternalCommandJobKind::new(), + args: ExternalCommandArgs::new(None, FormalAdditionalArgs {})?, + }; + Ok(JobArgsAndDependencies { args, dependencies }) +} + +pub fn try_assert_formal>, T: BundleType>( + test_name: impl std::fmt::Display, + module: M, + formal_mode: FormalMode, + formal_depth: u64, + solver: Option<&str>, + export_options: ExportOptions, +) -> eyre::Result<()> { + const APP_NAME: &'static str = "fayalite::testing::assert_formal"; + make_assert_formal_args( + &test_name, + formal_mode, + formal_depth, + solver, + export_options, + )? + .run( + |NoArgs {}| Ok(JobParams::new(module, APP_NAME)), + clap::Command::new(APP_NAME), // not actually used, so we can use an arbitrary value + ) +} + +#[track_caller] +pub fn assert_formal>, T: BundleType>( + test_name: impl std::fmt::Display, + module: M, + formal_mode: FormalMode, + formal_depth: u64, + solver: Option<&str>, + export_options: ExportOptions, +) { + try_assert_formal( + test_name, + module, + formal_mode, + formal_depth, + solver, + export_options, + ) + .expect("testing::assert_formal() failed"); } diff --git a/crates/fayalite/src/util.rs b/crates/fayalite/src/util.rs index e85bc9c..23a6852 100644 --- a/crates/fayalite/src/util.rs +++ b/crates/fayalite/src/util.rs @@ -36,8 +36,11 @@ pub use scoped_ref::ScopedRef; pub(crate) use misc::chain; #[doc(inline)] pub use misc::{ - BitSliceWriteWithBase, DebugAsDisplay, DebugAsRawString, MakeMutSlice, RcWriter, interned_bit, - iter_eq_by, slice_range, try_slice_range, + BitSliceWriteWithBase, DebugAsDisplay, DebugAsRawString, MakeMutSlice, RcWriter, + SerdeJsonEscapeIf, SerdeJsonEscapeIfFormatter, SerdeJsonEscapeIfTest, + SerdeJsonEscapeIfTestResult, interned_bit, iter_eq_by, serialize_to_json_ascii, + serialize_to_json_ascii_pretty, serialize_to_json_ascii_pretty_with_indent, slice_range, + try_slice_range, }; pub mod job_server; diff --git a/crates/fayalite/src/util/job_server.rs b/crates/fayalite/src/util/job_server.rs index 376ddc0..e58bba8 100644 --- a/crates/fayalite/src/util/job_server.rs +++ b/crates/fayalite/src/util/job_server.rs @@ -1,192 +1,156 @@ // SPDX-License-Identifier: LGPL-3.0-or-later // See Notices.txt for copyright information -use ctor::ctor; -use jobslot::{Acquired, Client}; +use ctor::{ctor, dtor}; +use jobslot::Client; use std::{ ffi::OsString, - mem, + io, mem, num::NonZeroUsize, - sync::{Condvar, Mutex, Once, OnceLock}, - thread::spawn, + sync::{Mutex, MutexGuard}, }; -fn get_or_make_client() -> &'static Client { - #[ctor] - static CLIENT: OnceLock = unsafe { - match Client::from_env() { - Some(client) => OnceLock::from(client), - None => OnceLock::new(), - } - }; +#[ctor] +static CLIENT: Mutex>> = unsafe { Mutex::new(Some(Client::from_env())) }; - CLIENT.get_or_init(|| { - let mut available_parallelism = None; - let mut args = std::env::args_os().skip(1); - while let Some(arg) = args.next() { - const TEST_THREADS_OPTION: &'static [u8] = b"--test-threads"; - if arg.as_encoded_bytes().starts_with(TEST_THREADS_OPTION) { - match arg.as_encoded_bytes().get(TEST_THREADS_OPTION.len()) { - Some(b'=') => { - let mut arg = arg.into_encoded_bytes(); - arg.drain(..=TEST_THREADS_OPTION.len()); - available_parallelism = Some(arg); - break; +#[dtor] +fn drop_client() { + drop( + match CLIENT.lock() { + Ok(v) => v, + Err(e) => e.into_inner(), + } + .take(), + ); +} + +fn get_or_make_client() -> Client { + CLIENT + .lock() + .expect("shouldn't have panicked") + .as_mut() + .expect("shutting down") + .get_or_insert_with(|| { + let mut available_parallelism = None; + let mut args = std::env::args_os().skip(1); + while let Some(arg) = args.next() { + const TEST_THREADS_OPTION: &'static [u8] = b"--test-threads"; + if arg.as_encoded_bytes().starts_with(TEST_THREADS_OPTION) { + match arg.as_encoded_bytes().get(TEST_THREADS_OPTION.len()) { + Some(b'=') => { + let mut arg = arg.into_encoded_bytes(); + arg.drain(..=TEST_THREADS_OPTION.len()); + available_parallelism = Some(arg); + break; + } + None => { + available_parallelism = args.next().map(OsString::into_encoded_bytes); + break; + } + _ => {} } - None => { - available_parallelism = args.next().map(OsString::into_encoded_bytes); - break; - } - _ => {} } } - } - let available_parallelism = if let Some(available_parallelism) = available_parallelism - .as_deref() - .and_then(|v| std::str::from_utf8(v).ok()) - .and_then(|v| v.parse().ok()) - { - available_parallelism - } else if let Ok(available_parallelism) = std::thread::available_parallelism() { - available_parallelism - } else { - NonZeroUsize::new(1).unwrap() - }; - Client::new_with_fifo(available_parallelism.get() - 1).expect("failed to create job server") - }) + let available_parallelism = if let Some(available_parallelism) = available_parallelism + .as_deref() + .and_then(|v| std::str::from_utf8(v).ok()) + .and_then(|v| v.parse().ok()) + { + available_parallelism + } else if let Ok(available_parallelism) = std::thread::available_parallelism() { + available_parallelism + } else { + NonZeroUsize::new(1).unwrap() + }; + Client::new_with_fifo(available_parallelism.get() - 1) + .expect("failed to create job server") + }) + .clone() } struct State { + obtained_count: usize, waiting_count: usize, - available: Vec, - implicit_available: bool, -} - -impl State { - fn total_available(&self) -> usize { - self.available.len() + self.implicit_available as usize - } - fn additional_waiting(&self) -> usize { - self.waiting_count.saturating_sub(self.total_available()) - } } static STATE: Mutex = Mutex::new(State { + obtained_count: 0, waiting_count: 0, - available: Vec::new(), - implicit_available: true, }); -static COND_VAR: Condvar = Condvar::new(); - -#[derive(Debug)] -enum AcquiredJobInner { - FromJobServer(Acquired), - ImplicitJob, -} #[derive(Debug)] pub struct AcquiredJob { - job: AcquiredJobInner, + client: Client, } impl AcquiredJob { - fn start_acquire_thread() { - static STARTED_THREAD: Once = Once::new(); - STARTED_THREAD.call_once(|| { - spawn(|| { - let mut acquired = None; - let client = get_or_make_client(); + pub fn acquire() -> io::Result { + let client = get_or_make_client(); + struct Waiting {} + + impl Waiting { + fn done(self) -> MutexGuard<'static, State> { + mem::forget(self); let mut state = STATE.lock().unwrap(); - loop { - state = if state.additional_waiting() == 0 { - if acquired.is_some() { - drop(state); - drop(acquired.take()); // drop Acquired outside of lock - STATE.lock().unwrap() - } else { - COND_VAR.wait(state).unwrap() - } - } else if acquired.is_some() { - // allocate space before moving Acquired to ensure we - // drop Acquired outside of the lock on panic - state.available.reserve(1); - state.available.push(acquired.take().unwrap()); - COND_VAR.notify_all(); - state - } else { - drop(state); - acquired = Some( - client - .acquire() - .expect("can't acquire token from job server"), - ); - STATE.lock().unwrap() - }; - } - }); - }); - } - fn acquire_inner(block: bool) -> Option { - Self::start_acquire_thread(); - let mut state = STATE.lock().unwrap(); - loop { - if let Some(acquired) = state.available.pop() { - return Some(Self { - job: AcquiredJobInner::FromJobServer(acquired), - }); + state.waiting_count -= 1; + state } - if state.implicit_available { - state.implicit_available = false; - return Some(Self { - job: AcquiredJobInner::ImplicitJob, - }); - } - if !block { - return None; - } - state.waiting_count += 1; - state = COND_VAR.wait(state).unwrap(); - state.waiting_count -= 1; } - } - pub fn try_acquire() -> Option { - Self::acquire_inner(false) - } - pub fn acquire() -> Self { - Self::acquire_inner(true).expect("failed to acquire token") + impl Drop for Waiting { + fn drop(&mut self) { + STATE.lock().unwrap().waiting_count -= 1; + } + } + let mut state = STATE.lock().unwrap(); + if state.obtained_count == 0 && state.waiting_count == 0 { + state.obtained_count = 1; // get implicit token + return Ok(Self { client }); + } + state.waiting_count += 1; + drop(state); + let waiting = Waiting {}; + client.acquire_raw()?; + state = waiting.done(); + state.obtained_count = state + .obtained_count + .checked_add(1) + .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "obtained_count overflowed"))?; + drop(state); + Ok(Self { client }) } pub fn run_command( &mut self, cmd: std::process::Command, f: impl FnOnce(&mut std::process::Command) -> std::io::Result, ) -> std::io::Result { - get_or_make_client().configure_make_and_run_with_fifo(cmd, f) + self.client.configure_make_and_run_with_fifo(cmd, f) } } impl Drop for AcquiredJob { fn drop(&mut self) { let mut state = STATE.lock().unwrap(); - match &self.job { - AcquiredJobInner::FromJobServer(_) => { - if state.waiting_count > state.available.len() + state.implicit_available as usize { - // allocate space before moving Acquired to ensure we - // drop Acquired outside of the lock on panic - state.available.reserve(1); - let AcquiredJobInner::FromJobServer(acquired) = - mem::replace(&mut self.job, AcquiredJobInner::ImplicitJob) - else { - unreachable!() - }; - state.available.push(acquired); - COND_VAR.notify_all(); + match &mut *state { + State { + obtained_count: 0, .. + } => unreachable!(), + State { + obtained_count: obtained_count @ 1, + waiting_count, + } => { + *obtained_count = 0; // drop implicit token + let any_waiting = *waiting_count != 0; + drop(state); + if any_waiting { + // we have the implicit token, but some other thread is trying to acquire a token, + // release the implicit token so they can acquire it. + let _ = self.client.release_raw(); // we're in drop, just ignore errors since we at least tried } } - AcquiredJobInner::ImplicitJob => { - state.implicit_available = true; - if state.waiting_count > state.available.len() { - COND_VAR.notify_all(); - } + State { obtained_count, .. } => { + *obtained_count = obtained_count.saturating_sub(1); + drop(state); + let _ = self.client.release_raw(); // we're in drop, just ignore errors since we at least tried } } } diff --git a/crates/fayalite/src/util/misc.rs b/crates/fayalite/src/util/misc.rs index cebbceb..6c9acee 100644 --- a/crates/fayalite/src/util/misc.rs +++ b/crates/fayalite/src/util/misc.rs @@ -5,6 +5,7 @@ use bitvec::{bits, order::Lsb0, slice::BitSlice, view::BitView}; use std::{ cell::Cell, fmt::{self, Debug, Write}, + io, ops::{Bound, Range, RangeBounds}, rc::Rc, sync::{Arc, OnceLock}, @@ -243,3 +244,323 @@ pub fn try_slice_range>(range: R, size: usize) -> Option>(range: R, size: usize) -> Range { try_slice_range(range, size).expect("range out of bounds") } + +pub trait SerdeJsonEscapeIfTest { + fn char_needs_escape(&mut self, ch: char) -> serde_json::Result; +} + +pub trait SerdeJsonEscapeIfTestResult { + fn to_result(self) -> serde_json::Result; +} + +impl SerdeJsonEscapeIfTestResult for bool { + fn to_result(self) -> serde_json::Result { + Ok(self) + } +} + +impl> SerdeJsonEscapeIfTestResult for Result { + fn to_result(self) -> serde_json::Result { + self.map_err(Into::into) + } +} + +impl R, R: SerdeJsonEscapeIfTestResult> SerdeJsonEscapeIfTest for T { + fn char_needs_escape(&mut self, ch: char) -> serde_json::Result { + self(ch).to_result() + } +} + +pub trait SerdeJsonEscapeIfFormatter: serde_json::ser::Formatter { + fn write_unicode_escape(&mut self, writer: &mut W, ch: char) -> io::Result<()> + where + W: ?Sized + io::Write, + { + for utf16 in ch.encode_utf16(&mut [0; 2]) { + write!(writer, "\\u{utf16:04x}")?; + } + Ok(()) + } +} + +impl SerdeJsonEscapeIfFormatter for serde_json::ser::CompactFormatter {} +impl SerdeJsonEscapeIfFormatter for serde_json::ser::PrettyFormatter<'_> {} + +pub struct SerdeJsonEscapeIf { + pub base: Base, + pub test: Test, +} + +impl serde_json::ser::Formatter + for SerdeJsonEscapeIf +{ + fn write_null(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_null(writer) + } + + fn write_bool(&mut self, writer: &mut W, value: bool) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_bool(writer, value) + } + + fn write_i8(&mut self, writer: &mut W, value: i8) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_i8(writer, value) + } + + fn write_i16(&mut self, writer: &mut W, value: i16) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_i16(writer, value) + } + + fn write_i32(&mut self, writer: &mut W, value: i32) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_i32(writer, value) + } + + fn write_i64(&mut self, writer: &mut W, value: i64) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_i64(writer, value) + } + + fn write_i128(&mut self, writer: &mut W, value: i128) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_i128(writer, value) + } + + fn write_u8(&mut self, writer: &mut W, value: u8) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_u8(writer, value) + } + + fn write_u16(&mut self, writer: &mut W, value: u16) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_u16(writer, value) + } + + fn write_u32(&mut self, writer: &mut W, value: u32) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_u32(writer, value) + } + + fn write_u64(&mut self, writer: &mut W, value: u64) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_u64(writer, value) + } + + fn write_u128(&mut self, writer: &mut W, value: u128) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_u128(writer, value) + } + + fn write_f32(&mut self, writer: &mut W, value: f32) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_f32(writer, value) + } + + fn write_f64(&mut self, writer: &mut W, value: f64) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_f64(writer, value) + } + + fn write_number_str(&mut self, writer: &mut W, value: &str) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_number_str(writer, value) + } + + fn begin_string(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_string(writer) + } + + fn end_string(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_string(writer) + } + + fn write_string_fragment(&mut self, writer: &mut W, mut fragment: &str) -> io::Result<()> + where + W: ?Sized + io::Write, + { + while let Some((next_escape_index, next_escape_char)) = fragment + .char_indices() + .find_map(|(index, ch)| match self.test.char_needs_escape(ch) { + Ok(false) => None, + Ok(true) => Some(Ok((index, ch))), + Err(e) => Some(Err(e)), + }) + .transpose()? + { + let (no_escapes, rest) = fragment.split_at(next_escape_index); + fragment = &rest[next_escape_char.len_utf8()..]; + self.base.write_string_fragment(writer, no_escapes)?; + self.base.write_unicode_escape(writer, next_escape_char)?; + } + self.base.write_string_fragment(writer, fragment) + } + + fn write_char_escape( + &mut self, + writer: &mut W, + char_escape: serde_json::ser::CharEscape, + ) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_char_escape(writer, char_escape) + } + + fn write_byte_array(&mut self, writer: &mut W, value: &[u8]) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_byte_array(writer, value) + } + + fn begin_array(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_array(writer) + } + + fn end_array(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_array(writer) + } + + fn begin_array_value(&mut self, writer: &mut W, first: bool) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_array_value(writer, first) + } + + fn end_array_value(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_array_value(writer) + } + + fn begin_object(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_object(writer) + } + + fn end_object(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_object(writer) + } + + fn begin_object_key(&mut self, writer: &mut W, first: bool) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_object_key(writer, first) + } + + fn end_object_key(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_object_key(writer) + } + + fn begin_object_value(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_object_value(writer) + } + + fn end_object_value(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_object_value(writer) + } + + fn write_raw_fragment(&mut self, writer: &mut W, fragment: &str) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_raw_fragment(writer, fragment) + } +} + +fn serialize_to_json_ascii_helper( + v: &S, + base: F, +) -> serde_json::Result { + let mut retval = Vec::new(); + v.serialize(&mut serde_json::ser::Serializer::with_formatter( + &mut retval, + SerdeJsonEscapeIf { + base, + test: |ch| ch < '\x20' || ch > '\x7F', + }, + ))?; + String::from_utf8(retval).map_err(|_| serde::ser::Error::custom("invalid UTF-8")) +} + +pub fn serialize_to_json_ascii(v: &T) -> serde_json::Result { + serialize_to_json_ascii_helper(v, serde_json::ser::CompactFormatter) +} + +pub fn serialize_to_json_ascii_pretty( + v: &T, +) -> serde_json::Result { + serialize_to_json_ascii_helper(v, serde_json::ser::PrettyFormatter::new()) +} + +pub fn serialize_to_json_ascii_pretty_with_indent( + v: &T, + indent: &str, +) -> serde_json::Result { + serialize_to_json_ascii_helper( + v, + serde_json::ser::PrettyFormatter::with_indent(indent.as_bytes()), + ) +} diff --git a/crates/fayalite/src/util/ready_valid.rs b/crates/fayalite/src/util/ready_valid.rs index ac08a64..06dc873 100644 --- a/crates/fayalite/src/util/ready_valid.rs +++ b/crates/fayalite/src/util/ready_valid.rs @@ -212,7 +212,7 @@ pub fn queue( mod tests { use super::*; use crate::{ - cli::FormalMode, firrtl::ExportOptions, + build::formal::FormalMode, firrtl::ExportOptions, module::transform::simplify_enums::SimplifyEnumsKind, testing::assert_formal, ty::StaticType, }; diff --git a/crates/fayalite/tests/formal.rs b/crates/fayalite/tests/formal.rs index 65264dc..e7d677d 100644 --- a/crates/fayalite/tests/formal.rs +++ b/crates/fayalite/tests/formal.rs @@ -3,7 +3,7 @@ //! Formal tests in Fayalite use fayalite::{ - cli::FormalMode, + build::formal::FormalMode, clock::{Clock, ClockDomain}, expr::{CastTo, HdlPartialEq}, firrtl::ExportOptions,