WIP refactoring to have JobKind be internal jobs

crates/fayalite/examples/blinky.rs

@@ -1,7 +1,10 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
 // See Notices.txt for copyright information
 use clap::Parser;
-use fayalite::{cli, prelude::*};
+use fayalite::{
+    build::{BuildSubcommand, DynJob, JobParams, RunBuild},
+    prelude::*,
+};
 
 #[hdl_module]
 fn blinky(clock_frequency: u64) {
@@ -38,10 +41,19 @@ struct Cli {
     #[arg(long, default_value = "1000000", value_parser = clap::value_parser!(u64).range(2..))]
     clock_frequency: u64,
     #[command(subcommand)]
-    cli: cli::Cli,
+    cli: BuildSubcommand,
 }
 
-fn main() -> cli::Result {
+impl RunBuild for Cli {
-    let cli = Cli::parse();
+    fn into_jobs<E: ?Sized + Extend<DynJob>>(
-    cli.cli.run(blinky(cli.clock_frequency))
+        self,
+        jobs: &mut E,
+        params: impl AsRef<JobParams>,
+    ) -> eyre::Result<()> {
+        self.cli.into_jobs(jobs, params)
+    }
+}
+
+fn main() {
+    Cli::main(|cli| Ok(JobParams::new(blinky(cli.clock_frequency), "blinky")));
 }

crates/fayalite/src/build/external.rs

@@ -2,22 +2,33 @@
 // See Notices.txt for copyright information
 
 use crate::{
-    build::{DynJob, EscapeForUnixShell, JobItem, JobItemName, JobKind},
+    build::{
+        CommandParams, GetBaseJob, JobAndDependencies, JobAndKind, JobArgs, JobArgsAndDependencies,
+        JobDependencies, JobItem, JobItemName, JobKind, JobParams, intern_known_utf8_path_buf,
+    },
     intern::{Intern, Interned},
     util::{job_server::AcquiredJob, streaming_read_utf8::streaming_read_utf8},
 };
 use base64::{Engine, prelude::BASE64_URL_SAFE_NO_PAD};
-use clap::builder::StyledStr;
+use clap::builder::OsStringValueParser;
-use eyre::{Context, ensure, eyre};
+use eyre::{Context, bail, ensure, eyre};
-use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error};
+use serde::{
+    Deserialize, Deserializer, Serialize, Serializer,
+    de::{DeserializeOwned, Error},
+};
 use std::{
     borrow::Cow,
     collections::BTreeMap,
-    env,
+    ffi::{OsStr, OsString},
-    fmt::{self, Write},
+    fmt,
-    mem,
+    hash::{Hash, Hasher},
+    io::Write,
+    marker::PhantomData,
+    path::{Path, PathBuf},
+    sync::OnceLock,
 };
 
+#[cfg(todo)]
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 enum TemplateArg {
     Literal(String),
@@ -25,6 +36,7 @@ enum TemplateArg {
     OutputPath { before: String, after: String },
 }
 
+#[cfg(todo)]
 impl TemplateArg {
     fn after_mut(&mut self) -> &mut String {
         match self {
@@ -114,14 +126,15 @@ impl From<String> for MaybeUtf8 {
 }
 
 #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
-pub struct ExternalJobCache {
+#[serde(rename = "ExternalJobCache")]
+pub struct ExternalJobCacheV2 {
     pub version: ExternalJobCacheVersion,
     pub inputs_hash: blake3::Hash,
     pub stdout_stderr: String,
     pub result: Result<BTreeMap<String, MaybeUtf8>, String>,
 }
 
-impl ExternalJobCache {
+impl ExternalJobCacheV2 {
     fn read_from_file(cache_json_path: Interned<str>) -> eyre::Result<Self> {
         let cache_str = std::fs::read_to_string(&*cache_json_path)
             .wrap_err_with(|| format!("can't read {cache_json_path}"))?;
@@ -136,8 +149,8 @@ impl ExternalJobCache {
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
 pub struct ExternalJobCaching {
-    pub cache_json_path: Interned<str>,
+    cache_json_path: Interned<str>,
-    pub run_even_if_cached: bool,
+    run_even_if_cached: bool,
 }
 
 #[derive(Default)]
@@ -180,20 +193,64 @@ impl JobCacheHasher {
     }
 }
 
+fn write_file_atomically_no_clobber<F: FnOnce() -> C, C: AsRef<[u8]>>(
+    path: impl AsRef<Path>,
+    containing_dir: impl AsRef<Path>,
+    contents: F,
+) -> std::io::Result<()> {
+    let path = path.as_ref();
+    let containing_dir = containing_dir.as_ref();
+    if !matches!(std::fs::exists(&path), Ok(true)) {
+        // use File::create_new rather than tempfile's code to get normal file permissions rather than mode 600 on Unix.
+        let mut file = tempfile::Builder::new()
+            .make_in(containing_dir, |path| std::fs::File::create_new(path))?;
+        file.write_all(contents().as_ref())?; // write all in one operation to avoid a bunch of tiny writes
+        file.into_temp_path().persist_noclobber(path)?;
+    }
+    Ok(())
+}
+
 impl ExternalJobCaching {
-    pub fn new(cache_json_path: Interned<str>) -> Self {
+    pub fn get_cache_dir_from_output_dir(output_dir: &str) -> PathBuf {
-        Self {
+        Path::join(output_dir.as_ref(), ".cache")
+    }
+    pub fn make_cache_dir(
+        cache_dir: impl AsRef<Path>,
+        application_name: &str,
+    ) -> std::io::Result<()> {
+        let cache_dir = cache_dir.as_ref();
+        std::fs::create_dir_all(cache_dir)?;
+        write_file_atomically_no_clobber(cache_dir.join("CACHEDIR.TAG"), cache_dir, || {
+            format!(
+                "Signature: 8a477f597d28d172789f06886806bc55\n\
+                # This file is a cache directory tag created by {application_name}.\n\
+                # For information about cache directory tags see https://bford.info/cachedir/\n"
+            )
+        })?;
+        write_file_atomically_no_clobber(cache_dir.join(".gitignore"), cache_dir, || {
+            format!(
+                "# This is a cache directory created by {application_name}.\n\
+                # ignore all files\n\
+                *\n"
+            )
+        })
+    }
+    pub fn new(
+        output_dir: &str,
+        application_name: &str,
+        json_file_stem: &str,
+        run_even_if_cached: bool,
+    ) -> std::io::Result<Self> {
+        let cache_dir = Self::get_cache_dir_from_output_dir(output_dir);
+        Self::make_cache_dir(&cache_dir, application_name)?;
+        let mut cache_json_path = cache_dir;
+        cache_json_path.push(json_file_stem);
+        cache_json_path.set_extension("json");
+        let cache_json_path = intern_known_utf8_path_buf(cache_json_path);
+        Ok(Self {
             cache_json_path,
-            run_even_if_cached: false,
+            run_even_if_cached,
-        }
+        })
-    }
-    #[track_caller]
-    pub fn from_path(cache_json_path: impl AsRef<std::path::Path>) -> Self {
-        let cache_json_path = cache_json_path.as_ref();
-        let Some(cache_json_path) = cache_json_path.as_os_str().to_str() else {
-            panic!("non-UTF-8 path to cache json: {cache_json_path:?}");
-        };
-        Self::new(cache_json_path.intern())
     }
     fn write_stdout_stderr(stdout_stderr: &str) {
         if stdout_stderr == "" {
@@ -215,12 +272,12 @@ impl ExternalJobCaching {
         if self.run_even_if_cached {
             return Err(());
         }
-        let Ok(ExternalJobCache {
+        let Ok(ExternalJobCacheV2 {
             version: ExternalJobCacheVersion::CURRENT,
             inputs_hash: cached_inputs_hash,
             stdout_stderr,
             result,
-        }) = ExternalJobCache::read_from_file(self.cache_json_path)
+        }) = ExternalJobCacheV2::read_from_file(self.cache_json_path)
         else {
             return Err(());
         };
@@ -253,7 +310,7 @@ impl ExternalJobCaching {
     fn make_command(
         command_line: Interned<[Interned<str>]>,
     ) -> eyre::Result<std::process::Command> {
-        ensure!(command_line.is_empty(), "command line must not be empty");
+        ensure!(!command_line.is_empty(), "command line must not be empty");
         let mut cmd = std::process::Command::new(&*command_line[0]);
         cmd.args(command_line[1..].iter().map(|arg| &**arg))
             .stdin(std::process::Stdio::null());
@@ -314,7 +371,7 @@ impl ExternalJobCaching {
                 .expect("spawn shouldn't fail");
             run_fn(cmd)
         });
-        ExternalJobCache {
+        ExternalJobCacheV2 {
             version: ExternalJobCacheVersion::CURRENT,
             inputs_hash,
             stdout_stderr,
@@ -350,6 +407,7 @@ impl ExternalJobCaching {
     }
 }
 
+#[cfg(todo)]
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct TemplatedExternalJobKind {
     template: Interned<[TemplateArg]>,
@@ -357,12 +415,14 @@ pub struct TemplatedExternalJobKind {
     caching: Option<ExternalJobCaching>,
 }
 
+#[cfg(todo)]
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 enum Token {
     Char(char),
     ArgSeparator,
 }
 
+#[cfg(todo)]
 impl Token {
     fn as_ident_start(self) -> Option<char> {
         match self {
@@ -380,12 +440,14 @@ impl Token {
     }
 }
 
+#[cfg(todo)]
 #[derive(Clone, Debug)]
 struct Tokens<'a> {
     current: std::str::Chars<'a>,
     rest: std::slice::Iter<'a, &'a str>,
 }
 
+#[cfg(todo)]
 impl<'a> Tokens<'a> {
     fn new(args: &'a [&'a str]) -> Self {
         Self {
@@ -395,6 +457,7 @@ impl<'a> Tokens<'a> {
     }
 }
 
+#[cfg(todo)]
 impl Iterator for Tokens<'_> {
     type Item = Token;
 
@@ -409,11 +472,13 @@ impl Iterator for Tokens<'_> {
     }
 }
 
+#[cfg(todo)]
 struct Parser<'a> {
     tokens: std::iter::Peekable<Tokens<'a>>,
     template: Vec<TemplateArg>,
 }
 
+#[cfg(todo)]
 impl<'a> Parser<'a> {
     fn new(args_template: &'a [&'a str]) -> Self {
         Self {
@@ -519,6 +584,7 @@ impl<'a> Parser<'a> {
     }
 }
 
+#[cfg(todo)]
 pub fn find_program<'a>(
     default_program_name: &'a str,
     program_path_env_var: Option<&str>,
@@ -535,6 +601,7 @@ pub fn find_program<'a>(
         .map_err(|program_path| eyre!("path to program is not valid UTF-8: {program_path:?}"))
 }
 
+#[cfg(todo)]
 #[derive(Clone, Debug)]
 enum ParseErrorKind {
     ExpectedVar,
@@ -542,15 +609,18 @@ enum ParseErrorKind {
     EachArgMustHaveAtMostOneVar,
 }
 
+#[cfg(todo)]
 #[derive(Clone, Debug)]
 pub struct TemplateParseError(ParseErrorKind);
 
+#[cfg(todo)]
 impl From<ParseErrorKind> for TemplateParseError {
     fn from(value: ParseErrorKind) -> Self {
         Self(value)
     }
 }
 
+#[cfg(todo)]
 impl fmt::Display for TemplateParseError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match &self.0 {
@@ -568,8 +638,10 @@ impl fmt::Display for TemplateParseError {
     }
 }
 
+#[cfg(todo)]
 impl std::error::Error for TemplateParseError {}
 
+#[cfg(todo)]
 impl TemplatedExternalJobKind {
     pub fn try_new(
         default_program_name: &str,
@@ -599,6 +671,7 @@ impl TemplatedExternalJobKind {
             Err(e) => panic!("{e}"),
         }
     }
+    #[cfg(todo)]
     fn usage(&self) -> StyledStr {
         let mut retval = String::from("Usage:");
         let mut last_input_index = 0usize;
@@ -632,6 +705,7 @@ impl TemplatedExternalJobKind {
         }
         retval.into()
     }
+    #[cfg(todo)]
     fn with_usage(&self, mut e: clap::Error) -> clap::Error {
         e.insert(
             clap::error::ContextKind::Usage,
@@ -641,6 +715,7 @@ impl TemplatedExternalJobKind {
     }
 }
 
+#[cfg(todo)]
 impl JobKind for TemplatedExternalJobKind {
     type Job = TemplatedExternalJob;
 
@@ -771,6 +846,7 @@ impl JobKind for TemplatedExternalJobKind {
     }
 }
 
+#[cfg(todo)]
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct TemplatedExternalJob {
     command_line: Interned<[Interned<str>]>,
@@ -778,6 +854,7 @@ pub struct TemplatedExternalJob {
     outputs: Interned<[JobItemName]>,
 }
 
+#[cfg(todo)]
 impl TemplatedExternalJob {
     pub fn try_add_direct_dependency(&mut self, new_dependency: DynJob) -> eyre::Result<()> {
         let mut added = false;
@@ -833,3 +910,525 @@ impl TemplatedExternalJob {
         self
     }
 }
+
+#[derive(Clone, Eq, Hash)]
+pub struct ExternalCommandJobKind<T: ExternalCommand>(PhantomData<T>);
+
+impl<T: ExternalCommand> fmt::Debug for ExternalCommandJobKind<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "ExternalCommandJobKind<{}>", std::any::type_name::<T>())
+    }
+}
+
+impl<T: ExternalCommand> PartialEq for ExternalCommandJobKind<T> {
+    fn eq(&self, _other: &Self) -> bool {
+        true
+    }
+}
+
+impl<T: ExternalCommand> Ord for ExternalCommandJobKind<T> {
+    fn cmp(&self, _other: &Self) -> std::cmp::Ordering {
+        std::cmp::Ordering::Equal
+    }
+}
+
+impl<T: ExternalCommand> PartialOrd for ExternalCommandJobKind<T> {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl<T: ExternalCommand> Default for ExternalCommandJobKind<T> {
+    fn default() -> Self {
+        Self(PhantomData)
+    }
+}
+
+impl<T: ExternalCommand> Copy for ExternalCommandJobKind<T> {}
+
+impl<T: ExternalCommand> ExternalCommandJobKind<T> {
+    pub const fn new() -> Self {
+        Self(PhantomData)
+    }
+}
+
+#[derive(Copy, Clone)]
+struct ExternalCommandProgramPathValueParser<T: ExternalCommand>(PhantomData<T>);
+
+fn parse_which_result(
+    which_result: which::Result<PathBuf>,
+    program_name: impl Into<OsString>,
+    program_path_arg_name: impl FnOnce() -> String,
+) -> Result<Interned<str>, ResolveProgramPathError> {
+    let which_result = match which_result {
+        Ok(v) => v,
+        Err(e) => {
+            return Err(ResolveProgramPathError {
+                inner: ResolveProgramPathErrorInner::Which(e),
+                program_name: program_name.into(),
+                program_path_arg_name: program_path_arg_name(),
+            });
+        }
+    };
+    Ok(str::intern_owned(
+        which_result
+            .into_os_string()
+            .into_string()
+            .map_err(|_| ResolveProgramPathError {
+                inner: ResolveProgramPathErrorInner::NotValidUtf8,
+                program_name: program_name.into(),
+                program_path_arg_name: program_path_arg_name(),
+            })?,
+    ))
+}
+
+impl<T: ExternalCommand> clap::builder::TypedValueParser
+    for ExternalCommandProgramPathValueParser<T>
+{
+    type Value = Interned<str>;
+
+    fn parse_ref(
+        &self,
+        cmd: &clap::Command,
+        arg: Option<&clap::Arg>,
+        value: &OsStr,
+    ) -> clap::error::Result<Self::Value> {
+        OsStringValueParser::new()
+            .try_map(|program_name| {
+                parse_which_result(which::which(&program_name), program_name, || {
+                    T::program_path_arg_name().into()
+                })
+            })
+            .parse_ref(cmd, arg, value)
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, Hash, Debug, clap::Args)]
+#[group(id = T::args_group_id())]
+#[non_exhaustive]
+pub struct ExternalCommandArgs<T: ExternalCommand> {
+    #[arg(
+        name = Interned::into_inner(T::program_path_arg_name()),
+        long = T::program_path_arg_name(),
+        value_name = T::program_path_arg_value_name(),
+        env = T::program_path_env_var_name().map(Interned::into_inner),
+        value_parser = ExternalCommandProgramPathValueParser::<T>(PhantomData),
+        default_value = T::default_program_name(),
+        value_hint = clap::ValueHint::CommandName,
+    )]
+    pub program_path: Interned<str>,
+    #[arg(
+        name = Interned::into_inner(T::run_even_if_cached_arg_name()),
+        long = T::run_even_if_cached_arg_name(),
+    )]
+    pub run_even_if_cached: bool,
+    #[command(flatten)]
+    pub additional_args: T::AdditionalArgs,
+}
+
+#[derive(Clone)]
+enum ResolveProgramPathErrorInner {
+    Which(which::Error),
+    NotValidUtf8,
+}
+
+impl fmt::Debug for ResolveProgramPathErrorInner {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Which(v) => v.fmt(f),
+            Self::NotValidUtf8 => f.write_str("NotValidUtf8"),
+        }
+    }
+}
+
+impl fmt::Display for ResolveProgramPathErrorInner {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Which(v) => v.fmt(f),
+            Self::NotValidUtf8 => f.write_str("path is not valid UTF-8"),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct ResolveProgramPathError {
+    inner: ResolveProgramPathErrorInner,
+    program_name: std::ffi::OsString,
+    program_path_arg_name: String,
+}
+
+impl fmt::Display for ResolveProgramPathError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let Self {
+            inner,
+            program_name,
+            program_path_arg_name,
+        } = self;
+        write!(
+            f,
+            "{program_path_arg_name}: failed to resolve {program_name:?} to a valid program: {inner}",
+        )
+    }
+}
+
+impl std::error::Error for ResolveProgramPathError {}
+
+pub fn resolve_program_path(
+    program_name: Option<&OsStr>,
+    default_program_name: impl AsRef<OsStr>,
+    program_path_env_var_name: Option<&OsStr>,
+) -> Result<Interned<str>, ResolveProgramPathError> {
+    let default_program_name = default_program_name.as_ref();
+    let owned_program_name;
+    let program_name = if let Some(program_name) = program_name {
+        program_name
+    } else if let Some(v) = program_path_env_var_name.and_then(std::env::var_os) {
+        owned_program_name = v;
+        &owned_program_name
+    } else {
+        default_program_name
+    };
+    parse_which_result(which::which(program_name), program_name, || {
+        default_program_name.display().to_string()
+    })
+}
+
+impl<T: ExternalCommand> ExternalCommandArgs<T> {
+    pub fn with_resolved_program_path(
+        program_path: Interned<str>,
+        additional_args: T::AdditionalArgs,
+    ) -> Self {
+        Self {
+            program_path,
+            run_even_if_cached: false,
+            additional_args,
+        }
+    }
+    pub fn new(
+        program_name: Option<&OsStr>,
+        additional_args: T::AdditionalArgs,
+    ) -> Result<Self, ResolveProgramPathError> {
+        Ok(Self::with_resolved_program_path(
+            resolve_program_path(
+                program_name,
+                T::default_program_name(),
+                T::program_path_env_var_name().as_ref().map(AsRef::as_ref),
+            )?,
+            additional_args,
+        ))
+    }
+}
+
+impl<T: ExternalCommand> JobArgs for ExternalCommandArgs<T> {
+    fn to_args<Args: Extend<Interned<str>> + ?Sized>(&self, args: &mut Args) {
+        let Self {
+            program_path,
+            run_even_if_cached,
+            ref additional_args,
+        } = *self;
+        args.extend([str::intern_owned(format!(
+            "--{}={program_path}",
+            T::program_path_arg_name()
+        ))]);
+        if run_even_if_cached {
+            args.extend([str::intern_owned(format!(
+                "--{}",
+                T::run_even_if_cached_arg_name()
+            ))]);
+        }
+        additional_args.to_args(args);
+    }
+}
+
+#[derive(Copy, Clone)]
+struct ExternalCommandJobParams {
+    command_params: CommandParams,
+    inputs: Interned<[JobItemName]>,
+    outputs: Interned<[JobItemName]>,
+    output_paths: Interned<[Interned<str>]>,
+}
+
+impl ExternalCommandJobParams {
+    fn new<T: ExternalCommand>(job: &ExternalCommandJob<T>) -> Self {
+        let output_paths = T::output_paths(job);
+        Self {
+            command_params: CommandParams {
+                command_line: Interned::from_iter(
+                    [job.program_path]
+                        .into_iter()
+                        .chain(T::command_line_args(job).iter().copied()),
+                ),
+                current_dir: T::current_dir(job),
+            },
+            inputs: T::inputs(job),
+            outputs: output_paths
+                .iter()
+                .map(|&path| JobItemName::Path { path })
+                .collect(),
+            output_paths,
+        }
+    }
+}
+
+#[derive(Deserialize, Serialize)]
+pub struct ExternalCommandJob<T: ExternalCommand> {
+    additional_job_data: T::AdditionalJobData,
+    program_path: Interned<str>,
+    output_dir: Interned<str>,
+    run_even_if_cached: bool,
+    #[serde(skip)]
+    params_cache: OnceLock<ExternalCommandJobParams>,
+}
+
+impl<T: ExternalCommand> Eq for ExternalCommandJob<T> {}
+
+impl<T: ExternalCommand<AdditionalJobData: Clone>> Clone for ExternalCommandJob<T> {
+    fn clone(&self) -> Self {
+        let Self {
+            ref additional_job_data,
+            program_path,
+            output_dir,
+            run_even_if_cached,
+            ref params_cache,
+        } = *self;
+        Self {
+            additional_job_data: additional_job_data.clone(),
+            program_path,
+            output_dir,
+            run_even_if_cached,
+            params_cache: params_cache.clone(),
+        }
+    }
+}
+
+impl<T: ExternalCommand> fmt::Debug for ExternalCommandJob<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let Self {
+            additional_job_data,
+            program_path,
+            output_dir,
+            run_even_if_cached,
+            params_cache: _,
+        } = self;
+        write!(f, "ExternalCommandJob<{}>", std::any::type_name::<T>())?;
+        f.debug_struct("")
+            .field("additional_job_data", additional_job_data)
+            .field("program_path", program_path)
+            .field("output_dir", output_dir)
+            .field("run_even_if_cached", run_even_if_cached)
+            .finish()
+    }
+}
+
+impl<T: ExternalCommand> PartialEq for ExternalCommandJob<T> {
+    fn eq(&self, other: &Self) -> bool {
+        let Self {
+            additional_job_data,
+            program_path,
+            output_dir,
+            run_even_if_cached,
+            params_cache: _,
+        } = self;
+        *additional_job_data == other.additional_job_data
+            && *program_path == other.program_path
+            && *output_dir == other.output_dir
+            && *run_even_if_cached == other.run_even_if_cached
+    }
+}
+
+impl<T: ExternalCommand> Hash for ExternalCommandJob<T> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        let Self {
+            additional_job_data,
+            program_path,
+            output_dir,
+            run_even_if_cached,
+            params_cache: _,
+        } = self;
+        additional_job_data.hash(state);
+        program_path.hash(state);
+        output_dir.hash(state);
+        run_even_if_cached.hash(state);
+    }
+}
+
+impl<T: ExternalCommand> ExternalCommandJob<T> {
+    pub fn additional_job_data(&self) -> &T::AdditionalJobData {
+        &self.additional_job_data
+    }
+    pub fn program_path(&self) -> Interned<str> {
+        self.program_path
+    }
+    pub fn output_dir(&self) -> Interned<str> {
+        self.output_dir
+    }
+    pub fn run_even_if_cached(&self) -> bool {
+        self.run_even_if_cached
+    }
+    fn params(&self) -> &ExternalCommandJobParams {
+        self.params_cache
+            .get_or_init(|| ExternalCommandJobParams::new(self))
+    }
+    pub fn command_params(&self) -> CommandParams {
+        self.params().command_params
+    }
+    pub fn inputs(&self) -> Interned<[JobItemName]> {
+        self.params().inputs
+    }
+    pub fn output_paths(&self) -> Interned<[Interned<str>]> {
+        self.params().output_paths
+    }
+    pub fn outputs(&self) -> Interned<[JobItemName]> {
+        self.params().outputs
+    }
+}
+
+pub trait ExternalCommand: 'static + Send + Sync + Hash + Eq + fmt::Debug + Sized + Clone {
+    type AdditionalArgs: JobArgs;
+    type AdditionalJobData: 'static
+        + Send
+        + Sync
+        + Hash
+        + Eq
+        + fmt::Debug
+        + Serialize
+        + DeserializeOwned;
+    type Dependencies: JobDependencies<JobsAndKinds: GetBaseJob>;
+    fn dependencies() -> Self::Dependencies;
+    fn args_to_jobs(
+        args: JobArgsAndDependencies<ExternalCommandJobKind<Self>>,
+        params: &JobParams,
+    ) -> eyre::Result<(
+        Self::AdditionalJobData,
+        <Self::Dependencies as JobDependencies>::JobsAndKinds,
+    )>;
+    fn inputs(job: &ExternalCommandJob<Self>) -> Interned<[JobItemName]>;
+    fn output_paths(job: &ExternalCommandJob<Self>) -> Interned<[Interned<str>]>;
+    fn command_line_args(job: &ExternalCommandJob<Self>) -> Interned<[Interned<str>]>;
+    fn current_dir(job: &ExternalCommandJob<Self>) -> Option<Interned<str>>;
+    fn job_kind_name() -> Interned<str>;
+    fn args_group_id() -> clap::Id {
+        Interned::into_inner(Self::job_kind_name()).into()
+    }
+    fn program_path_arg_name() -> Interned<str> {
+        Self::default_program_name()
+    }
+    fn program_path_arg_value_name() -> Interned<str> {
+        Intern::intern_owned(Self::program_path_arg_name().to_uppercase())
+    }
+    fn default_program_name() -> Interned<str>;
+    fn program_path_env_var_name() -> Option<Interned<str>> {
+        Some(Intern::intern_owned(
+            Self::program_path_arg_name()
+                .to_uppercase()
+                .replace('-', "_"),
+        ))
+    }
+    fn run_even_if_cached_arg_name() -> Interned<str> {
+        Intern::intern_owned(format!("{}-run-even-if-cached", Self::job_kind_name()))
+    }
+    fn subcommand_hidden() -> bool {
+        false
+    }
+}
+
+impl<T: ExternalCommand> JobKind for ExternalCommandJobKind<T> {
+    type Args = ExternalCommandArgs<T>;
+    type Job = ExternalCommandJob<T>;
+    type Dependencies = T::Dependencies;
+
+    fn dependencies(self) -> Self::Dependencies {
+        T::dependencies()
+    }
+
+    fn args_to_jobs(
+        args: JobArgsAndDependencies<Self>,
+        params: &JobParams,
+    ) -> eyre::Result<JobAndDependencies<Self>> {
+        let kind = args.args.kind;
+        let program_path = args.args.args.program_path;
+        let (additional_job_data, dependencies) = T::args_to_jobs(args, params)?;
+        let base_job = dependencies.base_job();
+        let job = ExternalCommandJob {
+            additional_job_data,
+            program_path,
+            output_dir: base_job.output_dir(),
+            run_even_if_cached: base_job.run_even_if_cached(),
+            params_cache: OnceLock::new(),
+        };
+        job.params(); // fill cache
+        Ok(JobAndDependencies {
+            job: JobAndKind { kind, job },
+            dependencies,
+        })
+    }
+
+    fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]> {
+        job.inputs()
+    }
+
+    fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> {
+        job.outputs()
+    }
+
+    fn name(self) -> Interned<str> {
+        T::job_kind_name()
+    }
+
+    fn external_command_params(self, job: &Self::Job) -> Option<CommandParams> {
+        Some(job.command_params())
+    }
+
+    fn run(
+        self,
+        job: &Self::Job,
+        inputs: &[JobItem],
+        params: &JobParams,
+        acquired_job: &mut AcquiredJob,
+    ) -> eyre::Result<Vec<JobItem>> {
+        assert!(inputs.iter().map(JobItem::name).eq(job.inputs()));
+        let CommandParams {
+            command_line,
+            current_dir,
+        } = job.command_params();
+        ExternalJobCaching::new(
+            &job.output_dir,
+            &params.application_name(),
+            &T::job_kind_name(),
+            job.run_even_if_cached,
+        )?
+        .run(
+            command_line,
+            inputs
+                .iter()
+                .flat_map(|item| match item {
+                    JobItem::Path { path } => std::slice::from_ref(path),
+                    JobItem::DynamicPaths {
+                        paths,
+                        source_job_name: _,
+                    } => paths,
+                })
+                .copied(),
+            job.output_paths(),
+            |mut cmd| {
+                if let Some(current_dir) = current_dir {
+                    cmd.current_dir(current_dir);
+                }
+                let status = acquired_job.run_command(cmd, |cmd| cmd.status())?;
+                if !status.success() {
+                    bail!("running {command_line:?} failed: {status}")
+                }
+                Ok(())
+            },
+        )?;
+        Ok(job
+            .output_paths()
+            .iter()
+            .map(|&path| JobItem::Path { path })
+            .collect())
+    }
+
+    fn subcommand_hidden(self) -> bool {
+        T::subcommand_hidden()
+    }
+}

crates/fayalite/src/build/firrtl.rs

@@ -2,81 +2,119 @@
 // See Notices.txt for copyright information
 
 use crate::{
-    build::{BaseArgs, DynJob, InternalJobTrait, JobItem, JobItemName},
+    build::{
+        BaseJob, BaseJobKind, CommandParams, JobAndDependencies, JobArgs, JobArgsAndDependencies,
+        JobItem, JobItemName, JobKind, JobKindAndDependencies, JobParams,
+    },
     firrtl::{ExportOptions, FileBackend},
     intern::{Intern, Interned},
     util::job_server::AcquiredJob,
 };
-use clap::Parser;
+use clap::Args;
-use std::{borrow::Cow, collections::BTreeMap};
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
 
-#[derive(Parser, Debug, Clone, Hash, PartialEq, Eq)]
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Debug)]
+pub struct FirrtlJobKind;
+
+#[derive(Args, Debug, Clone, Hash, PartialEq, Eq)]
+#[group(id = "Firrtl")]
 #[non_exhaustive]
 pub struct FirrtlArgs {
-    #[command(flatten)]
-    pub base: BaseArgs,
     #[command(flatten)]
     pub export_options: ExportOptions,
 }
 
-impl FirrtlArgs {
+impl JobArgs for FirrtlArgs {
+    fn to_args<Args: Extend<Interned<str>> + ?Sized>(&self, args: &mut Args) {
+        let Self { export_options } = self;
+        export_options.to_args(args);
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct Firrtl {
+    base: BaseJob,
+    export_options: ExportOptions,
+}
+
+impl Firrtl {
     fn make_firrtl_file_backend(&self) -> FileBackend {
         FileBackend {
-            dir_path: self.base.output.path().into(),
+            dir_path: PathBuf::from(&*self.base.output_dir()),
-            top_fir_file_stem: self.base.file_stem.clone(),
+            top_fir_file_stem: Some(String::from(&*self.base.file_stem())),
             circuit_name: None,
         }
     }
-    pub fn firrtl_file(&self) -> String {
+    pub fn firrtl_file(&self) -> Interned<str> {
         self.base.file_with_ext("fir")
     }
 }
 
-impl InternalJobTrait for FirrtlArgs {
+impl JobKind for FirrtlJobKind {
-    fn subcommand_name() -> Interned<str> {
+    type Args = FirrtlArgs;
-        "firrtl".intern()
+    type Job = Firrtl;
+    type Dependencies = JobKindAndDependencies<BaseJobKind>;
+
+    fn dependencies(self) -> Self::Dependencies {
+        JobKindAndDependencies::new(BaseJobKind)
     }
 
-    fn to_args(&self) -> Vec<Interned<str>> {
+    fn args_to_jobs(
-        let Self {
+        args: JobArgsAndDependencies<Self>,
-            base,
+        params: &JobParams,
+    ) -> eyre::Result<JobAndDependencies<Self>> {
+        args.args_to_jobs_simple(
+            params,
+            |_kind, FirrtlArgs { export_options }, dependencies| {
+                Ok(Firrtl {
+                    base: dependencies.job.job.clone(),
                     export_options,
-        } = self;
+                })
-        let mut retval = base.to_args();
-        retval.extend(export_options.to_args());
-        retval
-    }
-
-    fn inputs_and_direct_dependencies<'a>(
-        &'a self,
-    ) -> Cow<'a, BTreeMap<JobItemName, Option<DynJob>>> {
-        Cow::Owned(BTreeMap::from_iter([(
-            JobItemName::Module {
-                name: str::intern(&self.base.module_name),
             },
-            None,
+        )
-        )]))
     }
 
-    fn outputs(&self) -> Interned<[JobItemName]> {
+    fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]> {
-        [JobItemName::File {
+        [JobItemName::Path {
-            path: str::intern_owned(self.firrtl_file()),
+            path: job.base.output_dir(),
         }][..]
             .intern()
     }
 
+    fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> {
+        [JobItemName::Path {
+            path: job.firrtl_file(),
+        }][..]
+            .intern()
+    }
+
+    fn name(self) -> Interned<str> {
+        "firrtl".intern()
+    }
+
+    fn external_command_params(self, _job: &Self::Job) -> Option<CommandParams> {
+        None
+    }
+
     fn run(
-        &self,
+        self,
+        job: &Self::Job,
         inputs: &[JobItem],
+        params: &JobParams,
         _acquired_job: &mut AcquiredJob,
     ) -> eyre::Result<Vec<JobItem>> {
-        let [JobItem::Module { value: module }] = inputs else {
+        let [JobItem::Path { path: input_path }] = *inputs else {
-            panic!("wrong inputs, expected a single `Module`");
+            panic!("wrong inputs, expected a single `Path`");
         };
-        assert_eq!(*module.name(), *self.base.module_name);
+        assert_eq!(input_path, job.base.output_dir());
-        crate::firrtl::export(self.make_firrtl_file_backend(), module, self.export_options)?;
+        crate::firrtl::export(
-        Ok(vec![JobItem::File {
+            job.make_firrtl_file_backend(),
-            path: str::intern_owned(self.firrtl_file()),
+            params.main_module(),
+            job.export_options,
+        )?;
+        Ok(vec![JobItem::Path {
+            path: job.firrtl_file(),
         }])
     }
 }

crates/fayalite/src/build/graph.rs

@@ -0,0 +1,792 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// See Notices.txt for copyright information
+
+use crate::{
+    build::{DynJob, JobItem, JobItemName, JobParams, program_name_for_internal_jobs},
+    intern::Interned,
+    util::{HashMap, HashSet, job_server::AcquiredJob},
+};
+use petgraph::{
+    algo::{DfsSpace, kosaraju_scc, toposort},
+    graph::DiGraph,
+    visit::{GraphBase, Visitable},
+};
+use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error, ser::SerializeSeq};
+use std::{
+    cell::OnceCell,
+    collections::{BTreeMap, BTreeSet, VecDeque},
+    fmt::{self, Write},
+    panic,
+    rc::Rc,
+    sync::mpsc,
+    thread::{self, ScopedJoinHandle},
+};
+
+macro_rules! write_str {
+    ($s:expr, $($rest:tt)*) => {
+        write!($s, $($rest)*).expect("String::write_fmt can't fail")
+    };
+}
+
+#[derive(Clone, Debug)]
+enum JobGraphNode {
+    Job(DynJob),
+    Item {
+        #[allow(dead_code, reason = "name used for debugging")]
+        name: JobItemName,
+        source_job: Option<DynJob>,
+    },
+}
+
+type JobGraphInner = DiGraph<JobGraphNode, ()>;
+
+#[derive(Clone, Default)]
+pub struct JobGraph {
+    jobs: HashMap<DynJob, <JobGraphInner as GraphBase>::NodeId>,
+    items: HashMap<JobItemName, <JobGraphInner as GraphBase>::NodeId>,
+    graph: JobGraphInner,
+    topological_order: Vec<<JobGraphInner as GraphBase>::NodeId>,
+    space: DfsSpace<<JobGraphInner as GraphBase>::NodeId, <JobGraphInner as Visitable>::Map>,
+}
+
+impl fmt::Debug for JobGraph {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let Self {
+            jobs: _,
+            items: _,
+            graph,
+            topological_order,
+            space: _,
+        } = self;
+        f.debug_struct("JobGraph")
+            .field("graph", graph)
+            .field("topological_order", topological_order)
+            .finish_non_exhaustive()
+    }
+}
+
+#[derive(Clone, Debug)]
+pub enum JobGraphError {
+    CycleError {
+        job: DynJob,
+        output: JobItemName,
+    },
+    MultipleJobsCreateSameOutput {
+        output_item: JobItemName,
+        existing_job: DynJob,
+        new_job: DynJob,
+    },
+}
+
+impl std::error::Error for JobGraphError {}
+
+impl fmt::Display for JobGraphError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::CycleError { job, output } => write!(
+                f,
+                "job can't be added to job graph because it would introduce a cyclic dependency through this job output:\n\
+                {output:?}\n\
+                job:\n{job:?}",
+            ),
+            JobGraphError::MultipleJobsCreateSameOutput {
+                output_item,
+                existing_job,
+                new_job,
+            } => write!(
+                f,
+                "job can't be added to job graph because the new job has an output that is also produced by an existing job.\n\
+                conflicting output:\n\
+                {output_item:?}\n\
+                existing job:\n\
+                {existing_job:?}\n\
+                new job:\n\
+                {new_job:?}",
+            ),
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+enum EscapeForUnixShellState {
+    DollarSingleQuote,
+    SingleQuote,
+    Unquoted,
+}
+
+#[derive(Clone)]
+pub struct EscapeForUnixShell<'a> {
+    state: EscapeForUnixShellState,
+    prefix: [u8; 3],
+    bytes: &'a [u8],
+}
+
+impl<'a> fmt::Debug for EscapeForUnixShell<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(self, f)
+    }
+}
+
+impl<'a> fmt::Display for EscapeForUnixShell<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        for c in self.clone() {
+            f.write_char(c)?;
+        }
+        Ok(())
+    }
+}
+
+impl<'a> EscapeForUnixShell<'a> {
+    pub fn new(s: &'a str) -> Self {
+        Self::from_bytes(s.as_bytes())
+    }
+    fn make_prefix(bytes: &[u8]) -> [u8; 3] {
+        let mut prefix = [0; 3];
+        prefix[..bytes.len()].copy_from_slice(bytes);
+        prefix
+    }
+    pub fn from_bytes(bytes: &'a [u8]) -> Self {
+        let mut needs_single_quote = bytes.is_empty();
+        for &b in bytes {
+            match b {
+                b'!' | b'\'' | b'\"' | b' ' => needs_single_quote = true,
+                0..0x20 | 0x7F.. => {
+                    return Self {
+                        state: EscapeForUnixShellState::DollarSingleQuote,
+                        prefix: Self::make_prefix(b"$'"),
+                        bytes,
+                    };
+                }
+                _ => {}
+            }
+        }
+        if needs_single_quote {
+            Self {
+                state: EscapeForUnixShellState::SingleQuote,
+                prefix: Self::make_prefix(b"'"),
+                bytes,
+            }
+        } else {
+            Self {
+                state: EscapeForUnixShellState::Unquoted,
+                prefix: Self::make_prefix(b""),
+                bytes,
+            }
+        }
+    }
+}
+
+impl Iterator for EscapeForUnixShell<'_> {
+    type Item = char;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match &mut self.prefix {
+            [0, 0, 0] => {}
+            [0, 0, v] | // find first
+            [0, v, _] | // non-zero byte
+            [v, _, _] => {
+                let retval = *v as char;
+                *v = 0;
+                return Some(retval);
+            }
+        }
+        let Some(&next_byte) = self.bytes.split_off_first() else {
+            return match self.state {
+                EscapeForUnixShellState::DollarSingleQuote
+                | EscapeForUnixShellState::SingleQuote => {
+                    self.state = EscapeForUnixShellState::Unquoted;
+                    Some('\'')
+                }
+                EscapeForUnixShellState::Unquoted => None,
+            };
+        };
+        match self.state {
+            EscapeForUnixShellState::DollarSingleQuote => match next_byte {
+                b'\'' | b'\\' => {
+                    self.prefix = Self::make_prefix(&[next_byte]);
+                    Some('\\')
+                }
+                b'\t' => {
+                    self.prefix = Self::make_prefix(b"t");
+                    Some('\\')
+                }
+                b'\n' => {
+                    self.prefix = Self::make_prefix(b"n");
+                    Some('\\')
+                }
+                b'\r' => {
+                    self.prefix = Self::make_prefix(b"r");
+                    Some('\\')
+                }
+                0x20..=0x7E => Some(next_byte as char),
+                _ => {
+                    self.prefix = [
+                        b'x',
+                        char::from_digit(next_byte as u32 >> 4, 0x10).expect("known to be in range")
+                            as u8,
+                        char::from_digit(next_byte as u32 & 0xF, 0x10)
+                            .expect("known to be in range") as u8,
+                    ];
+                    Some('\\')
+                }
+            },
+            EscapeForUnixShellState::SingleQuote => {
+                if next_byte == b'\'' {
+                    self.prefix = Self::make_prefix(b"\\''");
+                    Some('\'')
+                } else {
+                    Some(next_byte as char)
+                }
+            }
+            EscapeForUnixShellState::Unquoted => match next_byte {
+                b' ' | b'!' | b'"' | b'#' | b'$' | b'&' | b'\'' | b'(' | b')' | b'*' | b','
+                | b';' | b'<' | b'>' | b'?' | b'[' | b'\\' | b']' | b'^' | b'`' | b'{' | b'|'
+                | b'}' | b'~' => {
+                    self.prefix = Self::make_prefix(&[next_byte]);
+                    Some('\\')
+                }
+                _ => Some(next_byte as char),
+            },
+        }
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+#[non_exhaustive]
+pub enum UnixMakefileEscapeKind {
+    NonRecipe,
+    RecipeWithoutShellEscaping,
+    RecipeWithShellEscaping,
+}
+
+#[derive(Copy, Clone)]
+pub struct EscapeForUnixMakefile<'a> {
+    s: &'a str,
+    kind: UnixMakefileEscapeKind,
+}
+
+impl<'a> fmt::Debug for EscapeForUnixMakefile<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(self, f)
+    }
+}
+
+impl<'a> fmt::Display for EscapeForUnixMakefile<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.do_write(f, fmt::Write::write_str, fmt::Write::write_char, |_, _| {
+            Ok(())
+        })
+    }
+}
+
+impl<'a> EscapeForUnixMakefile<'a> {
+    fn do_write<S: ?Sized, E>(
+        &self,
+        state: &mut S,
+        write_str: impl Fn(&mut S, &str) -> Result<(), E>,
+        write_char: impl Fn(&mut S, char) -> Result<(), E>,
+        add_variable: impl Fn(&mut S, &'static str) -> Result<(), E>,
+    ) -> Result<(), E> {
+        let escape_recipe_char = |c| match c {
+            '$' => write_str(state, "$$"),
+            '\0'..='\x1F' | '\x7F' => {
+                panic!("can't escape a control character for Unix Makefile: {c:?}");
+            }
+            _ => write_char(state, c),
+        };
+        match self.kind {
+            UnixMakefileEscapeKind::NonRecipe => self.s.chars().try_for_each(|c| match c {
+                '=' => {
+                    add_variable(state, "EQUALS = =")?;
+                    write_str(state, "$(EQUALS)")
+                }
+                ';' => panic!("can't escape a semicolon (;) for Unix Makefile"),
+                '$' => write_str(state, "$$"),
+                '\\' | ' ' | '#' | ':' | '%' | '*' | '?' | '[' | ']' | '~' => {
+                    write_char(state, '\\')?;
+                    write_char(state, c)
+                }
+                '\0'..='\x1F' | '\x7F' => {
+                    panic!("can't escape a control character for Unix Makefile: {c:?}");
+                }
+                _ => write_char(state, c),
+            }),
+            UnixMakefileEscapeKind::RecipeWithoutShellEscaping => {
+                self.s.chars().try_for_each(escape_recipe_char)
+            }
+            UnixMakefileEscapeKind::RecipeWithShellEscaping => {
+                EscapeForUnixShell::new(self.s).try_for_each(escape_recipe_char)
+            }
+        }
+    }
+    pub fn new(
+        s: &'a str,
+        kind: UnixMakefileEscapeKind,
+        needed_variables: &mut BTreeSet<&'static str>,
+    ) -> Self {
+        let retval = Self { s, kind };
+        let Ok(()) = retval.do_write(
+            needed_variables,
+            |_, _| Ok(()),
+            |_, _| Ok(()),
+            |needed_variables, variable| -> Result<(), std::convert::Infallible> {
+                needed_variables.insert(variable);
+                Ok(())
+            },
+        );
+        retval
+    }
+}
+
+impl JobGraph {
+    pub fn new() -> Self {
+        Self::default()
+    }
+    fn try_add_item_node(
+        &mut self,
+        name: JobItemName,
+        new_source_job: Option<DynJob>,
+        new_nodes: &mut HashSet<<JobGraphInner as GraphBase>::NodeId>,
+    ) -> Result<<JobGraphInner as GraphBase>::NodeId, JobGraphError> {
+        use hashbrown::hash_map::Entry;
+        match self.items.entry(name) {
+            Entry::Occupied(item_entry) => {
+                let node_id = *item_entry.get();
+                let JobGraphNode::Item {
+                    name: _,
+                    source_job,
+                } = &mut self.graph[node_id]
+                else {
+                    unreachable!("known to be an item");
+                };
+                if let Some(new_source_job) = new_source_job {
+                    if let Some(source_job) = source_job {
+                        return Err(JobGraphError::MultipleJobsCreateSameOutput {
+                            output_item: item_entry.key().clone(),
+                            existing_job: source_job.clone(),
+                            new_job: new_source_job,
+                        });
+                    } else {
+                        *source_job = Some(new_source_job);
+                    }
+                }
+                Ok(node_id)
+            }
+            Entry::Vacant(item_entry) => {
+                let node_id = self.graph.add_node(JobGraphNode::Item {
+                    name,
+                    source_job: new_source_job,
+                });
+                new_nodes.insert(node_id);
+                item_entry.insert(node_id);
+                Ok(node_id)
+            }
+        }
+    }
+    pub fn try_add_jobs<I: IntoIterator<Item = DynJob>>(
+        &mut self,
+        jobs: I,
+    ) -> Result<(), JobGraphError> {
+        use hashbrown::hash_map::Entry;
+        let jobs = jobs.into_iter();
+        struct RemoveNewNodesOnError<'a> {
+            this: &'a mut JobGraph,
+            new_nodes: HashSet<<JobGraphInner as GraphBase>::NodeId>,
+        }
+        impl Drop for RemoveNewNodesOnError<'_> {
+            fn drop(&mut self) {
+                for node in self.new_nodes.drain() {
+                    self.this.graph.remove_node(node);
+                }
+            }
+        }
+        let mut remove_new_nodes_on_error = RemoveNewNodesOnError {
+            this: self,
+            new_nodes: HashSet::with_capacity_and_hasher(jobs.size_hint().0, Default::default()),
+        };
+        let new_nodes = &mut remove_new_nodes_on_error.new_nodes;
+        let this = &mut *remove_new_nodes_on_error.this;
+        for job in jobs {
+            let Entry::Vacant(job_entry) = this.jobs.entry(job.clone()) else {
+                continue;
+            };
+            let job_node_id = this
+                .graph
+                .add_node(JobGraphNode::Job(job_entry.key().clone()));
+            new_nodes.insert(job_node_id);
+            job_entry.insert(job_node_id);
+            for name in job.outputs() {
+                let item_node_id = this.try_add_item_node(name, Some(job.clone()), new_nodes)?;
+                this.graph.add_edge(job_node_id, item_node_id, ());
+            }
+            for name in job.inputs() {
+                let item_node_id = this.try_add_item_node(name, None, new_nodes)?;
+                this.graph.add_edge(item_node_id, job_node_id, ());
+            }
+        }
+        match toposort(&this.graph, Some(&mut this.space)) {
+            Ok(v) => {
+                this.topological_order = v;
+                // no need to remove any of the new nodes on drop since we didn't encounter any errors
+                remove_new_nodes_on_error.new_nodes.clear();
+                Ok(())
+            }
+            Err(_) => {
+                // there's at least one cycle, find one!
+                let cycle = kosaraju_scc(&this.graph)
+                    .into_iter()
+                    .find_map(|scc| {
+                        if scc.len() <= 1 {
+                            // can't be a cycle since our graph is bipartite --
+                            // jobs only connect to items, never jobs to jobs or items to items
+                            None
+                        } else {
+                            Some(scc)
+                        }
+                    })
+                    .expect("we know there's a cycle");
+                let cycle_set = HashSet::from_iter(cycle.iter().copied());
+                let job = cycle
+                    .into_iter()
+                    .find_map(|node_id| {
+                        if let JobGraphNode::Job(job) = &this.graph[node_id] {
+                            Some(job.clone())
+                        } else {
+                            None
+                        }
+                    })
+                    .expect("a job must be part of the cycle");
+                let output = job
+                    .outputs()
+                    .into_iter()
+                    .find(|output| cycle_set.contains(&this.items[output]))
+                    .expect("an output must be part of the cycle");
+                Err(JobGraphError::CycleError { job, output })
+            }
+        }
+    }
+    #[track_caller]
+    pub fn add_jobs<I: IntoIterator<Item = DynJob>>(&mut self, jobs: I) {
+        match self.try_add_jobs(jobs) {
+            Ok(()) => {}
+            Err(e) => panic!("error: {e}"),
+        }
+    }
+    pub fn to_unix_makefile(&self) -> String {
+        self.to_unix_makefile_with_internal_program_prefix(&[program_name_for_internal_jobs()])
+    }
+    pub fn to_unix_makefile_with_internal_program_prefix(
+        &self,
+        internal_program_prefix: &[Interned<str>],
+    ) -> String {
+        let mut retval = String::new();
+        let mut needed_variables = BTreeSet::new();
+        let mut phony_targets = BTreeSet::new();
+        for &node_id in &self.topological_order {
+            let JobGraphNode::Job(job) = &self.graph[node_id] else {
+                continue;
+            };
+            let outputs = job.outputs();
+            if outputs.is_empty() {
+                retval.push_str(":");
+            } else {
+                for output in job.outputs() {
+                    match output {
+                        JobItemName::Path { path } => {
+                            write_str!(
+                                retval,
+                                "{} ",
+                                EscapeForUnixMakefile::new(
+                                    &path,
+                                    UnixMakefileEscapeKind::NonRecipe,
+                                    &mut needed_variables
+                                )
+                            );
+                        }
+                        JobItemName::DynamicPaths { source_job_name } => {
+                            write_str!(
+                                retval,
+                                "{} ",
+                                EscapeForUnixMakefile::new(
+                                    &source_job_name,
+                                    UnixMakefileEscapeKind::NonRecipe,
+                                    &mut needed_variables
+                                )
+                            );
+                            phony_targets.insert(Interned::into_inner(source_job_name));
+                        }
+                    }
+                }
+                if outputs.len() == 1 {
+                    retval.push_str(":");
+                } else {
+                    retval.push_str("&:");
+                }
+            }
+            for input in job.inputs() {
+                match input {
+                    JobItemName::Path { path } => {
+                        write_str!(
+                            retval,
+                            " {}",
+                            EscapeForUnixMakefile::new(
+                                &path,
+                                UnixMakefileEscapeKind::NonRecipe,
+                                &mut needed_variables
+                            )
+                        );
+                    }
+                    JobItemName::DynamicPaths { source_job_name } => {
+                        write_str!(
+                            retval,
+                            " {}",
+                            EscapeForUnixMakefile::new(
+                                &source_job_name,
+                                UnixMakefileEscapeKind::NonRecipe,
+                                &mut needed_variables
+                            )
+                        );
+                        phony_targets.insert(Interned::into_inner(source_job_name));
+                    }
+                }
+            }
+            retval.push_str("\n\t");
+            job.command_params_with_internal_program_prefix(internal_program_prefix)
+                .to_unix_shell_line(&mut retval, |arg, output| {
+                    write!(
+                        output,
+                        "{}",
+                        EscapeForUnixMakefile::new(
+                            arg,
+                            UnixMakefileEscapeKind::RecipeWithShellEscaping,
+                            &mut needed_variables
+                        )
+                    )
+                })
+                .expect("writing to String never fails");
+            retval.push_str("\n\n");
+        }
+        if !phony_targets.is_empty() {
+            retval.push_str("\n.PHONY:");
+            for phony_target in phony_targets {
+                write_str!(
+                    retval,
+                    " {}",
+                    EscapeForUnixMakefile::new(
+                        phony_target,
+                        UnixMakefileEscapeKind::NonRecipe,
+                        &mut needed_variables
+                    )
+                );
+            }
+            retval.push_str("\n");
+        }
+        if !needed_variables.is_empty() {
+            retval.insert_str(
+                0,
+                &String::from_iter(needed_variables.into_iter().map(|v| format!("{v}\n"))),
+            );
+        }
+        retval
+    }
+    pub fn to_unix_shell_script(&self) -> String {
+        self.to_unix_shell_script_with_internal_program_prefix(&[program_name_for_internal_jobs()])
+    }
+    pub fn to_unix_shell_script_with_internal_program_prefix(
+        &self,
+        internal_program_prefix: &[Interned<str>],
+    ) -> String {
+        let mut retval = String::from(
+            "#!/bin/sh\n\
+            set -ex\n",
+        );
+        for &node_id in &self.topological_order {
+            let JobGraphNode::Job(job) = &self.graph[node_id] else {
+                continue;
+            };
+            job.command_params_with_internal_program_prefix(internal_program_prefix)
+                .to_unix_shell_line(&mut retval, |arg, output| {
+                    write!(output, "{}", EscapeForUnixShell::new(&arg))
+                })
+                .expect("writing to String never fails");
+            retval.push_str("\n");
+        }
+        retval
+    }
+    pub fn run(&self, params: &JobParams) -> eyre::Result<()> {
+        // use scope to auto-join threads on errors
+        thread::scope(|scope| {
+            struct WaitingJobState {
+                job_node_id: <JobGraphInner as GraphBase>::NodeId,
+                job: DynJob,
+                inputs: BTreeMap<JobItemName, OnceCell<JobItem>>,
+            }
+            let mut ready_jobs = VecDeque::new();
+            let mut item_name_to_waiting_jobs_map = HashMap::<_, Vec<_>>::default();
+            for &node_id in &self.topological_order {
+                let JobGraphNode::Job(job) = &self.graph[node_id] else {
+                    continue;
+                };
+                let waiting_job = WaitingJobState {
+                    job_node_id: node_id,
+                    job: job.clone(),
+                    inputs: job
+                        .inputs()
+                        .iter()
+                        .map(|&name| (name, OnceCell::new()))
+                        .collect(),
+                };
+                if waiting_job.inputs.is_empty() {
+                    ready_jobs.push_back(waiting_job);
+                } else {
+                    let waiting_job = Rc::new(waiting_job);
+                    for &input_item in waiting_job.inputs.keys() {
+                        item_name_to_waiting_jobs_map
+                            .entry(input_item)
+                            .or_default()
+                            .push(waiting_job.clone());
+                    }
+                }
+            }
+            struct RunningJob<'scope> {
+                job: DynJob,
+                thread: ScopedJoinHandle<'scope, eyre::Result<Vec<JobItem>>>,
+            }
+            let mut running_jobs = HashMap::default();
+            let (finished_jobs_sender, finished_jobs_receiver) = mpsc::channel();
+            loop {
+                while let Some(finished_job) = finished_jobs_receiver.try_recv().ok() {
+                    let Some(RunningJob { job, thread }) = running_jobs.remove(&finished_job)
+                    else {
+                        unreachable!();
+                    };
+                    let output_items = thread.join().map_err(panic::resume_unwind)??;
+                    assert!(
+                        output_items.iter().map(JobItem::name).eq(job.outputs()),
+                        "job's run() method returned the wrong output items:\n\
+                        output items:\n\
+                        {output_items:?}\n\
+                        expected outputs:\n\
+                        {:?}\n\
+                        job:\n\
+                        {job:?}",
+                        job.outputs(),
+                    );
+                    for output_item in output_items {
+                        for waiting_job in item_name_to_waiting_jobs_map
+                            .remove(&output_item.name())
+                            .unwrap_or_default()
+                        {
+                            let Ok(()) =
+                                waiting_job.inputs[&output_item.name()].set(output_item.clone())
+                            else {
+                                unreachable!();
+                            };
+                            if let Some(waiting_job) = Rc::into_inner(waiting_job) {
+                                ready_jobs.push_back(waiting_job);
+                            }
+                        }
+                    }
+                }
+                if let Some(WaitingJobState {
+                    job_node_id,
+                    job,
+                    inputs,
+                }) = ready_jobs.pop_front()
+                {
+                    struct RunningJobInThread<'a> {
+                        job_node_id: <JobGraphInner as GraphBase>::NodeId,
+                        job: DynJob,
+                        inputs: Vec<JobItem>,
+                        params: &'a JobParams,
+                        acquired_job: AcquiredJob,
+                        finished_jobs_sender: mpsc::Sender<<JobGraphInner as GraphBase>::NodeId>,
+                    }
+                    impl RunningJobInThread<'_> {
+                        fn run(mut self) -> eyre::Result<Vec<JobItem>> {
+                            self.job
+                                .run(&self.inputs, self.params, &mut self.acquired_job)
+                        }
+                    }
+                    impl Drop for RunningJobInThread<'_> {
+                        fn drop(&mut self) {
+                            let _ = self.finished_jobs_sender.send(self.job_node_id);
+                        }
+                    }
+                    let name = job.kind().name();
+                    let running_job_in_thread = RunningJobInThread {
+                        job_node_id,
+                        job: job.clone(),
+                        inputs: Vec::from_iter(
+                            inputs
+                                .into_values()
+                                .map(|input| input.into_inner().expect("was set earlier")),
+                        ),
+                        params,
+                        acquired_job: AcquiredJob::acquire()?,
+                        finished_jobs_sender: finished_jobs_sender.clone(),
+                    };
+                    running_jobs.insert(
+                        job_node_id,
+                        RunningJob {
+                            job,
+                            thread: thread::Builder::new()
+                                .name(format!("job:{name}"))
+                                .spawn_scoped(scope, move || running_job_in_thread.run())
+                                .expect("failed to spawn thread for job"),
+                        },
+                    );
+                }
+                if running_jobs.is_empty() {
+                    assert!(item_name_to_waiting_jobs_map.is_empty());
+                    assert!(ready_jobs.is_empty());
+                    return Ok(());
+                }
+            }
+        })
+    }
+}
+
+impl Extend<DynJob> for JobGraph {
+    #[track_caller]
+    fn extend<T: IntoIterator<Item = DynJob>>(&mut self, iter: T) {
+        self.add_jobs(iter);
+    }
+}
+
+impl FromIterator<DynJob> for JobGraph {
+    #[track_caller]
+    fn from_iter<T: IntoIterator<Item = DynJob>>(iter: T) -> Self {
+        let mut retval = Self::new();
+        retval.add_jobs(iter);
+        retval
+    }
+}
+
+impl Serialize for JobGraph {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut serializer = serializer.serialize_seq(Some(self.jobs.len()))?;
+        for &node_id in &self.topological_order {
+            let JobGraphNode::Job(job) = &self.graph[node_id] else {
+                continue;
+            };
+            serializer.serialize_element(job)?;
+        }
+        serializer.end()
+    }
+}
+
+impl<'de> Deserialize<'de> for JobGraph {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let jobs = Vec::<DynJob>::deserialize(deserializer)?;
+        let mut retval = JobGraph::new();
+        retval.try_add_jobs(jobs).map_err(D::Error::custom)?;
+        Ok(retval)
+    }
+}

crates/fayalite/src/build/registry.rs

@@ -0,0 +1,341 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// See Notices.txt for copyright information
+
+use crate::{
+    build::{BUILT_IN_JOB_KINDS, DynJobKind, JobKind},
+    intern::Interned,
+};
+use std::{
+    borrow::Borrow,
+    cmp::Ordering,
+    collections::BTreeMap,
+    fmt,
+    sync::{Arc, OnceLock, RwLock, RwLockWriteGuard},
+};
+
+impl DynJobKind {
+    pub fn registry() -> JobKindRegistrySnapshot {
+        JobKindRegistrySnapshot(JobKindRegistry::get())
+    }
+    #[track_caller]
+    pub fn register(self) {
+        JobKindRegistry::register(JobKindRegistry::lock(), self);
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+struct InternedStrCompareAsStr(Interned<str>);
+
+impl fmt::Debug for InternedStrCompareAsStr {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl Ord for InternedStrCompareAsStr {
+    fn cmp(&self, other: &Self) -> Ordering {
+        str::cmp(&self.0, &other.0)
+    }
+}
+
+impl PartialOrd for InternedStrCompareAsStr {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Borrow<str> for InternedStrCompareAsStr {
+    fn borrow(&self) -> &str {
+        &self.0
+    }
+}
+
+#[derive(Clone, Debug)]
+struct JobKindRegistry {
+    job_kinds: BTreeMap<InternedStrCompareAsStr, DynJobKind>,
+}
+
+enum JobKindRegisterError {
+    SameName {
+        name: InternedStrCompareAsStr,
+        old_job_kind: DynJobKind,
+        new_job_kind: DynJobKind,
+    },
+}
+
+impl fmt::Display for JobKindRegisterError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::SameName {
+                name,
+                old_job_kind,
+                new_job_kind,
+            } => write!(
+                f,
+                "two different `JobKind` can't share the same name:\n\
+                {name:?}\n\
+                old job kind:\n\
+                {old_job_kind:?}\n\
+                new job kind:\n\
+                {new_job_kind:?}",
+            ),
+        }
+    }
+}
+
+trait JobKindRegistryRegisterLock {
+    type Locked;
+    fn lock(self) -> Self::Locked;
+    fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry;
+}
+
+impl JobKindRegistryRegisterLock for &'static RwLock<Arc<JobKindRegistry>> {
+    type Locked = RwLockWriteGuard<'static, Arc<JobKindRegistry>>;
+    fn lock(self) -> Self::Locked {
+        self.write().expect("shouldn't be poisoned")
+    }
+    fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry {
+        Arc::make_mut(locked)
+    }
+}
+
+impl JobKindRegistryRegisterLock for &'_ mut JobKindRegistry {
+    type Locked = Self;
+
+    fn lock(self) -> Self::Locked {
+        self
+    }
+
+    fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry {
+        locked
+    }
+}
+
+impl JobKindRegistry {
+    fn lock() -> &'static RwLock<Arc<Self>> {
+        static REGISTRY: OnceLock<RwLock<Arc<JobKindRegistry>>> = OnceLock::new();
+        REGISTRY.get_or_init(Default::default)
+    }
+    fn try_register<L: JobKindRegistryRegisterLock>(
+        lock: L,
+        job_kind: DynJobKind,
+    ) -> Result<(), JobKindRegisterError> {
+        use std::collections::btree_map::Entry;
+        let name = InternedStrCompareAsStr(job_kind.name());
+        // run user code only outside of lock
+        let mut locked = lock.lock();
+        let this = L::make_mut(&mut locked);
+        let result = match this.job_kinds.entry(name) {
+            Entry::Occupied(entry) => Err(JobKindRegisterError::SameName {
+                name,
+                old_job_kind: entry.get().clone(),
+                new_job_kind: job_kind,
+            }),
+            Entry::Vacant(entry) => {
+                entry.insert(job_kind);
+                Ok(())
+            }
+        };
+        drop(locked);
+        // outside of lock now, so we can test if it's the same DynJobKind
+        match result {
+            Err(JobKindRegisterError::SameName {
+                name: _,
+                old_job_kind,
+                new_job_kind,
+            }) if old_job_kind == new_job_kind => Ok(()),
+            result => result,
+        }
+    }
+    #[track_caller]
+    fn register<L: JobKindRegistryRegisterLock>(lock: L, job_kind: DynJobKind) {
+        match Self::try_register(lock, job_kind) {
+            Err(e) => panic!("{e}"),
+            Ok(()) => {}
+        }
+    }
+    fn get() -> Arc<Self> {
+        Self::lock().read().expect("shouldn't be poisoned").clone()
+    }
+}
+
+impl Default for JobKindRegistry {
+    fn default() -> Self {
+        let mut retval = Self {
+            job_kinds: BTreeMap::new(),
+        };
+        for job_kind in BUILT_IN_JOB_KINDS {
+            Self::register(&mut retval, job_kind());
+        }
+        retval
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct JobKindRegistrySnapshot(Arc<JobKindRegistry>);
+
+impl JobKindRegistrySnapshot {
+    pub fn get() -> Self {
+        JobKindRegistrySnapshot(JobKindRegistry::get())
+    }
+    pub fn get_by_name<'a>(&'a self, name: &str) -> Option<&'a DynJobKind> {
+        self.0.job_kinds.get(name)
+    }
+    pub fn iter_with_names(&self) -> JobKindRegistryIterWithNames<'_> {
+        JobKindRegistryIterWithNames(self.0.job_kinds.iter())
+    }
+    pub fn iter(&self) -> JobKindRegistryIter<'_> {
+        JobKindRegistryIter(self.0.job_kinds.values())
+    }
+}
+
+impl<'a> IntoIterator for &'a JobKindRegistrySnapshot {
+    type Item = &'a DynJobKind;
+    type IntoIter = JobKindRegistryIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.iter()
+    }
+}
+
+impl<'a> IntoIterator for &'a mut JobKindRegistrySnapshot {
+    type Item = &'a DynJobKind;
+    type IntoIter = JobKindRegistryIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.iter()
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct JobKindRegistryIter<'a>(
+    std::collections::btree_map::Values<'a, InternedStrCompareAsStr, DynJobKind>,
+);
+
+impl<'a> Iterator for JobKindRegistryIter<'a> {
+    type Item = &'a DynJobKind;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.0.next()
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.0.size_hint()
+    }
+
+    fn count(self) -> usize
+    where
+        Self: Sized,
+    {
+        self.0.count()
+    }
+
+    fn last(self) -> Option<Self::Item> {
+        self.0.last()
+    }
+
+    fn nth(&mut self, n: usize) -> Option<Self::Item> {
+        self.0.nth(n)
+    }
+
+    fn fold<B, F>(self, init: B, f: F) -> B
+    where
+        F: FnMut(B, Self::Item) -> B,
+    {
+        self.0.fold(init, f)
+    }
+}
+
+impl<'a> std::iter::FusedIterator for JobKindRegistryIter<'a> {}
+
+impl<'a> ExactSizeIterator for JobKindRegistryIter<'a> {}
+
+impl<'a> DoubleEndedIterator for JobKindRegistryIter<'a> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.0.next_back()
+    }
+
+    fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
+        self.0.nth_back(n)
+    }
+
+    fn rfold<B, F>(self, init: B, f: F) -> B
+    where
+        F: FnMut(B, Self::Item) -> B,
+    {
+        self.0.rfold(init, f)
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct JobKindRegistryIterWithNames<'a>(
+    std::collections::btree_map::Iter<'a, InternedStrCompareAsStr, DynJobKind>,
+);
+
+impl<'a> Iterator for JobKindRegistryIterWithNames<'a> {
+    type Item = (Interned<str>, &'a DynJobKind);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.0.next().map(|(name, job_kind)| (name.0, job_kind))
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.0.size_hint()
+    }
+
+    fn count(self) -> usize
+    where
+        Self: Sized,
+    {
+        self.0.count()
+    }
+
+    fn last(self) -> Option<Self::Item> {
+        self.0.last().map(|(name, job_kind)| (name.0, job_kind))
+    }
+
+    fn nth(&mut self, n: usize) -> Option<Self::Item> {
+        self.0.nth(n).map(|(name, job_kind)| (name.0, job_kind))
+    }
+
+    fn fold<B, F>(self, init: B, f: F) -> B
+    where
+        F: FnMut(B, Self::Item) -> B,
+    {
+        self.0
+            .map(|(name, job_kind)| (name.0, job_kind))
+            .fold(init, f)
+    }
+}
+
+impl<'a> std::iter::FusedIterator for JobKindRegistryIterWithNames<'a> {}
+
+impl<'a> ExactSizeIterator for JobKindRegistryIterWithNames<'a> {}
+
+impl<'a> DoubleEndedIterator for JobKindRegistryIterWithNames<'a> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.0
+            .next_back()
+            .map(|(name, job_kind)| (name.0, job_kind))
+    }
+
+    fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
+        self.0
+            .nth_back(n)
+            .map(|(name, job_kind)| (name.0, job_kind))
+    }
+
+    fn rfold<B, F>(self, init: B, f: F) -> B
+    where
+        F: FnMut(B, Self::Item) -> B,
+    {
+        self.0
+            .map(|(name, job_kind)| (name.0, job_kind))
+            .rfold(init, f)
+    }
+}
+
+#[track_caller]
+pub fn register_job_kind<K: JobKind>(kind: K) {
+    DynJobKind::new(kind).register();
+}

crates/fayalite/src/build/verilog.rs

@@ -0,0 +1,371 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// See Notices.txt for copyright information
+
+use crate::{
+    build::{
+        CommandParams, GetBaseJob, JobAndDependencies, JobArgs, JobArgsAndDependencies,
+        JobDependencies, JobItem, JobItemName, JobKind, JobKindAndDependencies, JobParams,
+        external::{ExternalCommand, ExternalCommandJob, ExternalCommandJobKind},
+        firrtl::FirrtlJobKind,
+        interned_known_utf8_method, interned_known_utf8_path_buf_method,
+    },
+    intern::{Intern, Interned},
+    util::job_server::AcquiredJob,
+};
+use clap::Args;
+use eyre::bail;
+use serde::{Deserialize, Serialize};
+use std::{fmt, mem};
+
+/// based on [LLVM Circt's recommended lowering options][lowering-options]
+///
+/// [lowering-options]: https://circt.llvm.org/docs/VerilogGeneration/#recommended-loweringoptions-by-target
+#[derive(clap::ValueEnum, Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[non_exhaustive]
+pub enum VerilogDialect {
+    Questa,
+    Spyglass,
+    Verilator,
+    Vivado,
+    Yosys,
+}
+
+impl fmt::Display for VerilogDialect {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(self.as_str())
+    }
+}
+
+impl VerilogDialect {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            VerilogDialect::Questa => "questa",
+            VerilogDialect::Spyglass => "spyglass",
+            VerilogDialect::Verilator => "verilator",
+            VerilogDialect::Vivado => "vivado",
+            VerilogDialect::Yosys => "yosys",
+        }
+    }
+    pub fn firtool_extra_args(self) -> &'static [&'static str] {
+        match self {
+            VerilogDialect::Questa => &["--lowering-options=emitWireInPorts"],
+            VerilogDialect::Spyglass => {
+                &["--lowering-options=explicitBitcast,disallowExpressionInliningInPorts"]
+            }
+            VerilogDialect::Verilator => &[
+                "--lowering-options=locationInfoStyle=wrapInAtSquareBracket,disallowLocalVariables",
+            ],
+            VerilogDialect::Vivado => &["--lowering-options=mitigateVivadoArrayIndexConstPropBug"],
+            VerilogDialect::Yosys => {
+                &["--lowering-options=disallowLocalVariables,disallowPackedArrays"]
+            }
+        }
+    }
+}
+
+#[derive(Args, Debug, Clone, PartialEq, Eq, Hash)]
+#[non_exhaustive]
+pub struct UnadjustedVerilogArgs {
+    #[arg(long = "firtool-extra-arg", value_name = "ARG")]
+    pub firtool_extra_args: Vec<String>,
+    /// adapt the generated Verilog for a particular toolchain
+    #[arg(long)]
+    pub verilog_dialect: Option<VerilogDialect>,
+    #[arg(long)]
+    pub verilog_debug: bool,
+}
+
+impl JobArgs for UnadjustedVerilogArgs {
+    fn to_args<Args: Extend<Interned<str>> + ?Sized>(&self, args: &mut Args) {
+        let Self {
+            ref firtool_extra_args,
+            verilog_dialect,
+            verilog_debug,
+        } = *self;
+        args.extend(
+            firtool_extra_args
+                .iter()
+                .map(|arg| str::intern_owned(format!("--firtool-extra-arg={arg}"))),
+        );
+        args.extend(verilog_dialect.map(|v| str::intern_owned(format!("--verilog-dialect={v}"))));
+        if verilog_debug {
+            args.extend(["--verilog-debug".intern()]);
+        }
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, Hash, Debug, Deserialize, Serialize)]
+pub struct UnadjustedVerilog {
+    firrtl_file: Interned<str>,
+    firrtl_file_name: Interned<str>,
+    unadjusted_verilog_file: Interned<str>,
+    unadjusted_verilog_file_name: Interned<str>,
+    firtool_extra_args: Interned<[Interned<str>]>,
+    verilog_dialect: Option<VerilogDialect>,
+    verilog_debug: bool,
+}
+
+impl UnadjustedVerilog {
+    pub fn firrtl_file(&self) -> Interned<str> {
+        self.firrtl_file
+    }
+    pub fn unadjusted_verilog_file(&self) -> Interned<str> {
+        self.unadjusted_verilog_file
+    }
+    pub fn firtool_extra_args(&self) -> Interned<[Interned<str>]> {
+        self.firtool_extra_args
+    }
+    pub fn verilog_dialect(&self) -> Option<VerilogDialect> {
+        self.verilog_dialect
+    }
+    pub fn verilog_debug(&self) -> bool {
+        self.verilog_debug
+    }
+}
+
+impl ExternalCommand for UnadjustedVerilog {
+    type AdditionalArgs = UnadjustedVerilogArgs;
+    type AdditionalJobData = UnadjustedVerilog;
+    type Dependencies = JobKindAndDependencies<FirrtlJobKind>;
+
+    fn dependencies() -> Self::Dependencies {
+        Default::default()
+    }
+
+    fn args_to_jobs(
+        args: JobArgsAndDependencies<ExternalCommandJobKind<Self>>,
+        params: &JobParams,
+    ) -> eyre::Result<(
+        Self::AdditionalJobData,
+        <Self::Dependencies as JobDependencies>::JobsAndKinds,
+    )> {
+        args.args_to_jobs_external_simple(params, |args, dependencies| {
+            let UnadjustedVerilogArgs {
+                firtool_extra_args,
+                verilog_dialect,
+                verilog_debug,
+            } = args.additional_args;
+            let unadjusted_verilog_file = dependencies
+                .dependencies
+                .job
+                .job
+                .file_with_ext("unadjusted.v");
+            Ok(UnadjustedVerilog {
+                firrtl_file: dependencies.job.job.firrtl_file(),
+                firrtl_file_name: interned_known_utf8_method(
+                    dependencies.job.job.firrtl_file(),
+                    |v| v.file_name().expect("known to have file name"),
+                ),
+                unadjusted_verilog_file,
+                unadjusted_verilog_file_name: interned_known_utf8_method(
+                    unadjusted_verilog_file,
+                    |v| v.file_name().expect("known to have file name"),
+                ),
+                firtool_extra_args: firtool_extra_args
+                    .into_iter()
+                    .map(str::intern_owned)
+                    .collect(),
+                verilog_dialect,
+                verilog_debug,
+            })
+        })
+    }
+
+    fn inputs(job: &ExternalCommandJob<Self>) -> Interned<[JobItemName]> {
+        [JobItemName::Path {
+            path: job.additional_job_data().firrtl_file,
+        }][..]
+            .intern()
+    }
+
+    fn output_paths(job: &ExternalCommandJob<Self>) -> Interned<[Interned<str>]> {
+        [job.additional_job_data().unadjusted_verilog_file][..].intern()
+    }
+
+    fn command_line_args(job: &ExternalCommandJob<Self>) -> Interned<[Interned<str>]> {
+        let UnadjustedVerilog {
+            firrtl_file: _,
+            firrtl_file_name,
+            unadjusted_verilog_file: _,
+            unadjusted_verilog_file_name,
+            firtool_extra_args,
+            verilog_dialect,
+            verilog_debug,
+        } = *job.additional_job_data();
+        let mut retval = vec![
+            firrtl_file_name,
+            "-o".intern(),
+            unadjusted_verilog_file_name,
+        ];
+        if verilog_debug {
+            retval.push("-g".intern());
+            retval.push("--preserve-values=all".intern());
+        }
+        if let Some(dialect) = verilog_dialect {
+            retval.extend(
+                dialect
+                    .firtool_extra_args()
+                    .iter()
+                    .copied()
+                    .map(str::intern),
+            );
+        }
+        retval.extend_from_slice(&firtool_extra_args);
+        Intern::intern_owned(retval)
+    }
+
+    fn current_dir(job: &ExternalCommandJob<Self>) -> Option<Interned<str>> {
+        Some(job.output_dir())
+    }
+
+    fn job_kind_name() -> Interned<str> {
+        "unadjusted-verilog".intern()
+    }
+
+    fn default_program_name() -> Interned<str> {
+        "firtool".intern()
+    }
+
+    fn subcommand_hidden() -> bool {
+        true
+    }
+
+    fn run_even_if_cached_arg_name() -> Interned<str> {
+        "firtool-run-even-if-cached".intern()
+    }
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct VerilogJobKind;
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Args)]
+#[non_exhaustive]
+pub struct VerilogJobArgs {}
+
+impl JobArgs for VerilogJobArgs {
+    fn to_args<Args: Extend<Interned<str>> + ?Sized>(&self, _args: &mut Args) {
+        let Self {} = self;
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct VerilogJob {
+    output_dir: Interned<str>,
+    unadjusted_verilog_file: Interned<str>,
+    main_verilog_file: Interned<str>,
+}
+
+impl VerilogJob {
+    pub fn output_dir(&self) -> Interned<str> {
+        self.output_dir
+    }
+    pub fn unadjusted_verilog_file(&self) -> Interned<str> {
+        self.unadjusted_verilog_file
+    }
+    pub fn main_verilog_file(&self) -> Interned<str> {
+        self.main_verilog_file
+    }
+}
+
+impl JobKind for VerilogJobKind {
+    type Args = VerilogJobArgs;
+    type Job = VerilogJob;
+    type Dependencies = JobKindAndDependencies<ExternalCommandJobKind<UnadjustedVerilog>>;
+
+    fn dependencies(self) -> Self::Dependencies {
+        Default::default()
+    }
+
+    fn args_to_jobs(
+        args: JobArgsAndDependencies<Self>,
+        params: &JobParams,
+    ) -> eyre::Result<JobAndDependencies<Self>> {
+        args.args_to_jobs_simple(params, |_kind, args, dependencies| {
+            let VerilogJobArgs {} = args;
+            Ok(VerilogJob {
+                output_dir: dependencies.base_job().output_dir(),
+                unadjusted_verilog_file: dependencies
+                    .job
+                    .job
+                    .additional_job_data()
+                    .unadjusted_verilog_file(),
+                main_verilog_file: dependencies.base_job().file_with_ext("v"),
+            })
+        })
+    }
+
+    fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]> {
+        [JobItemName::Path {
+            path: job.unadjusted_verilog_file,
+        }][..]
+            .intern()
+    }
+
+    fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> {
+        [
+            JobItemName::Path {
+                path: job.main_verilog_file,
+            },
+            JobItemName::DynamicPaths {
+                source_job_name: self.name(),
+            },
+        ][..]
+            .intern()
+    }
+
+    fn name(self) -> Interned<str> {
+        "verilog".intern()
+    }
+
+    fn external_command_params(self, _job: &Self::Job) -> Option<CommandParams> {
+        None
+    }
+
+    fn run(
+        self,
+        job: &Self::Job,
+        inputs: &[JobItem],
+        _params: &JobParams,
+        _acquired_job: &mut AcquiredJob,
+    ) -> eyre::Result<Vec<JobItem>> {
+        assert!(inputs.iter().map(JobItem::name).eq(self.inputs(job)));
+        let input = std::fs::read_to_string(job.unadjusted_verilog_file())?;
+        let file_separator_prefix = "\n// ----- 8< ----- FILE \"";
+        let file_separator_suffix = "\" ----- 8< -----\n\n";
+        let mut input = &*input;
+        let main_verilog_file = job.main_verilog_file();
+        let mut file_name = Some(main_verilog_file);
+        let mut additional_outputs = Vec::new();
+        loop {
+            let (chunk, next_file_name) = if let Some((chunk, rest)) =
+                input.split_once(file_separator_prefix)
+            {
+                let Some((next_file_name, rest)) = rest.split_once(file_separator_suffix) else {
+                    bail!(
+                        "parsing firtool's output failed: found {file_separator_prefix:?} but no {file_separator_suffix:?}"
+                    );
+                };
+                input = rest;
+                let next_file_name =
+                    interned_known_utf8_path_buf_method(job.output_dir, |v| v.join(next_file_name));
+                additional_outputs.push(next_file_name);
+                (chunk, Some(next_file_name))
+            } else {
+                (mem::take(&mut input), None)
+            };
+            let Some(file_name) = mem::replace(&mut file_name, next_file_name) else {
+                break;
+            };
+            std::fs::write(&file_name, chunk)?;
+        }
+        Ok(vec![
+            JobItem::Path {
+                path: main_verilog_file,
+            },
+            JobItem::DynamicPaths {
+                paths: additional_outputs,
+                source_job_name: self.name(),
+            },
+        ])
+    }
+}

crates/fayalite/src/cli.rs

@@ -50,7 +50,7 @@ impl From<io::Error> for CliError {
 pub trait RunPhase<Arg> {
     type Output;
     fn run(&self, arg: Arg) -> Result<Self::Output> {
-        self.run_with_job(arg, &mut AcquiredJob::acquire())
+        self.run_with_job(arg, &mut AcquiredJob::acquire()?)
     }
     fn run_with_job(&self, arg: Arg, acquired_job: &mut AcquiredJob) -> Result<Self::Output>;
 }

crates/fayalite/src/firrtl.rs

@@ -7,6 +7,7 @@ use crate::{
         DocStringAnnotation, DontTouchAnnotation, SVAttributeAnnotation,
     },
     array::Array,
+    build::JobArgs,
     bundle::{Bundle, BundleField, BundleType},
     clock::Clock,
     enum_::{Enum, EnumType, EnumVariant},
@@ -42,7 +43,7 @@ use crate::{
 use bitvec::slice::BitSlice;
 use clap::value_parser;
 use num_traits::Signed;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use std::{
     cell::{Cell, RefCell},
     cmp::Ordering,
@@ -2749,14 +2750,23 @@ impl clap::builder::TypedValueParser for OptionSimplifyEnumsKindValueParser {
 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
 pub struct ExportOptionsPrivate(());
 
-#[derive(clap::Parser, Copy, Clone, PartialEq, Eq, Hash)]
+impl ExportOptionsPrivate {
+    fn private_new() -> Self {
+        Self(())
+    }
+}
+
+#[derive(clap::Parser, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct ExportOptions {
     #[clap(long = "no-simplify-memories", action = clap::ArgAction::SetFalse)]
+    #[serde(default = "ExportOptions::default_simplify_memories")]
     pub simplify_memories: bool,
     #[clap(long, value_parser = OptionSimplifyEnumsKindValueParser, default_value = "replace-with-bundle-of-uints")]
+    #[serde(default = "ExportOptions::default_simplify_enums")]
     pub simplify_enums: std::option::Option<SimplifyEnumsKind>, // use std::option::Option instead of Option to avoid clap mis-parsing
     #[doc(hidden)]
     #[clap(skip = ExportOptionsPrivate(()))]
+    #[serde(skip, default = "ExportOptionsPrivate::private_new")]
     /// `#[non_exhaustive]` except allowing struct update syntax
     pub __private: ExportOptionsPrivate,
 }
@@ -2767,16 +2777,15 @@ impl fmt::Debug for ExportOptions {
     }
 }
 
-impl ExportOptions {
+impl JobArgs for ExportOptions {
-    pub fn to_args(&self) -> Vec<Interned<str>> {
+    fn to_args<Args: Extend<Interned<str>> + ?Sized>(&self, args: &mut Args) {
         let Self {
             simplify_memories,
             simplify_enums,
             __private: ExportOptionsPrivate(()),
-        } = self;
+        } = *self;
-        let mut retval = Vec::new();
+        if !simplify_memories {
-        if !*simplify_memories {
+            args.extend(["--no-simplify-memories".intern()]);
-            retval.push("--no-simplify-memories".intern());
         }
         let simplify_enums = simplify_enums.map(|v| {
             clap::ValueEnum::to_possible_value(&v).expect("there are no skipped variants")
@@ -2785,10 +2794,18 @@ impl ExportOptions {
             None => OptionSimplifyEnumsKindValueParser::NONE_NAME,
             Some(v) => v.get_name(),
         };
-        retval.push(str::intern_owned(format!(
+        args.extend([str::intern_owned(format!(
             "--simplify-enums={simplify_enums}"
-        )));
+        ))]);
-        retval
+    }
+}
+
+impl ExportOptions {
+    fn default_simplify_memories() -> bool {
+        true
+    }
+    fn default_simplify_enums() -> Option<SimplifyEnumsKind> {
+        Some(SimplifyEnumsKind::ReplaceWithBundleOfUInts)
     }
     fn debug_fmt(
         &self,
@@ -2846,8 +2863,8 @@ impl ExportOptions {
 impl Default for ExportOptions {
     fn default() -> Self {
         Self {
-            simplify_memories: true,
+            simplify_memories: Self::default_simplify_memories(),
-            simplify_enums: Some(SimplifyEnumsKind::ReplaceWithBundleOfUInts),
+            simplify_enums: Self::default_simplify_enums(),
             __private: ExportOptionsPrivate(()),
         }
     }

crates/fayalite/src/intern.rs

@@ -9,11 +9,13 @@ use std::{
     any::{Any, TypeId},
     borrow::{Borrow, Cow},
     cmp::Ordering,
+    ffi::OsStr,
     fmt,
     hash::{BuildHasher, Hash, Hasher},
     iter::FusedIterator,
     marker::PhantomData,
     ops::Deref,
+    path::Path,
     sync::{Mutex, RwLock},
 };
 
@@ -416,6 +418,12 @@ forward_fmt_trait!(Pointer);
 forward_fmt_trait!(UpperExp);
 forward_fmt_trait!(UpperHex);
 
+impl<T: ?Sized + 'static + Send + Sync> AsRef<T> for Interned<T> {
+    fn as_ref(&self) -> &T {
+        self
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct InternedSliceIter<T: Clone + 'static + Send + Sync> {
     slice: Interned<[T]>,
@@ -485,6 +493,51 @@ where
     }
 }
 
+impl<I> FromIterator<I> for Interned<str>
+where
+    String: FromIterator<I>,
+{
+    fn from_iter<T: IntoIterator<Item = I>>(iter: T) -> Self {
+        str::intern_owned(FromIterator::from_iter(iter))
+    }
+}
+
+impl AsRef<OsStr> for Interned<str> {
+    fn as_ref(&self) -> &OsStr {
+        str::as_ref(self)
+    }
+}
+
+impl AsRef<Path> for Interned<str> {
+    fn as_ref(&self) -> &Path {
+        str::as_ref(self)
+    }
+}
+
+impl From<Interned<str>> for clap::builder::Str {
+    fn from(value: Interned<str>) -> Self {
+        Interned::into_inner(value).into()
+    }
+}
+
+impl From<Interned<str>> for clap::builder::OsStr {
+    fn from(value: Interned<str>) -> Self {
+        Interned::into_inner(value).into()
+    }
+}
+
+impl From<Interned<str>> for clap::builder::StyledStr {
+    fn from(value: Interned<str>) -> Self {
+        Interned::into_inner(value).into()
+    }
+}
+
+impl From<Interned<str>> for clap::Id {
+    fn from(value: Interned<str>) -> Self {
+        Interned::into_inner(value).into()
+    }
+}
+
 impl<T: 'static + Clone + Send + Sync> From<Interned<[T]>> for Vec<T> {
     fn from(value: Interned<[T]>) -> Self {
         Vec::from(&*value)

crates/fayalite/src/module/transform/simplify_enums.rs

@@ -1,3 +1,5 @@
+use serde::{Deserialize, Serialize};
+
 // SPDX-License-Identifier: LGPL-3.0-or-later
 // See Notices.txt for copyright information
 use crate::{
@@ -955,12 +957,15 @@ impl Folder for State {
     }
 }
 
-#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, clap::ValueEnum)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, clap::ValueEnum, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
 pub enum SimplifyEnumsKind {
     SimplifyToEnumsWithNoBody,
     #[clap(name = "replace-with-bundle-of-uints")]
+    #[serde(rename = "replace-with-bundle-of-uints")]
     ReplaceWithBundleOfUInts,
     #[clap(name = "replace-with-uint")]
+    #[serde(rename = "replace-with-uint")]
     ReplaceWithUInt,
 }
 

crates/fayalite/src/util.rs

@@ -36,8 +36,11 @@ pub use scoped_ref::ScopedRef;
 pub(crate) use misc::chain;
 #[doc(inline)]
 pub use misc::{
-    BitSliceWriteWithBase, DebugAsDisplay, DebugAsRawString, MakeMutSlice, RcWriter, interned_bit,
+    BitSliceWriteWithBase, DebugAsDisplay, DebugAsRawString, MakeMutSlice, RcWriter,
-    iter_eq_by, slice_range, try_slice_range,
+    SerdeJsonEscapeIf, SerdeJsonEscapeIfFormatter, SerdeJsonEscapeIfTest,
+    SerdeJsonEscapeIfTestResult, interned_bit, iter_eq_by, serialize_to_json_ascii,
+    serialize_to_json_ascii_pretty, serialize_to_json_ascii_pretty_with_indent, slice_range,
+    try_slice_range,
 };
 
 pub mod job_server;

crates/fayalite/src/util/job_server.rs

@@ -1,26 +1,36 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
 // See Notices.txt for copyright information
 
-use ctor::ctor;
+use ctor::{ctor, dtor};
-use jobslot::{Acquired, Client};
+use jobslot::Client;
 use std::{
     ffi::OsString,
-    mem,
+    io, mem,
     num::NonZeroUsize,
-    sync::{Condvar, Mutex, Once, OnceLock},
+    sync::{Mutex, MutexGuard},
-    thread::spawn,
 };
 
-fn get_or_make_client() -> &'static Client {
+#[ctor]
-    #[ctor]
+static CLIENT: Mutex<Option<Option<Client>>> = unsafe { Mutex::new(Some(Client::from_env())) };
-    static CLIENT: OnceLock<Client> = unsafe {
-        match Client::from_env() {
-            Some(client) => OnceLock::from(client),
-            None => OnceLock::new(),
-        }
-    };
 
-    CLIENT.get_or_init(|| {
+#[dtor]
+fn drop_client() {
+    drop(
+        match CLIENT.lock() {
+            Ok(v) => v,
+            Err(e) => e.into_inner(),
+        }
+        .take(),
+    );
+}
+
+fn get_or_make_client() -> Client {
+    CLIENT
+        .lock()
+        .expect("shouldn't have panicked")
+        .as_mut()
+        .expect("shutting down")
+        .get_or_insert_with(|| {
             let mut available_parallelism = None;
             let mut args = std::env::args_os().skip(1);
             while let Some(arg) = args.next() {
@@ -52,141 +62,95 @@ fn get_or_make_client() -> &'static Client {
             } else {
                 NonZeroUsize::new(1).unwrap()
             };
-        Client::new_with_fifo(available_parallelism.get() - 1).expect("failed to create job server")
+            Client::new_with_fifo(available_parallelism.get() - 1)
+                .expect("failed to create job server")
         })
+        .clone()
 }
 
 struct State {
+    obtained_count: usize,
     waiting_count: usize,
-    available: Vec<Acquired>,
-    implicit_available: bool,
-}
-
-impl State {
-    fn total_available(&self) -> usize {
-        self.available.len() + self.implicit_available as usize
-    }
-    fn additional_waiting(&self) -> usize {
-        self.waiting_count.saturating_sub(self.total_available())
-    }
 }
 
 static STATE: Mutex<State> = Mutex::new(State {
+    obtained_count: 0,
     waiting_count: 0,
-    available: Vec::new(),
-    implicit_available: true,
 });
-static COND_VAR: Condvar = Condvar::new();
-
-#[derive(Debug)]
-enum AcquiredJobInner {
-    FromJobServer(Acquired),
-    ImplicitJob,
-}
 
 #[derive(Debug)]
 pub struct AcquiredJob {
-    job: AcquiredJobInner,
+    client: Client,
 }
 
 impl AcquiredJob {
-    fn start_acquire_thread() {
+    pub fn acquire() -> io::Result<Self> {
-        static STARTED_THREAD: Once = Once::new();
-        STARTED_THREAD.call_once(|| {
-            spawn(|| {
-                let mut acquired = None;
         let client = get_or_make_client();
+        struct Waiting {}
+
+        impl Waiting {
+            fn done(self) -> MutexGuard<'static, State> {
+                mem::forget(self);
                 let mut state = STATE.lock().unwrap();
-                loop {
+                state.waiting_count -= 1;
-                    state = if state.additional_waiting() == 0 {
-                        if acquired.is_some() {
-                            drop(state);
-                            drop(acquired.take()); // drop Acquired outside of lock
-                            STATE.lock().unwrap()
-                        } else {
-                            COND_VAR.wait(state).unwrap()
-                        }
-                    } else if acquired.is_some() {
-                        // allocate space before moving Acquired to ensure we
-                        // drop Acquired outside of the lock on panic
-                        state.available.reserve(1);
-                        state.available.push(acquired.take().unwrap());
-                        COND_VAR.notify_all();
                 state
-                    } else {
-                        drop(state);
-                        acquired = Some(
-                            client
-                                .acquire()
-                                .expect("can't acquire token from job server"),
-                        );
-                        STATE.lock().unwrap()
-                    };
             }
-            });
-        });
         }
-    fn acquire_inner(block: bool) -> Option<Self> {
+        impl Drop for Waiting {
-        Self::start_acquire_thread();
+            fn drop(&mut self) {
+                STATE.lock().unwrap().waiting_count -= 1;
+            }
+        }
         let mut state = STATE.lock().unwrap();
-        loop {
+        if state.obtained_count == 0 && state.waiting_count == 0 {
-            if let Some(acquired) = state.available.pop() {
+            state.obtained_count = 1; // get implicit token
-                return Some(Self {
+            return Ok(Self { client });
-                    job: AcquiredJobInner::FromJobServer(acquired),
-                });
-            }
-            if state.implicit_available {
-                state.implicit_available = false;
-                return Some(Self {
-                    job: AcquiredJobInner::ImplicitJob,
-                });
-            }
-            if !block {
-                return None;
         }
         state.waiting_count += 1;
-            state = COND_VAR.wait(state).unwrap();
+        drop(state);
-            state.waiting_count -= 1;
+        let waiting = Waiting {};
-        }
+        client.acquire_raw()?;
-    }
+        state = waiting.done();
-    pub fn try_acquire() -> Option<Self> {
+        state.obtained_count = state
-        Self::acquire_inner(false)
+            .obtained_count
-    }
+            .checked_add(1)
-    pub fn acquire() -> Self {
+            .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "obtained_count overflowed"))?;
-        Self::acquire_inner(true).expect("failed to acquire token")
+        drop(state);
+        Ok(Self { client })
     }
     pub fn run_command<R>(
         &mut self,
         cmd: std::process::Command,
         f: impl FnOnce(&mut std::process::Command) -> std::io::Result<R>,
     ) -> std::io::Result<R> {
-        get_or_make_client().configure_make_and_run_with_fifo(cmd, f)
+        self.client.configure_make_and_run_with_fifo(cmd, f)
     }
 }
 
 impl Drop for AcquiredJob {
     fn drop(&mut self) {
         let mut state = STATE.lock().unwrap();
-        match &self.job {
+        match &mut *state {
-            AcquiredJobInner::FromJobServer(_) => {
+            State {
-                if state.waiting_count > state.available.len() + state.implicit_available as usize {
+                obtained_count: 0, ..
-                    // allocate space before moving Acquired to ensure we
+            } => unreachable!(),
-                    // drop Acquired outside of the lock on panic
+            State {
-                    state.available.reserve(1);
+                obtained_count: obtained_count @ 1,
-                    let AcquiredJobInner::FromJobServer(acquired) =
+                waiting_count,
-                        mem::replace(&mut self.job, AcquiredJobInner::ImplicitJob)
+            } => {
-                    else {
+                *obtained_count = 0; // drop implicit token
-                        unreachable!()
+                let any_waiting = *waiting_count != 0;
-                    };
+                drop(state);
-                    state.available.push(acquired);
+                if any_waiting {
-                    COND_VAR.notify_all();
+                    // we have the implicit token, but some other thread is trying to acquire a token,
+                    // release the implicit token so they can acquire it.
+                    let _ = self.client.release_raw(); // we're in drop, just ignore errors since we at least tried
                 }
             }
-            AcquiredJobInner::ImplicitJob => {
+            State { obtained_count, .. } => {
-                state.implicit_available = true;
+                *obtained_count = obtained_count.saturating_sub(1);
-                if state.waiting_count > state.available.len() {
+                drop(state);
-                    COND_VAR.notify_all();
+                let _ = self.client.release_raw(); // we're in drop, just ignore errors since we at least tried
-                }
             }
         }
     }

crates/fayalite/src/util/misc.rs

@@ -5,6 +5,7 @@ use bitvec::{bits, order::Lsb0, slice::BitSlice, view::BitView};
 use std::{
     cell::Cell,
     fmt::{self, Debug, Write},
+    io,
     ops::{Bound, Range, RangeBounds},
     rc::Rc,
     sync::{Arc, OnceLock},
@@ -243,3 +244,323 @@ pub fn try_slice_range<R: RangeBounds<usize>>(range: R, size: usize) -> Option<R
 pub fn slice_range<R: RangeBounds<usize>>(range: R, size: usize) -> Range<usize> {
     try_slice_range(range, size).expect("range out of bounds")
 }
+
+pub trait SerdeJsonEscapeIfTest {
+    fn char_needs_escape(&mut self, ch: char) -> serde_json::Result<bool>;
+}
+
+pub trait SerdeJsonEscapeIfTestResult {
+    fn to_result(self) -> serde_json::Result<bool>;
+}
+
+impl SerdeJsonEscapeIfTestResult for bool {
+    fn to_result(self) -> serde_json::Result<bool> {
+        Ok(self)
+    }
+}
+
+impl<E: Into<serde_json::Error>> SerdeJsonEscapeIfTestResult for Result<bool, E> {
+    fn to_result(self) -> serde_json::Result<bool> {
+        self.map_err(Into::into)
+    }
+}
+
+impl<T: ?Sized + FnMut(char) -> R, R: SerdeJsonEscapeIfTestResult> SerdeJsonEscapeIfTest for T {
+    fn char_needs_escape(&mut self, ch: char) -> serde_json::Result<bool> {
+        self(ch).to_result()
+    }
+}
+
+pub trait SerdeJsonEscapeIfFormatter: serde_json::ser::Formatter {
+    fn write_unicode_escape<W>(&mut self, writer: &mut W, ch: char) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        for utf16 in ch.encode_utf16(&mut [0; 2]) {
+            write!(writer, "\\u{utf16:04x}")?;
+        }
+        Ok(())
+    }
+}
+
+impl SerdeJsonEscapeIfFormatter for serde_json::ser::CompactFormatter {}
+impl SerdeJsonEscapeIfFormatter for serde_json::ser::PrettyFormatter<'_> {}
+
+pub struct SerdeJsonEscapeIf<Test, Base = serde_json::ser::CompactFormatter> {
+    pub base: Base,
+    pub test: Test,
+}
+
+impl<Test: SerdeJsonEscapeIfTest, Base: SerdeJsonEscapeIfFormatter> serde_json::ser::Formatter
+    for SerdeJsonEscapeIf<Test, Base>
+{
+    fn write_null<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_null(writer)
+    }
+
+    fn write_bool<W>(&mut self, writer: &mut W, value: bool) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_bool(writer, value)
+    }
+
+    fn write_i8<W>(&mut self, writer: &mut W, value: i8) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_i8(writer, value)
+    }
+
+    fn write_i16<W>(&mut self, writer: &mut W, value: i16) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_i16(writer, value)
+    }
+
+    fn write_i32<W>(&mut self, writer: &mut W, value: i32) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_i32(writer, value)
+    }
+
+    fn write_i64<W>(&mut self, writer: &mut W, value: i64) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_i64(writer, value)
+    }
+
+    fn write_i128<W>(&mut self, writer: &mut W, value: i128) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_i128(writer, value)
+    }
+
+    fn write_u8<W>(&mut self, writer: &mut W, value: u8) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_u8(writer, value)
+    }
+
+    fn write_u16<W>(&mut self, writer: &mut W, value: u16) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_u16(writer, value)
+    }
+
+    fn write_u32<W>(&mut self, writer: &mut W, value: u32) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_u32(writer, value)
+    }
+
+    fn write_u64<W>(&mut self, writer: &mut W, value: u64) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_u64(writer, value)
+    }
+
+    fn write_u128<W>(&mut self, writer: &mut W, value: u128) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_u128(writer, value)
+    }
+
+    fn write_f32<W>(&mut self, writer: &mut W, value: f32) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_f32(writer, value)
+    }
+
+    fn write_f64<W>(&mut self, writer: &mut W, value: f64) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_f64(writer, value)
+    }
+
+    fn write_number_str<W>(&mut self, writer: &mut W, value: &str) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_number_str(writer, value)
+    }
+
+    fn begin_string<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_string(writer)
+    }
+
+    fn end_string<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_string(writer)
+    }
+
+    fn write_string_fragment<W>(&mut self, writer: &mut W, mut fragment: &str) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        while let Some((next_escape_index, next_escape_char)) = fragment
+            .char_indices()
+            .find_map(|(index, ch)| match self.test.char_needs_escape(ch) {
+                Ok(false) => None,
+                Ok(true) => Some(Ok((index, ch))),
+                Err(e) => Some(Err(e)),
+            })
+            .transpose()?
+        {
+            let (no_escapes, rest) = fragment.split_at(next_escape_index);
+            fragment = &rest[next_escape_char.len_utf8()..];
+            self.base.write_string_fragment(writer, no_escapes)?;
+            self.base.write_unicode_escape(writer, next_escape_char)?;
+        }
+        self.base.write_string_fragment(writer, fragment)
+    }
+
+    fn write_char_escape<W>(
+        &mut self,
+        writer: &mut W,
+        char_escape: serde_json::ser::CharEscape,
+    ) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_char_escape(writer, char_escape)
+    }
+
+    fn write_byte_array<W>(&mut self, writer: &mut W, value: &[u8]) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_byte_array(writer, value)
+    }
+
+    fn begin_array<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_array(writer)
+    }
+
+    fn end_array<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_array(writer)
+    }
+
+    fn begin_array_value<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_array_value(writer, first)
+    }
+
+    fn end_array_value<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_array_value(writer)
+    }
+
+    fn begin_object<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_object(writer)
+    }
+
+    fn end_object<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_object(writer)
+    }
+
+    fn begin_object_key<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_object_key(writer, first)
+    }
+
+    fn end_object_key<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_object_key(writer)
+    }
+
+    fn begin_object_value<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_object_value(writer)
+    }
+
+    fn end_object_value<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_object_value(writer)
+    }
+
+    fn write_raw_fragment<W>(&mut self, writer: &mut W, fragment: &str) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_raw_fragment(writer, fragment)
+    }
+}
+
+fn serialize_to_json_ascii_helper<F: SerdeJsonEscapeIfFormatter, S: serde::Serialize + ?Sized>(
+    v: &S,
+    base: F,
+) -> serde_json::Result<String> {
+    let mut retval = Vec::new();
+    v.serialize(&mut serde_json::ser::Serializer::with_formatter(
+        &mut retval,
+        SerdeJsonEscapeIf {
+            base,
+            test: |ch| ch < '\x20' || ch > '\x7F',
+        },
+    ))?;
+    String::from_utf8(retval).map_err(|_| serde::ser::Error::custom("invalid UTF-8"))
+}
+
+pub fn serialize_to_json_ascii<T: serde::Serialize + ?Sized>(v: &T) -> serde_json::Result<String> {
+    serialize_to_json_ascii_helper(v, serde_json::ser::CompactFormatter)
+}
+
+pub fn serialize_to_json_ascii_pretty<T: serde::Serialize + ?Sized>(
+    v: &T,
+) -> serde_json::Result<String> {
+    serialize_to_json_ascii_helper(v, serde_json::ser::PrettyFormatter::new())
+}
+
+pub fn serialize_to_json_ascii_pretty_with_indent<T: serde::Serialize + ?Sized>(
+    v: &T,
+    indent: &str,
+) -> serde_json::Result<String> {
+    serialize_to_json_ascii_helper(
+        v,
+        serde_json::ser::PrettyFormatter::with_indent(indent.as_bytes()),
+    )
+}