From 267f37d1d301fdafa13465f4a9e659a7ea237f93 Mon Sep 17 00:00:00 2001 From: Jacob Lifshay Date: Sun, 28 Sep 2025 23:05:24 -0700 Subject: [PATCH] WIP refactoring to have JobKind be internal jobs --- crates/fayalite/src/build.rs | 2718 ++++++++--------- crates/fayalite/src/build/external.rs | 517 +++- crates/fayalite/src/build/firrtl.rs | 120 +- crates/fayalite/src/build/graph.rs | 762 +++++ crates/fayalite/src/build/registry.rs | 341 +++ crates/fayalite/src/cli.rs | 2 +- crates/fayalite/src/firrtl.rs | 43 +- crates/fayalite/src/intern.rs | 42 + .../src/module/transform/simplify_enums.rs | 7 +- crates/fayalite/src/util.rs | 7 +- crates/fayalite/src/util/job_server.rs | 254 +- crates/fayalite/src/util/misc.rs | 321 ++ 12 files changed, 3446 insertions(+), 1688 deletions(-) create mode 100644 crates/fayalite/src/build/graph.rs create mode 100644 crates/fayalite/src/build/registry.rs diff --git a/crates/fayalite/src/build.rs b/crates/fayalite/src/build.rs index 9005f785..23416117 100644 --- a/crates/fayalite/src/build.rs +++ b/crates/fayalite/src/build.rs @@ -5,74 +5,59 @@ use crate::{ bundle::Bundle, intern::{Intern, Interned}, module::Module, - util::{HashMap, HashSet, job_server::AcquiredJob}, + util::job_server::AcquiredJob, }; -use hashbrown::hash_map::Entry; -use petgraph::{ - algo::{DfsSpace, kosaraju_scc, toposort}, - graph::DiGraph, - visit::{GraphBase, Visitable}, +use serde::{ + Deserialize, Deserializer, Serialize, Serializer, + de::{DeserializeOwned, Error as _}, + ser::Error as _, }; -use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error, ser::SerializeSeq}; use std::{ any::{Any, TypeId}, borrow::Cow, - cell::OnceCell, cmp::Ordering, - collections::{BTreeMap, BTreeSet, VecDeque}, - fmt::{self, Write}, + fmt, hash::{Hash, Hasher}, - marker::PhantomData, - panic, - rc::Rc, - sync::{Arc, OnceLock, RwLock, RwLockWriteGuard, mpsc}, - thread::{self, ScopedJoinHandle}, + sync::{Arc, OnceLock}, }; use tempfile::TempDir; pub mod external; pub mod firrtl; - -macro_rules! write_str { - ($s:expr, $($rest:tt)*) => { - String::write_fmt(&mut $s, format_args!($($rest)*)).expect("String::write_fmt can't fail") - }; -} +pub mod graph; +pub mod registry; #[derive(Clone, Hash, PartialEq, Eq, Debug)] +#[non_exhaustive] pub enum JobItem { - Module { value: Module }, - File { path: Interned }, + Path { path: Interned }, } impl JobItem { pub fn name(&self) -> JobItemName { match self { - JobItem::Module { value } => JobItemName::Module { name: value.name() }, - &JobItem::File { path } => JobItemName::File { path }, + &JobItem::Path { path } => JobItemName::Path { path }, } } } #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] pub enum JobItemName { - Module { name: Interned }, - File { path: Interned }, + #[non_exhaustive] + Path { path: Interned }, } impl JobItemName { fn as_ref(&self) -> JobItemNameRef<'_> { match self { - JobItemName::Module { name } => JobItemNameRef::Module { name }, - JobItemName::File { path } => JobItemNameRef::File { path }, + JobItemName::Path { path } => JobItemNameRef::Path { path }, } } } #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] enum JobItemNameRef<'a> { - Module { name: &'a str }, - File { path: &'a str }, + Path { path: &'a str }, } /// ordered by string contents, not by `Interned` @@ -93,45 +78,315 @@ impl Ord for JobItemName { } } -pub trait JobKind: 'static + Send + Sync + Hash + Eq + fmt::Debug { - type Job: 'static + Send + Sync + Hash + Eq + fmt::Debug; - fn inputs_and_direct_dependencies<'a>( - &'a self, - job: &'a Self::Job, - ) -> Cow<'a, BTreeMap>>; - fn outputs(&self, job: &Self::Job) -> Interned<[JobItemName]>; - /// gets the part of the command line that is common for all members of this job kind -- usually the executable name/path and any global options and/or subcommands - fn command_line_prefix(&self) -> Interned<[Interned]>; - fn to_command_line(&self, job: &Self::Job) -> Interned<[Interned]>; - /// return the subcommand if this is an internal JobKind - fn subcommand(&self) -> Option; - /// Parse from [`ArgMatches`], this should only be called with the results of parsing [`subcommand()`]. - /// If [`subcommand()`] returned [`None`], you should not call this function since it will panic. - /// - /// [`ArgMatches`]: clap::ArgMatches - /// [`subcommand()`]: JobKind::subcommand - fn from_arg_matches(&self, matches: &mut clap::ArgMatches) -> clap::error::Result; - fn debug_name(&self, job: &Self::Job) -> String { - let name = self - .command_line_prefix() - .last() - .copied() - .or_else(|| self.to_command_line(job).first().copied()) - .unwrap_or_default(); - let name = match name.rsplit_once(['/', '\\']) { - Some((_, name)) if name.trim() != "" => name, - _ => &*name, - }; - format!("job:{name}") +pub trait JobArgs: clap::Args + 'static + Send + Sync + Hash + Eq + fmt::Debug + Clone { + fn to_args> + ?Sized>(&self, args: &mut Args); +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct JobKindAndArgs { + pub kind: K, + pub args: K::Args, +} + +impl JobKindAndArgs { + pub fn args_to_jobs( + self, + dependencies: ::KindsAndArgs, + params: &JobParams, + ) -> eyre::Result> { + K::args_to_jobs( + JobArgsAndDependencies { + args: self, + dependencies, + }, + params, + ) } - fn parse_command_line( - &self, - command_line: Interned<[Interned]>, - ) -> clap::error::Result; +} + +impl> Copy for JobKindAndArgs {} + +impl From> for DynJobArgs { + fn from(value: JobKindAndArgs) -> Self { + let JobKindAndArgs { kind, args } = value; + DynJobArgs::new(kind, args) + } +} + +impl TryFrom for JobKindAndArgs { + type Error = DynJobArgs; + fn try_from(value: DynJobArgs) -> Result { + value.downcast() + } +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct JobAndKind { + pub kind: K, + pub job: K::Job, +} + +impl> Clone for JobAndKind { + fn clone(&self) -> Self { + Self { + kind: self.kind.clone(), + job: self.job.clone(), + } + } +} + +impl> Copy for JobAndKind {} + +impl From> for DynJob { + fn from(value: JobAndKind) -> Self { + let JobAndKind { kind, job } = value; + DynJob::new(kind, job) + } +} + +impl> TryFrom for JobAndKind { + type Error = DynJob; + fn try_from(value: DynJob) -> Result { + value.downcast() + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub struct JobKindAndDependencies { + pub kind: K, + pub dependencies: K::Dependencies, +} + +impl JobKindAndDependencies { + pub fn new(kind: K) -> Self { + Self { + kind, + dependencies: kind.dependencies(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct JobAndDependencies { + pub job: JobAndKind, + pub dependencies: ::JobsAndKinds, +} + +impl Clone for JobAndDependencies +where + K::Job: Clone, + ::JobsAndKinds: Clone, +{ + fn clone(&self) -> Self { + Self { + job: self.job.clone(), + dependencies: self.dependencies.clone(), + } + } +} + +impl Copy for JobAndDependencies +where + K::Job: Copy, + ::JobsAndKinds: Copy, +{ +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct JobArgsAndDependencies { + pub args: JobKindAndArgs, + pub dependencies: ::KindsAndArgs, +} + +impl Copy for JobArgsAndDependencies +where + K::Args: Copy, + ::KindsAndArgs: Copy, +{ +} + +impl JobArgsAndDependencies { + pub fn args_to_jobs(self, params: &JobParams) -> eyre::Result> { + K::args_to_jobs(self, params) + } +} + +impl>, D: JobKind> JobArgsAndDependencies { + pub fn args_to_jobs_simple( + self, + params: &JobParams, + f: F, + ) -> eyre::Result> + where + F: FnOnce(K, K::Args, &mut JobAndDependencies) -> eyre::Result, + { + let Self { + args: JobKindAndArgs { kind, args }, + dependencies, + } = self; + let mut dependencies = dependencies.args_to_jobs(params)?; + let job = f(kind, args, &mut dependencies)?; + Ok(JobAndDependencies { + job: JobAndKind { kind, job }, + dependencies, + }) + } +} + +pub trait JobDependencies: 'static + Send + Sync + Hash + Eq + fmt::Debug + Copy { + type KindsAndArgs: 'static + Send + Sync + Hash + Eq + fmt::Debug + Clone; + type JobsAndKinds: 'static + Send + Sync + Hash + Eq + fmt::Debug; + fn kinds_dyn_extend>(self, dyn_kinds: &mut E); + fn kinds_dyn(self) -> Vec { + let mut retval = Vec::new(); + self.kinds_dyn_extend(&mut retval); + retval + } + fn into_dyn_jobs_extend>(jobs: Self::JobsAndKinds, dyn_jobs: &mut E); + fn into_dyn_jobs(jobs: Self::JobsAndKinds) -> Vec { + let mut retval = Vec::new(); + Self::into_dyn_jobs_extend(jobs, &mut retval); + retval + } + #[track_caller] + fn from_dyn_args_prefix>( + args: &mut I, + ) -> Self::KindsAndArgs; + #[track_caller] + fn from_dyn_args>(args: I) -> Self::KindsAndArgs { + let mut iter = args.into_iter(); + let retval = Self::from_dyn_args_prefix(&mut iter); + if iter.next().is_some() { + panic!("wrong number of dependencies"); + } + retval + } +} + +impl JobDependencies for JobKindAndDependencies { + type KindsAndArgs = JobArgsAndDependencies; + type JobsAndKinds = JobAndDependencies; + + fn kinds_dyn_extend>(self, dyn_kinds: &mut E) { + let Self { kind, dependencies } = self; + dependencies.kinds_dyn_extend(dyn_kinds); + dyn_kinds.extend([DynJobKind::new(kind)]); + } + + fn into_dyn_jobs_extend>( + jobs: Self::JobsAndKinds, + dyn_jobs: &mut E, + ) { + let JobAndDependencies { job, dependencies } = jobs; + K::Dependencies::into_dyn_jobs_extend(dependencies, dyn_jobs); + dyn_jobs.extend([job.into()]); + } + + #[track_caller] + fn from_dyn_args_prefix>( + args: &mut I, + ) -> Self::KindsAndArgs { + let dependencies = K::Dependencies::from_dyn_args_prefix(args); + let Some(args) = args.next() else { + panic!("wrong number of dependencies"); + }; + match args.downcast() { + Ok(args) => JobArgsAndDependencies { args, dependencies }, + Err(args) => { + panic!( + "wrong type of dependency, expected {} got:\n{args:?}", + std::any::type_name::() + ) + } + } + } +} + +macro_rules! impl_job_dependencies { + (@impl $(($v:ident: $T:ident),)*) => { + impl<$($T: JobDependencies),*> JobDependencies for ($($T,)*) { + type KindsAndArgs = ($($T::KindsAndArgs,)*); + type JobsAndKinds = ($($T::JobsAndKinds,)*); + + fn kinds_dyn_extend>(self, dyn_kinds: &mut E) { + #![allow(unused_variables)] + let ($($v,)*) = self; + $($T::kinds_dyn_extend($v, dyn_kinds);)* + } + + fn into_dyn_jobs_extend>( + jobs: Self::JobsAndKinds, + dyn_jobs: &mut E, + ) { + #![allow(unused_variables)] + let ($($v,)*) = jobs; + $($T::into_dyn_jobs_extend($v, dyn_jobs);)* + } + + #[track_caller] + fn from_dyn_args_prefix>( + args: &mut I, + ) -> Self::KindsAndArgs { + #![allow(unused_variables)] + $(let $v = $T::from_dyn_args_prefix(args);)* + ($($v,)*) + } + } + }; + ($($first:tt, $($rest:tt,)*)?) => { + impl_job_dependencies!(@impl $($first, $($rest,)*)?); + $(impl_job_dependencies!($($rest,)*);)? + }; +} + +impl_job_dependencies! { + (v0: T0), + (v1: T1), + (v2: T2), + (v3: T3), + (v4: T4), + (v5: T5), + (v6: T6), + (v7: T7), + (v8: T8), + (v9: T9), + (v10: T10), + (v11: T11), +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub struct JobParams { + main_module: Module, +} + +impl JobParams { + pub fn new(main_module: Module) -> Self { + Self { main_module } + } + pub fn main_module(&self) -> &Module { + &self.main_module + } +} + +pub trait JobKind: 'static + Send + Sync + Hash + Eq + fmt::Debug + Sized + Copy { + type Args: JobArgs; + type Job: 'static + Send + Sync + Hash + Eq + fmt::Debug + Serialize + DeserializeOwned; + type Dependencies: JobDependencies; + fn dependencies(self) -> Self::Dependencies; + fn args_to_jobs( + args: JobArgsAndDependencies, + params: &JobParams, + ) -> eyre::Result>; + fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]>; + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]>; + fn name(self) -> Interned; + fn external_command_line(self, job: &Self::Job) -> Option]>>; fn run( - &self, + self, job: &Self::Job, inputs: &[JobItem], + params: &JobParams, acquired_job: &mut AcquiredJob, ) -> eyre::Result>; } @@ -141,16 +396,20 @@ trait DynJobKindTrait: 'static + Send + Sync + fmt::Debug { fn as_arc_any(self: Arc) -> Arc; fn eq_dyn(&self, other: &dyn DynJobKindTrait) -> bool; fn hash_dyn(&self, state: &mut dyn Hasher); - fn command_line_prefix_dyn(&self) -> Interned<[Interned]>; - fn subcommand_dyn(&self) -> Option; + fn dependencies_kinds_dyn(&self) -> Vec; + fn args_group_id_dyn(&self) -> Option; + fn augment_args_dyn(&self, cmd: clap::Command) -> clap::Command; + fn augment_args_for_update_dyn(&self, cmd: clap::Command) -> clap::Command; fn from_arg_matches_dyn( - self: Arc, + &self, matches: &mut clap::ArgMatches, - ) -> clap::error::Result; - fn parse_command_line_dyn( + ) -> clap::error::Result; + fn name_dyn(&self) -> Interned; + fn deserialize_job_from_json_str(self: Arc, json: &str) -> serde_json::Result; + fn deserialize_job_from_json_value( self: Arc, - command_line: Interned<[Interned]>, - ) -> clap::error::Result; + json: &serde_json::Value, + ) -> serde_json::Result; } impl DynJobKindTrait for T { @@ -165,36 +424,53 @@ impl DynJobKindTrait for T { fn eq_dyn(&self, other: &dyn DynJobKindTrait) -> bool { other .as_any() - .downcast_ref::() + .downcast_ref::() .is_some_and(|other| self == other) } fn hash_dyn(&self, mut state: &mut dyn Hasher) { - self.hash(&mut state) + self.hash(&mut state); } - fn command_line_prefix_dyn(&self) -> Interned<[Interned]> { - self.command_line_prefix() + fn dependencies_kinds_dyn(&self) -> Vec { + self.dependencies().kinds_dyn() } - fn subcommand_dyn(&self) -> Option { - self.subcommand() + fn args_group_id_dyn(&self) -> Option { + ::group_id() + } + + fn augment_args_dyn(&self, cmd: clap::Command) -> clap::Command { + ::augment_args(cmd) + } + + fn augment_args_for_update_dyn(&self, cmd: clap::Command) -> clap::Command { + ::augment_args_for_update(cmd) } fn from_arg_matches_dyn( - self: Arc, + &self, matches: &mut clap::ArgMatches, - ) -> clap::error::Result { - let job = self.from_arg_matches(matches)?; - Ok(DynJob::from_arc(self, job)) + ) -> clap::error::Result { + Ok(DynJobArgs::new( + *self, + ::from_arg_matches_mut(matches)?, + )) } - fn parse_command_line_dyn( - self: Arc, - command_line: Interned<[Interned]>, - ) -> clap::error::Result { - let job = self.parse_command_line(command_line)?; - Ok(DynJob::from_arc(self, job)) + fn name_dyn(&self) -> Interned { + self.name() + } + + fn deserialize_job_from_json_str(self: Arc, json: &str) -> serde_json::Result { + Ok(DynJob::from_arc(self, serde_json::from_str(json)?)) + } + + fn deserialize_job_from_json_value( + self: Arc, + json: &serde_json::Value, + ) -> serde_json::Result { + Ok(DynJob::from_arc(self, Deserialize::deserialize(json)?)) } } @@ -203,31 +479,19 @@ pub struct DynJobKind(Arc); impl DynJobKind { pub fn from_arc(job_kind: Arc) -> Self { - if TypeId::of::() == TypeId::of::() { - Self::clone( - &Arc::downcast::(job_kind.as_arc_any()) - .ok() - .expect("already checked type"), - ) - } else { - Self(job_kind) - } + Self(job_kind) } pub fn new(job_kind: T) -> Self { - if let Some(job_kind) = DynJobKindTrait::as_any(&job_kind).downcast_ref::() { - job_kind.clone() - } else { - Self(Arc::new(job_kind)) - } + Self(Arc::new(job_kind)) } pub fn type_id(&self) -> TypeId { DynJobKindTrait::as_any(&*self.0).type_id() } - pub fn downcast_ref(&self) -> Option<&T> { - DynJobKindTrait::as_any(&*self.0).downcast_ref() + pub fn downcast(&self) -> Option { + DynJobKindTrait::as_any(&*self.0).downcast_ref().copied() } pub fn downcast_arc(self) -> Result, Self> { - if self.downcast_ref::().is_some() { + if self.downcast::().is_some() { Ok(Arc::downcast::(self.0.as_arc_any()) .ok() .expect("already checked type")) @@ -235,18 +499,55 @@ impl DynJobKind { Err(self) } } - pub fn registry() -> JobKindRegistrySnapshot { - JobKindRegistrySnapshot(JobKindRegistry::get()) + pub fn dependencies_kinds(&self) -> Vec { + DynJobKindTrait::dependencies_kinds_dyn(&*self.0) } - #[track_caller] - pub fn register(self) { - JobKindRegistry::register(JobKindRegistry::lock(), self); + pub fn args_group_id(&self) -> Option { + DynJobKindTrait::args_group_id_dyn(&*self.0) + } + pub fn augment_args(&self, cmd: clap::Command) -> clap::Command { + DynJobKindTrait::augment_args_dyn(&*self.0, cmd) + } + pub fn augment_args_for_update(&self, cmd: clap::Command) -> clap::Command { + DynJobKindTrait::augment_args_for_update_dyn(&*self.0, cmd) + } + pub fn from_arg_matches( + &self, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result { + DynJobKindTrait::from_arg_matches_dyn(&*self.0, matches) + } + pub fn name(&self) -> Interned { + DynJobKindTrait::name_dyn(&*self.0) + } + pub fn deserialize_job_from_json_str(self, json: &str) -> serde_json::Result { + DynJobKindTrait::deserialize_job_from_json_str(self.0, json) + } + pub fn deserialize_job_from_json_value( + self, + json: &serde_json::Value, + ) -> serde_json::Result { + DynJobKindTrait::deserialize_job_from_json_value(self.0, json) + } + pub fn make_subcommand(&self) -> clap::Command { + let mut subcommand = clap::Command::new(Interned::into_inner(self.name())); + for dependency in self.dependencies_kinds() { + subcommand = dependency.augment_args(subcommand); + } + self.augment_args(subcommand) + } + pub fn make_subcommand_for_update(&self) -> clap::Command { + let mut subcommand = clap::Command::new(Interned::into_inner(self.name())); + for dependency in self.dependencies_kinds() { + subcommand = dependency.augment_args_for_update(subcommand); + } + self.augment_args_for_update(subcommand) } } impl Hash for DynJobKind { fn hash(&self, state: &mut H) { - DynJobKindTrait::as_any(&*self.0).type_id().hash(state); + self.type_id().hash(state); DynJobKindTrait::hash_dyn(&*self.0, state); } } @@ -270,7 +571,7 @@ impl Serialize for DynJobKind { where S: Serializer, { - self.command_line_prefix().serialize(serializer) + self.name().serialize(serializer) } } @@ -279,303 +580,361 @@ impl<'de> Deserialize<'de> for DynJobKind { where D: Deserializer<'de>, { - let command_line_prefix: Cow<'_, [Interned]> = Cow::deserialize(deserializer)?; - match Self::registry().get_by_command_line_prefix(&command_line_prefix) { + let name = Cow::::deserialize(deserializer)?; + match Self::registry().get_by_name(&name) { Some(retval) => Ok(retval.clone()), None => Err(D::Error::custom(format_args!( - "unknown job kind: command line prefix not found in registry: {command_line_prefix:?}" + "unknown job kind: name not found in registry: {name:?}" ))), } } } -#[derive(Clone, Debug)] -struct JobKindRegistry { - command_line_prefix_to_job_kind_map: HashMap<&'static [Interned], DynJobKind>, - job_kinds: Vec, - subcommand_names: BTreeMap<&'static str, DynJobKind>, +#[derive(Copy, Clone, Debug, Default)] +pub struct DynJobKindValueParser; + +#[derive(Clone, PartialEq, Eq, Hash)] +struct DynJobKindValueEnum { + name: Interned, + job_kind: DynJobKind, } -enum JobKindRegisterError { - SameCommandLinePrefix { - command_line_prefix: &'static [Interned], - old_job_kind: DynJobKind, - new_job_kind: DynJobKind, - }, - CommandLinePrefixDoesNotMatchSubcommandName { - command_line_prefix: &'static [Interned], - expected: [Interned; 2], - job_kind: DynJobKind, - }, +impl clap::ValueEnum for DynJobKindValueEnum { + fn value_variants<'a>() -> &'a [Self] { + Interned::into_inner( + registry::JobKindRegistrySnapshot::get() + .iter_with_names() + .map(|(name, job_kind)| Self { + name, + job_kind: job_kind.clone(), + }) + .collect(), + ) + } + + fn to_possible_value(&self) -> Option { + Some(clap::builder::PossibleValue::new(Interned::into_inner( + self.name, + ))) + } } -impl fmt::Display for JobKindRegisterError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::SameCommandLinePrefix { - command_line_prefix, - old_job_kind, - new_job_kind, - } => write!( - f, - "two different `DynJobKind` can't share the same `command_line_prefix` of:\n\ - {command_line_prefix:?}\n\ - old job kind:\n\ - {old_job_kind:?}\n\ - new job kind:\n\ - {new_job_kind:?}", - ), - Self::CommandLinePrefixDoesNotMatchSubcommandName { - command_line_prefix, - expected, - job_kind, - } => write!( - f, - "`JobKind::subcommand()` returned `Some` but the `command_line_prefix` is not as expected\n\ - (it should be `[program_name_for_internal_jobs(), subcommand_name]`):\n\ - command_line_prefix:\n\ - {command_line_prefix:?}\n\ - expected:\n\ - {expected:?}\n\ - job kind:\n\ - {job_kind:?}", - ), +impl clap::builder::TypedValueParser for DynJobKindValueParser { + type Value = DynJobKind; + + fn parse_ref( + &self, + cmd: &clap::Command, + arg: Option<&clap::Arg>, + value: &std::ffi::OsStr, + ) -> clap::error::Result { + clap::builder::EnumValueParser::::new() + .parse_ref(cmd, arg, value) + .map(|v| v.job_kind) + } + + fn possible_values( + &self, + ) -> Option + '_>> { + static ENUM_VALUE_PARSER: OnceLock> = + OnceLock::new(); + ENUM_VALUE_PARSER + .get_or_init(clap::builder::EnumValueParser::::new) + .possible_values() + } +} + +impl clap::builder::ValueParserFactory for DynJobKind { + type Parser = DynJobKindValueParser; + + fn value_parser() -> Self::Parser { + DynJobKindValueParser::default() + } +} + +trait DynExtendInternedStr { + fn extend_from_slice(&mut self, items: &[Interned]); +} + +impl Extend> for dyn DynExtendInternedStr + '_ { + fn extend>>(&mut self, iter: T) { + let mut buf = [Interned::default(); 64]; + let mut buf_len = 0; + iter.into_iter().for_each(|item| { + buf[buf_len] = item; + buf_len += 1; + if buf_len == buf.len() { + ::extend_from_slice(self, &buf); + buf_len = 0; + } + }); + if buf_len > 0 { + ::extend_from_slice( + self, + &buf[..buf_len], + ); } } } -trait JobKindRegistryRegisterLock { - type Locked; - fn lock(self) -> Self::Locked; - fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry; -} - -impl JobKindRegistryRegisterLock for &'static RwLock> { - type Locked = RwLockWriteGuard<'static, Arc>; - fn lock(self) -> Self::Locked { - self.write().expect("shouldn't be poisoned") - } - fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry { - Arc::make_mut(locked) +impl>> DynExtendInternedStr for T { + fn extend_from_slice(&mut self, items: &[Interned]) { + self.extend(items.iter().copied()); } } -impl JobKindRegistryRegisterLock for &'_ mut JobKindRegistry { - type Locked = Self; +#[derive(PartialEq, Eq, Hash, Clone)] +struct DynJobArgsInner(JobKindAndArgs); - fn lock(self) -> Self::Locked { +impl fmt::Debug for DynJobArgsInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self(JobKindAndArgs { kind, args }) = self; + f.debug_struct("DynJobArgs") + .field("kind", kind) + .field("args", args) + .finish() + } +} + +trait DynJobArgsTrait: 'static + Send + Sync + fmt::Debug { + fn as_any(&self) -> &dyn Any; + fn as_arc_any(self: Arc) -> Arc; + fn kind_type_id(&self) -> TypeId; + fn eq_dyn(&self, other: &dyn DynJobArgsTrait) -> bool; + fn hash_dyn(&self, state: &mut dyn Hasher); + fn kind(&self) -> DynJobKind; + fn to_args(&self, args: &mut dyn DynExtendInternedStr); + fn to_args_extend_vec(&self, args: Vec>) -> Vec>; + fn clone_into_arc(&self) -> Arc; + fn update_from_arg_matches( + &mut self, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result<()>; + #[track_caller] + fn args_to_jobs( + self: Arc, + dependencies_args: Vec, + params: &JobParams, + ) -> eyre::Result<(DynJob, Vec)>; +} + +impl DynJobArgsTrait for DynJobArgsInner { + fn as_any(&self) -> &dyn Any { self } - fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry { - locked - } -} - -impl JobKindRegistry { - fn lock() -> &'static RwLock> { - static REGISTRY: OnceLock>> = OnceLock::new(); - REGISTRY.get_or_init(Default::default) - } - fn try_register( - lock: L, - job_kind: DynJobKind, - ) -> Result<(), JobKindRegisterError> { - let command_line_prefix = Interned::into_inner(job_kind.command_line_prefix()); - let subcommand_name = job_kind - .subcommand() - .map(|subcommand| subcommand.get_name().intern()); - if let Some(subcommand_name) = subcommand_name { - let expected = [program_name_for_internal_jobs(), subcommand_name]; - if command_line_prefix != &expected { - return Err( - JobKindRegisterError::CommandLinePrefixDoesNotMatchSubcommandName { - command_line_prefix, - expected, - job_kind, - }, - ); - } - } - // run user code only outside of lock - let mut locked = lock.lock(); - let this = L::make_mut(&mut locked); - let result = match this - .command_line_prefix_to_job_kind_map - .entry(command_line_prefix) - { - Entry::Occupied(entry) => Err(JobKindRegisterError::SameCommandLinePrefix { - command_line_prefix, - old_job_kind: entry.get().clone(), - new_job_kind: job_kind, - }), - Entry::Vacant(entry) => { - this.job_kinds.push(job_kind.clone()); - if let Some(subcommand_name) = subcommand_name { - this.subcommand_names - .insert(Interned::into_inner(subcommand_name), job_kind.clone()); - } - entry.insert(job_kind); - Ok(()) - } - }; - drop(locked); - // outside of lock now, so we can test if it's the same DynJobKind - match result { - Err(JobKindRegisterError::SameCommandLinePrefix { - command_line_prefix: _, - old_job_kind, - new_job_kind, - }) if old_job_kind == new_job_kind => Ok(()), - result => result, - } - } - #[track_caller] - fn register(lock: L, job_kind: DynJobKind) { - match Self::try_register(lock, job_kind) { - Err(e) => panic!("{e}"), - Ok(()) => {} - } - } - fn get() -> Arc { - Self::lock().read().expect("shouldn't be poisoned").clone() - } -} - -impl Default for JobKindRegistry { - fn default() -> Self { - let mut retval = Self { - command_line_prefix_to_job_kind_map: HashMap::default(), - job_kinds: Vec::new(), - subcommand_names: BTreeMap::new(), - }; - for job_kind in [] { - Self::register(&mut retval, job_kind); - } - retval - } -} - -#[derive(Clone, Debug)] -pub struct JobKindRegistrySnapshot(Arc); - -impl JobKindRegistrySnapshot { - pub fn get() -> Self { - JobKindRegistrySnapshot(JobKindRegistry::get()) - } - pub fn get_by_command_line_prefix<'a>( - &'a self, - command_line_prefix: &[Interned], - ) -> Option<&'a DynJobKind> { - self.0 - .command_line_prefix_to_job_kind_map - .get(command_line_prefix) - } - pub fn job_kinds(&self) -> &[DynJobKind] { - &self.0.job_kinds - } -} - -#[derive(Clone, Debug, Hash, PartialEq, Eq)] -pub struct AnyInternalJob(pub DynJob); - -impl clap::Subcommand for AnyInternalJob { - fn augment_subcommands(mut cmd: clap::Command) -> clap::Command { - for job_kind in JobKindRegistrySnapshot::get().0.subcommand_names.values() { - let Some(subcommand) = job_kind.subcommand() else { - // shouldn't happen, ignore it - continue; - }; - cmd = cmd.subcommand(subcommand); - } - cmd + fn as_arc_any(self: Arc) -> Arc { + self } - fn augment_subcommands_for_update(cmd: clap::Command) -> clap::Command { - Self::augment_subcommands(cmd) + fn kind_type_id(&self) -> TypeId { + TypeId::of::() } - fn has_subcommand(name: &str) -> bool { - JobKindRegistrySnapshot::get() - .0 - .subcommand_names - .contains_key(name) - } -} - -impl clap::FromArgMatches for AnyInternalJob { - fn from_arg_matches(matches: &clap::ArgMatches) -> Result { - Self::from_arg_matches_mut(&mut matches.clone()) + fn eq_dyn(&self, other: &dyn DynJobArgsTrait) -> bool { + other + .as_any() + .downcast_ref::() + .is_some_and(|other| self == other) } - fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result { - if let Some((name, mut matches)) = matches.remove_subcommand() { - let job_kind_registry_snapshot = JobKindRegistrySnapshot::get(); - if let Some(job_kind) = job_kind_registry_snapshot.0.subcommand_names.get(&*name) { - Ok(Self(job_kind.from_arg_matches(&mut matches)?)) - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::InvalidSubcommand, - format!("the subcommand '{name}' wasn't recognized"), - )) - } - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::MissingSubcommand, - "a subcommand is required but one was not provided", - )) - } + fn hash_dyn(&self, mut state: &mut dyn Hasher) { + self.hash(&mut state); } - fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> { - *self = Self::from_arg_matches(matches)?; - Ok(()) + fn kind(&self) -> DynJobKind { + DynJobKind::new(self.0.kind) } - fn update_from_arg_matches_mut( + fn to_args(&self, args: &mut dyn DynExtendInternedStr) { + self.0.args.to_args(args) + } + + fn to_args_extend_vec(&self, mut args: Vec>) -> Vec> { + self.0.args.to_args(&mut args); + args + } + + fn clone_into_arc(&self) -> Arc { + Arc::new(self.clone()) + } + + fn update_from_arg_matches( &mut self, matches: &mut clap::ArgMatches, - ) -> Result<(), clap::Error> { - *self = Self::from_arg_matches_mut(matches)?; - Ok(()) + ) -> clap::error::Result<()> { + clap::FromArgMatches::update_from_arg_matches_mut(&mut self.0.args, matches) + } + + #[track_caller] + fn args_to_jobs( + self: Arc, + dependencies_args: Vec, + params: &JobParams, + ) -> eyre::Result<(DynJob, Vec)> { + let JobAndDependencies { job, dependencies } = JobArgsAndDependencies { + args: Arc::unwrap_or_clone(self).0, + dependencies: K::Dependencies::from_dyn_args(dependencies_args), + } + .args_to_jobs(params)?; + Ok((job.into(), K::Dependencies::into_dyn_jobs(dependencies))) + } +} + +#[derive(Clone)] +pub struct DynJobArgs(Arc); + +impl DynJobArgs { + pub fn new(kind: K, args: K::Args) -> Self { + Self(Arc::new(DynJobArgsInner(JobKindAndArgs { kind, args }))) + } + pub fn kind_type_id(&self) -> TypeId { + DynJobArgsTrait::kind_type_id(&*self.0) + } + pub fn downcast_ref(&self) -> Option<(&K, &K::Args)> { + let DynJobArgsInner::(JobKindAndArgs { kind, args }) = + DynJobArgsTrait::as_any(&*self.0).downcast_ref()?; + Some((kind, args)) + } + pub fn downcast(self) -> Result, Self> { + if self.downcast_ref::().is_some() { + let this = Arc::downcast::>(self.0.as_arc_any()) + .ok() + .expect("already checked type"); + Ok(Arc::unwrap_or_clone(this).0) + } else { + Err(self) + } + } + pub fn kind(&self) -> DynJobKind { + DynJobArgsTrait::kind(&*self.0) + } + pub fn to_args>>(&self, args: &mut Args) { + DynJobArgsTrait::to_args(&*self.0, args); + } + pub fn to_args_vec(&self) -> Vec> { + self.to_args_extend_vec(Vec::new()) + } + pub fn to_args_extend_vec(&self, args: Vec>) -> Vec> { + DynJobArgsTrait::to_args_extend_vec(&*self.0, args) + } + fn make_mut(&mut self) -> &mut dyn DynJobArgsTrait { + // can't just return the reference if the first get_mut returns Some since + // as of rustc 1.90.0 this causes a false-positive lifetime error. + if Arc::get_mut(&mut self.0).is_none() { + self.0 = DynJobArgsTrait::clone_into_arc(&*self.0); + } + Arc::get_mut(&mut self.0).expect("clone_into_arc returns a new arc with a ref-count of 1") + } + pub fn update_from_arg_matches( + &mut self, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result<()> { + DynJobArgsTrait::update_from_arg_matches(self.make_mut(), matches) + } + pub fn args_to_jobs( + self, + dependencies_args: Vec, + params: &JobParams, + ) -> eyre::Result<(DynJob, Vec)> { + DynJobArgsTrait::args_to_jobs(self.0, dependencies_args, params) + } +} + +impl Hash for DynJobArgs { + fn hash(&self, state: &mut H) { + self.kind_type_id().hash(state); + DynJobArgsTrait::hash_dyn(&*self.0, state); + } +} + +impl PartialEq for DynJobArgs { + fn eq(&self, other: &Self) -> bool { + DynJobArgsTrait::eq_dyn(&*self.0, &*other.0) + } +} + +impl Eq for DynJobArgs {} + +impl fmt::Debug for DynJobArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[derive(PartialEq, Eq, Hash)] +struct DynJobInner { + kind: Arc, + job: T::Job, + inputs: Interned<[JobItemName]>, + outputs: Interned<[JobItemName]>, + external_command_line: Option]>>, +} + +impl> Clone for DynJobInner { + fn clone(&self) -> Self { + Self { + kind: self.kind.clone(), + job: self.job.clone(), + inputs: self.inputs, + outputs: self.outputs, + external_command_line: self.external_command_line, + } + } +} + +impl fmt::Debug for DynJobInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + kind, + job, + inputs, + outputs, + external_command_line, + } = self; + f.debug_struct("DynJob") + .field("kind", kind) + .field("job", job) + .field("inputs", inputs) + .field("outputs", outputs) + .field("external_command_line", external_command_line) + .finish() } } trait DynJobTrait: 'static + Send + Sync + fmt::Debug { fn as_any(&self) -> &dyn Any; + fn as_arc_any(self: Arc) -> Arc; fn eq_dyn(&self, other: &dyn DynJobTrait) -> bool; fn hash_dyn(&self, state: &mut dyn Hasher); fn kind_type_id(&self) -> TypeId; fn kind(&self) -> DynJobKind; - fn inputs_and_direct_dependencies<'a>(&'a self) -> &'a BTreeMap>; + fn inputs(&self) -> Interned<[JobItemName]>; fn outputs(&self) -> Interned<[JobItemName]>; - fn to_command_line(&self) -> Interned<[Interned]>; - fn debug_name(&self) -> String; - fn run(&self, inputs: &[JobItem], acquired_job: &mut AcquiredJob) - -> eyre::Result>; + fn external_command_line(&self) -> Option]>>; + fn serialize_to_json_ascii(&self) -> serde_json::Result; + fn serialize_to_json_value(&self) -> serde_json::Result; + fn run( + &self, + inputs: &[JobItem], + params: &JobParams, + acquired_job: &mut AcquiredJob, + ) -> eyre::Result>; } -mod inner { - use super::*; - - #[derive(Debug, PartialEq, Eq, Hash)] - pub(crate) struct DynJob { - pub(crate) kind: Arc, - pub(crate) job: T::Job, - pub(crate) inputs_and_direct_dependencies: BTreeMap>, - pub(crate) outputs: Interned<[JobItemName]>, - } -} - -impl DynJobTrait for inner::DynJob { +impl DynJobTrait for DynJobInner { fn as_any(&self) -> &dyn Any { self } + fn as_arc_any(self: Arc) -> Arc { + self + } + fn eq_dyn(&self, other: &dyn DynJobTrait) -> bool { other .as_any() - .downcast_ref::>() + .downcast_ref::() .is_some_and(|other| self == other) } @@ -591,28 +950,33 @@ impl DynJobTrait for inner::DynJob { DynJobKind(self.kind.clone()) } - fn inputs_and_direct_dependencies<'a>(&'a self) -> &'a BTreeMap> { - &self.inputs_and_direct_dependencies + fn inputs(&self) -> Interned<[JobItemName]> { + self.inputs } fn outputs(&self) -> Interned<[JobItemName]> { self.outputs } - fn to_command_line(&self) -> Interned<[Interned]> { - self.kind.to_command_line(&self.job) + fn external_command_line(&self) -> Option]>> { + self.external_command_line } - fn debug_name(&self) -> String { - self.kind.debug_name(&self.job) + fn serialize_to_json_ascii(&self) -> serde_json::Result { + crate::util::serialize_to_json_ascii(&self.job) + } + + fn serialize_to_json_value(&self) -> serde_json::Result { + serde_json::to_value(&self.job) } fn run( &self, inputs: &[JobItem], + params: &JobParams, acquired_job: &mut AcquiredJob, ) -> eyre::Result> { - self.kind.run(&self.job, inputs, acquired_job) + self.kind.run(&self.job, inputs, params, acquired_job) } } @@ -620,65 +984,95 @@ impl DynJobTrait for inner::DynJob { pub struct DynJob(Arc); impl DynJob { - fn new_unchecked(job_kind: Arc, job: T::Job) -> Self { - let inputs_and_direct_dependencies = - job_kind.inputs_and_direct_dependencies(&job).into_owned(); + pub fn from_arc(job_kind: Arc, job: T::Job) -> Self { + let inputs = job_kind.inputs(&job); let outputs = job_kind.outputs(&job); - Self(Arc::new(inner::DynJob { + let external_command_line = job_kind.external_command_line(&job); + Self(Arc::new(DynJobInner { kind: job_kind, job, - inputs_and_direct_dependencies, + inputs, outputs, + external_command_line, })) } - pub fn from_arc(job_kind: Arc, job: T::Job) -> Self { - if TypeId::of::() == TypeId::of::() { - ::downcast_ref::(&job) - .expect("already checked type") - .clone() - } else { - Self::new_unchecked(job_kind, job) - } - } pub fn new(job_kind: T, job: T::Job) -> Self { - if TypeId::of::() == TypeId::of::() { - ::downcast_ref::(&job) - .expect("already checked type") - .clone() - } else { - Self::new_unchecked(Arc::new(job_kind), job) - } + Self::from_arc(Arc::new(job_kind), job) } pub fn kind_type_id(&self) -> TypeId { self.0.kind_type_id() } - pub fn downcast(&self) -> Option<(&T, &T::Job)> { - let inner::DynJob { kind, job, .. } = self.0.as_any().downcast_ref()?; + pub fn downcast_ref(&self) -> Option<(&K, &K::Job)> { + let DynJobInner { kind, job, .. } = self.0.as_any().downcast_ref()?; Some((kind, job)) } + pub fn downcast>(self) -> Result, Self> { + if self.kind_type_id() == TypeId::of::() { + let DynJobInner { kind, job, .. } = Arc::unwrap_or_clone( + self.0 + .as_arc_any() + .downcast::>() + .expect("already checked type"), + ); + Ok(JobAndKind { kind: *kind, job }) + } else { + Err(self) + } + } pub fn kind(&self) -> DynJobKind { DynJobTrait::kind(&*self.0) } - pub fn inputs_and_direct_dependencies<'a>( - &'a self, - ) -> &'a BTreeMap> { - DynJobTrait::inputs_and_direct_dependencies(&*self.0) + pub fn inputs(&self) -> Interned<[JobItemName]> { + DynJobTrait::inputs(&*self.0) } pub fn outputs(&self) -> Interned<[JobItemName]> { DynJobTrait::outputs(&*self.0) } - pub fn to_command_line(&self) -> Interned<[Interned]> { - DynJobTrait::to_command_line(&*self.0) + pub fn serialize_to_json_ascii(&self) -> serde_json::Result { + DynJobTrait::serialize_to_json_ascii(&*self.0) } - pub fn debug_name(&self) -> String { - DynJobTrait::debug_name(&*self.0) + pub fn serialize_to_json_value(&self) -> serde_json::Result { + DynJobTrait::serialize_to_json_value(&*self.0) + } + pub fn external_command_line(&self) -> Option]>> { + DynJobTrait::external_command_line(&*self.0) + } + #[track_caller] + pub fn internal_command_line_with_program_prefix( + &self, + internal_program_prefix: &[Interned], + ) -> Interned<[Interned]> { + let mut retval = internal_program_prefix.to_vec(); + match RunSingleJob::try_add_subcommand(self, &mut retval) { + Ok(()) => Intern::intern_owned(retval), + Err(e) => panic!("Serializing job {:?} failed: {e}", self.kind().name()), + } + } + #[track_caller] + pub fn internal_command_line(&self) -> Interned<[Interned]> { + self.internal_command_line_with_program_prefix(&[program_name_for_internal_jobs()]) + } + #[track_caller] + pub fn command_line_with_internal_program_prefix( + &self, + internal_program_prefix: &[Interned], + ) -> Interned<[Interned]> { + match self.external_command_line() { + Some(v) => v, + None => self.internal_command_line_with_program_prefix(internal_program_prefix), + } + } + #[track_caller] + pub fn command_line(&self) -> Interned<[Interned]> { + self.command_line_with_internal_program_prefix(&[program_name_for_internal_jobs()]) } pub fn run( &self, inputs: &[JobItem], + params: &JobParams, acquired_job: &mut AcquiredJob, ) -> eyre::Result> { - DynJobTrait::run(&*self.0, inputs, acquired_job) + DynJobTrait::run(&*self.0, inputs, params, acquired_job) } } @@ -700,7 +1094,7 @@ impl Hash for DynJob { #[serde(rename = "DynJob")] struct DynJobSerde { kind: DynJobKind, - command_line: Interned<[Interned]>, + job: serde_json::Value, } impl Serialize for DynJob { @@ -710,7 +1104,7 @@ impl Serialize for DynJob { { DynJobSerde { kind: self.kind(), - command_line: self.to_command_line(), + job: self.serialize_to_json_value().map_err(S::Error::custom)?, } .serialize(serializer) } @@ -721,1082 +1115,82 @@ impl<'de> Deserialize<'de> for DynJob { where D: Deserializer<'de>, { - let DynJobSerde { kind, command_line } = Deserialize::deserialize(deserializer)?; - kind.parse_command_line(command_line) + let DynJobSerde { kind, job } = Deserialize::deserialize(deserializer)?; + kind.deserialize_job_from_json_value(&job) .map_err(D::Error::custom) } } -impl JobKind for DynJobKind { - type Job = DynJob; +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct RunSingleJob(pub DynJob); - fn inputs_and_direct_dependencies<'a>( - &'a self, - job: &'a Self::Job, - ) -> Cow<'a, BTreeMap>> { - Cow::Borrowed(job.inputs_and_direct_dependencies()) - } - - fn outputs(&self, job: &Self::Job) -> Interned<[JobItemName]> { - job.outputs() - } - - fn command_line_prefix(&self) -> Interned<[Interned]> { - self.0.command_line_prefix_dyn() - } - - fn to_command_line(&self, job: &Self::Job) -> Interned<[Interned]> { - job.to_command_line() - } - - fn subcommand(&self) -> Option { - self.0.subcommand_dyn() - } - - fn from_arg_matches(&self, matches: &mut clap::ArgMatches) -> clap::error::Result { - self.0.clone().from_arg_matches_dyn(matches) - } - - fn parse_command_line( - &self, - command_line: Interned<[Interned]>, - ) -> clap::error::Result { - self.0.clone().parse_command_line_dyn(command_line) - } - - fn run( - &self, - job: &Self::Job, - inputs: &[JobItem], - acquired_job: &mut AcquiredJob, - ) -> eyre::Result> { - job.run(inputs, acquired_job) - } -} - -#[derive(Clone, Debug)] -enum JobGraphNode { - Job(DynJob), - Item { - #[allow(dead_code, reason = "name used for debugging")] - name: JobItemName, - source_job: Option, - }, -} - -type JobGraphInner = DiGraph; - -#[derive(Clone, Default)] -pub struct JobGraph { - jobs: HashMap::NodeId>, - items: HashMap::NodeId>, - graph: JobGraphInner, - topological_order: Vec<::NodeId>, - space: DfsSpace<::NodeId, ::Map>, -} - -impl fmt::Debug for JobGraph { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Self { - jobs: _, - items: _, - graph, - topological_order, - space: _, - } = self; - f.debug_struct("JobGraph") - .field("graph", graph) - .field("topological_order", topological_order) - .finish_non_exhaustive() - } -} - -#[derive(Clone, Debug)] -pub enum JobGraphError { - CycleError { - job: DynJob, - output: JobItemName, - }, - MultipleJobsCreateSameOutput { - output_item: JobItemName, - existing_job: DynJob, - new_job: DynJob, - }, -} - -impl std::error::Error for JobGraphError {} - -impl fmt::Display for JobGraphError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::CycleError { job, output } => write!( - f, - "job can't be added to job graph because it would introduce a cyclic dependency through this job output:\n\ - {output:?}\n\ - job:\n{job:?}", - ), - JobGraphError::MultipleJobsCreateSameOutput { - output_item, - existing_job, - new_job, - } => write!( - f, - "job can't be added to job graph because the new job has an output that is also produced by an existing job.\n\ - conflicting output:\n\ - {output_item:?}\n\ - existing job:\n\ - {existing_job:?}\n\ - new job:\n\ - {new_job:?}", - ), - } - } -} - -#[derive(Copy, Clone, Debug)] -enum EscapeForUnixShellState { - DollarSingleQuote, - SingleQuote, - Unquoted, -} - -#[derive(Clone)] -pub struct EscapeForUnixShell<'a> { - state: EscapeForUnixShellState, - prefix: [u8; 3], - bytes: &'a [u8], -} - -impl<'a> fmt::Debug for EscapeForUnixShell<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self, f) - } -} - -impl<'a> fmt::Display for EscapeForUnixShell<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - for c in self.clone() { - f.write_char(c)?; - } +impl RunSingleJob { + pub const SUBCOMMAND_NAME: &'static str = "run-single-job"; + pub fn try_add_subcommand>>( + job: &DynJob, + subcommand_line: &mut SL, + ) -> serde_json::Result<()> { + subcommand_line.extend([ + Self::SUBCOMMAND_NAME.intern(), + job.kind().name(), + Intern::intern_owned(job.serialize_to_json_ascii()?), + ]); Ok(()) } } -impl<'a> EscapeForUnixShell<'a> { - pub fn new(s: &'a str) -> Self { - Self::from_bytes(s.as_bytes()) - } - fn make_prefix(bytes: &[u8]) -> [u8; 3] { - let mut prefix = [0; 3]; - prefix[..bytes.len()].copy_from_slice(bytes); - prefix - } - pub fn from_bytes(bytes: &'a [u8]) -> Self { - let mut needs_single_quote = bytes.is_empty(); - for &b in bytes { - match b { - b'!' | b'\'' | b'\"' | b' ' => needs_single_quote = true, - 0..0x20 | 0x7F.. => { - return Self { - state: EscapeForUnixShellState::DollarSingleQuote, - prefix: Self::make_prefix(b"$'"), - bytes, - }; - } - _ => {} - } - } - if needs_single_quote { - Self { - state: EscapeForUnixShellState::SingleQuote, - prefix: Self::make_prefix(b"'"), - bytes, - } - } else { - Self { - state: EscapeForUnixShellState::Unquoted, - prefix: Self::make_prefix(b""), - bytes, - } - } +impl TryFrom for RunSingleJob { + type Error = clap::Error; + + fn try_from(value: RunSingleJobClap) -> Result { + let RunSingleJobClap::RunSingleJob { + name: job_kind, + json, + } = value; + let name = job_kind.name(); + job_kind + .deserialize_job_from_json_str(&json) + .map_err(|e| { + clap::Error::raw( + clap::error::ErrorKind::ValueValidation, + format_args!("failed to parse job {name} from JSON: {e}"), + ) + }) + .map(Self) } } -impl Iterator for EscapeForUnixShell<'_> { - type Item = char; - - fn next(&mut self) -> Option { - match &mut self.prefix { - [0, 0, 0] => {} - [0, 0, v] | // find first - [0, v, _] | // non-zero byte - [v, _, _] => { - let retval = *v as char; - *v = 0; - return Some(retval); - } - } - let Some(&next_byte) = self.bytes.split_off_first() else { - return match self.state { - EscapeForUnixShellState::DollarSingleQuote - | EscapeForUnixShellState::SingleQuote => { - self.state = EscapeForUnixShellState::Unquoted; - Some('\'') - } - EscapeForUnixShellState::Unquoted => None, - }; - }; - match self.state { - EscapeForUnixShellState::DollarSingleQuote => match next_byte { - b'\'' | b'\\' => { - self.prefix = Self::make_prefix(&[next_byte]); - Some('\\') - } - b'\t' => { - self.prefix = Self::make_prefix(b"t"); - Some('\\') - } - b'\n' => { - self.prefix = Self::make_prefix(b"n"); - Some('\\') - } - b'\r' => { - self.prefix = Self::make_prefix(b"r"); - Some('\\') - } - 0x20..=0x7E => Some(next_byte as char), - _ => { - self.prefix = [ - b'x', - char::from_digit(next_byte as u32 >> 4, 0x10).expect("known to be in range") - as u8, - char::from_digit(next_byte as u32 & 0xF, 0x10) - .expect("known to be in range") as u8, - ]; - Some('\\') - } - }, - EscapeForUnixShellState::SingleQuote => { - if next_byte == b'\'' { - self.prefix = Self::make_prefix(b"\\''"); - Some('\'') - } else { - Some(next_byte as char) - } - } - EscapeForUnixShellState::Unquoted => match next_byte { - b' ' | b'!' | b'"' | b'#' | b'$' | b'&' | b'\'' | b'(' | b')' | b'*' | b',' - | b';' | b'<' | b'>' | b'?' | b'[' | b'\\' | b']' | b'^' | b'`' | b'{' | b'|' - | b'}' | b'~' => { - self.prefix = Self::make_prefix(&[next_byte]); - Some('\\') - } - _ => Some(next_byte as char), - }, - } - } +#[derive(clap::Subcommand)] +enum RunSingleJobClap { + #[command(name = RunSingleJob::SUBCOMMAND_NAME)] + RunSingleJob { name: DynJobKind, json: String }, } -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] -#[non_exhaustive] -pub enum UnixMakefileEscapeKind { - NonRecipe, - RecipeWithoutShellEscaping, - RecipeWithShellEscaping, -} - -#[derive(Copy, Clone)] -pub struct EscapeForUnixMakefile<'a> { - s: &'a str, - kind: UnixMakefileEscapeKind, -} - -impl<'a> fmt::Debug for EscapeForUnixMakefile<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self, f) - } -} - -impl<'a> fmt::Display for EscapeForUnixMakefile<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.do_write(f, fmt::Write::write_str, fmt::Write::write_char, |_, _| { - Ok(()) - }) - } -} - -impl<'a> EscapeForUnixMakefile<'a> { - fn do_write( - &self, - state: &mut S, - write_str: impl Fn(&mut S, &str) -> Result<(), E>, - write_char: impl Fn(&mut S, char) -> Result<(), E>, - add_variable: impl Fn(&mut S, &'static str) -> Result<(), E>, - ) -> Result<(), E> { - let escape_recipe_char = |c| match c { - '$' => write_str(state, "$$"), - '\0'..='\x1F' | '\x7F' => { - panic!("can't escape a control character for Unix Makefile: {c:?}"); - } - _ => write_char(state, c), - }; - match self.kind { - UnixMakefileEscapeKind::NonRecipe => self.s.chars().try_for_each(|c| match c { - '=' => { - add_variable(state, "EQUALS = =")?; - write_str(state, "$(EQUALS)") - } - ';' => panic!("can't escape a semicolon (;) for Unix Makefile"), - '$' => write_str(state, "$$"), - '\\' | ' ' | '#' | ':' | '%' | '*' | '?' | '[' | ']' | '~' => { - write_char(state, '\\')?; - write_char(state, c) - } - '\0'..='\x1F' | '\x7F' => { - panic!("can't escape a control character for Unix Makefile: {c:?}"); - } - _ => write_char(state, c), - }), - UnixMakefileEscapeKind::RecipeWithoutShellEscaping => { - self.s.chars().try_for_each(escape_recipe_char) - } - UnixMakefileEscapeKind::RecipeWithShellEscaping => { - EscapeForUnixShell::new(self.s).try_for_each(escape_recipe_char) - } - } - } - pub fn new( - s: &'a str, - kind: UnixMakefileEscapeKind, - needed_variables: &mut BTreeSet<&'static str>, - ) -> Self { - let retval = Self { s, kind }; - let Ok(()) = retval.do_write( - needed_variables, - |_, _| Ok(()), - |_, _| Ok(()), - |needed_variables, variable| -> Result<(), std::convert::Infallible> { - needed_variables.insert(variable); - Ok(()) - }, - ); - retval - } -} - -impl JobGraph { - pub fn new() -> Self { - Self::default() - } - fn try_add_item_node( - &mut self, - name: JobItemName, - new_source_job: Option, - new_nodes: &mut HashSet<::NodeId>, - ) -> Result<::NodeId, JobGraphError> { - match self.items.entry(name) { - Entry::Occupied(item_entry) => { - let node_id = *item_entry.get(); - let JobGraphNode::Item { - name: _, - source_job, - } = &mut self.graph[node_id] - else { - unreachable!("known to be an item"); - }; - if let Some(new_source_job) = new_source_job { - if let Some(source_job) = source_job { - return Err(JobGraphError::MultipleJobsCreateSameOutput { - output_item: item_entry.key().clone(), - existing_job: source_job.clone(), - new_job: new_source_job, - }); - } else { - *source_job = Some(new_source_job); - } - } - Ok(node_id) - } - Entry::Vacant(item_entry) => { - let node_id = self.graph.add_node(JobGraphNode::Item { - name, - source_job: new_source_job, - }); - new_nodes.insert(node_id); - item_entry.insert(node_id); - Ok(node_id) - } - } - } - pub fn try_add_jobs>( - &mut self, - jobs: I, - ) -> Result<(), JobGraphError> { - let jobs = jobs.into_iter(); - struct RemoveNewNodesOnError<'a> { - this: &'a mut JobGraph, - new_nodes: HashSet<::NodeId>, - } - impl Drop for RemoveNewNodesOnError<'_> { - fn drop(&mut self) { - for node in self.new_nodes.drain() { - self.this.graph.remove_node(node); - } - } - } - let mut remove_new_nodes_on_error = RemoveNewNodesOnError { - this: self, - new_nodes: HashSet::with_capacity_and_hasher(jobs.size_hint().0, Default::default()), - }; - let new_nodes = &mut remove_new_nodes_on_error.new_nodes; - let this = &mut *remove_new_nodes_on_error.this; - let mut worklist = Vec::from_iter(jobs); - while let Some(job) = worklist.pop() { - let Entry::Vacant(job_entry) = this.jobs.entry(job.clone()) else { - continue; - }; - let job_node_id = this - .graph - .add_node(JobGraphNode::Job(job_entry.key().clone())); - new_nodes.insert(job_node_id); - job_entry.insert(job_node_id); - for name in job.outputs() { - let item_node_id = this.try_add_item_node(name, Some(job.clone()), new_nodes)?; - this.graph.add_edge(job_node_id, item_node_id, ()); - } - for (&name, direct_dependency) in job.inputs_and_direct_dependencies() { - worklist.extend(direct_dependency.clone()); - let item_node_id = this.try_add_item_node(name, None, new_nodes)?; - this.graph.add_edge(item_node_id, job_node_id, ()); - } - } - match toposort(&this.graph, Some(&mut this.space)) { - Ok(v) => { - this.topological_order = v; - // no need to remove any of the new nodes on drop since we didn't encounter any errors - remove_new_nodes_on_error.new_nodes.clear(); - Ok(()) - } - Err(_) => { - // there's at least one cycle, find one! - let cycle = kosaraju_scc(&this.graph) - .into_iter() - .find_map(|scc| { - if scc.len() <= 1 { - // can't be a cycle since our graph is bipartite -- - // jobs only connect to items, never jobs to jobs or items to items - None - } else { - Some(scc) - } - }) - .expect("we know there's a cycle"); - let cycle_set = HashSet::from_iter(cycle.iter().copied()); - let job = cycle - .into_iter() - .find_map(|node_id| { - if let JobGraphNode::Job(job) = &this.graph[node_id] { - Some(job.clone()) - } else { - None - } - }) - .expect("a job must be part of the cycle"); - let output = job - .outputs() - .into_iter() - .find(|output| cycle_set.contains(&this.items[output])) - .expect("an output must be part of the cycle"); - Err(JobGraphError::CycleError { job, output }) - } - } - } - #[track_caller] - pub fn add_jobs>(&mut self, jobs: I) { - match self.try_add_jobs(jobs) { - Ok(()) => {} - Err(e) => panic!("error: {e}"), - } - } - pub fn to_unix_makefile(&self) -> String { - let mut retval = String::new(); - let mut needed_variables = BTreeSet::new(); - for &node_id in &self.topological_order { - let JobGraphNode::Job(job) = &self.graph[node_id] else { - continue; - }; - for (index, output) in job.outputs().into_iter().enumerate() { - match output { - JobItemName::Module { .. } => continue, - JobItemName::File { path } => { - if index != 0 { - retval.push_str(" "); - } - write_str!( - retval, - "{}", - EscapeForUnixMakefile::new( - &path, - UnixMakefileEscapeKind::NonRecipe, - &mut needed_variables - ) - ); - } - } - } - retval.push_str(":"); - for input in job.inputs_and_direct_dependencies().keys() { - match input { - JobItemName::Module { .. } => continue, - JobItemName::File { path } => { - write_str!( - retval, - " {}", - EscapeForUnixMakefile::new( - &path, - UnixMakefileEscapeKind::NonRecipe, - &mut needed_variables - ) - ); - } - } - } - retval.push_str("\n\t"); - for (index, arg) in job.to_command_line().into_iter().enumerate() { - if index != 0 { - retval.push_str(" "); - } - write_str!( - retval, - "{}", - EscapeForUnixMakefile::new( - &arg, - UnixMakefileEscapeKind::RecipeWithShellEscaping, - &mut needed_variables - ) - ); - } - retval.push_str("\n\n"); - } - if !needed_variables.is_empty() { - retval.insert_str( - 0, - &String::from_iter(needed_variables.into_iter().map(|v| format!("{v}\n"))), - ); - } - retval - } - pub fn to_unix_shell_script(&self) -> String { - let mut retval = String::from( - "#!/bin/sh\n\ - set -ex\n", - ); - for &node_id in &self.topological_order { - let JobGraphNode::Job(job) = &self.graph[node_id] else { - continue; - }; - for (index, arg) in job.to_command_line().into_iter().enumerate() { - if index != 0 { - retval.push_str(" "); - } - write_str!(retval, "{}", EscapeForUnixShell::new(&arg)); - } - retval.push_str("\n"); - } - retval - } - pub fn run(&self) -> eyre::Result<()> { - // use scope to auto-join threads on errors - thread::scope(|scope| { - struct WaitingJobState { - job_node_id: ::NodeId, - job: DynJob, - inputs: BTreeMap>, - } - let mut ready_jobs = VecDeque::new(); - let mut item_name_to_waiting_jobs_map = HashMap::<_, Vec<_>>::default(); - for &node_id in &self.topological_order { - let JobGraphNode::Job(job) = &self.graph[node_id] else { - continue; - }; - let waiting_job = WaitingJobState { - job_node_id: node_id, - job: job.clone(), - inputs: job - .inputs_and_direct_dependencies() - .keys() - .map(|&name| (name, OnceCell::new())) - .collect(), - }; - if waiting_job.inputs.is_empty() { - ready_jobs.push_back(waiting_job); - } else { - let waiting_job = Rc::new(waiting_job); - for &input_item in waiting_job.inputs.keys() { - item_name_to_waiting_jobs_map - .entry(input_item) - .or_default() - .push(waiting_job.clone()); - } - } - } - struct RunningJob<'scope> { - job: DynJob, - thread: ScopedJoinHandle<'scope, eyre::Result>>, - } - let mut running_jobs = HashMap::default(); - let (finished_jobs_sender, finished_jobs_receiver) = mpsc::channel(); - loop { - while let Some(finished_job) = finished_jobs_receiver.try_recv().ok() { - let Some(RunningJob { job, thread }) = running_jobs.remove(&finished_job) - else { - unreachable!(); - }; - let output_items = thread.join().map_err(panic::resume_unwind)??; - assert!( - output_items.iter().map(JobItem::name).eq(job.outputs()), - "job's run() method returned the wrong output items:\n\ - output items:\n\ - {output_items:?}\n\ - expected outputs:\n\ - {:?}\n\ - job:\n\ - {job:?}", - job.outputs(), - ); - for output_item in output_items { - for waiting_job in item_name_to_waiting_jobs_map - .remove(&output_item.name()) - .unwrap_or_default() - { - let Ok(()) = - waiting_job.inputs[&output_item.name()].set(output_item.clone()) - else { - unreachable!(); - }; - if let Some(waiting_job) = Rc::into_inner(waiting_job) { - ready_jobs.push_back(waiting_job); - } - } - } - } - if let Some(WaitingJobState { - job_node_id, - job, - inputs, - }) = ready_jobs.pop_front() - { - struct RunningJobInThread { - job_node_id: ::NodeId, - job: DynJob, - inputs: Vec, - acquired_job: AcquiredJob, - finished_jobs_sender: mpsc::Sender<::NodeId>, - } - impl RunningJobInThread { - fn run(mut self) -> eyre::Result> { - self.job.run(&self.inputs, &mut self.acquired_job) - } - } - impl Drop for RunningJobInThread { - fn drop(&mut self) { - let _ = self.finished_jobs_sender.send(self.job_node_id); - } - } - let name = job.debug_name(); - let running_job_in_thread = RunningJobInThread { - job_node_id, - job: job.clone(), - inputs: Vec::from_iter( - inputs - .into_values() - .map(|input| input.into_inner().expect("was set earlier")), - ), - acquired_job: AcquiredJob::acquire(), - finished_jobs_sender: finished_jobs_sender.clone(), - }; - running_jobs.insert( - job_node_id, - RunningJob { - job, - thread: thread::Builder::new() - .name(name) - .spawn_scoped(scope, move || running_job_in_thread.run()) - .expect("failed to spawn thread for job"), - }, - ); - } - if running_jobs.is_empty() { - assert!(item_name_to_waiting_jobs_map.is_empty()); - assert!(ready_jobs.is_empty()); - return Ok(()); - } - } - }) - } -} - -impl Extend for JobGraph { - #[track_caller] - fn extend>(&mut self, iter: T) { - self.add_jobs(iter); - } -} - -impl FromIterator for JobGraph { - #[track_caller] - fn from_iter>(iter: T) -> Self { - let mut retval = Self::new(); - retval.add_jobs(iter); - retval - } -} - -impl Serialize for JobGraph { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut serializer = serializer.serialize_seq(Some(self.jobs.len()))?; - for &node_id in &self.topological_order { - let JobGraphNode::Job(job) = &self.graph[node_id] else { - continue; - }; - serializer.serialize_element(job)?; - } - serializer.end() - } -} - -impl<'de> Deserialize<'de> for JobGraph { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let jobs = Vec::::deserialize(deserializer)?; - let mut retval = JobGraph::new(); - retval.try_add_jobs(jobs).map_err(D::Error::custom)?; - Ok(retval) - } -} - -pub fn program_name_for_internal_jobs() -> Interned { - static PROGRAM_NAME: OnceLock> = OnceLock::new(); - *PROGRAM_NAME - .get_or_init(|| str::intern_owned(std::env::args().next().expect("can't get program name"))) -} - -pub trait InternalJobTrait: clap::Args + 'static + fmt::Debug + Eq + Hash + Send + Sync { - fn subcommand_name() -> Interned; - fn to_args(&self) -> Vec>; - fn inputs_and_direct_dependencies<'a>( - &'a self, - ) -> Cow<'a, BTreeMap>>; - fn outputs(&self) -> Interned<[JobItemName]>; - fn run(&self, inputs: &[JobItem], acquired_job: &mut AcquiredJob) - -> eyre::Result>; -} - -#[derive(Hash, PartialEq, Eq)] -pub struct InternalJobKind(PhantomData Job>); - -impl Clone for InternalJobKind { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for InternalJobKind {} - -impl InternalJobKind { - pub const fn new() -> Self { - Self(PhantomData) - } -} - -impl Default for InternalJobKind { - fn default() -> Self { - Self::new() - } -} - -impl fmt::Debug for InternalJobKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "InternalJobKind<{}>", std::any::type_name::()) - } -} - -impl JobKind for InternalJobKind { - type Job = InternalJob; - - fn inputs_and_direct_dependencies<'a>( - &'a self, - job: &'a Self::Job, - ) -> Cow<'a, BTreeMap>> { - job.0.inputs_and_direct_dependencies() - } - - fn outputs(&self, job: &Self::Job) -> Interned<[JobItemName]> { - job.0.outputs() - } - - fn command_line_prefix(&self) -> Interned<[Interned]> { - [program_name_for_internal_jobs(), Job::subcommand_name()][..].intern() - } - - fn subcommand(&self) -> Option { - Some(Job::augment_args(clap::Command::new(Interned::into_inner( - Job::subcommand_name(), - )))) - } - - fn from_arg_matches(&self, matches: &mut clap::ArgMatches) -> clap::error::Result { - clap::FromArgMatches::from_arg_matches_mut(matches) - } - - fn to_command_line(&self, job: &Self::Job) -> Interned<[Interned]> { - Interned::from_iter( - self.command_line_prefix() - .iter() - .copied() - .chain(job.0.to_args()), - ) - } - - fn parse_command_line( - &self, - command_line: Interned<[Interned]>, - ) -> clap::error::Result { - let cmd = clap::Command::new(Interned::into_inner(program_name_for_internal_jobs())); - let mut matches = as clap::Subcommand>::augment_subcommands(cmd) - .subcommand_required(true) - .arg_required_else_help(true) - .try_get_matches_from(command_line.iter().map(|arg| &**arg))?; - as clap::FromArgMatches>::from_arg_matches_mut(&mut matches) - } - - fn run( - &self, - job: &Self::Job, - inputs: &[JobItem], - acquired_job: &mut AcquiredJob, - ) -> eyre::Result> { - job.0.run(inputs, acquired_job) - } -} - -#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Default, PartialOrd, Ord)] -pub struct InternalJob(pub Job); - -impl clap::FromArgMatches for InternalJob { - fn from_arg_matches(matches: &clap::ArgMatches) -> Result { - Self::from_arg_matches_mut(&mut matches.clone()) - } - - fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result { - if let Some((name, mut matches)) = matches.remove_subcommand() { - if *name == *Job::subcommand_name() { - Ok(Self(Job::from_arg_matches_mut(&mut matches)?)) - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::InvalidSubcommand, - format!("the subcommand '{name}' wasn't recognized"), - )) - } - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::MissingSubcommand, - "a subcommand is required but one was not provided", - )) - } - } - - fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> { - self.update_from_arg_matches_mut(&mut matches.clone()) - } - - fn update_from_arg_matches_mut( - &mut self, - matches: &mut clap::ArgMatches, - ) -> Result<(), clap::Error> { - if let Some((name, mut matches)) = matches.remove_subcommand() { - if *name == *Job::subcommand_name() { - self.0.update_from_arg_matches_mut(&mut matches)?; - Ok(()) - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::InvalidSubcommand, - format!("the subcommand '{name}' wasn't recognized"), - )) - } - } else { - Err(clap::Error::raw( - clap::error::ErrorKind::MissingSubcommand, - "a subcommand is required but one was not provided", - )) - } - } -} - -impl clap::Subcommand for InternalJob { +impl clap::Subcommand for RunSingleJob { fn augment_subcommands(cmd: clap::Command) -> clap::Command { - cmd.subcommand( - InternalJobKind::::new() - .subcommand() - .expect("known to return Some"), - ) + RunSingleJobClap::augment_subcommands(cmd) } fn augment_subcommands_for_update(cmd: clap::Command) -> clap::Command { - cmd.subcommand(Job::augment_args_for_update(clap::Command::new( - Interned::into_inner(Job::subcommand_name()), - ))) + RunSingleJobClap::augment_subcommands(cmd) } fn has_subcommand(name: &str) -> bool { - *name == *Job::subcommand_name() + RunSingleJobClap::has_subcommand(name) } } -#[derive(clap::Args)] -#[clap(id = "OutputDir")] -struct OutputDirArgs { - /// the directory to put the generated main output file and associated files in - #[arg(short, long, value_hint = clap::ValueHint::DirPath, required = true)] - output: Option, - #[arg(long, env = "FAYALITE_KEEP_TEMP_DIR")] - keep_temp_dir: bool, -} - -#[derive(Debug, Clone)] -pub struct OutputDir { - output: String, - temp_dir: Option>, - keep_temp_dir: bool, -} - -impl Eq for OutputDir {} - -impl AsRef for OutputDir { - fn as_ref(&self) -> &str { - self.path() - } -} - -impl AsRef for OutputDir { - fn as_ref(&self) -> &std::path::Path { - self.path().as_ref() - } -} - -impl OutputDir { - pub fn path(&self) -> &str { - &self.output - } - pub fn new(output: String) -> Self { - Self { - output, - temp_dir: None, - keep_temp_dir: false, - } - } - pub fn with_keep_temp_dir(output: String, keep_temp_dir: bool) -> Self { - Self { - output, - temp_dir: None, - keep_temp_dir, - } - } - pub fn temp(keep_temp_dir: bool) -> std::io::Result { - let temp_dir = TempDir::new()?; - let output = String::from(temp_dir.path().as_os_str().to_str().ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidFilename, - format!( - "temporary directory path is not valid UTF-8: {:?}", - temp_dir.path() - ), - ) - })?); - let temp_dir = if keep_temp_dir { - println!( - "created temporary directory: {}", - temp_dir.into_path().display() - ); - None - } else { - Some(Arc::new(temp_dir)) - }; - Ok(Self { - output, - temp_dir, - keep_temp_dir, - }) - } - pub fn to_args(&self) -> Vec> { - let Self { - output, - temp_dir: _, - keep_temp_dir, - } = self; - let mut retval = Vec::new(); - retval.push(str::intern_owned(format!("--output={output}"))); - if *keep_temp_dir { - retval.push("--keep-temp-dir".intern()); - } - retval - } - fn compare_key(&self) -> (&str, bool, bool) { - let Self { - output, - temp_dir, - keep_temp_dir, - } = self; - (output, temp_dir.is_some(), *keep_temp_dir) - } -} - -impl PartialEq for OutputDir { - fn eq(&self, other: &Self) -> bool { - self.compare_key() == other.compare_key() - } -} - -impl Hash for OutputDir { - fn hash(&self, state: &mut H) { - self.compare_key().hash(state); - } -} - -impl TryFrom for OutputDir { - type Error = clap::Error; - - fn try_from(value: OutputDirArgs) -> Result { - let OutputDirArgs { - output, - keep_temp_dir, - } = value; - match output { - Some(output) => Ok(Self::with_keep_temp_dir(output, keep_temp_dir)), - None => Ok(Self::temp(keep_temp_dir)?), - } - } -} - -impl clap::FromArgMatches for OutputDir { +impl clap::FromArgMatches for RunSingleJob { fn from_arg_matches(matches: &clap::ArgMatches) -> clap::error::Result { - OutputDirArgs::from_arg_matches(matches)?.try_into() + RunSingleJobClap::from_arg_matches(matches)?.try_into() } - fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> clap::error::Result { - OutputDirArgs::from_arg_matches_mut(matches)?.try_into() + RunSingleJobClap::from_arg_matches_mut(matches)?.try_into() } - fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> clap::error::Result<()> { *self = Self::from_arg_matches(matches)?; Ok(()) } - fn update_from_arg_matches_mut( &mut self, matches: &mut clap::ArgMatches, @@ -1806,53 +1200,431 @@ impl clap::FromArgMatches for OutputDir { } } -impl clap::Args for OutputDir { - fn group_id() -> Option { - OutputDirArgs::group_id() - } +#[derive(Clone, PartialEq, Eq, Hash, clap::Subcommand)] +pub enum BuildSubcommand { + #[clap(flatten)] + RunSingleJob(RunSingleJob), + #[clap(flatten)] + Job(AnyJobSubcommand), +} - fn augment_args(cmd: clap::Command) -> clap::Command { - OutputDirArgs::augment_args(cmd) - } +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct AnyJobSubcommand { + pub args: DynJobArgs, + pub dependencies_args: Vec, +} - fn augment_args_for_update(cmd: clap::Command) -> clap::Command { - OutputDirArgs::augment_args_for_update(cmd) +impl AnyJobSubcommand { + pub fn from_subcommand_arg_matches( + job_kind: &DynJobKind, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result { + let dependencies = job_kind.dependencies_kinds(); + let dependencies_args = Result::from_iter( + dependencies + .into_iter() + .map(|dependency| dependency.from_arg_matches(matches)), + )?; + Ok(Self { + args: job_kind.clone().from_arg_matches(matches)?, + dependencies_args, + }) + } + pub fn update_from_subcommand_arg_matches( + &mut self, + job_kind: &DynJobKind, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result<()> { + if *job_kind == self.args.kind() { + for dependency in &mut self.dependencies_args { + dependency.update_from_arg_matches(matches)?; + } + self.args.update_from_arg_matches(matches) + } else { + let dependencies = job_kind.dependencies_kinds(); + let dependencies_args = Result::from_iter( + dependencies + .into_iter() + .map(|dependency| dependency.from_arg_matches(matches)), + )?; + *self = Self { + args: job_kind.clone().from_arg_matches(matches)?, + dependencies_args, + }; + Ok(()) + } } } -#[derive(clap::Parser, Debug, Clone, Hash, PartialEq, Eq)] -#[non_exhaustive] -pub struct BaseArgs { +impl clap::Subcommand for AnyJobSubcommand { + fn augment_subcommands(mut cmd: clap::Command) -> clap::Command { + let snapshot = registry::JobKindRegistrySnapshot::get(); + for job_kind in &snapshot { + cmd = cmd.subcommand(job_kind.make_subcommand()); + } + cmd + } + + fn augment_subcommands_for_update(mut cmd: clap::Command) -> clap::Command { + let snapshot = registry::JobKindRegistrySnapshot::get(); + for job_kind in &snapshot { + cmd = cmd.subcommand(job_kind.make_subcommand_for_update()); + } + cmd + } + + fn has_subcommand(name: &str) -> bool { + registry::JobKindRegistrySnapshot::get() + .get_by_name(name) + .is_some() + } +} + +impl clap::FromArgMatches for AnyJobSubcommand { + fn from_arg_matches(matches: &clap::ArgMatches) -> clap::error::Result { + Self::from_arg_matches_mut(&mut matches.clone()) + } + + fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> clap::error::Result { + if let Some((name, mut matches)) = matches.remove_subcommand() { + let job_kind_registry_snapshot = registry::JobKindRegistrySnapshot::get(); + if let Some(job_kind) = job_kind_registry_snapshot.get_by_name(&name) { + Self::from_subcommand_arg_matches(job_kind, &mut matches) + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::InvalidSubcommand, + format!("the subcommand '{name}' wasn't recognized"), + )) + } + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::MissingSubcommand, + "a subcommand is required but one was not provided", + )) + } + } + + fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> clap::error::Result<()> { + Self::update_from_arg_matches_mut(self, &mut matches.clone()) + } + + fn update_from_arg_matches_mut( + &mut self, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result<()> { + if let Some((name, mut matches)) = matches.remove_subcommand() { + let job_kind_registry_snapshot = registry::JobKindRegistrySnapshot::get(); + if let Some(job_kind) = job_kind_registry_snapshot.get_by_name(&name) { + self.update_from_subcommand_arg_matches(job_kind, &mut matches) + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::InvalidSubcommand, + format!("the subcommand '{name}' wasn't recognized"), + )) + } + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::MissingSubcommand, + "a subcommand is required but one was not provided", + )) + } + } +} + +pub fn program_name_for_internal_jobs() -> Interned { + static PROGRAM_NAME: OnceLock> = OnceLock::new(); + *PROGRAM_NAME + .get_or_init(|| str::intern_owned(std::env::args().next().expect("can't get program name"))) +} + +#[derive(clap::Args, PartialEq, Eq, Hash, Debug, Clone)] +#[group(id = "CreateOutputDir")] +pub struct CreateOutputDirArgs { /// the directory to put the generated main output file and associated files in + #[arg(short, long, value_hint = clap::ValueHint::DirPath)] + pub output: Option, + #[arg(long, env = "FAYALITE_KEEP_TEMP_DIR")] + pub keep_temp_dir: bool, +} + +impl JobArgs for CreateOutputDirArgs { + fn to_args> + ?Sized>(&self, args: &mut Args) { + let Self { + output, + keep_temp_dir, + } = self; + if let Some(output) = output { + args.extend([Intern::intern_owned(format!("--output={output}"))]); + } + if *keep_temp_dir { + args.extend(["--keep-temp-dir".intern()]); + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CreateOutputDir { + output_dir: Interned, + #[serde(skip)] + temp_dir: Option>, +} + +impl Eq for CreateOutputDir {} + +impl PartialEq for CreateOutputDir { + fn eq(&self, other: &Self) -> bool { + self.compare_key() == other.compare_key() + } +} + +impl Hash for CreateOutputDir { + fn hash(&self, state: &mut H) { + self.compare_key().hash(state); + } +} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Debug)] +pub struct CreateOutputDirJobKind; + +impl JobKind for CreateOutputDirJobKind { + type Args = CreateOutputDirArgs; + type Job = CreateOutputDir; + type Dependencies = (); + + fn dependencies(self) -> Self::Dependencies { + () + } + + fn args_to_jobs( + args: JobArgsAndDependencies, + _params: &JobParams, + ) -> eyre::Result> { + let JobArgsAndDependencies { + args: + JobKindAndArgs { + kind, + args: + CreateOutputDirArgs { + output, + keep_temp_dir, + }, + }, + dependencies: (), + } = args; + let (output_dir, temp_dir) = if let Some(output) = output { + (Intern::intern_owned(output), None) + } else { + // we create the temp dir here rather than in run so other + // jobs can have their paths based on the chosen temp dir + let temp_dir = TempDir::new()?; + let output_dir = temp_dir + .path() + .as_os_str() + .to_str() + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidFilename, + format!( + "temporary directory path is not valid UTF-8: {:?}", + temp_dir.path() + ), + ) + })? + .intern(); + let temp_dir = if keep_temp_dir { + // use TempDir::into_path() to no longer automatically delete the temp dir + let temp_dir_path = temp_dir.into_path(); + println!("created temporary directory: {}", temp_dir_path.display()); + None + } else { + Some(Arc::new(temp_dir)) + }; + (output_dir, temp_dir) + }; + Ok(JobAndDependencies { + job: JobAndKind { + kind, + job: CreateOutputDir { + output_dir, + temp_dir, + }, + }, + dependencies: (), + }) + } + + fn inputs(self, _job: &Self::Job) -> Interned<[JobItemName]> { + Interned::default() + } + + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + [JobItemName::Path { + path: job.output_dir, + }][..] + .intern() + } + + fn name(self) -> Interned { + "create-output-dir".intern() + } + + fn external_command_line(self, job: &Self::Job) -> Option]>> { + Some( + [ + "mkdir".intern(), + "-p".intern(), + "--".intern(), + job.output_dir, + ][..] + .intern(), + ) + } + + fn run( + self, + job: &Self::Job, + inputs: &[JobItem], + _params: &JobParams, + _acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + let [] = inputs else { + panic!("invalid inputs for CreateOutputDir"); + }; + std::fs::create_dir_all(&*job.output_dir)?; + Ok(vec![JobItem::Path { + path: job.output_dir, + }]) + } +} + +impl CreateOutputDir { + pub fn output_dir(&self) -> Interned { + self.output_dir + } + fn compare_key(&self) -> (&str, bool) { + let Self { + output_dir, + temp_dir, + } = self; + (output_dir, temp_dir.is_some()) + } +} + +#[derive(clap::Args, Debug, Clone, Hash, PartialEq, Eq)] +#[group(id = "BaseJob")] +#[non_exhaustive] +pub struct BaseJobArgs { + /// rather than having CreateOutputDir be a normal dependency, it's nested in BaseJob to avoid a cyclic dependency #[command(flatten)] - pub output: OutputDir, + pub create_output_dir_args: CreateOutputDirArgs, /// the stem of the generated main output file, e.g. to get foo.v, pass --file-stem=foo #[arg(long)] pub file_stem: Option, - pub module_name: String, } -impl BaseArgs { - pub fn to_args(&self) -> Vec> { +impl JobArgs for BaseJobArgs { + fn to_args> + ?Sized>(&self, args: &mut Args) { let Self { - output, + create_output_dir_args, file_stem, - module_name, } = self; - let mut retval = output.to_args(); + create_output_dir_args.to_args(args); if let Some(file_stem) = file_stem { - retval.push(str::intern_owned(format!("--file-stem={file_stem}"))); + args.extend([Intern::intern_owned(format!("--file-stem={file_stem}"))]); } - retval.push(str::intern(module_name)); - retval - } - pub fn file_with_ext(&self, ext: &str) -> String { - let mut retval = std::path::Path::new(self.output.path()) - .join(self.file_stem.as_ref().unwrap_or(&self.module_name)); - retval.set_extension(ext); - retval - .into_os_string() - .into_string() - .expect("known to be UTF-8") + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct BaseJob { + /// rather than having CreateOutputDir be a normal dependency, it's nested in BaseJob to avoid a cyclic dependency + #[serde(flatten)] + create_output_dir: CreateOutputDir, + file_stem: Interned, +} + +impl BaseJob { + pub fn output_dir(&self) -> Interned { + self.create_output_dir.output_dir() + } + pub fn file_stem(&self) -> Interned { + self.file_stem + } + pub fn file_with_ext(&self, ext: &str) -> Interned { + let mut retval = std::path::Path::new(&self.output_dir()).join(self.file_stem()); + retval.set_extension(ext); + Intern::intern_owned( + retval + .into_os_string() + .into_string() + .expect("known to be UTF-8"), + ) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)] +pub struct BaseJobKind; + +impl JobKind for BaseJobKind { + type Args = BaseJobArgs; + type Job = BaseJob; + type Dependencies = (); + + fn dependencies(self) -> Self::Dependencies { + () + } + + fn args_to_jobs( + args: JobArgsAndDependencies, + params: &JobParams, + ) -> eyre::Result> { + let BaseJobArgs { + create_output_dir_args, + file_stem, + } = args.args.args; + let create_output_dir_args = JobKindAndArgs { + kind: CreateOutputDirJobKind, + args: create_output_dir_args, + }; + let create_output_dir = create_output_dir_args.args_to_jobs((), params)?.job.job; + let file_stem = file_stem + .map(Intern::intern_owned) + .unwrap_or(params.main_module().name()); + Ok(JobAndDependencies { + job: JobAndKind { + kind: BaseJobKind, + job: BaseJob { + create_output_dir, + file_stem, + }, + }, + dependencies: (), + }) + } + + fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + CreateOutputDirJobKind.inputs(&job.create_output_dir) + } + + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + CreateOutputDirJobKind.outputs(&job.create_output_dir) + } + + fn name(self) -> Interned { + "base-job".intern() + } + + fn external_command_line(self, job: &Self::Job) -> Option]>> { + CreateOutputDirJobKind.external_command_line(&job.create_output_dir) + } + + fn run( + self, + job: &Self::Job, + inputs: &[JobItem], + params: &JobParams, + acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + CreateOutputDirJobKind.run(&job.create_output_dir, inputs, params, acquired_job) } } diff --git a/crates/fayalite/src/build/external.rs b/crates/fayalite/src/build/external.rs index 360ad6e7..36281e19 100644 --- a/crates/fayalite/src/build/external.rs +++ b/crates/fayalite/src/build/external.rs @@ -2,22 +2,33 @@ // See Notices.txt for copyright information use crate::{ - build::{DynJob, EscapeForUnixShell, JobItem, JobItemName, JobKind}, + build::{ + BaseJob, JobAndDependencies, JobAndKind, JobArgs, JobArgsAndDependencies, JobDependencies, + JobItem, JobItemName, JobKind, JobParams, + }, intern::{Intern, Interned}, util::{job_server::AcquiredJob, streaming_read_utf8::streaming_read_utf8}, }; use base64::{Engine, prelude::BASE64_URL_SAFE_NO_PAD}; -use clap::builder::StyledStr; +use clap::builder::OsStringValueParser; use eyre::{Context, ensure, eyre}; -use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error}; +use serde::{ + Deserialize, Deserializer, Serialize, Serializer, + de::{DeserializeOwned, Error}, +}; use std::{ borrow::Cow, collections::BTreeMap, - env, - fmt::{self, Write}, - mem, + ffi::{OsStr, OsString}, + fmt, + hash::{Hash, Hasher}, + io::Write, + marker::PhantomData, + path::{Path, PathBuf}, + sync::OnceLock, }; +#[cfg(todo)] #[derive(Clone, Debug, PartialEq, Eq, Hash)] enum TemplateArg { Literal(String), @@ -25,6 +36,7 @@ enum TemplateArg { OutputPath { before: String, after: String }, } +#[cfg(todo)] impl TemplateArg { fn after_mut(&mut self) -> &mut String { match self { @@ -114,14 +126,15 @@ impl From for MaybeUtf8 { } #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] -pub struct ExternalJobCache { +#[serde(rename = "ExternalJobCache")] +pub struct ExternalJobCacheV2 { pub version: ExternalJobCacheVersion, pub inputs_hash: blake3::Hash, pub stdout_stderr: String, pub result: Result, String>, } -impl ExternalJobCache { +impl ExternalJobCacheV2 { fn read_from_file(cache_json_path: Interned) -> eyre::Result { let cache_str = std::fs::read_to_string(&*cache_json_path) .wrap_err_with(|| format!("can't read {cache_json_path}"))?; @@ -181,6 +194,43 @@ impl JobCacheHasher { } impl ExternalJobCaching { + pub fn get_cache_dir_from_output_dir(output_dir: Interned) -> PathBuf { + Path::join(output_dir.as_ref(), ".cache") + } + pub fn make_cache_dir( + cache_dir: impl AsRef, + application_name: &str, + ) -> std::io::Result<()> { + let cache_dir = cache_dir.as_ref(); + std::fs::create_dir_all(cache_dir)?; + let write_atomically = |name: &str, contents: fmt::Arguments<'_>| { + let path = cache_dir.join(name); + if !matches!(std::fs::exists(&path), Ok(true)) { + // use File::create_new rather than tempfile's code to get normal file permissions rather than mode 600 on Unix. + let mut file = tempfile::Builder::new() + .make_in(cache_dir, |path| std::fs::File::create_new(path))?; + file.write_all(contents.to_string().as_bytes())?; // use to_string to avoid a bunch of tiny writes + file.into_temp_path().persist_noclobber(path)?; + } + Ok(()) + }; + write_atomically( + "CACHEDIR.TAG", + format_args!( + "Signature: 8a477f597d28d172789f06886806bc55\n\ + # This file is a cache directory tag created by {application_name}.\n\ + # For information about cache directory tags see https://bford.info/cachedir/" + ), + )?; + write_atomically( + ".gitignore", + format_args!( + "# This is a cache directory created by {application_name}.\n\ + # ignore all files\n\ + *" + ), + ) + } pub fn new(cache_json_path: Interned) -> Self { Self { cache_json_path, @@ -188,7 +238,7 @@ impl ExternalJobCaching { } } #[track_caller] - pub fn from_path(cache_json_path: impl AsRef) -> Self { + pub fn from_path(cache_json_path: impl AsRef) -> Self { let cache_json_path = cache_json_path.as_ref(); let Some(cache_json_path) = cache_json_path.as_os_str().to_str() else { panic!("non-UTF-8 path to cache json: {cache_json_path:?}"); @@ -215,12 +265,12 @@ impl ExternalJobCaching { if self.run_even_if_cached { return Err(()); } - let Ok(ExternalJobCache { + let Ok(ExternalJobCacheV2 { version: ExternalJobCacheVersion::CURRENT, inputs_hash: cached_inputs_hash, stdout_stderr, result, - }) = ExternalJobCache::read_from_file(self.cache_json_path) + }) = ExternalJobCacheV2::read_from_file(self.cache_json_path) else { return Err(()); }; @@ -314,7 +364,7 @@ impl ExternalJobCaching { .expect("spawn shouldn't fail"); run_fn(cmd) }); - ExternalJobCache { + ExternalJobCacheV2 { version: ExternalJobCacheVersion::CURRENT, inputs_hash, stdout_stderr, @@ -350,6 +400,7 @@ impl ExternalJobCaching { } } +#[cfg(todo)] #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct TemplatedExternalJobKind { template: Interned<[TemplateArg]>, @@ -357,12 +408,14 @@ pub struct TemplatedExternalJobKind { caching: Option, } +#[cfg(todo)] #[derive(Copy, Clone, Debug, PartialEq, Eq)] enum Token { Char(char), ArgSeparator, } +#[cfg(todo)] impl Token { fn as_ident_start(self) -> Option { match self { @@ -380,12 +433,14 @@ impl Token { } } +#[cfg(todo)] #[derive(Clone, Debug)] struct Tokens<'a> { current: std::str::Chars<'a>, rest: std::slice::Iter<'a, &'a str>, } +#[cfg(todo)] impl<'a> Tokens<'a> { fn new(args: &'a [&'a str]) -> Self { Self { @@ -395,6 +450,7 @@ impl<'a> Tokens<'a> { } } +#[cfg(todo)] impl Iterator for Tokens<'_> { type Item = Token; @@ -409,11 +465,13 @@ impl Iterator for Tokens<'_> { } } +#[cfg(todo)] struct Parser<'a> { tokens: std::iter::Peekable>, template: Vec, } +#[cfg(todo)] impl<'a> Parser<'a> { fn new(args_template: &'a [&'a str]) -> Self { Self { @@ -519,6 +577,7 @@ impl<'a> Parser<'a> { } } +#[cfg(todo)] pub fn find_program<'a>( default_program_name: &'a str, program_path_env_var: Option<&str>, @@ -535,6 +594,7 @@ pub fn find_program<'a>( .map_err(|program_path| eyre!("path to program is not valid UTF-8: {program_path:?}")) } +#[cfg(todo)] #[derive(Clone, Debug)] enum ParseErrorKind { ExpectedVar, @@ -542,15 +602,18 @@ enum ParseErrorKind { EachArgMustHaveAtMostOneVar, } +#[cfg(todo)] #[derive(Clone, Debug)] pub struct TemplateParseError(ParseErrorKind); +#[cfg(todo)] impl From for TemplateParseError { fn from(value: ParseErrorKind) -> Self { Self(value) } } +#[cfg(todo)] impl fmt::Display for TemplateParseError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match &self.0 { @@ -568,8 +631,10 @@ impl fmt::Display for TemplateParseError { } } +#[cfg(todo)] impl std::error::Error for TemplateParseError {} +#[cfg(todo)] impl TemplatedExternalJobKind { pub fn try_new( default_program_name: &str, @@ -599,6 +664,7 @@ impl TemplatedExternalJobKind { Err(e) => panic!("{e}"), } } + #[cfg(todo)] fn usage(&self) -> StyledStr { let mut retval = String::from("Usage:"); let mut last_input_index = 0usize; @@ -632,6 +698,7 @@ impl TemplatedExternalJobKind { } retval.into() } + #[cfg(todo)] fn with_usage(&self, mut e: clap::Error) -> clap::Error { e.insert( clap::error::ContextKind::Usage, @@ -641,6 +708,7 @@ impl TemplatedExternalJobKind { } } +#[cfg(todo)] impl JobKind for TemplatedExternalJobKind { type Job = TemplatedExternalJob; @@ -771,6 +839,7 @@ impl JobKind for TemplatedExternalJobKind { } } +#[cfg(todo)] #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct TemplatedExternalJob { command_line: Interned<[Interned]>, @@ -778,6 +847,7 @@ pub struct TemplatedExternalJob { outputs: Interned<[JobItemName]>, } +#[cfg(todo)] impl TemplatedExternalJob { pub fn try_add_direct_dependency(&mut self, new_dependency: DynJob) -> eyre::Result<()> { let mut added = false; @@ -833,3 +903,426 @@ impl TemplatedExternalJob { self } } + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Default)] +pub struct ExternalCommandJobKind(PhantomData); + +impl ExternalCommandJobKind { + pub const fn new() -> Self { + Self(PhantomData) + } +} + +#[derive(Copy, Clone)] +struct ExternalCommandProgramPathValueParser(PhantomData); + +fn parse_which_result( + which_result: which::Result, + program_name: impl Into, + program_path_arg_name: impl FnOnce() -> String, +) -> Result, ResolveProgramPathError> { + let which_result = match which_result { + Ok(v) => v, + Err(e) => { + return Err(ResolveProgramPathError { + inner: ResolveProgramPathErrorInner::Which(e), + program_name: program_name.into(), + program_path_arg_name: program_path_arg_name(), + }); + } + }; + Ok(str::intern_owned( + which_result + .into_os_string() + .into_string() + .map_err(|_| ResolveProgramPathError { + inner: ResolveProgramPathErrorInner::NotValidUtf8, + program_name: program_name.into(), + program_path_arg_name: program_path_arg_name(), + })?, + )) +} + +#[derive(Clone)] +struct WhichResultValueParser; + +impl clap::builder::TypedValueParser for WhichResultValueParser { + type Value = which::Result; + + fn parse_ref( + &self, + _cmd: &clap::Command, + _arg: Option<&clap::Arg>, + value: &OsStr, + ) -> clap::error::Result { + Ok(which::which(value)) + } +} + +impl clap::builder::TypedValueParser + for ExternalCommandProgramPathValueParser +{ + type Value = Interned; + + fn parse_ref( + &self, + cmd: &clap::Command, + arg: Option<&clap::Arg>, + value: &OsStr, + ) -> clap::error::Result { + OsStringValueParser::new() + .try_map(|program_name| { + parse_which_result(which::which(&program_name), program_name, || { + T::program_path_arg_name().into() + }) + }) + .parse_ref(cmd, arg, value) + } +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug, clap::Args)] +#[group(id = T::args_group_id())] +#[non_exhaustive] +pub struct ExternalCommandArgs { + #[arg( + name = Interned::into_inner(T::program_path_arg_name()), + long = T::program_path_arg_name(), + value_name = T::program_path_arg_value_name(), + env = T::program_path_env_var_name().map(Interned::into_inner), + value_parser = ExternalCommandProgramPathValueParser::(PhantomData), + default_value = T::default_program_name(), + value_hint = clap::ValueHint::CommandName, + )] + pub program_path: Interned, + #[arg( + name = Interned::into_inner(T::run_even_if_cached_arg_name()), + long = T::run_even_if_cached_arg_name(), + )] + pub run_even_if_cached: bool, + #[command(flatten)] + pub additional_args: T::AdditionalArgs, +} + +#[derive(Clone)] +enum ResolveProgramPathErrorInner { + Which(which::Error), + NotValidUtf8, +} + +impl fmt::Debug for ResolveProgramPathErrorInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Which(v) => v.fmt(f), + Self::NotValidUtf8 => f.write_str("NotValidUtf8"), + } + } +} + +impl fmt::Display for ResolveProgramPathErrorInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Which(v) => v.fmt(f), + Self::NotValidUtf8 => f.write_str("path is not valid UTF-8"), + } + } +} + +#[derive(Clone, Debug)] +pub struct ResolveProgramPathError { + inner: ResolveProgramPathErrorInner, + program_name: std::ffi::OsString, + program_path_arg_name: String, +} + +impl fmt::Display for ResolveProgramPathError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + inner, + program_name, + program_path_arg_name, + } = self; + write!( + f, + "{program_path_arg_name}: failed to resolve {program_name:?} to a valid program: {inner}", + ) + } +} + +impl std::error::Error for ResolveProgramPathError {} + +pub fn resolve_program_path( + program_name: Option<&OsStr>, + default_program_name: impl AsRef, + program_path_env_var_name: Option<&OsStr>, +) -> Result, ResolveProgramPathError> { + let default_program_name = default_program_name.as_ref(); + let owned_program_name; + let program_name = if let Some(program_name) = program_name { + program_name + } else if let Some(v) = program_path_env_var_name.and_then(std::env::var_os) { + owned_program_name = v; + &owned_program_name + } else { + default_program_name + }; + parse_which_result(which::which(program_name), program_name, || { + default_program_name.display().to_string() + }) +} + +impl ExternalCommandArgs { + pub fn with_resolved_program_path( + program_path: Interned, + additional_args: T::AdditionalArgs, + ) -> Self { + Self { + program_path, + run_even_if_cached: false, + additional_args, + } + } + pub fn new( + program_name: Option<&OsStr>, + additional_args: T::AdditionalArgs, + ) -> Result { + Ok(Self::with_resolved_program_path( + resolve_program_path( + program_name, + T::default_program_name(), + T::program_path_env_var_name().as_ref().map(AsRef::as_ref), + )?, + additional_args, + )) + } +} + +impl JobArgs for ExternalCommandArgs { + fn to_args> + ?Sized>(&self, args: &mut Args) { + let Self { + program_path, + run_even_if_cached, + ref additional_args, + } = *self; + args.extend([str::intern_owned(format!( + "--{}={program_path}", + T::program_path_arg_name() + ))]); + if run_even_if_cached { + args.extend([str::intern_owned(format!( + "--{}", + T::run_even_if_cached_arg_name() + ))]); + } + additional_args.to_args(args); + } +} + +#[derive(Copy, Clone)] +struct ExternalCommandJobParams { + command_line: Interned<[Interned]>, + inputs: Interned<[JobItemName]>, + outputs: Interned<[JobItemName]>, +} + +impl ExternalCommandJobParams { + fn new(job: &ExternalCommandJob) -> Self { + Self { + command_line: Interned::from_iter( + [job.program_path] + .into_iter() + .chain(T::command_line_args(job).iter().copied()), + ), + inputs: T::inputs(job), + outputs: T::outputs(job), + } + } +} + +#[derive(Deserialize, Serialize)] +pub struct ExternalCommandJob { + additional_job_data: T::AdditionalJobData, + program_path: Interned, + #[serde(skip)] + params_cache: OnceLock, +} + +impl Eq for ExternalCommandJob {} + +impl> Clone for ExternalCommandJob { + fn clone(&self) -> Self { + let Self { + ref additional_job_data, + program_path, + ref params_cache, + } = *self; + Self { + additional_job_data: additional_job_data.clone(), + program_path, + params_cache: params_cache.clone(), + } + } +} + +impl fmt::Debug for ExternalCommandJob { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + additional_job_data, + program_path, + params_cache: _, + } = self; + write!(f, "ExternalCommandJob<{}>", std::any::type_name::())?; + f.debug_struct("") + .field("additional_job_data", additional_job_data) + .field("program_path", program_path) + .finish() + } +} + +impl PartialEq for ExternalCommandJob { + fn eq(&self, other: &Self) -> bool { + let Self { + additional_job_data, + program_path, + params_cache: _, + } = self; + *additional_job_data == other.additional_job_data && *program_path == other.program_path + } +} + +impl Hash for ExternalCommandJob { + fn hash(&self, state: &mut H) { + let Self { + additional_job_data, + program_path, + params_cache: _, + } = self; + additional_job_data.hash(state); + program_path.hash(state); + } +} + +impl ExternalCommandJob { + pub fn additional_job_data(&self) -> &T::AdditionalJobData { + &self.additional_job_data + } + pub fn program_path(&self) -> Interned { + self.program_path + } + fn params(&self) -> &ExternalCommandJobParams { + self.params_cache + .get_or_init(|| ExternalCommandJobParams::new(self)) + } + pub fn command_line(&self) -> Interned<[Interned]> { + self.params().command_line + } + pub fn inputs(&self) -> Interned<[JobItemName]> { + self.params().inputs + } + pub fn outputs(&self) -> Interned<[JobItemName]> { + self.params().outputs + } +} + +pub trait ExternalCommand: 'static + Send + Sync + Hash + Eq + fmt::Debug + Sized + Copy { + type AdditionalArgs: JobArgs; + type AdditionalJobData: 'static + + Send + + Sync + + Hash + + Eq + + fmt::Debug + + Serialize + + DeserializeOwned; + type Dependencies: JobDependencies; + fn dependencies() -> Self::Dependencies; + fn base_job(dependencies: &::JobsAndKinds) -> &BaseJob; + fn args_to_jobs( + args: JobArgsAndDependencies>, + params: &JobParams, + ) -> eyre::Result<( + Self::AdditionalJobData, + ::JobsAndKinds, + )>; + fn inputs(job: &ExternalCommandJob) -> Interned<[JobItemName]>; + fn outputs(job: &ExternalCommandJob) -> Interned<[JobItemName]>; + fn command_line_args(job: &ExternalCommandJob) -> Interned<[Interned]>; + fn job_kind_name() -> Interned; + fn args_group_id() -> clap::Id { + Interned::into_inner(Self::job_kind_name()).into() + } + fn program_path_arg_name() -> Interned { + Self::default_program_name() + } + fn program_path_arg_value_name() -> Interned { + Intern::intern_owned(Self::program_path_arg_name().to_uppercase()) + } + fn default_program_name() -> Interned; + fn program_path_env_var_name() -> Option> { + Some(Intern::intern_owned( + Self::program_path_arg_name() + .to_uppercase() + .replace('-', "_"), + )) + } + fn run_even_if_cached_arg_name() -> Interned { + Intern::intern_owned(format!("{}-run-even-if-cached", Self::job_kind_name())) + } +} + +impl JobKind for ExternalCommandJobKind { + type Args = ExternalCommandArgs; + type Job = ExternalCommandJob; + type Dependencies = T::Dependencies; + + fn dependencies(self) -> Self::Dependencies { + T::dependencies() + } + + fn args_to_jobs( + args: JobArgsAndDependencies, + params: &JobParams, + ) -> eyre::Result> { + let kind = args.args.kind; + let program_path = args.args.args.program_path; + let (additional_job_data, dependencies) = T::args_to_jobs(args, params)?; + let job = ExternalCommandJob { + additional_job_data, + program_path, + params_cache: OnceLock::new(), + }; + job.params(); // fill cache + Ok(JobAndDependencies { + job: JobAndKind { kind, job }, + dependencies, + }) + } + + fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + job.inputs() + } + + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + job.outputs() + } + + fn name(self) -> Interned { + T::job_kind_name() + } + + fn external_command_line(self, job: &Self::Job) -> Option]>> { + Some(job.command_line()) + } + + fn run( + self, + job: &Self::Job, + inputs: &[JobItem], + params: &JobParams, + acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + ExternalJobCaching { + cache_json_path: todo!(), + run_even_if_cached: todo!(), + } + } +} diff --git a/crates/fayalite/src/build/firrtl.rs b/crates/fayalite/src/build/firrtl.rs index 7c9998f1..20f2ee7a 100644 --- a/crates/fayalite/src/build/firrtl.rs +++ b/crates/fayalite/src/build/firrtl.rs @@ -2,81 +2,119 @@ // See Notices.txt for copyright information use crate::{ - build::{BaseArgs, DynJob, InternalJobTrait, JobItem, JobItemName}, + build::{ + BaseJob, BaseJobKind, JobAndDependencies, JobArgs, JobArgsAndDependencies, JobItem, + JobItemName, JobKind, JobKindAndDependencies, JobParams, + }, firrtl::{ExportOptions, FileBackend}, intern::{Intern, Interned}, util::job_server::AcquiredJob, }; -use clap::Parser; -use std::{borrow::Cow, collections::BTreeMap}; +use clap::Args; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; -#[derive(Parser, Debug, Clone, Hash, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Debug)] +pub struct FirrtlJobKind; + +#[derive(Args, Debug, Clone, Hash, PartialEq, Eq)] +#[group(id = "Firrtl")] #[non_exhaustive] pub struct FirrtlArgs { - #[command(flatten)] - pub base: BaseArgs, #[command(flatten)] pub export_options: ExportOptions, } -impl FirrtlArgs { +impl JobArgs for FirrtlArgs { + fn to_args> + ?Sized>(&self, args: &mut Args) { + let Self { export_options } = self; + export_options.to_args(args); + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Firrtl { + base: BaseJob, + export_options: ExportOptions, +} + +impl Firrtl { fn make_firrtl_file_backend(&self) -> FileBackend { FileBackend { - dir_path: self.base.output.path().into(), - top_fir_file_stem: self.base.file_stem.clone(), + dir_path: PathBuf::from(&*self.base.output_dir()), + top_fir_file_stem: Some(String::from(&*self.base.file_stem())), circuit_name: None, } } - pub fn firrtl_file(&self) -> String { + pub fn firrtl_file(&self) -> Interned { self.base.file_with_ext("fir") } } -impl InternalJobTrait for FirrtlArgs { - fn subcommand_name() -> Interned { - "firrtl".intern() +impl JobKind for FirrtlJobKind { + type Args = FirrtlArgs; + type Job = Firrtl; + type Dependencies = JobKindAndDependencies; + + fn dependencies(self) -> Self::Dependencies { + JobKindAndDependencies::new(BaseJobKind) } - fn to_args(&self) -> Vec> { - let Self { - base, - export_options, - } = self; - let mut retval = base.to_args(); - retval.extend(export_options.to_args()); - retval - } - - fn inputs_and_direct_dependencies<'a>( - &'a self, - ) -> Cow<'a, BTreeMap>> { - Cow::Owned(BTreeMap::from_iter([( - JobItemName::Module { - name: str::intern(&self.base.module_name), + fn args_to_jobs( + args: JobArgsAndDependencies, + params: &JobParams, + ) -> eyre::Result> { + args.args_to_jobs_simple( + params, + |_kind, FirrtlArgs { export_options }, dependencies| { + Ok(Firrtl { + base: dependencies.job.job.clone(), + export_options, + }) }, - None, - )])) + ) } - fn outputs(&self) -> Interned<[JobItemName]> { - [JobItemName::File { - path: str::intern_owned(self.firrtl_file()), + fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + [JobItemName::Path { + path: job.base.output_dir(), }][..] .intern() } + fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> { + [JobItemName::Path { + path: job.firrtl_file(), + }][..] + .intern() + } + + fn name(self) -> Interned { + "firrtl".intern() + } + + fn external_command_line(self, _job: &Self::Job) -> Option]>> { + None + } + fn run( - &self, + self, + job: &Self::Job, inputs: &[JobItem], + params: &JobParams, _acquired_job: &mut AcquiredJob, ) -> eyre::Result> { - let [JobItem::Module { value: module }] = inputs else { - panic!("wrong inputs, expected a single `Module`"); + let [JobItem::Path { path: input_path }] = *inputs else { + panic!("wrong inputs, expected a single `Path`"); }; - assert_eq!(*module.name(), *self.base.module_name); - crate::firrtl::export(self.make_firrtl_file_backend(), module, self.export_options)?; - Ok(vec![JobItem::File { - path: str::intern_owned(self.firrtl_file()), + assert_eq!(input_path, job.base.output_dir()); + crate::firrtl::export( + job.make_firrtl_file_backend(), + params.main_module(), + job.export_options, + )?; + Ok(vec![JobItem::Path { + path: job.firrtl_file(), }]) } } diff --git a/crates/fayalite/src/build/graph.rs b/crates/fayalite/src/build/graph.rs new file mode 100644 index 00000000..ba9c5d5c --- /dev/null +++ b/crates/fayalite/src/build/graph.rs @@ -0,0 +1,762 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// See Notices.txt for copyright information + +use crate::{ + build::{DynJob, JobItem, JobItemName, JobParams, program_name_for_internal_jobs}, + intern::Interned, + util::{HashMap, HashSet, job_server::AcquiredJob}, +}; +use petgraph::{ + algo::{DfsSpace, kosaraju_scc, toposort}, + graph::DiGraph, + visit::{GraphBase, Visitable}, +}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error, ser::SerializeSeq}; +use std::{ + cell::OnceCell, + collections::{BTreeMap, BTreeSet, VecDeque}, + fmt::{self, Write}, + panic, + rc::Rc, + sync::mpsc, + thread::{self, ScopedJoinHandle}, +}; + +macro_rules! write_str { + ($s:expr, $($rest:tt)*) => { + write!($s, $($rest)*).expect("String::write_fmt can't fail") + }; +} + +#[derive(Clone, Debug)] +enum JobGraphNode { + Job(DynJob), + Item { + #[allow(dead_code, reason = "name used for debugging")] + name: JobItemName, + source_job: Option, + }, +} + +type JobGraphInner = DiGraph; + +#[derive(Clone, Default)] +pub struct JobGraph { + jobs: HashMap::NodeId>, + items: HashMap::NodeId>, + graph: JobGraphInner, + topological_order: Vec<::NodeId>, + space: DfsSpace<::NodeId, ::Map>, +} + +impl fmt::Debug for JobGraph { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + jobs: _, + items: _, + graph, + topological_order, + space: _, + } = self; + f.debug_struct("JobGraph") + .field("graph", graph) + .field("topological_order", topological_order) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug)] +pub enum JobGraphError { + CycleError { + job: DynJob, + output: JobItemName, + }, + MultipleJobsCreateSameOutput { + output_item: JobItemName, + existing_job: DynJob, + new_job: DynJob, + }, +} + +impl std::error::Error for JobGraphError {} + +impl fmt::Display for JobGraphError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CycleError { job, output } => write!( + f, + "job can't be added to job graph because it would introduce a cyclic dependency through this job output:\n\ + {output:?}\n\ + job:\n{job:?}", + ), + JobGraphError::MultipleJobsCreateSameOutput { + output_item, + existing_job, + new_job, + } => write!( + f, + "job can't be added to job graph because the new job has an output that is also produced by an existing job.\n\ + conflicting output:\n\ + {output_item:?}\n\ + existing job:\n\ + {existing_job:?}\n\ + new job:\n\ + {new_job:?}", + ), + } + } +} + +#[derive(Copy, Clone, Debug)] +enum EscapeForUnixShellState { + DollarSingleQuote, + SingleQuote, + Unquoted, +} + +#[derive(Clone)] +pub struct EscapeForUnixShell<'a> { + state: EscapeForUnixShellState, + prefix: [u8; 3], + bytes: &'a [u8], +} + +impl<'a> fmt::Debug for EscapeForUnixShell<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl<'a> fmt::Display for EscapeForUnixShell<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for c in self.clone() { + f.write_char(c)?; + } + Ok(()) + } +} + +impl<'a> EscapeForUnixShell<'a> { + pub fn new(s: &'a str) -> Self { + Self::from_bytes(s.as_bytes()) + } + fn make_prefix(bytes: &[u8]) -> [u8; 3] { + let mut prefix = [0; 3]; + prefix[..bytes.len()].copy_from_slice(bytes); + prefix + } + pub fn from_bytes(bytes: &'a [u8]) -> Self { + let mut needs_single_quote = bytes.is_empty(); + for &b in bytes { + match b { + b'!' | b'\'' | b'\"' | b' ' => needs_single_quote = true, + 0..0x20 | 0x7F.. => { + return Self { + state: EscapeForUnixShellState::DollarSingleQuote, + prefix: Self::make_prefix(b"$'"), + bytes, + }; + } + _ => {} + } + } + if needs_single_quote { + Self { + state: EscapeForUnixShellState::SingleQuote, + prefix: Self::make_prefix(b"'"), + bytes, + } + } else { + Self { + state: EscapeForUnixShellState::Unquoted, + prefix: Self::make_prefix(b""), + bytes, + } + } + } +} + +impl Iterator for EscapeForUnixShell<'_> { + type Item = char; + + fn next(&mut self) -> Option { + match &mut self.prefix { + [0, 0, 0] => {} + [0, 0, v] | // find first + [0, v, _] | // non-zero byte + [v, _, _] => { + let retval = *v as char; + *v = 0; + return Some(retval); + } + } + let Some(&next_byte) = self.bytes.split_off_first() else { + return match self.state { + EscapeForUnixShellState::DollarSingleQuote + | EscapeForUnixShellState::SingleQuote => { + self.state = EscapeForUnixShellState::Unquoted; + Some('\'') + } + EscapeForUnixShellState::Unquoted => None, + }; + }; + match self.state { + EscapeForUnixShellState::DollarSingleQuote => match next_byte { + b'\'' | b'\\' => { + self.prefix = Self::make_prefix(&[next_byte]); + Some('\\') + } + b'\t' => { + self.prefix = Self::make_prefix(b"t"); + Some('\\') + } + b'\n' => { + self.prefix = Self::make_prefix(b"n"); + Some('\\') + } + b'\r' => { + self.prefix = Self::make_prefix(b"r"); + Some('\\') + } + 0x20..=0x7E => Some(next_byte as char), + _ => { + self.prefix = [ + b'x', + char::from_digit(next_byte as u32 >> 4, 0x10).expect("known to be in range") + as u8, + char::from_digit(next_byte as u32 & 0xF, 0x10) + .expect("known to be in range") as u8, + ]; + Some('\\') + } + }, + EscapeForUnixShellState::SingleQuote => { + if next_byte == b'\'' { + self.prefix = Self::make_prefix(b"\\''"); + Some('\'') + } else { + Some(next_byte as char) + } + } + EscapeForUnixShellState::Unquoted => match next_byte { + b' ' | b'!' | b'"' | b'#' | b'$' | b'&' | b'\'' | b'(' | b')' | b'*' | b',' + | b';' | b'<' | b'>' | b'?' | b'[' | b'\\' | b']' | b'^' | b'`' | b'{' | b'|' + | b'}' | b'~' => { + self.prefix = Self::make_prefix(&[next_byte]); + Some('\\') + } + _ => Some(next_byte as char), + }, + } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +#[non_exhaustive] +pub enum UnixMakefileEscapeKind { + NonRecipe, + RecipeWithoutShellEscaping, + RecipeWithShellEscaping, +} + +#[derive(Copy, Clone)] +pub struct EscapeForUnixMakefile<'a> { + s: &'a str, + kind: UnixMakefileEscapeKind, +} + +impl<'a> fmt::Debug for EscapeForUnixMakefile<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl<'a> fmt::Display for EscapeForUnixMakefile<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.do_write(f, fmt::Write::write_str, fmt::Write::write_char, |_, _| { + Ok(()) + }) + } +} + +impl<'a> EscapeForUnixMakefile<'a> { + fn do_write( + &self, + state: &mut S, + write_str: impl Fn(&mut S, &str) -> Result<(), E>, + write_char: impl Fn(&mut S, char) -> Result<(), E>, + add_variable: impl Fn(&mut S, &'static str) -> Result<(), E>, + ) -> Result<(), E> { + let escape_recipe_char = |c| match c { + '$' => write_str(state, "$$"), + '\0'..='\x1F' | '\x7F' => { + panic!("can't escape a control character for Unix Makefile: {c:?}"); + } + _ => write_char(state, c), + }; + match self.kind { + UnixMakefileEscapeKind::NonRecipe => self.s.chars().try_for_each(|c| match c { + '=' => { + add_variable(state, "EQUALS = =")?; + write_str(state, "$(EQUALS)") + } + ';' => panic!("can't escape a semicolon (;) for Unix Makefile"), + '$' => write_str(state, "$$"), + '\\' | ' ' | '#' | ':' | '%' | '*' | '?' | '[' | ']' | '~' => { + write_char(state, '\\')?; + write_char(state, c) + } + '\0'..='\x1F' | '\x7F' => { + panic!("can't escape a control character for Unix Makefile: {c:?}"); + } + _ => write_char(state, c), + }), + UnixMakefileEscapeKind::RecipeWithoutShellEscaping => { + self.s.chars().try_for_each(escape_recipe_char) + } + UnixMakefileEscapeKind::RecipeWithShellEscaping => { + EscapeForUnixShell::new(self.s).try_for_each(escape_recipe_char) + } + } + } + pub fn new( + s: &'a str, + kind: UnixMakefileEscapeKind, + needed_variables: &mut BTreeSet<&'static str>, + ) -> Self { + let retval = Self { s, kind }; + let Ok(()) = retval.do_write( + needed_variables, + |_, _| Ok(()), + |_, _| Ok(()), + |needed_variables, variable| -> Result<(), std::convert::Infallible> { + needed_variables.insert(variable); + Ok(()) + }, + ); + retval + } +} + +impl JobGraph { + pub fn new() -> Self { + Self::default() + } + fn try_add_item_node( + &mut self, + name: JobItemName, + new_source_job: Option, + new_nodes: &mut HashSet<::NodeId>, + ) -> Result<::NodeId, JobGraphError> { + use hashbrown::hash_map::Entry; + match self.items.entry(name) { + Entry::Occupied(item_entry) => { + let node_id = *item_entry.get(); + let JobGraphNode::Item { + name: _, + source_job, + } = &mut self.graph[node_id] + else { + unreachable!("known to be an item"); + }; + if let Some(new_source_job) = new_source_job { + if let Some(source_job) = source_job { + return Err(JobGraphError::MultipleJobsCreateSameOutput { + output_item: item_entry.key().clone(), + existing_job: source_job.clone(), + new_job: new_source_job, + }); + } else { + *source_job = Some(new_source_job); + } + } + Ok(node_id) + } + Entry::Vacant(item_entry) => { + let node_id = self.graph.add_node(JobGraphNode::Item { + name, + source_job: new_source_job, + }); + new_nodes.insert(node_id); + item_entry.insert(node_id); + Ok(node_id) + } + } + } + pub fn try_add_jobs>( + &mut self, + jobs: I, + ) -> Result<(), JobGraphError> { + use hashbrown::hash_map::Entry; + let jobs = jobs.into_iter(); + struct RemoveNewNodesOnError<'a> { + this: &'a mut JobGraph, + new_nodes: HashSet<::NodeId>, + } + impl Drop for RemoveNewNodesOnError<'_> { + fn drop(&mut self) { + for node in self.new_nodes.drain() { + self.this.graph.remove_node(node); + } + } + } + let mut remove_new_nodes_on_error = RemoveNewNodesOnError { + this: self, + new_nodes: HashSet::with_capacity_and_hasher(jobs.size_hint().0, Default::default()), + }; + let new_nodes = &mut remove_new_nodes_on_error.new_nodes; + let this = &mut *remove_new_nodes_on_error.this; + for job in jobs { + let Entry::Vacant(job_entry) = this.jobs.entry(job.clone()) else { + continue; + }; + let job_node_id = this + .graph + .add_node(JobGraphNode::Job(job_entry.key().clone())); + new_nodes.insert(job_node_id); + job_entry.insert(job_node_id); + for name in job.outputs() { + let item_node_id = this.try_add_item_node(name, Some(job.clone()), new_nodes)?; + this.graph.add_edge(job_node_id, item_node_id, ()); + } + for name in job.inputs() { + let item_node_id = this.try_add_item_node(name, None, new_nodes)?; + this.graph.add_edge(item_node_id, job_node_id, ()); + } + } + match toposort(&this.graph, Some(&mut this.space)) { + Ok(v) => { + this.topological_order = v; + // no need to remove any of the new nodes on drop since we didn't encounter any errors + remove_new_nodes_on_error.new_nodes.clear(); + Ok(()) + } + Err(_) => { + // there's at least one cycle, find one! + let cycle = kosaraju_scc(&this.graph) + .into_iter() + .find_map(|scc| { + if scc.len() <= 1 { + // can't be a cycle since our graph is bipartite -- + // jobs only connect to items, never jobs to jobs or items to items + None + } else { + Some(scc) + } + }) + .expect("we know there's a cycle"); + let cycle_set = HashSet::from_iter(cycle.iter().copied()); + let job = cycle + .into_iter() + .find_map(|node_id| { + if let JobGraphNode::Job(job) = &this.graph[node_id] { + Some(job.clone()) + } else { + None + } + }) + .expect("a job must be part of the cycle"); + let output = job + .outputs() + .into_iter() + .find(|output| cycle_set.contains(&this.items[output])) + .expect("an output must be part of the cycle"); + Err(JobGraphError::CycleError { job, output }) + } + } + } + #[track_caller] + pub fn add_jobs>(&mut self, jobs: I) { + match self.try_add_jobs(jobs) { + Ok(()) => {} + Err(e) => panic!("error: {e}"), + } + } + pub fn to_unix_makefile(&self) -> String { + self.to_unix_makefile_with_internal_program_prefix(&[program_name_for_internal_jobs()]) + } + pub fn to_unix_makefile_with_internal_program_prefix( + &self, + internal_program_prefix: &[Interned], + ) -> String { + let mut retval = String::new(); + let mut needed_variables = BTreeSet::new(); + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + let outputs = job.outputs(); + if outputs.is_empty() { + retval.push_str(":"); + } else { + for output in job.outputs() { + match output { + JobItemName::Path { path } => { + write_str!( + retval, + "{} ", + EscapeForUnixMakefile::new( + &path, + UnixMakefileEscapeKind::NonRecipe, + &mut needed_variables + ) + ); + } + } + } + if outputs.len() == 1 { + retval.push_str(":"); + } else { + retval.push_str("&:"); + } + } + for input in job.inputs() { + match input { + JobItemName::Path { path } => { + write_str!( + retval, + " {}", + EscapeForUnixMakefile::new( + &path, + UnixMakefileEscapeKind::NonRecipe, + &mut needed_variables + ) + ); + } + } + } + retval.push_str("\n\t"); + for (index, arg) in job + .command_line_with_internal_program_prefix(internal_program_prefix) + .into_iter() + .enumerate() + { + if index != 0 { + retval.push_str(" "); + } + write_str!( + retval, + "{}", + EscapeForUnixMakefile::new( + &arg, + UnixMakefileEscapeKind::RecipeWithShellEscaping, + &mut needed_variables + ) + ); + } + retval.push_str("\n\n"); + } + if !needed_variables.is_empty() { + retval.insert_str( + 0, + &String::from_iter(needed_variables.into_iter().map(|v| format!("{v}\n"))), + ); + } + retval + } + pub fn to_unix_shell_script(&self) -> String { + self.to_unix_shell_script_with_internal_program_prefix(&[program_name_for_internal_jobs()]) + } + pub fn to_unix_shell_script_with_internal_program_prefix( + &self, + internal_program_prefix: &[Interned], + ) -> String { + let mut retval = String::from( + "#!/bin/sh\n\ + set -ex\n", + ); + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + for (index, arg) in job + .command_line_with_internal_program_prefix(internal_program_prefix) + .into_iter() + .enumerate() + { + if index != 0 { + retval.push_str(" "); + } + write_str!(retval, "{}", EscapeForUnixShell::new(&arg)); + } + retval.push_str("\n"); + } + retval + } + pub fn run(&self, params: &JobParams) -> eyre::Result<()> { + // use scope to auto-join threads on errors + thread::scope(|scope| { + struct WaitingJobState { + job_node_id: ::NodeId, + job: DynJob, + inputs: BTreeMap>, + } + let mut ready_jobs = VecDeque::new(); + let mut item_name_to_waiting_jobs_map = HashMap::<_, Vec<_>>::default(); + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + let waiting_job = WaitingJobState { + job_node_id: node_id, + job: job.clone(), + inputs: job + .inputs() + .iter() + .map(|&name| (name, OnceCell::new())) + .collect(), + }; + if waiting_job.inputs.is_empty() { + ready_jobs.push_back(waiting_job); + } else { + let waiting_job = Rc::new(waiting_job); + for &input_item in waiting_job.inputs.keys() { + item_name_to_waiting_jobs_map + .entry(input_item) + .or_default() + .push(waiting_job.clone()); + } + } + } + struct RunningJob<'scope> { + job: DynJob, + thread: ScopedJoinHandle<'scope, eyre::Result>>, + } + let mut running_jobs = HashMap::default(); + let (finished_jobs_sender, finished_jobs_receiver) = mpsc::channel(); + loop { + while let Some(finished_job) = finished_jobs_receiver.try_recv().ok() { + let Some(RunningJob { job, thread }) = running_jobs.remove(&finished_job) + else { + unreachable!(); + }; + let output_items = thread.join().map_err(panic::resume_unwind)??; + assert!( + output_items.iter().map(JobItem::name).eq(job.outputs()), + "job's run() method returned the wrong output items:\n\ + output items:\n\ + {output_items:?}\n\ + expected outputs:\n\ + {:?}\n\ + job:\n\ + {job:?}", + job.outputs(), + ); + for output_item in output_items { + for waiting_job in item_name_to_waiting_jobs_map + .remove(&output_item.name()) + .unwrap_or_default() + { + let Ok(()) = + waiting_job.inputs[&output_item.name()].set(output_item.clone()) + else { + unreachable!(); + }; + if let Some(waiting_job) = Rc::into_inner(waiting_job) { + ready_jobs.push_back(waiting_job); + } + } + } + } + if let Some(WaitingJobState { + job_node_id, + job, + inputs, + }) = ready_jobs.pop_front() + { + struct RunningJobInThread<'a> { + job_node_id: ::NodeId, + job: DynJob, + inputs: Vec, + params: &'a JobParams, + acquired_job: AcquiredJob, + finished_jobs_sender: mpsc::Sender<::NodeId>, + } + impl RunningJobInThread<'_> { + fn run(mut self) -> eyre::Result> { + self.job + .run(&self.inputs, self.params, &mut self.acquired_job) + } + } + impl Drop for RunningJobInThread<'_> { + fn drop(&mut self) { + let _ = self.finished_jobs_sender.send(self.job_node_id); + } + } + let name = job.kind().name(); + let running_job_in_thread = RunningJobInThread { + job_node_id, + job: job.clone(), + inputs: Vec::from_iter( + inputs + .into_values() + .map(|input| input.into_inner().expect("was set earlier")), + ), + params, + acquired_job: AcquiredJob::acquire()?, + finished_jobs_sender: finished_jobs_sender.clone(), + }; + running_jobs.insert( + job_node_id, + RunningJob { + job, + thread: thread::Builder::new() + .name(format!("job:{name}")) + .spawn_scoped(scope, move || running_job_in_thread.run()) + .expect("failed to spawn thread for job"), + }, + ); + } + if running_jobs.is_empty() { + assert!(item_name_to_waiting_jobs_map.is_empty()); + assert!(ready_jobs.is_empty()); + return Ok(()); + } + } + }) + } +} + +impl Extend for JobGraph { + #[track_caller] + fn extend>(&mut self, iter: T) { + self.add_jobs(iter); + } +} + +impl FromIterator for JobGraph { + #[track_caller] + fn from_iter>(iter: T) -> Self { + let mut retval = Self::new(); + retval.add_jobs(iter); + retval + } +} + +impl Serialize for JobGraph { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut serializer = serializer.serialize_seq(Some(self.jobs.len()))?; + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + serializer.serialize_element(job)?; + } + serializer.end() + } +} + +impl<'de> Deserialize<'de> for JobGraph { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let jobs = Vec::::deserialize(deserializer)?; + let mut retval = JobGraph::new(); + retval.try_add_jobs(jobs).map_err(D::Error::custom)?; + Ok(retval) + } +} diff --git a/crates/fayalite/src/build/registry.rs b/crates/fayalite/src/build/registry.rs new file mode 100644 index 00000000..9ea5400f --- /dev/null +++ b/crates/fayalite/src/build/registry.rs @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// See Notices.txt for copyright information + +use crate::{ + build::{DynJobKind, JobKind}, + intern::Interned, +}; +use std::{ + borrow::Borrow, + cmp::Ordering, + collections::BTreeMap, + fmt, + sync::{Arc, OnceLock, RwLock, RwLockWriteGuard}, +}; + +impl DynJobKind { + pub fn registry() -> JobKindRegistrySnapshot { + JobKindRegistrySnapshot(JobKindRegistry::get()) + } + #[track_caller] + pub fn register(self) { + JobKindRegistry::register(JobKindRegistry::lock(), self); + } +} + +#[derive(Copy, Clone, PartialEq, Eq)] +struct InternedStrCompareAsStr(Interned); + +impl fmt::Debug for InternedStrCompareAsStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl Ord for InternedStrCompareAsStr { + fn cmp(&self, other: &Self) -> Ordering { + str::cmp(&self.0, &other.0) + } +} + +impl PartialOrd for InternedStrCompareAsStr { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Borrow for InternedStrCompareAsStr { + fn borrow(&self) -> &str { + &self.0 + } +} + +#[derive(Clone, Debug)] +struct JobKindRegistry { + job_kinds: BTreeMap, +} + +enum JobKindRegisterError { + SameName { + name: InternedStrCompareAsStr, + old_job_kind: DynJobKind, + new_job_kind: DynJobKind, + }, +} + +impl fmt::Display for JobKindRegisterError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SameName { + name, + old_job_kind, + new_job_kind, + } => write!( + f, + "two different `JobKind` can't share the same name:\n\ + {name:?}\n\ + old job kind:\n\ + {old_job_kind:?}\n\ + new job kind:\n\ + {new_job_kind:?}", + ), + } + } +} + +trait JobKindRegistryRegisterLock { + type Locked; + fn lock(self) -> Self::Locked; + fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry; +} + +impl JobKindRegistryRegisterLock for &'static RwLock> { + type Locked = RwLockWriteGuard<'static, Arc>; + fn lock(self) -> Self::Locked { + self.write().expect("shouldn't be poisoned") + } + fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry { + Arc::make_mut(locked) + } +} + +impl JobKindRegistryRegisterLock for &'_ mut JobKindRegistry { + type Locked = Self; + + fn lock(self) -> Self::Locked { + self + } + + fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry { + locked + } +} + +impl JobKindRegistry { + fn lock() -> &'static RwLock> { + static REGISTRY: OnceLock>> = OnceLock::new(); + REGISTRY.get_or_init(Default::default) + } + fn try_register( + lock: L, + job_kind: DynJobKind, + ) -> Result<(), JobKindRegisterError> { + use std::collections::btree_map::Entry; + let name = InternedStrCompareAsStr(job_kind.name()); + // run user code only outside of lock + let mut locked = lock.lock(); + let this = L::make_mut(&mut locked); + let result = match this.job_kinds.entry(name) { + Entry::Occupied(entry) => Err(JobKindRegisterError::SameName { + name, + old_job_kind: entry.get().clone(), + new_job_kind: job_kind, + }), + Entry::Vacant(entry) => { + entry.insert(job_kind); + Ok(()) + } + }; + drop(locked); + // outside of lock now, so we can test if it's the same DynJobKind + match result { + Err(JobKindRegisterError::SameName { + name: _, + old_job_kind, + new_job_kind, + }) if old_job_kind == new_job_kind => Ok(()), + result => result, + } + } + #[track_caller] + fn register(lock: L, job_kind: DynJobKind) { + match Self::try_register(lock, job_kind) { + Err(e) => panic!("{e}"), + Ok(()) => {} + } + } + fn get() -> Arc { + Self::lock().read().expect("shouldn't be poisoned").clone() + } +} + +impl Default for JobKindRegistry { + fn default() -> Self { + let mut retval = Self { + job_kinds: BTreeMap::new(), + }; + for job_kind in [] { + Self::register(&mut retval, job_kind); + } + retval + } +} + +#[derive(Clone, Debug)] +pub struct JobKindRegistrySnapshot(Arc); + +impl JobKindRegistrySnapshot { + pub fn get() -> Self { + JobKindRegistrySnapshot(JobKindRegistry::get()) + } + pub fn get_by_name<'a>(&'a self, name: &str) -> Option<&'a DynJobKind> { + self.0.job_kinds.get(name) + } + pub fn iter_with_names(&self) -> JobKindRegistryIterWithNames<'_> { + JobKindRegistryIterWithNames(self.0.job_kinds.iter()) + } + pub fn iter(&self) -> JobKindRegistryIter<'_> { + JobKindRegistryIter(self.0.job_kinds.values()) + } +} + +impl<'a> IntoIterator for &'a JobKindRegistrySnapshot { + type Item = &'a DynJobKind; + type IntoIter = JobKindRegistryIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a> IntoIterator for &'a mut JobKindRegistrySnapshot { + type Item = &'a DynJobKind; + type IntoIter = JobKindRegistryIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +#[derive(Clone, Debug)] +pub struct JobKindRegistryIter<'a>( + std::collections::btree_map::Values<'a, InternedStrCompareAsStr, DynJobKind>, +); + +impl<'a> Iterator for JobKindRegistryIter<'a> { + type Item = &'a DynJobKind; + + fn next(&mut self) -> Option { + self.0.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } + + fn count(self) -> usize + where + Self: Sized, + { + self.0.count() + } + + fn last(self) -> Option { + self.0.last() + } + + fn nth(&mut self, n: usize) -> Option { + self.0.nth(n) + } + + fn fold(self, init: B, f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + self.0.fold(init, f) + } +} + +impl<'a> std::iter::FusedIterator for JobKindRegistryIter<'a> {} + +impl<'a> ExactSizeIterator for JobKindRegistryIter<'a> {} + +impl<'a> DoubleEndedIterator for JobKindRegistryIter<'a> { + fn next_back(&mut self) -> Option { + self.0.next_back() + } + + fn nth_back(&mut self, n: usize) -> Option { + self.0.nth_back(n) + } + + fn rfold(self, init: B, f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + self.0.rfold(init, f) + } +} + +#[derive(Clone, Debug)] +pub struct JobKindRegistryIterWithNames<'a>( + std::collections::btree_map::Iter<'a, InternedStrCompareAsStr, DynJobKind>, +); + +impl<'a> Iterator for JobKindRegistryIterWithNames<'a> { + type Item = (Interned, &'a DynJobKind); + + fn next(&mut self) -> Option { + self.0.next().map(|(name, job_kind)| (name.0, job_kind)) + } + + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } + + fn count(self) -> usize + where + Self: Sized, + { + self.0.count() + } + + fn last(self) -> Option { + self.0.last().map(|(name, job_kind)| (name.0, job_kind)) + } + + fn nth(&mut self, n: usize) -> Option { + self.0.nth(n).map(|(name, job_kind)| (name.0, job_kind)) + } + + fn fold(self, init: B, f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + self.0 + .map(|(name, job_kind)| (name.0, job_kind)) + .fold(init, f) + } +} + +impl<'a> std::iter::FusedIterator for JobKindRegistryIterWithNames<'a> {} + +impl<'a> ExactSizeIterator for JobKindRegistryIterWithNames<'a> {} + +impl<'a> DoubleEndedIterator for JobKindRegistryIterWithNames<'a> { + fn next_back(&mut self) -> Option { + self.0 + .next_back() + .map(|(name, job_kind)| (name.0, job_kind)) + } + + fn nth_back(&mut self, n: usize) -> Option { + self.0 + .nth_back(n) + .map(|(name, job_kind)| (name.0, job_kind)) + } + + fn rfold(self, init: B, f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + self.0 + .map(|(name, job_kind)| (name.0, job_kind)) + .rfold(init, f) + } +} + +#[track_caller] +pub fn register_job_kind(kind: K) { + DynJobKind::new(kind).register(); +} diff --git a/crates/fayalite/src/cli.rs b/crates/fayalite/src/cli.rs index 85095da7..f8394f7e 100644 --- a/crates/fayalite/src/cli.rs +++ b/crates/fayalite/src/cli.rs @@ -50,7 +50,7 @@ impl From for CliError { pub trait RunPhase { type Output; fn run(&self, arg: Arg) -> Result { - self.run_with_job(arg, &mut AcquiredJob::acquire()) + self.run_with_job(arg, &mut AcquiredJob::acquire()?) } fn run_with_job(&self, arg: Arg, acquired_job: &mut AcquiredJob) -> Result; } diff --git a/crates/fayalite/src/firrtl.rs b/crates/fayalite/src/firrtl.rs index 5fa66446..07910f4a 100644 --- a/crates/fayalite/src/firrtl.rs +++ b/crates/fayalite/src/firrtl.rs @@ -7,6 +7,7 @@ use crate::{ DocStringAnnotation, DontTouchAnnotation, SVAttributeAnnotation, }, array::Array, + build::JobArgs, bundle::{Bundle, BundleField, BundleType}, clock::Clock, enum_::{Enum, EnumType, EnumVariant}, @@ -42,7 +43,7 @@ use crate::{ use bitvec::slice::BitSlice; use clap::value_parser; use num_traits::Signed; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::{ cell::{Cell, RefCell}, cmp::Ordering, @@ -2749,14 +2750,23 @@ impl clap::builder::TypedValueParser for OptionSimplifyEnumsKindValueParser { #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct ExportOptionsPrivate(()); -#[derive(clap::Parser, Copy, Clone, PartialEq, Eq, Hash)] +impl ExportOptionsPrivate { + fn private_new() -> Self { + Self(()) + } +} + +#[derive(clap::Parser, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct ExportOptions { #[clap(long = "no-simplify-memories", action = clap::ArgAction::SetFalse)] + #[serde(default = "ExportOptions::default_simplify_memories")] pub simplify_memories: bool, #[clap(long, value_parser = OptionSimplifyEnumsKindValueParser, default_value = "replace-with-bundle-of-uints")] + #[serde(default = "ExportOptions::default_simplify_enums")] pub simplify_enums: std::option::Option, // use std::option::Option instead of Option to avoid clap mis-parsing #[doc(hidden)] #[clap(skip = ExportOptionsPrivate(()))] + #[serde(skip, default = "ExportOptionsPrivate::private_new")] /// `#[non_exhaustive]` except allowing struct update syntax pub __private: ExportOptionsPrivate, } @@ -2767,16 +2777,15 @@ impl fmt::Debug for ExportOptions { } } -impl ExportOptions { - pub fn to_args(&self) -> Vec> { +impl JobArgs for ExportOptions { + fn to_args> + ?Sized>(&self, args: &mut Args) { let Self { simplify_memories, simplify_enums, __private: ExportOptionsPrivate(()), - } = self; - let mut retval = Vec::new(); - if !*simplify_memories { - retval.push("--no-simplify-memories".intern()); + } = *self; + if !simplify_memories { + args.extend(["--no-simplify-memories".intern()]); } let simplify_enums = simplify_enums.map(|v| { clap::ValueEnum::to_possible_value(&v).expect("there are no skipped variants") @@ -2785,10 +2794,18 @@ impl ExportOptions { None => OptionSimplifyEnumsKindValueParser::NONE_NAME, Some(v) => v.get_name(), }; - retval.push(str::intern_owned(format!( + args.extend([str::intern_owned(format!( "--simplify-enums={simplify_enums}" - ))); - retval + ))]); + } +} + +impl ExportOptions { + fn default_simplify_memories() -> bool { + true + } + fn default_simplify_enums() -> Option { + Some(SimplifyEnumsKind::ReplaceWithBundleOfUInts) } fn debug_fmt( &self, @@ -2846,8 +2863,8 @@ impl ExportOptions { impl Default for ExportOptions { fn default() -> Self { Self { - simplify_memories: true, - simplify_enums: Some(SimplifyEnumsKind::ReplaceWithBundleOfUInts), + simplify_memories: Self::default_simplify_memories(), + simplify_enums: Self::default_simplify_enums(), __private: ExportOptionsPrivate(()), } } diff --git a/crates/fayalite/src/intern.rs b/crates/fayalite/src/intern.rs index af91f0a8..584f33d8 100644 --- a/crates/fayalite/src/intern.rs +++ b/crates/fayalite/src/intern.rs @@ -485,6 +485,48 @@ where } } +impl FromIterator for Interned +where + String: FromIterator, +{ + fn from_iter>(iter: T) -> Self { + str::intern_owned(FromIterator::from_iter(iter)) + } +} + +impl AsRef for Interned +where + str: AsRef, +{ + fn as_ref(&self) -> &T { + str::as_ref(self) + } +} + +impl From> for clap::builder::Str { + fn from(value: Interned) -> Self { + Interned::into_inner(value).into() + } +} + +impl From> for clap::builder::OsStr { + fn from(value: Interned) -> Self { + Interned::into_inner(value).into() + } +} + +impl From> for clap::builder::StyledStr { + fn from(value: Interned) -> Self { + Interned::into_inner(value).into() + } +} + +impl From> for clap::Id { + fn from(value: Interned) -> Self { + Interned::into_inner(value).into() + } +} + impl From> for Vec { fn from(value: Interned<[T]>) -> Self { Vec::from(&*value) diff --git a/crates/fayalite/src/module/transform/simplify_enums.rs b/crates/fayalite/src/module/transform/simplify_enums.rs index ccdecf6a..dd21e493 100644 --- a/crates/fayalite/src/module/transform/simplify_enums.rs +++ b/crates/fayalite/src/module/transform/simplify_enums.rs @@ -1,3 +1,5 @@ +use serde::{Deserialize, Serialize}; + // SPDX-License-Identifier: LGPL-3.0-or-later // See Notices.txt for copyright information use crate::{ @@ -955,12 +957,15 @@ impl Folder for State { } } -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, clap::ValueEnum)] +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, clap::ValueEnum, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] pub enum SimplifyEnumsKind { SimplifyToEnumsWithNoBody, #[clap(name = "replace-with-bundle-of-uints")] + #[serde(rename = "replace-with-bundle-of-uints")] ReplaceWithBundleOfUInts, #[clap(name = "replace-with-uint")] + #[serde(rename = "replace-with-uint")] ReplaceWithUInt, } diff --git a/crates/fayalite/src/util.rs b/crates/fayalite/src/util.rs index e85bc9c1..23a68521 100644 --- a/crates/fayalite/src/util.rs +++ b/crates/fayalite/src/util.rs @@ -36,8 +36,11 @@ pub use scoped_ref::ScopedRef; pub(crate) use misc::chain; #[doc(inline)] pub use misc::{ - BitSliceWriteWithBase, DebugAsDisplay, DebugAsRawString, MakeMutSlice, RcWriter, interned_bit, - iter_eq_by, slice_range, try_slice_range, + BitSliceWriteWithBase, DebugAsDisplay, DebugAsRawString, MakeMutSlice, RcWriter, + SerdeJsonEscapeIf, SerdeJsonEscapeIfFormatter, SerdeJsonEscapeIfTest, + SerdeJsonEscapeIfTestResult, interned_bit, iter_eq_by, serialize_to_json_ascii, + serialize_to_json_ascii_pretty, serialize_to_json_ascii_pretty_with_indent, slice_range, + try_slice_range, }; pub mod job_server; diff --git a/crates/fayalite/src/util/job_server.rs b/crates/fayalite/src/util/job_server.rs index 376ddc0b..e58bba8b 100644 --- a/crates/fayalite/src/util/job_server.rs +++ b/crates/fayalite/src/util/job_server.rs @@ -1,192 +1,156 @@ // SPDX-License-Identifier: LGPL-3.0-or-later // See Notices.txt for copyright information -use ctor::ctor; -use jobslot::{Acquired, Client}; +use ctor::{ctor, dtor}; +use jobslot::Client; use std::{ ffi::OsString, - mem, + io, mem, num::NonZeroUsize, - sync::{Condvar, Mutex, Once, OnceLock}, - thread::spawn, + sync::{Mutex, MutexGuard}, }; -fn get_or_make_client() -> &'static Client { - #[ctor] - static CLIENT: OnceLock = unsafe { - match Client::from_env() { - Some(client) => OnceLock::from(client), - None => OnceLock::new(), - } - }; +#[ctor] +static CLIENT: Mutex>> = unsafe { Mutex::new(Some(Client::from_env())) }; - CLIENT.get_or_init(|| { - let mut available_parallelism = None; - let mut args = std::env::args_os().skip(1); - while let Some(arg) = args.next() { - const TEST_THREADS_OPTION: &'static [u8] = b"--test-threads"; - if arg.as_encoded_bytes().starts_with(TEST_THREADS_OPTION) { - match arg.as_encoded_bytes().get(TEST_THREADS_OPTION.len()) { - Some(b'=') => { - let mut arg = arg.into_encoded_bytes(); - arg.drain(..=TEST_THREADS_OPTION.len()); - available_parallelism = Some(arg); - break; +#[dtor] +fn drop_client() { + drop( + match CLIENT.lock() { + Ok(v) => v, + Err(e) => e.into_inner(), + } + .take(), + ); +} + +fn get_or_make_client() -> Client { + CLIENT + .lock() + .expect("shouldn't have panicked") + .as_mut() + .expect("shutting down") + .get_or_insert_with(|| { + let mut available_parallelism = None; + let mut args = std::env::args_os().skip(1); + while let Some(arg) = args.next() { + const TEST_THREADS_OPTION: &'static [u8] = b"--test-threads"; + if arg.as_encoded_bytes().starts_with(TEST_THREADS_OPTION) { + match arg.as_encoded_bytes().get(TEST_THREADS_OPTION.len()) { + Some(b'=') => { + let mut arg = arg.into_encoded_bytes(); + arg.drain(..=TEST_THREADS_OPTION.len()); + available_parallelism = Some(arg); + break; + } + None => { + available_parallelism = args.next().map(OsString::into_encoded_bytes); + break; + } + _ => {} } - None => { - available_parallelism = args.next().map(OsString::into_encoded_bytes); - break; - } - _ => {} } } - } - let available_parallelism = if let Some(available_parallelism) = available_parallelism - .as_deref() - .and_then(|v| std::str::from_utf8(v).ok()) - .and_then(|v| v.parse().ok()) - { - available_parallelism - } else if let Ok(available_parallelism) = std::thread::available_parallelism() { - available_parallelism - } else { - NonZeroUsize::new(1).unwrap() - }; - Client::new_with_fifo(available_parallelism.get() - 1).expect("failed to create job server") - }) + let available_parallelism = if let Some(available_parallelism) = available_parallelism + .as_deref() + .and_then(|v| std::str::from_utf8(v).ok()) + .and_then(|v| v.parse().ok()) + { + available_parallelism + } else if let Ok(available_parallelism) = std::thread::available_parallelism() { + available_parallelism + } else { + NonZeroUsize::new(1).unwrap() + }; + Client::new_with_fifo(available_parallelism.get() - 1) + .expect("failed to create job server") + }) + .clone() } struct State { + obtained_count: usize, waiting_count: usize, - available: Vec, - implicit_available: bool, -} - -impl State { - fn total_available(&self) -> usize { - self.available.len() + self.implicit_available as usize - } - fn additional_waiting(&self) -> usize { - self.waiting_count.saturating_sub(self.total_available()) - } } static STATE: Mutex = Mutex::new(State { + obtained_count: 0, waiting_count: 0, - available: Vec::new(), - implicit_available: true, }); -static COND_VAR: Condvar = Condvar::new(); - -#[derive(Debug)] -enum AcquiredJobInner { - FromJobServer(Acquired), - ImplicitJob, -} #[derive(Debug)] pub struct AcquiredJob { - job: AcquiredJobInner, + client: Client, } impl AcquiredJob { - fn start_acquire_thread() { - static STARTED_THREAD: Once = Once::new(); - STARTED_THREAD.call_once(|| { - spawn(|| { - let mut acquired = None; - let client = get_or_make_client(); + pub fn acquire() -> io::Result { + let client = get_or_make_client(); + struct Waiting {} + + impl Waiting { + fn done(self) -> MutexGuard<'static, State> { + mem::forget(self); let mut state = STATE.lock().unwrap(); - loop { - state = if state.additional_waiting() == 0 { - if acquired.is_some() { - drop(state); - drop(acquired.take()); // drop Acquired outside of lock - STATE.lock().unwrap() - } else { - COND_VAR.wait(state).unwrap() - } - } else if acquired.is_some() { - // allocate space before moving Acquired to ensure we - // drop Acquired outside of the lock on panic - state.available.reserve(1); - state.available.push(acquired.take().unwrap()); - COND_VAR.notify_all(); - state - } else { - drop(state); - acquired = Some( - client - .acquire() - .expect("can't acquire token from job server"), - ); - STATE.lock().unwrap() - }; - } - }); - }); - } - fn acquire_inner(block: bool) -> Option { - Self::start_acquire_thread(); - let mut state = STATE.lock().unwrap(); - loop { - if let Some(acquired) = state.available.pop() { - return Some(Self { - job: AcquiredJobInner::FromJobServer(acquired), - }); + state.waiting_count -= 1; + state } - if state.implicit_available { - state.implicit_available = false; - return Some(Self { - job: AcquiredJobInner::ImplicitJob, - }); - } - if !block { - return None; - } - state.waiting_count += 1; - state = COND_VAR.wait(state).unwrap(); - state.waiting_count -= 1; } - } - pub fn try_acquire() -> Option { - Self::acquire_inner(false) - } - pub fn acquire() -> Self { - Self::acquire_inner(true).expect("failed to acquire token") + impl Drop for Waiting { + fn drop(&mut self) { + STATE.lock().unwrap().waiting_count -= 1; + } + } + let mut state = STATE.lock().unwrap(); + if state.obtained_count == 0 && state.waiting_count == 0 { + state.obtained_count = 1; // get implicit token + return Ok(Self { client }); + } + state.waiting_count += 1; + drop(state); + let waiting = Waiting {}; + client.acquire_raw()?; + state = waiting.done(); + state.obtained_count = state + .obtained_count + .checked_add(1) + .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "obtained_count overflowed"))?; + drop(state); + Ok(Self { client }) } pub fn run_command( &mut self, cmd: std::process::Command, f: impl FnOnce(&mut std::process::Command) -> std::io::Result, ) -> std::io::Result { - get_or_make_client().configure_make_and_run_with_fifo(cmd, f) + self.client.configure_make_and_run_with_fifo(cmd, f) } } impl Drop for AcquiredJob { fn drop(&mut self) { let mut state = STATE.lock().unwrap(); - match &self.job { - AcquiredJobInner::FromJobServer(_) => { - if state.waiting_count > state.available.len() + state.implicit_available as usize { - // allocate space before moving Acquired to ensure we - // drop Acquired outside of the lock on panic - state.available.reserve(1); - let AcquiredJobInner::FromJobServer(acquired) = - mem::replace(&mut self.job, AcquiredJobInner::ImplicitJob) - else { - unreachable!() - }; - state.available.push(acquired); - COND_VAR.notify_all(); + match &mut *state { + State { + obtained_count: 0, .. + } => unreachable!(), + State { + obtained_count: obtained_count @ 1, + waiting_count, + } => { + *obtained_count = 0; // drop implicit token + let any_waiting = *waiting_count != 0; + drop(state); + if any_waiting { + // we have the implicit token, but some other thread is trying to acquire a token, + // release the implicit token so they can acquire it. + let _ = self.client.release_raw(); // we're in drop, just ignore errors since we at least tried } } - AcquiredJobInner::ImplicitJob => { - state.implicit_available = true; - if state.waiting_count > state.available.len() { - COND_VAR.notify_all(); - } + State { obtained_count, .. } => { + *obtained_count = obtained_count.saturating_sub(1); + drop(state); + let _ = self.client.release_raw(); // we're in drop, just ignore errors since we at least tried } } } diff --git a/crates/fayalite/src/util/misc.rs b/crates/fayalite/src/util/misc.rs index cebbceb5..6c9aceef 100644 --- a/crates/fayalite/src/util/misc.rs +++ b/crates/fayalite/src/util/misc.rs @@ -5,6 +5,7 @@ use bitvec::{bits, order::Lsb0, slice::BitSlice, view::BitView}; use std::{ cell::Cell, fmt::{self, Debug, Write}, + io, ops::{Bound, Range, RangeBounds}, rc::Rc, sync::{Arc, OnceLock}, @@ -243,3 +244,323 @@ pub fn try_slice_range>(range: R, size: usize) -> Option>(range: R, size: usize) -> Range { try_slice_range(range, size).expect("range out of bounds") } + +pub trait SerdeJsonEscapeIfTest { + fn char_needs_escape(&mut self, ch: char) -> serde_json::Result; +} + +pub trait SerdeJsonEscapeIfTestResult { + fn to_result(self) -> serde_json::Result; +} + +impl SerdeJsonEscapeIfTestResult for bool { + fn to_result(self) -> serde_json::Result { + Ok(self) + } +} + +impl> SerdeJsonEscapeIfTestResult for Result { + fn to_result(self) -> serde_json::Result { + self.map_err(Into::into) + } +} + +impl R, R: SerdeJsonEscapeIfTestResult> SerdeJsonEscapeIfTest for T { + fn char_needs_escape(&mut self, ch: char) -> serde_json::Result { + self(ch).to_result() + } +} + +pub trait SerdeJsonEscapeIfFormatter: serde_json::ser::Formatter { + fn write_unicode_escape(&mut self, writer: &mut W, ch: char) -> io::Result<()> + where + W: ?Sized + io::Write, + { + for utf16 in ch.encode_utf16(&mut [0; 2]) { + write!(writer, "\\u{utf16:04x}")?; + } + Ok(()) + } +} + +impl SerdeJsonEscapeIfFormatter for serde_json::ser::CompactFormatter {} +impl SerdeJsonEscapeIfFormatter for serde_json::ser::PrettyFormatter<'_> {} + +pub struct SerdeJsonEscapeIf { + pub base: Base, + pub test: Test, +} + +impl serde_json::ser::Formatter + for SerdeJsonEscapeIf +{ + fn write_null(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_null(writer) + } + + fn write_bool(&mut self, writer: &mut W, value: bool) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_bool(writer, value) + } + + fn write_i8(&mut self, writer: &mut W, value: i8) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_i8(writer, value) + } + + fn write_i16(&mut self, writer: &mut W, value: i16) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_i16(writer, value) + } + + fn write_i32(&mut self, writer: &mut W, value: i32) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_i32(writer, value) + } + + fn write_i64(&mut self, writer: &mut W, value: i64) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_i64(writer, value) + } + + fn write_i128(&mut self, writer: &mut W, value: i128) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_i128(writer, value) + } + + fn write_u8(&mut self, writer: &mut W, value: u8) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_u8(writer, value) + } + + fn write_u16(&mut self, writer: &mut W, value: u16) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_u16(writer, value) + } + + fn write_u32(&mut self, writer: &mut W, value: u32) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_u32(writer, value) + } + + fn write_u64(&mut self, writer: &mut W, value: u64) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_u64(writer, value) + } + + fn write_u128(&mut self, writer: &mut W, value: u128) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_u128(writer, value) + } + + fn write_f32(&mut self, writer: &mut W, value: f32) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_f32(writer, value) + } + + fn write_f64(&mut self, writer: &mut W, value: f64) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_f64(writer, value) + } + + fn write_number_str(&mut self, writer: &mut W, value: &str) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_number_str(writer, value) + } + + fn begin_string(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_string(writer) + } + + fn end_string(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_string(writer) + } + + fn write_string_fragment(&mut self, writer: &mut W, mut fragment: &str) -> io::Result<()> + where + W: ?Sized + io::Write, + { + while let Some((next_escape_index, next_escape_char)) = fragment + .char_indices() + .find_map(|(index, ch)| match self.test.char_needs_escape(ch) { + Ok(false) => None, + Ok(true) => Some(Ok((index, ch))), + Err(e) => Some(Err(e)), + }) + .transpose()? + { + let (no_escapes, rest) = fragment.split_at(next_escape_index); + fragment = &rest[next_escape_char.len_utf8()..]; + self.base.write_string_fragment(writer, no_escapes)?; + self.base.write_unicode_escape(writer, next_escape_char)?; + } + self.base.write_string_fragment(writer, fragment) + } + + fn write_char_escape( + &mut self, + writer: &mut W, + char_escape: serde_json::ser::CharEscape, + ) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_char_escape(writer, char_escape) + } + + fn write_byte_array(&mut self, writer: &mut W, value: &[u8]) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_byte_array(writer, value) + } + + fn begin_array(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_array(writer) + } + + fn end_array(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_array(writer) + } + + fn begin_array_value(&mut self, writer: &mut W, first: bool) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_array_value(writer, first) + } + + fn end_array_value(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_array_value(writer) + } + + fn begin_object(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_object(writer) + } + + fn end_object(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_object(writer) + } + + fn begin_object_key(&mut self, writer: &mut W, first: bool) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_object_key(writer, first) + } + + fn end_object_key(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_object_key(writer) + } + + fn begin_object_value(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.begin_object_value(writer) + } + + fn end_object_value(&mut self, writer: &mut W) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.end_object_value(writer) + } + + fn write_raw_fragment(&mut self, writer: &mut W, fragment: &str) -> io::Result<()> + where + W: ?Sized + io::Write, + { + self.base.write_raw_fragment(writer, fragment) + } +} + +fn serialize_to_json_ascii_helper( + v: &S, + base: F, +) -> serde_json::Result { + let mut retval = Vec::new(); + v.serialize(&mut serde_json::ser::Serializer::with_formatter( + &mut retval, + SerdeJsonEscapeIf { + base, + test: |ch| ch < '\x20' || ch > '\x7F', + }, + ))?; + String::from_utf8(retval).map_err(|_| serde::ser::Error::custom("invalid UTF-8")) +} + +pub fn serialize_to_json_ascii(v: &T) -> serde_json::Result { + serialize_to_json_ascii_helper(v, serde_json::ser::CompactFormatter) +} + +pub fn serialize_to_json_ascii_pretty( + v: &T, +) -> serde_json::Result { + serialize_to_json_ascii_helper(v, serde_json::ser::PrettyFormatter::new()) +} + +pub fn serialize_to_json_ascii_pretty_with_indent( + v: &T, + indent: &str, +) -> serde_json::Result { + serialize_to_json_ascii_helper( + v, + serde_json::ser::PrettyFormatter::with_indent(indent.as_bytes()), + ) +}