From cec0eb410ede4a86fcaa3139e97a730ed1319784 Mon Sep 17 00:00:00 2001 From: Jacob Lifshay Date: Tue, 16 Sep 2025 01:40:42 -0700 Subject: [PATCH] WIP adding FPGA support -- build module should be complete --- crates/fayalite/src/build.rs | 1622 +++++++++++++++++++++++++ crates/fayalite/src/build/external.rs | 426 +++++++ crates/fayalite/src/lib.rs | 2 + crates/fayalite/src/target.rs | 202 +++ 4 files changed, 2252 insertions(+) create mode 100644 crates/fayalite/src/build.rs create mode 100644 crates/fayalite/src/build/external.rs create mode 100644 crates/fayalite/src/target.rs diff --git a/crates/fayalite/src/build.rs b/crates/fayalite/src/build.rs new file mode 100644 index 0000000..7729a8f --- /dev/null +++ b/crates/fayalite/src/build.rs @@ -0,0 +1,1622 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// See Notices.txt for copyright information + +use crate::{ + bundle::Bundle, + intern::{Intern, Interned}, + module::Module, + util::{HashMap, HashSet, job_server::AcquiredJob}, +}; +use clap::{FromArgMatches, Subcommand}; +use hashbrown::hash_map::Entry; +use petgraph::{ + algo::{DfsSpace, kosaraju_scc, toposort}, + graph::DiGraph, + visit::{GraphBase, Visitable}, +}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error, ser::SerializeSeq}; +use std::{ + any::{Any, TypeId}, + borrow::Cow, + cell::OnceCell, + collections::{BTreeMap, BTreeSet, VecDeque}, + fmt::{self, Write}, + hash::{Hash, Hasher}, + iter, + marker::PhantomData, + mem, panic, + rc::Rc, + sync::{Arc, OnceLock, RwLock, RwLockWriteGuard, mpsc}, + thread::{self, ScopedJoinHandle}, +}; + +mod external; + +pub use external::{ + TemplateParseError, TemplatedExternalJob, TemplatedExternalJobKind, find_program, +}; + +macro_rules! write_str { + ($s:expr, $($rest:tt)*) => { + String::write_fmt(&mut $s, format_args!($($rest)*)).expect("String::write_fmt can't fail") + }; +} + +#[derive(Clone, Hash, PartialEq, Eq, Debug)] +pub enum JobItem { + Module { value: Module }, + File { path: Interned }, +} + +impl JobItem { + pub fn name(&self) -> JobItemName { + match self { + JobItem::Module { value } => JobItemName::Module { name: value.name() }, + &JobItem::File { path } => JobItemName::File { path }, + } + } +} + +#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] +pub enum JobItemName { + Module { name: Interned }, + File { path: Interned }, +} + +pub struct CommandLine {} + +pub trait JobKind: 'static + Send + Sync + Hash + Eq + fmt::Debug { + type Job: 'static + Send + Sync + Hash + Eq + fmt::Debug; + fn inputs(&self, job: &Self::Job) -> Interned<[JobItemName]>; + fn outputs(&self, job: &Self::Job) -> Interned<[JobItemName]>; + /// gets the part of the command line that is common for all members of this job kind -- usually the executable name/path and any global options and/or subcommands + fn command_line_prefix(&self) -> Interned<[Interned]>; + fn to_command_line(&self, job: &Self::Job) -> Interned<[Interned]>; + /// return the subcommand if this is an internal JobKind + fn subcommand(&self) -> Option; + /// Parse from [`ArgMatches`], this should only be called with the results of parsing [`subcommand()`]. + /// If [`subcommand()`] returned [`None`], you should not call this function since it will panic. + /// + /// [`ArgMatches`]: clap::ArgMatches + /// [`subcommand()`]: JobKind::subcommand + fn from_arg_matches(&self, matches: &mut clap::ArgMatches) -> clap::error::Result; + fn debug_name(&self, job: &Self::Job) -> String { + let name = self + .command_line_prefix() + .last() + .copied() + .or_else(|| self.to_command_line(job).first().copied()) + .unwrap_or_default(); + let name = match name.rsplit_once(['/', '\\']) { + Some((_, name)) if name.trim() != "" => name, + _ => &*name, + }; + format!("job:{name}") + } + fn parse_command_line( + &self, + command_line: Interned<[Interned]>, + ) -> clap::error::Result; + fn run( + &self, + job: &Self::Job, + inputs: &[JobItem], + acquired_job: &mut AcquiredJob, + ) -> eyre::Result>; +} + +trait DynJobKindTrait: 'static + Send + Sync + fmt::Debug { + fn as_any(&self) -> &dyn Any; + fn as_arc_any(self: Arc) -> Arc; + fn eq_dyn(&self, other: &dyn DynJobKindTrait) -> bool; + fn hash_dyn(&self, state: &mut dyn Hasher); + fn command_line_prefix_dyn(&self) -> Interned<[Interned]>; + fn subcommand_dyn(&self) -> Option; + fn from_arg_matches_dyn( + self: Arc, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result; + fn parse_command_line_dyn( + self: Arc, + command_line: Interned<[Interned]>, + ) -> clap::error::Result; +} + +impl DynJobKindTrait for T { + fn as_any(&self) -> &dyn Any { + self + } + + fn as_arc_any(self: Arc) -> Arc { + self + } + + fn eq_dyn(&self, other: &dyn DynJobKindTrait) -> bool { + other + .as_any() + .downcast_ref::() + .is_some_and(|other| self == other) + } + + fn hash_dyn(&self, mut state: &mut dyn Hasher) { + self.hash(&mut state) + } + + fn command_line_prefix_dyn(&self) -> Interned<[Interned]> { + self.command_line_prefix() + } + + fn subcommand_dyn(&self) -> Option { + self.subcommand() + } + + fn from_arg_matches_dyn( + self: Arc, + matches: &mut clap::ArgMatches, + ) -> clap::error::Result { + let job = self.from_arg_matches(matches)?; + let inputs = self.inputs(&job); + let outputs = self.outputs(&job); + Ok(DynJob(Arc::new(inner::DynJob { + kind: self, + job, + inputs, + outputs, + }))) + } + + fn parse_command_line_dyn( + self: Arc, + command_line: Interned<[Interned]>, + ) -> clap::error::Result { + let job = self.parse_command_line(command_line)?; + let inputs = self.inputs(&job); + let outputs = self.outputs(&job); + Ok(DynJob(Arc::new(inner::DynJob { + kind: self, + job, + inputs, + outputs, + }))) + } +} + +#[derive(Clone)] +pub struct DynJobKind(Arc); + +impl DynJobKind { + pub fn from_arc(job_kind: Arc) -> Self { + if TypeId::of::() == TypeId::of::() { + Self::clone( + &Arc::downcast::(job_kind.as_arc_any()) + .ok() + .expect("already checked type"), + ) + } else { + Self(job_kind) + } + } + pub fn new(job_kind: T) -> Self { + if let Some(job_kind) = DynJobKindTrait::as_any(&job_kind).downcast_ref::() { + job_kind.clone() + } else { + Self(Arc::new(job_kind)) + } + } + pub fn type_id(&self) -> TypeId { + DynJobKindTrait::as_any(&*self.0).type_id() + } + pub fn downcast_ref(&self) -> Option<&T> { + DynJobKindTrait::as_any(&*self.0).downcast_ref() + } + pub fn downcast_arc(self) -> Result, Self> { + if self.downcast_ref::().is_some() { + Ok(Arc::downcast::(self.0.as_arc_any()) + .ok() + .expect("already checked type")) + } else { + Err(self) + } + } + pub fn registry() -> JobKindRegistrySnapshot { + JobKindRegistrySnapshot(JobKindRegistry::get()) + } + #[track_caller] + pub fn register(self) { + JobKindRegistry::register(JobKindRegistry::lock(), self); + } +} + +impl Hash for DynJobKind { + fn hash(&self, state: &mut H) { + DynJobKindTrait::as_any(&*self.0).type_id().hash(state); + DynJobKindTrait::hash_dyn(&*self.0, state); + } +} + +impl PartialEq for DynJobKind { + fn eq(&self, other: &Self) -> bool { + DynJobKindTrait::eq_dyn(&*self.0, &*other.0) + } +} + +impl Eq for DynJobKind {} + +impl fmt::Debug for DynJobKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl Serialize for DynJobKind { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.command_line_prefix().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for DynJobKind { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let command_line_prefix: Cow<'_, [Interned]> = Cow::deserialize(deserializer)?; + match Self::registry().get_by_command_line_prefix(&command_line_prefix) { + Some(retval) => Ok(retval.clone()), + None => Err(D::Error::custom(format_args!( + "unknown job kind: command line prefix not found in registry: {command_line_prefix:?}" + ))), + } + } +} + +#[derive(Clone, Debug)] +struct JobKindRegistry { + command_line_prefix_to_job_kind_map: HashMap<&'static [Interned], DynJobKind>, + job_kinds: Vec, + subcommand_names: BTreeMap<&'static str, DynJobKind>, +} + +enum JobKindRegisterError { + SameCommandLinePrefix { + command_line_prefix: &'static [Interned], + old_job_kind: DynJobKind, + new_job_kind: DynJobKind, + }, + CommandLinePrefixDoesNotMatchSubcommandName { + command_line_prefix: &'static [Interned], + expected: [Interned; 2], + job_kind: DynJobKind, + }, +} + +impl fmt::Display for JobKindRegisterError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SameCommandLinePrefix { + command_line_prefix, + old_job_kind, + new_job_kind, + } => write!( + f, + "two different `DynJobKind` can't share the same `command_line_prefix` of:\n\ + {command_line_prefix:?}\n\ + old job kind:\n\ + {old_job_kind:?}\n\ + new job kind:\n\ + {new_job_kind:?}", + ), + Self::CommandLinePrefixDoesNotMatchSubcommandName { + command_line_prefix, + expected, + job_kind, + } => write!( + f, + "`JobKind::subcommand()` returned `Some` but the `command_line_prefix` is not as expected\n\ + (it should be `[program_name_for_internal_jobs(), subcommand_name]`):\n\ + command_line_prefix:\n\ + {command_line_prefix:?}\n\ + expected:\n\ + {expected:?}\n\ + job kind:\n\ + {job_kind:?}", + ), + } + } +} + +trait JobKindRegistryRegisterLock { + type Locked; + fn lock(self) -> Self::Locked; + fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry; +} + +impl JobKindRegistryRegisterLock for &'static RwLock> { + type Locked = RwLockWriteGuard<'static, Arc>; + fn lock(self) -> Self::Locked { + self.write().expect("shouldn't be poisoned") + } + fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry { + Arc::make_mut(locked) + } +} + +impl JobKindRegistryRegisterLock for &'_ mut JobKindRegistry { + type Locked = Self; + + fn lock(self) -> Self::Locked { + self + } + + fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry { + locked + } +} + +impl JobKindRegistry { + fn lock() -> &'static RwLock> { + static REGISTRY: OnceLock>> = OnceLock::new(); + REGISTRY.get_or_init(Default::default) + } + fn try_register( + lock: L, + job_kind: DynJobKind, + ) -> Result<(), JobKindRegisterError> { + let command_line_prefix = Interned::into_inner(job_kind.command_line_prefix()); + let subcommand_name = job_kind + .subcommand() + .map(|subcommand| subcommand.get_name().intern()); + if let Some(subcommand_name) = subcommand_name { + let expected = [program_name_for_internal_jobs(), subcommand_name]; + if command_line_prefix != &expected { + return Err( + JobKindRegisterError::CommandLinePrefixDoesNotMatchSubcommandName { + command_line_prefix, + expected, + job_kind, + }, + ); + } + } + // run user code only outside of lock + let mut locked = lock.lock(); + let this = L::make_mut(&mut locked); + let result = match this + .command_line_prefix_to_job_kind_map + .entry(command_line_prefix) + { + Entry::Occupied(entry) => Err(JobKindRegisterError::SameCommandLinePrefix { + command_line_prefix, + old_job_kind: entry.get().clone(), + new_job_kind: job_kind, + }), + Entry::Vacant(entry) => { + this.job_kinds.push(job_kind.clone()); + if let Some(subcommand_name) = subcommand_name { + this.subcommand_names + .insert(Interned::into_inner(subcommand_name), job_kind.clone()); + } + entry.insert(job_kind); + Ok(()) + } + }; + drop(locked); + // outside of lock now, so we can test if it's the same DynJobKind + match result { + Err(JobKindRegisterError::SameCommandLinePrefix { + command_line_prefix: _, + old_job_kind, + new_job_kind, + }) if old_job_kind == new_job_kind => Ok(()), + result => result, + } + } + #[track_caller] + fn register(lock: L, job_kind: DynJobKind) { + match Self::try_register(lock, job_kind) { + Err(e) => panic!("{e}"), + Ok(()) => {} + } + } + fn get() -> Arc { + Self::lock().read().expect("shouldn't be poisoned").clone() + } +} + +impl Default for JobKindRegistry { + fn default() -> Self { + let mut retval = Self { + command_line_prefix_to_job_kind_map: HashMap::default(), + job_kinds: Vec::new(), + subcommand_names: BTreeMap::new(), + }; + for job_kind in [] { + Self::register(&mut retval, job_kind); + } + retval + } +} + +#[derive(Clone, Debug)] +pub struct JobKindRegistrySnapshot(Arc); + +impl JobKindRegistrySnapshot { + pub fn get() -> Self { + JobKindRegistrySnapshot(JobKindRegistry::get()) + } + pub fn get_by_command_line_prefix<'a>( + &'a self, + command_line_prefix: &[Interned], + ) -> Option<&'a DynJobKind> { + self.0 + .command_line_prefix_to_job_kind_map + .get(command_line_prefix) + } + pub fn job_kinds(&self) -> &[DynJobKind] { + &self.0.job_kinds + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub struct AnyInternalJob(pub DynJob); + +impl clap::Subcommand for AnyInternalJob { + fn augment_subcommands(mut cmd: clap::Command) -> clap::Command { + for job_kind in JobKindRegistrySnapshot::get().0.subcommand_names.values() { + let Some(subcommand) = job_kind.subcommand() else { + // shouldn't happen, ignore it + continue; + }; + cmd = cmd.subcommand(subcommand); + } + cmd + } + + fn augment_subcommands_for_update(cmd: clap::Command) -> clap::Command { + Self::augment_subcommands(cmd) + } + + fn has_subcommand(name: &str) -> bool { + JobKindRegistrySnapshot::get() + .0 + .subcommand_names + .contains_key(name) + } +} + +impl clap::FromArgMatches for AnyInternalJob { + fn from_arg_matches(matches: &clap::ArgMatches) -> Result { + Self::from_arg_matches_mut(&mut matches.clone()) + } + + fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result { + if let Some((name, mut matches)) = matches.remove_subcommand() { + let job_kind_registry_snapshot = JobKindRegistrySnapshot::get(); + if let Some(job_kind) = job_kind_registry_snapshot.0.subcommand_names.get(&*name) { + Ok(Self(job_kind.from_arg_matches(&mut matches)?)) + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::InvalidSubcommand, + format!("the subcommand '{name}' wasn't recognized"), + )) + } + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::MissingSubcommand, + "a subcommand is required but one was not provided", + )) + } + } + + fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> { + *self = Self::from_arg_matches(matches)?; + Ok(()) + } + + fn update_from_arg_matches_mut( + &mut self, + matches: &mut clap::ArgMatches, + ) -> Result<(), clap::Error> { + *self = Self::from_arg_matches_mut(matches)?; + Ok(()) + } +} + +trait DynJobTrait: 'static + Send + Sync + fmt::Debug { + fn as_any(&self) -> &dyn Any; + fn eq_dyn(&self, other: &dyn DynJobTrait) -> bool; + fn hash_dyn(&self, state: &mut dyn Hasher); + fn kind_type_id(&self) -> TypeId; + fn kind(&self) -> DynJobKind; + fn inputs(&self) -> Interned<[JobItemName]>; + fn outputs(&self) -> Interned<[JobItemName]>; + fn to_command_line(&self) -> Interned<[Interned]>; + fn debug_name(&self) -> String; + fn run(&self, inputs: &[JobItem], acquired_job: &mut AcquiredJob) + -> eyre::Result>; +} + +mod inner { + use super::*; + + #[derive(Debug, PartialEq, Eq, Hash)] + pub(crate) struct DynJob { + pub(crate) kind: Arc, + pub(crate) job: T::Job, + pub(crate) inputs: Interned<[JobItemName]>, + pub(crate) outputs: Interned<[JobItemName]>, + } +} + +impl DynJobTrait for inner::DynJob { + fn as_any(&self) -> &dyn Any { + self + } + + fn eq_dyn(&self, other: &dyn DynJobTrait) -> bool { + other + .as_any() + .downcast_ref::>() + .is_some_and(|other| self == other) + } + + fn hash_dyn(&self, mut state: &mut dyn Hasher) { + self.hash(&mut state); + } + + fn kind_type_id(&self) -> TypeId { + TypeId::of::() + } + + fn kind(&self) -> DynJobKind { + DynJobKind(self.kind.clone()) + } + + fn inputs(&self) -> Interned<[JobItemName]> { + self.inputs + } + + fn outputs(&self) -> Interned<[JobItemName]> { + self.outputs + } + + fn to_command_line(&self) -> Interned<[Interned]> { + self.kind.to_command_line(&self.job) + } + + fn debug_name(&self) -> String { + self.kind.debug_name(&self.job) + } + + fn run( + &self, + inputs: &[JobItem], + acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + self.kind.run(&self.job, inputs, acquired_job) + } +} + +#[derive(Clone, Debug)] +pub struct DynJob(Arc); + +impl DynJob { + pub fn from_arc(job_kind: Arc, job: T::Job) -> Self { + if TypeId::of::() == TypeId::of::() { + ::downcast_ref::(&job) + .expect("already checked type") + .clone() + } else { + let inputs = job_kind.inputs(&job); + let outputs = job_kind.outputs(&job); + Self(Arc::new(inner::DynJob { + kind: job_kind, + job, + inputs, + outputs, + })) + } + } + pub fn new(job_kind: T, job: T::Job) -> Self { + if TypeId::of::() == TypeId::of::() { + ::downcast_ref::(&job) + .expect("already checked type") + .clone() + } else { + let inputs = job_kind.inputs(&job); + let outputs = job_kind.outputs(&job); + Self(Arc::new(inner::DynJob { + kind: Arc::new(job_kind), + job, + inputs, + outputs, + })) + } + } + pub fn kind_type_id(&self) -> TypeId { + self.0.kind_type_id() + } + pub fn downcast(&self) -> Option<(&T, &T::Job)> { + let inner::DynJob { kind, job, .. } = self.0.as_any().downcast_ref()?; + Some((kind, job)) + } + pub fn kind(&self) -> DynJobKind { + DynJobTrait::kind(&*self.0) + } + pub fn inputs(&self) -> Interned<[JobItemName]> { + DynJobTrait::inputs(&*self.0) + } + pub fn outputs(&self) -> Interned<[JobItemName]> { + DynJobTrait::outputs(&*self.0) + } + pub fn to_command_line(&self) -> Interned<[Interned]> { + DynJobTrait::to_command_line(&*self.0) + } + pub fn debug_name(&self) -> String { + DynJobTrait::debug_name(&*self.0) + } + pub fn run( + &self, + inputs: &[JobItem], + acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + DynJobTrait::run(&*self.0, inputs, acquired_job) + } +} + +impl Eq for DynJob {} + +impl PartialEq for DynJob { + fn eq(&self, other: &Self) -> bool { + DynJobTrait::eq_dyn(&*self.0, &*other.0) + } +} + +impl Hash for DynJob { + fn hash(&self, state: &mut H) { + DynJobTrait::hash_dyn(&*self.0, state); + } +} + +#[derive(Serialize, Deserialize)] +#[serde(rename = "DynJob")] +struct DynJobSerde { + kind: DynJobKind, + command_line: Interned<[Interned]>, +} + +impl Serialize for DynJob { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + DynJobSerde { + kind: self.kind(), + command_line: self.to_command_line(), + } + .serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for DynJob { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let DynJobSerde { kind, command_line } = Deserialize::deserialize(deserializer)?; + kind.parse_command_line(command_line) + .map_err(D::Error::custom) + } +} + +impl JobKind for DynJobKind { + type Job = DynJob; + + fn inputs(&self, job: &Self::Job) -> Interned<[JobItemName]> { + job.inputs() + } + + fn outputs(&self, job: &Self::Job) -> Interned<[JobItemName]> { + job.outputs() + } + + fn command_line_prefix(&self) -> Interned<[Interned]> { + self.0.command_line_prefix_dyn() + } + + fn to_command_line(&self, job: &Self::Job) -> Interned<[Interned]> { + job.to_command_line() + } + + fn subcommand(&self) -> Option { + self.0.subcommand_dyn() + } + + fn from_arg_matches(&self, matches: &mut clap::ArgMatches) -> clap::error::Result { + self.0.clone().from_arg_matches_dyn(matches) + } + + fn parse_command_line( + &self, + command_line: Interned<[Interned]>, + ) -> clap::error::Result { + self.0.clone().parse_command_line_dyn(command_line) + } + + fn run( + &self, + job: &Self::Job, + inputs: &[JobItem], + acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + job.run(inputs, acquired_job) + } +} + +#[derive(Clone, Debug)] +enum JobGraphNode { + Job(DynJob), + Item { + #[allow(dead_code, reason = "used for Debug")] + name: JobItemName, + source_job: Option, + }, +} + +type JobGraphInner = DiGraph; + +#[derive(Clone, Default)] +pub struct JobGraph { + jobs: HashMap::NodeId>, + items: HashMap::NodeId>, + graph: JobGraphInner, + topological_order: Vec<::NodeId>, + space: DfsSpace<::NodeId, ::Map>, +} + +impl fmt::Debug for JobGraph { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + jobs: _, + items: _, + graph, + topological_order, + space: _, + } = self; + f.debug_struct("JobGraph") + .field("graph", graph) + .field("topological_order", topological_order) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug)] +pub enum JobGraphError { + CycleError { + job: DynJob, + output: JobItemName, + }, + MultipleJobsCreateSameOutput { + output_item: JobItemName, + existing_job: DynJob, + new_job: DynJob, + }, +} + +impl std::error::Error for JobGraphError {} + +impl fmt::Display for JobGraphError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CycleError { job, output } => write!( + f, + "job can't be added to job graph because it would introduce a cyclic dependency through this job output:\n\ + {output:?}\n\ + job:\n{job:?}", + ), + JobGraphError::MultipleJobsCreateSameOutput { + output_item, + existing_job, + new_job, + } => write!( + f, + "job can't be added to job graph because the new job has an output that is also produced by an existing job.\n\ + conflicting output:\n\ + {output_item:?}\n\ + existing job:\n\ + {existing_job:?}\n\ + new job:\n\ + {new_job:?}", + ), + } + } +} + +#[derive(Copy, Clone, Debug)] +enum EscapeForUnixShellState { + DollarSingleQuote, + SingleQuote, + Unquoted, +} + +#[derive(Clone)] +pub struct EscapeForUnixShell<'a> { + state: EscapeForUnixShellState, + prefix: [u8; 3], + bytes: &'a [u8], +} + +impl<'a> fmt::Debug for EscapeForUnixShell<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl<'a> fmt::Display for EscapeForUnixShell<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for c in self.clone() { + f.write_char(c)?; + } + Ok(()) + } +} + +impl<'a> EscapeForUnixShell<'a> { + pub fn new(s: &'a str) -> Self { + Self::from_bytes(s.as_bytes()) + } + fn make_prefix(bytes: &[u8]) -> [u8; 3] { + let mut prefix = [0; 3]; + prefix[..bytes.len()].copy_from_slice(bytes); + prefix + } + pub fn from_bytes(bytes: &'a [u8]) -> Self { + let mut needs_single_quote = bytes.is_empty(); + for &b in bytes { + match b { + b'!' | b'\'' | b'\"' | b' ' => needs_single_quote = true, + 0..0x20 | 0x7F.. => { + return Self { + state: EscapeForUnixShellState::DollarSingleQuote, + prefix: Self::make_prefix(b"$'"), + bytes, + }; + } + _ => {} + } + } + if needs_single_quote { + Self { + state: EscapeForUnixShellState::SingleQuote, + prefix: Self::make_prefix(b"'"), + bytes, + } + } else { + Self { + state: EscapeForUnixShellState::Unquoted, + prefix: Self::make_prefix(b""), + bytes, + } + } + } +} + +impl Iterator for EscapeForUnixShell<'_> { + type Item = char; + + fn next(&mut self) -> Option { + match &mut self.prefix { + [0, 0, 0] => {} + [0, 0, v] | // find first + [0, v, _] | // non-zero byte + [v, _, _] => { + let retval = *v as char; + *v = 0; + return Some(retval); + } + } + let Some(&next_byte) = self.bytes.split_off_first() else { + return match self.state { + EscapeForUnixShellState::DollarSingleQuote + | EscapeForUnixShellState::SingleQuote => { + self.state = EscapeForUnixShellState::Unquoted; + Some('\'') + } + EscapeForUnixShellState::Unquoted => None, + }; + }; + match self.state { + EscapeForUnixShellState::DollarSingleQuote => match next_byte { + b'\'' | b'\\' => { + self.prefix = Self::make_prefix(&[next_byte]); + Some('\\') + } + b'\t' => { + self.prefix = Self::make_prefix(b"t"); + Some('\\') + } + b'\n' => { + self.prefix = Self::make_prefix(b"n"); + Some('\\') + } + b'\r' => { + self.prefix = Self::make_prefix(b"r"); + Some('\\') + } + 0x20..=0x7E => Some(next_byte as char), + _ => { + self.prefix = [ + b'x', + char::from_digit(next_byte as u32 >> 4, 0x10).expect("known to be in range") + as u8, + char::from_digit(next_byte as u32 & 0xF, 0x10) + .expect("known to be in range") as u8, + ]; + Some('\\') + } + }, + EscapeForUnixShellState::SingleQuote => { + if next_byte == b'\'' { + self.prefix = Self::make_prefix(b"\\''"); + Some('\'') + } else { + Some(next_byte as char) + } + } + EscapeForUnixShellState::Unquoted => match next_byte { + b' ' | b'!' | b'"' | b'#' | b'$' | b'&' | b'\'' | b'(' | b')' | b'*' | b',' + | b';' | b'<' | b'>' | b'?' | b'[' | b'\\' | b']' | b'^' | b'`' | b'{' | b'|' + | b'}' | b'~' => { + self.prefix = Self::make_prefix(&[next_byte]); + Some('\\') + } + _ => Some(next_byte as char), + }, + } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +#[non_exhaustive] +pub enum UnixMakefileEscapeKind { + NonRecipe, + RecipeWithoutShellEscaping, + RecipeWithShellEscaping, +} + +#[derive(Copy, Clone)] +pub struct EscapeForUnixMakefile<'a> { + s: &'a str, + kind: UnixMakefileEscapeKind, +} + +impl<'a> fmt::Debug for EscapeForUnixMakefile<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl<'a> fmt::Display for EscapeForUnixMakefile<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.do_write(f, fmt::Write::write_str, fmt::Write::write_char, |_, _| { + Ok(()) + }) + } +} + +impl<'a> EscapeForUnixMakefile<'a> { + fn do_write( + &self, + state: &mut S, + write_str: impl Fn(&mut S, &str) -> Result<(), E>, + write_char: impl Fn(&mut S, char) -> Result<(), E>, + add_variable: impl Fn(&mut S, &'static str) -> Result<(), E>, + ) -> Result<(), E> { + let escape_recipe_char = |c| match c { + '$' => write_str(state, "$$"), + '\0'..='\x1F' | '\x7F' => { + panic!("can't escape a control character for Unix Makefile: {c:?}"); + } + _ => write_char(state, c), + }; + match self.kind { + UnixMakefileEscapeKind::NonRecipe => self.s.chars().try_for_each(|c| match c { + '=' => { + add_variable(state, "EQUALS = =")?; + write_str(state, "$(EQUALS)") + } + ';' => panic!("can't escape a semicolon (;) for Unix Makefile"), + '$' => write_str(state, "$$"), + '\\' | ' ' | '#' | ':' | '%' | '*' | '?' | '[' | ']' | '~' => { + write_char(state, '\\')?; + write_char(state, c) + } + '\0'..='\x1F' | '\x7F' => { + panic!("can't escape a control character for Unix Makefile: {c:?}"); + } + _ => write_char(state, c), + }), + UnixMakefileEscapeKind::RecipeWithoutShellEscaping => { + self.s.chars().try_for_each(escape_recipe_char) + } + UnixMakefileEscapeKind::RecipeWithShellEscaping => { + EscapeForUnixShell::new(self.s).try_for_each(escape_recipe_char) + } + } + } + pub fn new( + s: &'a str, + kind: UnixMakefileEscapeKind, + needed_variables: &mut BTreeSet<&'static str>, + ) -> Self { + let retval = Self { s, kind }; + let Ok(()) = retval.do_write( + needed_variables, + |_, _| Ok(()), + |_, _| Ok(()), + |needed_variables, variable| -> Result<(), std::convert::Infallible> { + needed_variables.insert(variable); + Ok(()) + }, + ); + retval + } +} + +impl JobGraph { + pub fn new() -> Self { + Self::default() + } + pub fn try_add_jobs>( + &mut self, + jobs: I, + ) -> Result<(), JobGraphError> { + let jobs = jobs.into_iter(); + struct RemoveNewNodesOnError<'a> { + this: &'a mut JobGraph, + new_nodes: HashSet<::NodeId>, + } + impl Drop for RemoveNewNodesOnError<'_> { + fn drop(&mut self) { + for node in self.new_nodes.drain() { + self.this.graph.remove_node(node); + } + } + } + let mut remove_new_nodes_on_error = RemoveNewNodesOnError { + this: self, + new_nodes: HashSet::with_capacity_and_hasher(jobs.size_hint().0, Default::default()), + }; + let new_nodes = &mut remove_new_nodes_on_error.new_nodes; + let this = &mut *remove_new_nodes_on_error.this; + for job in jobs { + let Entry::Vacant(job_entry) = this.jobs.entry(job) else { + continue; + }; + let job_node_id = this + .graph + .add_node(JobGraphNode::Job(job_entry.key().clone())); + new_nodes.insert(job_node_id); + let job_entry = job_entry.insert_entry(job_node_id); + for (item, is_output) in job_entry + .key() + .inputs() + .iter() + .zip(iter::repeat(false)) + .chain(job_entry.key().outputs().iter().zip(iter::repeat(true))) + { + let item_node_id; + match this.items.entry(*item) { + Entry::Occupied(item_entry) => { + item_node_id = *item_entry.get(); + if is_output { + let JobGraphNode::Item { + name: _, + source_job, + } = &mut this.graph[item_node_id] + else { + unreachable!("known to be an item"); + }; + if let Some(source_job) = source_job { + return Err(JobGraphError::MultipleJobsCreateSameOutput { + output_item: item_entry.key().clone(), + existing_job: source_job.clone(), + new_job: job_entry.key().clone(), + }); + } else { + *source_job = Some(job_entry.key().clone()); + } + } + } + Entry::Vacant(item_entry) => { + item_node_id = this.graph.add_node(JobGraphNode::Item { + name: *item, + source_job: is_output.then(|| job_entry.key().clone()), + }); + new_nodes.insert(item_node_id); + item_entry.insert(item_node_id); + } + } + let mut source = item_node_id; + let mut dest = job_node_id; + if is_output { + mem::swap(&mut source, &mut dest); + } + this.graph.add_edge(source, dest, ()); + } + } + match toposort(&this.graph, Some(&mut this.space)) { + Ok(v) => { + this.topological_order = v; + // no need to remove any of the new nodes on drop since we didn't encounter any errors + remove_new_nodes_on_error.new_nodes.clear(); + Ok(()) + } + Err(_) => { + // there's at least one cycle, find one! + let cycle = kosaraju_scc(&this.graph) + .into_iter() + .find_map(|scc| { + if scc.len() <= 1 { + // can't be a cycle since our graph is bipartite -- + // jobs only connect to items, never jobs to jobs or items to items + None + } else { + Some(scc) + } + }) + .expect("we know there's a cycle"); + let cycle_set = HashSet::from_iter(cycle.iter().copied()); + let job = cycle + .into_iter() + .find_map(|node_id| { + if let JobGraphNode::Job(job) = &this.graph[node_id] { + Some(job.clone()) + } else { + None + } + }) + .expect("a job must be part of the cycle"); + let output = job + .outputs() + .into_iter() + .find(|output| cycle_set.contains(&this.items[output])) + .expect("an output must be part of the cycle"); + Err(JobGraphError::CycleError { job, output }) + } + } + } + #[track_caller] + pub fn add_jobs>(&mut self, jobs: I) { + match self.try_add_jobs(jobs) { + Ok(()) => {} + Err(e) => panic!("error: {e}"), + } + } + pub fn to_unix_makefile(&self) -> String { + let mut retval = String::new(); + let mut needed_variables = BTreeSet::new(); + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + for (index, output) in job.outputs().into_iter().enumerate() { + match output { + JobItemName::Module { .. } => continue, + JobItemName::File { path } => { + if index != 0 { + retval.push_str(" "); + } + write_str!( + retval, + "{}", + EscapeForUnixMakefile::new( + &path, + UnixMakefileEscapeKind::NonRecipe, + &mut needed_variables + ) + ); + } + } + } + retval.push_str(":"); + for input in job.inputs() { + match input { + JobItemName::Module { .. } => continue, + JobItemName::File { path } => { + write_str!( + retval, + " {}", + EscapeForUnixMakefile::new( + &path, + UnixMakefileEscapeKind::NonRecipe, + &mut needed_variables + ) + ); + } + } + } + retval.push_str("\n\t"); + for (index, arg) in job.to_command_line().into_iter().enumerate() { + if index != 0 { + retval.push_str(" "); + } + write_str!( + retval, + "{}", + EscapeForUnixMakefile::new( + &arg, + UnixMakefileEscapeKind::RecipeWithShellEscaping, + &mut needed_variables + ) + ); + } + retval.push_str("\n\n"); + } + if !needed_variables.is_empty() { + retval.insert_str( + 0, + &String::from_iter(needed_variables.into_iter().map(|v| format!("{v}\n"))), + ); + } + retval + } + pub fn to_unix_shell_script(&self) -> String { + let mut retval = String::from( + "#!/bin/sh\n\ + set -ex\n", + ); + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + for (index, arg) in job.to_command_line().into_iter().enumerate() { + if index != 0 { + retval.push_str(" "); + } + write_str!(retval, "{}", EscapeForUnixShell::new(&arg)); + } + retval.push_str("\n"); + } + retval + } + pub fn run(&self) -> eyre::Result<()> { + // use scope to auto-join threads on errors + thread::scope(|scope| { + struct WaitingJobState { + job_node_id: ::NodeId, + job: DynJob, + inputs: Vec>, + } + let mut ready_jobs = VecDeque::new(); + let mut item_name_to_waiting_jobs_map = HashMap::<_, Vec<_>>::default(); + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + let inputs = job.inputs(); + let waiting_job = WaitingJobState { + job_node_id: node_id, + job: job.clone(), + inputs: inputs.iter().map(|_| OnceCell::new()).collect(), + }; + if inputs.is_empty() { + ready_jobs.push_back(waiting_job); + } else { + let waiting_job = Rc::new(waiting_job); + for (input_index, input_item) in inputs.into_iter().enumerate() { + item_name_to_waiting_jobs_map + .entry(input_item) + .or_default() + .push((input_index, waiting_job.clone())); + } + } + } + struct RunningJob<'scope> { + job: DynJob, + thread: ScopedJoinHandle<'scope, eyre::Result>>, + } + let mut running_jobs = HashMap::default(); + let (finished_jobs_sender, finished_jobs_receiver) = mpsc::channel(); + loop { + while let Some(finished_job) = finished_jobs_receiver.try_recv().ok() { + let Some(RunningJob { job, thread }) = running_jobs.remove(&finished_job) + else { + unreachable!(); + }; + let output_items = thread.join().map_err(panic::resume_unwind)??; + let output_names = job.outputs(); + assert_eq!( + output_items.len(), + output_names.len(), + "job's run() method returned the wrong number of output items: {job:?}" + ); + for (output_item, output_name) in output_items.into_iter().zip(output_names) { + for (input_index, waiting_job) in item_name_to_waiting_jobs_map + .remove(&output_name) + .unwrap_or_default() + { + let Ok(()) = waiting_job.inputs[input_index].set(output_item.clone()) + else { + unreachable!(); + }; + if let Some(waiting_job) = Rc::into_inner(waiting_job) { + ready_jobs.push_back(waiting_job); + } + } + } + } + if let Some(WaitingJobState { + job_node_id, + job, + inputs, + }) = ready_jobs.pop_front() + { + struct RunningJobInThread { + job_node_id: ::NodeId, + job: DynJob, + inputs: Vec, + acquired_job: AcquiredJob, + finished_jobs_sender: mpsc::Sender<::NodeId>, + } + impl RunningJobInThread { + fn run(mut self) -> eyre::Result> { + self.job.run(&self.inputs, &mut self.acquired_job) + } + } + impl Drop for RunningJobInThread { + fn drop(&mut self) { + let _ = self.finished_jobs_sender.send(self.job_node_id); + } + } + let name = job.debug_name(); + let running_job_in_thread = RunningJobInThread { + job_node_id, + job: job.clone(), + inputs: Vec::from_iter( + inputs + .into_iter() + .map(|input| input.into_inner().expect("was set earlier")), + ), + acquired_job: AcquiredJob::acquire(), + finished_jobs_sender: finished_jobs_sender.clone(), + }; + running_jobs.insert( + job_node_id, + RunningJob { + job, + thread: thread::Builder::new() + .name(name) + .spawn_scoped(scope, move || running_job_in_thread.run()) + .expect("failed to spawn thread for job"), + }, + ); + } + if running_jobs.is_empty() { + assert!(item_name_to_waiting_jobs_map.is_empty()); + assert!(ready_jobs.is_empty()); + return Ok(()); + } + } + }) + } +} + +impl Extend for JobGraph { + #[track_caller] + fn extend>(&mut self, iter: T) { + self.add_jobs(iter); + } +} + +impl FromIterator for JobGraph { + #[track_caller] + fn from_iter>(iter: T) -> Self { + let mut retval = Self::new(); + retval.add_jobs(iter); + retval + } +} + +impl Serialize for JobGraph { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut serializer = serializer.serialize_seq(Some(self.jobs.len()))?; + for &node_id in &self.topological_order { + let JobGraphNode::Job(job) = &self.graph[node_id] else { + continue; + }; + serializer.serialize_element(job)?; + } + serializer.end() + } +} + +impl<'de> Deserialize<'de> for JobGraph { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let jobs = Vec::::deserialize(deserializer)?; + let mut retval = JobGraph::new(); + retval.try_add_jobs(jobs).map_err(D::Error::custom)?; + Ok(retval) + } +} + +pub fn program_name_for_internal_jobs() -> Interned { + static PROGRAM_NAME: OnceLock> = OnceLock::new(); + *PROGRAM_NAME + .get_or_init(|| str::intern_owned(std::env::args().next().expect("can't get program name"))) +} + +pub trait InternalJobTrait: clap::Args + 'static + fmt::Debug + Eq + Hash + Send + Sync { + fn subcommand_name() -> Interned; + fn to_args(&self) -> Vec>; + fn inputs(&self) -> Interned<[JobItemName]>; + fn outputs(&self) -> Interned<[JobItemName]>; + fn run(&self, inputs: &[JobItem], acquired_job: &mut AcquiredJob) + -> eyre::Result>; +} + +#[derive(Hash, PartialEq, Eq)] +pub struct InternalJobKind(PhantomData Job>); + +impl Clone for InternalJobKind { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for InternalJobKind {} + +impl InternalJobKind { + pub const fn new() -> Self { + Self(PhantomData) + } +} + +impl Default for InternalJobKind { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for InternalJobKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "InternalJobKind<{}>", std::any::type_name::()) + } +} + +impl JobKind for InternalJobKind { + type Job = InternalJob; + + fn inputs(&self, job: &Self::Job) -> Interned<[JobItemName]> { + job.0.inputs() + } + + fn outputs(&self, job: &Self::Job) -> Interned<[JobItemName]> { + job.0.outputs() + } + + fn command_line_prefix(&self) -> Interned<[Interned]> { + [program_name_for_internal_jobs(), Job::subcommand_name()][..].intern() + } + + fn subcommand(&self) -> Option { + Some(Job::augment_args(clap::Command::new(Interned::into_inner( + Job::subcommand_name(), + )))) + } + + fn from_arg_matches(&self, matches: &mut clap::ArgMatches) -> clap::error::Result { + InternalJob::::from_arg_matches_mut(matches) + } + + fn to_command_line(&self, job: &Self::Job) -> Interned<[Interned]> { + Interned::from_iter( + self.command_line_prefix() + .iter() + .copied() + .chain(job.0.to_args()), + ) + } + + fn parse_command_line( + &self, + command_line: Interned<[Interned]>, + ) -> clap::error::Result { + let cmd = clap::Command::new(Interned::into_inner(program_name_for_internal_jobs())); + let mut matches = InternalJob::::augment_subcommands(cmd) + .subcommand_required(true) + .arg_required_else_help(true) + .try_get_matches_from(command_line.iter().map(|arg| &**arg))?; + InternalJob::::from_arg_matches_mut(&mut matches) + } + + fn run( + &self, + job: &Self::Job, + inputs: &[JobItem], + acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + job.0.run(inputs, acquired_job) + } +} + +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Default, PartialOrd, Ord)] +pub struct InternalJob(pub Job); + +impl clap::FromArgMatches for InternalJob { + fn from_arg_matches(matches: &clap::ArgMatches) -> Result { + Self::from_arg_matches_mut(&mut matches.clone()) + } + + fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result { + if let Some((name, mut matches)) = matches.remove_subcommand() { + if *name == *Job::subcommand_name() { + Ok(Self(Job::from_arg_matches_mut(&mut matches)?)) + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::InvalidSubcommand, + format!("the subcommand '{name}' wasn't recognized"), + )) + } + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::MissingSubcommand, + "a subcommand is required but one was not provided", + )) + } + } + + fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> { + self.update_from_arg_matches_mut(&mut matches.clone()) + } + + fn update_from_arg_matches_mut( + &mut self, + matches: &mut clap::ArgMatches, + ) -> Result<(), clap::Error> { + if let Some((name, mut matches)) = matches.remove_subcommand() { + if *name == *Job::subcommand_name() { + self.0.update_from_arg_matches_mut(&mut matches)?; + Ok(()) + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::InvalidSubcommand, + format!("the subcommand '{name}' wasn't recognized"), + )) + } + } else { + Err(clap::Error::raw( + clap::error::ErrorKind::MissingSubcommand, + "a subcommand is required but one was not provided", + )) + } + } +} + +impl clap::Subcommand for InternalJob { + fn augment_subcommands(cmd: clap::Command) -> clap::Command { + cmd.subcommand( + InternalJobKind::::new() + .subcommand() + .expect("known to return Some"), + ) + } + + fn augment_subcommands_for_update(cmd: clap::Command) -> clap::Command { + cmd.subcommand(Job::augment_args_for_update(clap::Command::new( + Interned::into_inner(Job::subcommand_name()), + ))) + } + + fn has_subcommand(name: &str) -> bool { + *name == *Job::subcommand_name() + } +} diff --git a/crates/fayalite/src/build/external.rs b/crates/fayalite/src/build/external.rs new file mode 100644 index 0000000..4ca4549 --- /dev/null +++ b/crates/fayalite/src/build/external.rs @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// See Notices.txt for copyright information + +use crate::{ + build::{EscapeForUnixShell, JobItem, JobItemName, JobKind}, + intern::{Intern, Interned}, + util::job_server::AcquiredJob, +}; +use clap::builder::StyledStr; +use eyre::{Context, eyre}; +use std::{ + env, + fmt::{self, Write}, + mem, +}; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +enum TemplateArg { + Literal(String), + InputPath { before: String, after: String }, + OutputPath { before: String, after: String }, +} + +impl TemplateArg { + fn after_mut(&mut self) -> &mut String { + match self { + TemplateArg::Literal(after) + | TemplateArg::InputPath { after, .. } + | TemplateArg::OutputPath { after, .. } => after, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct TemplatedExternalJobKind { + template: Interned<[TemplateArg]>, + command_line_prefix: Interned<[Interned]>, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum Token { + Char(char), + ArgSeparator, +} + +impl Token { + fn as_ident_start(self) -> Option { + match self { + Self::Char(ch @ '_') => Some(ch), + Self::Char(ch) if ch.is_alphabetic() => Some(ch), + Self::Char(_) | Self::ArgSeparator => None, + } + } + fn as_ident_continue(self) -> Option { + match self { + Self::Char(ch @ '_') => Some(ch), + Self::Char(ch) if ch.is_alphanumeric() => Some(ch), + Self::Char(_) | Self::ArgSeparator => None, + } + } +} + +#[derive(Clone, Debug)] +struct Tokens<'a> { + current: std::str::Chars<'a>, + rest: std::slice::Iter<'a, &'a str>, +} + +impl<'a> Tokens<'a> { + fn new(args: &'a [&'a str]) -> Self { + Self { + current: "".chars(), + rest: args.iter(), + } + } +} + +impl Iterator for Tokens<'_> { + type Item = Token; + + fn next(&mut self) -> Option { + match self.current.next() { + Some(c) => Some(Token::Char(c)), + None => { + self.current = self.rest.next()?.chars(); + Some(Token::ArgSeparator) + } + } + } +} + +struct Parser<'a> { + tokens: std::iter::Peekable>, + template: Vec, +} + +impl<'a> Parser<'a> { + fn new(args_template: &'a [&'a str]) -> Self { + Self { + tokens: Tokens::new(args_template).peekable(), + template: vec![TemplateArg::Literal(String::new())], // placeholder for program path + } + } + fn parse_var(&mut self) -> Result<(), ParseErrorKind> { + let last_arg = self.template.last_mut().expect("known to be non-empty"); + let TemplateArg::Literal(before) = last_arg else { + return Err(ParseErrorKind::EachArgMustHaveAtMostOneVar); + }; + let before = mem::take(before); + self.tokens + .next_if_eq(&Token::Char('$')) + .ok_or(ParseErrorKind::ExpectedVar)?; + self.tokens + .next_if_eq(&Token::Char('{')) + .ok_or(ParseErrorKind::ExpectedVar)?; + let mut var_name = String::new(); + self.tokens + .next_if(|token| { + token.as_ident_start().is_some_and(|ch| { + var_name.push(ch); + true + }) + }) + .ok_or(ParseErrorKind::ExpectedVar)?; + while let Some(_) = self.tokens.next_if(|token| { + token.as_ident_continue().is_some_and(|ch| { + var_name.push(ch); + true + }) + }) {} + self.tokens + .next_if_eq(&Token::Char('}')) + .ok_or(ParseErrorKind::ExpectedVar)?; + let after = String::new(); + *last_arg = match &*var_name { + "input" => TemplateArg::InputPath { before, after }, + "output" => TemplateArg::OutputPath { before, after }, + "" => return Err(ParseErrorKind::ExpectedVar), + _ => { + return Err(ParseErrorKind::UnknownIdentifierExpectedInputOrOutput( + var_name, + )); + } + }; + Ok(()) + } + fn parse(&mut self) -> Result<(), ParseErrorKind> { + while let Some(&peek) = self.tokens.peek() { + match peek { + Token::ArgSeparator => { + self.template.push(TemplateArg::Literal(String::new())); + let _ = self.tokens.next(); + } + Token::Char('$') => self.parse_var()?, + Token::Char(ch) => { + self.template + .last_mut() + .expect("known to be non-empty") + .after_mut() + .push(ch); + let _ = self.tokens.next(); + } + } + } + Ok(()) + } + fn finish(self, program_path: String) -> TemplatedExternalJobKind { + let Self { + mut tokens, + mut template, + } = self; + assert!( + tokens.next().is_none(), + "parse() must be called before finish()" + ); + assert_eq!(template[0], TemplateArg::Literal(String::new())); + *template[0].after_mut() = program_path; + let template: Interned<[_]> = Intern::intern_owned(template); + let mut command_line_prefix = Vec::new(); + for arg in &template { + match arg { + TemplateArg::Literal(arg) => command_line_prefix.push(str::intern(arg)), + TemplateArg::InputPath { before, after: _ } + | TemplateArg::OutputPath { before, after: _ } => { + command_line_prefix.push(str::intern(before)); + break; + } + } + } + TemplatedExternalJobKind { + template, + command_line_prefix: Intern::intern_owned(command_line_prefix), + } + } +} + +pub fn find_program<'a>( + default_program_name: &'a str, + program_path_env_var: Option<&str>, +) -> eyre::Result { + let var = program_path_env_var + .and_then(env::var_os) + .filter(|v| !v.is_empty()); + let program_path = var.as_deref().unwrap_or(default_program_name.as_ref()); + let program_path = which::which(program_path) + .wrap_err_with(|| format!("can't find program {program_path:?}"))?; + program_path + .into_os_string() + .into_string() + .map_err(|program_path| eyre!("path to program is not valid UTF-8: {program_path:?}")) +} + +#[derive(Clone, Debug)] +enum ParseErrorKind { + ExpectedVar, + UnknownIdentifierExpectedInputOrOutput(String), + EachArgMustHaveAtMostOneVar, +} + +#[derive(Clone, Debug)] +pub struct TemplateParseError(ParseErrorKind); + +impl From for TemplateParseError { + fn from(value: ParseErrorKind) -> Self { + Self(value) + } +} + +impl fmt::Display for TemplateParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.0 { + ParseErrorKind::ExpectedVar => { + f.write_str("expected `${{ident}}` for some identifier `ident`") + } + ParseErrorKind::UnknownIdentifierExpectedInputOrOutput(ident) => write!( + f, + "unknown identifier: expected `input` or `output`: {ident:?}", + ), + ParseErrorKind::EachArgMustHaveAtMostOneVar => { + f.write_str("each argument must have at most one variable") + } + } + } +} + +impl std::error::Error for TemplateParseError {} + +impl TemplatedExternalJobKind { + pub fn try_new( + default_program_name: &str, + program_path_env_var: Option<&str>, + args_template: &[&str], + ) -> Result, TemplateParseError> { + let mut parser = Parser::new(args_template); + parser.parse()?; + Ok(find_program(default_program_name, program_path_env_var) + .map(|program_path| parser.finish(program_path))) + } + #[track_caller] + pub fn new( + default_program_name: &str, + program_path_env_var: Option<&str>, + args_template: &[&str], + ) -> eyre::Result { + match Self::try_new(default_program_name, program_path_env_var, args_template) { + Ok(retval) => retval, + Err(e) => panic!("{e}"), + } + } + fn usage(&self) -> StyledStr { + let mut retval = String::from("Usage:"); + let mut last_input_index = 0usize; + let mut last_output_index = 0usize; + for arg in &self.template { + let mut write_arg = |before: &str, middle: fmt::Arguments<'_>, after: &str| { + retval.push_str(" "); + let start_len = retval.len(); + if before != "" { + write!(retval, "{}", EscapeForUnixShell::new(before)).expect("won't error"); + } + retval.write_fmt(middle).expect("won't error"); + if after != "" { + write!(retval, "{}", EscapeForUnixShell::new(after)).expect("won't error"); + } + if retval.len() == start_len { + write!(retval, "{}", EscapeForUnixShell::new("")).expect("won't error"); + } + }; + match arg { + TemplateArg::Literal(s) => write_arg(s, format_args!(""), ""), + TemplateArg::InputPath { before, after } => { + last_input_index += 1; + write_arg(before, format_args!(""), after); + } + TemplateArg::OutputPath { before, after } => { + last_output_index += 1; + write_arg(before, format_args!(""), after); + } + } + } + retval.into() + } + fn with_usage(&self, mut e: clap::Error) -> clap::Error { + e.insert( + clap::error::ContextKind::Usage, + clap::error::ContextValue::StyledStr(self.usage()), + ); + e + } +} + +impl JobKind for TemplatedExternalJobKind { + type Job = TemplatedExternalJob; + + fn inputs(&self, job: &Self::Job) -> Interned<[JobItemName]> { + job.inputs + } + + fn outputs(&self, job: &Self::Job) -> Interned<[JobItemName]> { + job.outputs + } + + fn command_line_prefix(&self) -> Interned<[Interned]> { + self.command_line_prefix + } + + fn to_command_line(&self, job: &Self::Job) -> Interned<[Interned]> { + job.command_line + } + + fn subcommand(&self) -> Option { + None + } + + fn from_arg_matches(&self, _matches: &mut clap::ArgMatches) -> Result { + panic!( + "a TemplatedExternalJob is not a subcommand of this program -- TemplatedExternalJobKind::subcommand() always returns None" + ); + } + + fn parse_command_line( + &self, + command_line: Interned<[Interned]>, + ) -> clap::error::Result { + let mut inputs = Vec::new(); + let mut outputs = Vec::new(); + let mut command_line_iter = command_line.iter(); + for template_arg in &self.template { + let Some(command_line_arg) = command_line_iter.next() else { + return Err(self.with_usage(clap::Error::new( + clap::error::ErrorKind::MissingRequiredArgument, + ))); + }; + let match_io = |before: &str, after: &str| -> clap::error::Result<_> { + Ok(JobItemName::File { + path: command_line_arg + .strip_prefix(before) + .and_then(|s| s.strip_suffix(after)) + .ok_or_else(|| { + self.with_usage(clap::Error::new( + clap::error::ErrorKind::MissingRequiredArgument, + )) + })? + .intern(), + }) + }; + match template_arg { + TemplateArg::Literal(template_arg) => { + if **command_line_arg != **template_arg { + return Err(self.with_usage(clap::Error::new( + clap::error::ErrorKind::MissingRequiredArgument, + ))); + } + } + TemplateArg::InputPath { before, after } => inputs.push(match_io(before, after)?), + TemplateArg::OutputPath { before, after } => outputs.push(match_io(before, after)?), + } + } + if let Some(_) = command_line_iter.next() { + Err(self.with_usage(clap::Error::new(clap::error::ErrorKind::UnknownArgument))) + } else { + Ok(TemplatedExternalJob { + command_line, + inputs: Intern::intern_owned(inputs), + outputs: Intern::intern_owned(outputs), + }) + } + } + + fn run( + &self, + job: &Self::Job, + inputs: &[JobItem], + acquired_job: &mut AcquiredJob, + ) -> eyre::Result> { + assert!(inputs.iter().map(JobItem::name).eq(job.inputs)); + let mut cmd: std::process::Command = std::process::Command::new(&*job.command_line[0]); + cmd.args(job.command_line[1..].iter().map(|arg| &**arg)); + acquired_job + .run_command(cmd, |cmd: &mut std::process::Command| { + let status = cmd.status()?; + if status.success() { + Ok(()) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("process exited with status: {status}"), + )) + } + }) + .wrap_err_with(|| format!("when trying to run: {:?}", job.command_line))?; + Ok(Vec::from_iter(job.outputs.iter().map( + |&output| match output { + JobItemName::Module { .. } => unreachable!(), + JobItemName::File { path } => JobItem::File { path }, + }, + ))) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct TemplatedExternalJob { + command_line: Interned<[Interned]>, + inputs: Interned<[JobItemName]>, + outputs: Interned<[JobItemName]>, +} diff --git a/crates/fayalite/src/lib.rs b/crates/fayalite/src/lib.rs index 932464b..3a67c5c 100644 --- a/crates/fayalite/src/lib.rs +++ b/crates/fayalite/src/lib.rs @@ -87,6 +87,7 @@ pub mod _docs; pub mod annotations; pub mod array; +pub mod build; pub mod bundle; pub mod cli; pub mod clock; @@ -104,6 +105,7 @@ pub mod reg; pub mod reset; pub mod sim; pub mod source_location; +pub mod target; pub mod testing; pub mod ty; pub mod util; diff --git a/crates/fayalite/src/target.rs b/crates/fayalite/src/target.rs new file mode 100644 index 0000000..33ffee9 --- /dev/null +++ b/crates/fayalite/src/target.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// See Notices.txt for copyright information + +use crate::{intern::Interned, util::job_server::AcquiredJob}; +use std::{ + any::Any, + fmt, + iter::FusedIterator, + sync::{Arc, Mutex}, +}; + +pub trait Peripheral: Any + Send + Sync + fmt::Debug {} + +pub trait Tool: Any + Send + Sync + fmt::Debug { + fn name(&self) -> Interned; + fn run(&self, acquired_job: &mut AcquiredJob); +} + +pub trait Target: Any + Send + Sync + fmt::Debug { + fn name(&self) -> Interned; + fn peripherals(&self) -> Interned<[Interned]>; +} + +#[derive(Clone)] +struct TargetsMap(Vec<(Interned, Interned)>); + +impl TargetsMap { + fn sort(&mut self) { + self.0.sort_by(|(k1, _), (k2, _)| str::cmp(k1, k2)); + self.0.dedup_by_key(|(k, _)| *k); + } + fn from_unsorted_vec(unsorted_vec: Vec<(Interned, Interned)>) -> Self { + let mut retval = Self(unsorted_vec); + retval.sort(); + retval + } + fn extend_from_unsorted_slice(&mut self, additional: &[(Interned, Interned)]) { + self.0.extend_from_slice(additional); + self.sort(); + } +} + +impl Default for TargetsMap { + fn default() -> Self { + Self::from_unsorted_vec(vec![ + // TODO: add default targets here + ]) + } +} + +fn access_targets>) -> R, R>(f: F) -> R { + static TARGETS: Mutex>> = Mutex::new(None); + let mut targets_lock = TARGETS.lock().expect("shouldn't be poisoned"); + f(&mut targets_lock) +} + +pub fn add_targets>>(additional: I) { + // run iterator and target methods outside of lock + let additional = Vec::from_iter(additional.into_iter().map(|v| (v.name(), v))); + access_targets(|targets| { + Arc::make_mut(targets.get_or_insert_default()).extend_from_unsorted_slice(&additional); + }); +} + +pub fn targets() -> TargetsSnapshot { + access_targets(|targets| match targets { + Some(targets) => TargetsSnapshot { + targets: targets.clone(), + }, + None => { + let new_targets = Arc::::default(); + *targets = Some(new_targets.clone()); + TargetsSnapshot { + targets: new_targets, + } + } + }) +} + +#[derive(Clone)] +pub struct TargetsSnapshot { + targets: Arc, +} + +impl TargetsSnapshot { + pub fn get(&self, key: &str) -> Option> { + let index = self + .targets + .0 + .binary_search_by_key(&key, |(k, _v)| k) + .ok()?; + Some(self.targets.0[index].1) + } + pub fn iter(&self) -> TargetsIter { + self.into_iter() + } + pub fn len(&self) -> usize { + self.targets.0.len() + } +} + +impl fmt::Debug for TargetsSnapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("TargetsSnapshot ")?; + f.debug_map().entries(self).finish() + } +} + +impl IntoIterator for &'_ mut TargetsSnapshot { + type Item = (Interned, Interned); + type IntoIter = TargetsIter; + + fn into_iter(self) -> Self::IntoIter { + self.clone().into_iter() + } +} + +impl IntoIterator for &'_ TargetsSnapshot { + type Item = (Interned, Interned); + type IntoIter = TargetsIter; + + fn into_iter(self) -> Self::IntoIter { + self.clone().into_iter() + } +} + +impl IntoIterator for TargetsSnapshot { + type Item = (Interned, Interned); + type IntoIter = TargetsIter; + + fn into_iter(self) -> Self::IntoIter { + TargetsIter { + indexes: 0..self.targets.0.len(), + targets: self.targets, + } + } +} + +#[derive(Clone)] +pub struct TargetsIter { + targets: Arc, + indexes: std::ops::Range, +} + +impl fmt::Debug for TargetsIter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("TargetsIter ")?; + f.debug_map().entries(self.clone()).finish() + } +} + +impl Iterator for TargetsIter { + type Item = (Interned, Interned); + + fn next(&mut self) -> Option { + Some(self.targets.0[self.indexes.next()?]) + } + + fn size_hint(&self) -> (usize, Option) { + self.indexes.size_hint() + } + + fn count(self) -> usize { + self.indexes.len() + } + + fn last(mut self) -> Option { + self.next_back() + } + + fn nth(&mut self, n: usize) -> Option { + Some(self.targets.0[self.indexes.nth(n)?]) + } + + fn fold B>(self, init: B, mut f: F) -> B { + self.indexes + .fold(init, move |retval, index| f(retval, self.targets.0[index])) + } +} + +impl FusedIterator for TargetsIter {} + +impl DoubleEndedIterator for TargetsIter { + fn next_back(&mut self) -> Option { + Some(self.targets.0[self.indexes.next_back()?]) + } + + fn nth_back(&mut self, n: usize) -> Option { + Some(self.targets.0[self.indexes.nth_back(n)?]) + } + + fn rfold B>(self, init: B, mut f: F) -> B { + self.indexes + .rfold(init, move |retval, index| f(retval, self.targets.0[index])) + } +} + +impl ExactSizeIterator for TargetsIter { + fn len(&self) -> usize { + self.indexes.len() + } +}