programmerjake/fayalite
1
0
Fork 0
forked from libre-chip/fayalite
Code Activity

WIP refactoring to have JobKind be internal jobs

Browse source
This commit is contained in:
Jacob Lifshay 2025-09-28 23:05:24 -07:00
parent a823f8485b
commit 93b0cc2341
Signed by: programmerjake
SSH key fingerprint: SHA256:HnFTLGpSm4Q4Fj502oCFisjZSoakwEuTsJJMSke63RQ
11 changed files with 2921 additions and 1695 deletions
Download patch file Download diff file Expand all files Collapse all files

2691
crates/fayalite/src/build.rs
View file

File diff suppressed because it is too large Load diff

44
crates/fayalite/src/build/external.rs
View file

@ -2,22 +2,15 @@
// See Notices.txt for copyright information
use crate::{
build::{DynJob, EscapeForUnixShell, JobItem, JobItemName, JobKind},
intern::{Intern, Interned},
util::{job_server::AcquiredJob, streaming_read_utf8::streaming_read_utf8},
util::streaming_read_utf8::streaming_read_utf8,
};
use base64::{Engine, prelude::BASE64_URL_SAFE_NO_PAD};
use clap::builder::StyledStr;
use eyre::{Context, ensure, eyre};
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error};
use std::{
borrow::Cow,
collections::BTreeMap,
env,
fmt::{self, Write},
mem,
};
use std::{borrow::Cow, collections::BTreeMap, env};
#[cfg(todo)]
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
enum TemplateArg {
Literal(String),
@ -25,6 +18,7 @@ enum TemplateArg {
OutputPath { before: String, after: String },
}
#[cfg(todo)]
impl TemplateArg {
fn after_mut(&mut self) -> &mut String {
match self {
@ -114,14 +108,15 @@ impl From<String> for MaybeUtf8 {
}
#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub struct ExternalJobCache {
#[serde(rename = "ExternalJobCache")]
pub struct ExternalJobCacheV2 {
pub version: ExternalJobCacheVersion,
pub inputs_hash: blake3::Hash,
pub stdout_stderr: String,
pub result: Result<BTreeMap<String, MaybeUtf8>, String>,
}
impl ExternalJobCache {
impl ExternalJobCacheV2 {
fn read_from_file(cache_json_path: Interned<str>) -> eyre::Result<Self> {
let cache_str = std::fs::read_to_string(&*cache_json_path)
.wrap_err_with(|| format!("can't read {cache_json_path}"))?;
@ -215,12 +210,12 @@ impl ExternalJobCaching {
if self.run_even_if_cached {
return Err(());
}
let Ok(ExternalJobCache {
let Ok(ExternalJobCacheV2 {
version: ExternalJobCacheVersion::CURRENT,
inputs_hash: cached_inputs_hash,
stdout_stderr,
result,
}) = ExternalJobCache::read_from_file(self.cache_json_path)
}) = ExternalJobCacheV2::read_from_file(self.cache_json_path)
else {
return Err(());
};
@ -314,7 +309,7 @@ impl ExternalJobCaching {
.expect("spawn shouldn't fail");
run_fn(cmd)
});
ExternalJobCache {
ExternalJobCacheV2 {
version: ExternalJobCacheVersion::CURRENT,
inputs_hash,
stdout_stderr,
@ -350,6 +345,7 @@ impl ExternalJobCaching {
}
}
#[cfg(todo)]
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct TemplatedExternalJobKind {
template: Interned<[TemplateArg]>,
@ -357,12 +353,14 @@ pub struct TemplatedExternalJobKind {
caching: Option<ExternalJobCaching>,
}
#[cfg(todo)]
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
enum Token {
Char(char),
ArgSeparator,
}
#[cfg(todo)]
impl Token {
fn as_ident_start(self) -> Option<char> {
match self {
@ -380,12 +378,14 @@ impl Token {
}
}
#[cfg(todo)]
#[derive(Clone, Debug)]
struct Tokens<'a> {
current: std::str::Chars<'a>,
rest: std::slice::Iter<'a, &'a str>,
}
#[cfg(todo)]
impl<'a> Tokens<'a> {
fn new(args: &'a [&'a str]) -> Self {
Self {
@ -395,6 +395,7 @@ impl<'a> Tokens<'a> {
}
}
#[cfg(todo)]
impl Iterator for Tokens<'_> {
type Item = Token;
@ -409,11 +410,13 @@ impl Iterator for Tokens<'_> {
}
}
#[cfg(todo)]
struct Parser<'a> {
tokens: std::iter::Peekable<Tokens<'a>>,
template: Vec<TemplateArg>,
}
#[cfg(todo)]
impl<'a> Parser<'a> {
fn new(args_template: &'a [&'a str]) -> Self {
Self {
@ -535,6 +538,7 @@ pub fn find_program<'a>(
.map_err(|program_path| eyre!("path to program is not valid UTF-8: {program_path:?}"))
}
#[cfg(todo)]
#[derive(Clone, Debug)]
enum ParseErrorKind {
ExpectedVar,
@ -542,15 +546,18 @@ enum ParseErrorKind {
EachArgMustHaveAtMostOneVar,
}
#[cfg(todo)]
#[derive(Clone, Debug)]
pub struct TemplateParseError(ParseErrorKind);
#[cfg(todo)]
impl From<ParseErrorKind> for TemplateParseError {
fn from(value: ParseErrorKind) -> Self {
Self(value)
}
}
#[cfg(todo)]
impl fmt::Display for TemplateParseError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match &self.0 {
@ -568,8 +575,10 @@ impl fmt::Display for TemplateParseError {
}
}
#[cfg(todo)]
impl std::error::Error for TemplateParseError {}
#[cfg(todo)]
impl TemplatedExternalJobKind {
pub fn try_new(
default_program_name: &str,
@ -599,6 +608,7 @@ impl TemplatedExternalJobKind {
Err(e) => panic!("{e}"),
}
}
#[cfg(todo)]
fn usage(&self) -> StyledStr {
let mut retval = String::from("Usage:");
let mut last_input_index = 0usize;
@ -632,6 +642,7 @@ impl TemplatedExternalJobKind {
}
retval.into()
}
#[cfg(todo)]
fn with_usage(&self, mut e: clap::Error) -> clap::Error {
e.insert(
clap::error::ContextKind::Usage,
@ -641,6 +652,7 @@ impl TemplatedExternalJobKind {
}
}
#[cfg(todo)]
impl JobKind for TemplatedExternalJobKind {
type Job = TemplatedExternalJob;
@ -771,6 +783,7 @@ impl JobKind for TemplatedExternalJobKind {
}
}
#[cfg(todo)]
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct TemplatedExternalJob {
command_line: Interned<[Interned<str>]>,
@ -778,6 +791,7 @@ pub struct TemplatedExternalJob {
outputs: Interned<[JobItemName]>,
}
#[cfg(todo)]
impl TemplatedExternalJob {
pub fn try_add_direct_dependency(&mut self, new_dependency: DynJob) -> eyre::Result<()> {
let mut added = false;

118
crates/fayalite/src/build/firrtl.rs
View file

@ -2,81 +2,119 @@
// See Notices.txt for copyright information
use crate::{
build::{BaseArgs, DynJob, InternalJobTrait, JobItem, JobItemName},
build::{
BaseJob, BaseJobKind, JobAndDependencies, JobArgs, JobArgsAndDependencies, JobItem,
JobItemName, JobKind, JobKindAndDependencies, JobParams,
},
firrtl::{ExportOptions, FileBackend},
intern::{Intern, Interned},
util::job_server::AcquiredJob,
};
use clap::Parser;
use std::{borrow::Cow, collections::BTreeMap};
use clap::Args;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
#[derive(Parser, Debug, Clone, Hash, PartialEq, Eq)]
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Debug)]
pub struct FirrtlJobKind;
#[derive(Args, Debug, Clone, Hash, PartialEq, Eq)]
#[group(id = "Firrtl")]
#[non_exhaustive]
pub struct FirrtlArgs {
#[command(flatten)]
pub base: BaseArgs,
#[command(flatten)]
pub export_options: ExportOptions,
}
impl FirrtlArgs {
impl JobArgs for FirrtlArgs {
fn to_args<Args: Extend<Interned<str>> + ?Sized>(&self, args: &mut Args) {
let Self { export_options } = self;
export_options.to_args(args);
}
}
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct Firrtl {
base: BaseJob,
export_options: ExportOptions,
}
impl Firrtl {
fn make_firrtl_file_backend(&self) -> FileBackend {
FileBackend {
dir_path: self.base.output.path().into(),
top_fir_file_stem: self.base.file_stem.clone(),
dir_path: PathBuf::from(&*self.base.output_dir()),
top_fir_file_stem: Some(String::from(&*self.base.file_stem())),
circuit_name: None,
}
}
pub fn firrtl_file(&self) -> String {
pub fn firrtl_file(&self) -> Interned<str> {
self.base.file_with_ext("fir")
}
}
impl InternalJobTrait for FirrtlArgs {
fn subcommand_name() -> Interned<str> {
"firrtl".intern()
impl JobKind for FirrtlJobKind {
type Args = FirrtlArgs;
type Job = Firrtl;
type Dependencies = JobKindAndDependencies<BaseJobKind>;
fn dependencies(self) -> Self::Dependencies {
JobKindAndDependencies::new(BaseJobKind)
}
fn to_args(&self) -> Vec<Interned<str>> {
let Self {
base,
fn args_to_jobs(
args: JobArgsAndDependencies<Self>,
params: &JobParams,
) -> eyre::Result<JobAndDependencies<Self>> {
args.args_to_jobs_simple(
params,
|_kind, FirrtlArgs { export_options }, dependencies| {
Ok(Firrtl {
base: dependencies.job.job.clone(),
export_options,
} = self;
let mut retval = base.to_args();
retval.extend(export_options.to_args());
retval
}
fn inputs_and_direct_dependencies<'a>(
&'a self,
) -> Cow<'a, BTreeMap<JobItemName, Option<DynJob>>> {
Cow::Owned(BTreeMap::from_iter([(
JobItemName::Module {
name: str::intern(&self.base.module_name),
})
},
None,
)]))
)
}
fn outputs(&self) -> Interned<[JobItemName]> {
[JobItemName::File {
path: str::intern_owned(self.firrtl_file()),
fn inputs(self, job: &Self::Job) -> Interned<[JobItemName]> {
[JobItemName::Path {
path: job.base.output_dir(),
}][..]
.intern()
}
fn outputs(self, job: &Self::Job) -> Interned<[JobItemName]> {
[JobItemName::Path {
path: job.firrtl_file(),
}][..]
.intern()
}
fn name(self) -> Interned<str> {
"firrtl".intern()
}
fn external_command_line(self, _job: &Self::Job) -> Option<Interned<[Interned<str>]>> {
None
}
fn run(
&self,
self,
job: &Self::Job,
inputs: &[JobItem],
params: &JobParams,
_acquired_job: &mut AcquiredJob,
) -> eyre::Result<Vec<JobItem>> {
let [JobItem::Module { value: module }] = inputs else {
panic!("wrong inputs, expected a single `Module`");
let [JobItem::Path { path: input_path }] = *inputs else {
panic!("wrong inputs, expected a single `Path`");
};
assert_eq!(*module.name(), *self.base.module_name);
crate::firrtl::export(self.make_firrtl_file_backend(), module, self.export_options)?;
Ok(vec![JobItem::File {
path: str::intern_owned(self.firrtl_file()),
assert_eq!(input_path, job.base.output_dir());
crate::firrtl::export(
job.make_firrtl_file_backend(),
params.main_module(),
job.export_options,
)?;
Ok(vec![JobItem::Path {
path: job.firrtl_file(),
}])
}
}

762
crates/fayalite/src/build/graph.rs Normal file
View file

@ -0,0 +1,762 @@
// SPDX-License-Identifier: LGPL-3.0-or-later
// See Notices.txt for copyright information
use crate::{
build::{DynJob, JobItem, JobItemName, JobParams, program_name_for_internal_jobs},
intern::Interned,
util::{HashMap, HashSet, job_server::AcquiredJob},
};
use petgraph::{
algo::{DfsSpace, kosaraju_scc, toposort},
graph::DiGraph,
visit::{GraphBase, Visitable},
};
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error, ser::SerializeSeq};
use std::{
cell::OnceCell,
collections::{BTreeMap, BTreeSet, VecDeque},
fmt::{self, Write},
panic,
rc::Rc,
sync::mpsc,
thread::{self, ScopedJoinHandle},
};
macro_rules! write_str {
($s:expr, $($rest:tt)*) => {
write!($s, $($rest)*).expect("String::write_fmt can't fail")
};
}
#[derive(Clone, Debug)]
enum JobGraphNode {
Job(DynJob),
Item {
#[allow(dead_code, reason = "name used for debugging")]
name: JobItemName,
source_job: Option<DynJob>,
},
}
type JobGraphInner = DiGraph<JobGraphNode, ()>;
#[derive(Clone, Default)]
pub struct JobGraph {
jobs: HashMap<DynJob, <JobGraphInner as GraphBase>::NodeId>,
items: HashMap<JobItemName, <JobGraphInner as GraphBase>::NodeId>,
graph: JobGraphInner,
topological_order: Vec<<JobGraphInner as GraphBase>::NodeId>,
space: DfsSpace<<JobGraphInner as GraphBase>::NodeId, <JobGraphInner as Visitable>::Map>,
}
impl fmt::Debug for JobGraph {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let Self {
jobs: _,
items: _,
graph,
topological_order,
space: _,
} = self;
f.debug_struct("JobGraph")
.field("graph", graph)
.field("topological_order", topological_order)
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug)]
pub enum JobGraphError {
CycleError {
job: DynJob,
output: JobItemName,
},
MultipleJobsCreateSameOutput {
output_item: JobItemName,
existing_job: DynJob,
new_job: DynJob,
},
}
impl std::error::Error for JobGraphError {}
impl fmt::Display for JobGraphError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::CycleError { job, output } => write!(
f,
"job can't be added to job graph because it would introduce a cyclic dependency through this job output:\n\
{output:?}\n\
job:\n{job:?}",
),
JobGraphError::MultipleJobsCreateSameOutput {
output_item,
existing_job,
new_job,
} => write!(
f,
"job can't be added to job graph because the new job has an output that is also produced by an existing job.\n\
conflicting output:\n\
{output_item:?}\n\
existing job:\n\
{existing_job:?}\n\
new job:\n\
{new_job:?}",
),
}
}
}
#[derive(Copy, Clone, Debug)]
enum EscapeForUnixShellState {
DollarSingleQuote,
SingleQuote,
Unquoted,
}
#[derive(Clone)]
pub struct EscapeForUnixShell<'a> {
state: EscapeForUnixShellState,
prefix: [u8; 3],
bytes: &'a [u8],
}
impl<'a> fmt::Debug for EscapeForUnixShell<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(self, f)
}
}
impl<'a> fmt::Display for EscapeForUnixShell<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
for c in self.clone() {
f.write_char(c)?;
}
Ok(())
}
}
impl<'a> EscapeForUnixShell<'a> {
pub fn new(s: &'a str) -> Self {
Self::from_bytes(s.as_bytes())
}
fn make_prefix(bytes: &[u8]) -> [u8; 3] {
let mut prefix = [0; 3];
prefix[..bytes.len()].copy_from_slice(bytes);
prefix
}
pub fn from_bytes(bytes: &'a [u8]) -> Self {
let mut needs_single_quote = bytes.is_empty();
for &b in bytes {
match b {
b'!' | b'\'' | b'\"' | b' ' => needs_single_quote = true,
0..0x20 | 0x7F.. => {
return Self {
state: EscapeForUnixShellState::DollarSingleQuote,
prefix: Self::make_prefix(b"$'"),
bytes,
};
}
_ => {}
}
}
if needs_single_quote {
Self {
state: EscapeForUnixShellState::SingleQuote,
prefix: Self::make_prefix(b"'"),
bytes,
}
} else {
Self {
state: EscapeForUnixShellState::Unquoted,
prefix: Self::make_prefix(b""),
bytes,
}
}
}
}
impl Iterator for EscapeForUnixShell<'_> {
type Item = char;
fn next(&mut self) -> Option<Self::Item> {
match &mut self.prefix {
[0, 0, 0] => {}
[0, 0, v] | // find first
[0, v, _] | // non-zero byte
[v, _, _] => {
let retval = *v as char;
*v = 0;
return Some(retval);
}
}
let Some(&next_byte) = self.bytes.split_off_first() else {
return match self.state {
EscapeForUnixShellState::DollarSingleQuote
| EscapeForUnixShellState::SingleQuote => {
self.state = EscapeForUnixShellState::Unquoted;
Some('\'')
}
EscapeForUnixShellState::Unquoted => None,
};
};
match self.state {
EscapeForUnixShellState::DollarSingleQuote => match next_byte {
b'\'' | b'\\' => {
self.prefix = Self::make_prefix(&[next_byte]);
Some('\\')
}
b'\t' => {
self.prefix = Self::make_prefix(b"t");
Some('\\')
}
b'\n' => {
self.prefix = Self::make_prefix(b"n");
Some('\\')
}
b'\r' => {
self.prefix = Self::make_prefix(b"r");
Some('\\')
}
0x20..=0x7E => Some(next_byte as char),
_ => {
self.prefix = [
b'x',
char::from_digit(next_byte as u32 >> 4, 0x10).expect("known to be in range")
as u8,
char::from_digit(next_byte as u32 & 0xF, 0x10)
.expect("known to be in range") as u8,
];
Some('\\')
}
},
EscapeForUnixShellState::SingleQuote => {
if next_byte == b'\'' {
self.prefix = Self::make_prefix(b"\\''");
Some('\'')
} else {
Some(next_byte as char)
}
}
EscapeForUnixShellState::Unquoted => match next_byte {
b' ' | b'!' | b'"' | b'#' | b'$' | b'&' | b'\'' | b'(' | b')' | b'*' | b','
| b';' | b'<' | b'>' | b'?' | b'[' | b'\\' | b']' | b'^' | b'`' | b'{' | b'|'
| b'}' | b'~' => {
self.prefix = Self::make_prefix(&[next_byte]);
Some('\\')
}
_ => Some(next_byte as char),
},
}
}
}
#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
#[non_exhaustive]
pub enum UnixMakefileEscapeKind {
NonRecipe,
RecipeWithoutShellEscaping,
RecipeWithShellEscaping,
}
#[derive(Copy, Clone)]
pub struct EscapeForUnixMakefile<'a> {
s: &'a str,
kind: UnixMakefileEscapeKind,
}
impl<'a> fmt::Debug for EscapeForUnixMakefile<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(self, f)
}
}
impl<'a> fmt::Display for EscapeForUnixMakefile<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.do_write(f, fmt::Write::write_str, fmt::Write::write_char, |_, _| {
Ok(())
})
}
}
impl<'a> EscapeForUnixMakefile<'a> {
fn do_write<S: ?Sized, E>(
&self,
state: &mut S,
write_str: impl Fn(&mut S, &str) -> Result<(), E>,
write_char: impl Fn(&mut S, char) -> Result<(), E>,
add_variable: impl Fn(&mut S, &'static str) -> Result<(), E>,
) -> Result<(), E> {
let escape_recipe_char = |c| match c {
'$' => write_str(state, "$$"),
'\0'..='\x1F' | '\x7F' => {
panic!("can't escape a control character for Unix Makefile: {c:?}");
}
_ => write_char(state, c),
};
match self.kind {
UnixMakefileEscapeKind::NonRecipe => self.s.chars().try_for_each(|c| match c {
'=' => {
add_variable(state, "EQUALS = =")?;
write_str(state, "$(EQUALS)")
}
';' => panic!("can't escape a semicolon (;) for Unix Makefile"),
'$' => write_str(state, "$$"),
'\\' | ' ' | '#' | ':' | '%' | '*' | '?' | '[' | ']' | '~' => {
write_char(state, '\\')?;
write_char(state, c)
}
'\0'..='\x1F' | '\x7F' => {
panic!("can't escape a control character for Unix Makefile: {c:?}");
}
_ => write_char(state, c),
}),
UnixMakefileEscapeKind::RecipeWithoutShellEscaping => {
self.s.chars().try_for_each(escape_recipe_char)
}
UnixMakefileEscapeKind::RecipeWithShellEscaping => {
EscapeForUnixShell::new(self.s).try_for_each(escape_recipe_char)
}
}
}
pub fn new(
s: &'a str,
kind: UnixMakefileEscapeKind,
needed_variables: &mut BTreeSet<&'static str>,
) -> Self {
let retval = Self { s, kind };
let Ok(()) = retval.do_write(
needed_variables,
|_, _| Ok(()),
|_, _| Ok(()),
|needed_variables, variable| -> Result<(), std::convert::Infallible> {
needed_variables.insert(variable);
Ok(())
},
);
retval
}
}
impl JobGraph {
pub fn new() -> Self {
Self::default()
}
fn try_add_item_node(
&mut self,
name: JobItemName,
new_source_job: Option<DynJob>,
new_nodes: &mut HashSet<<JobGraphInner as GraphBase>::NodeId>,
) -> Result<<JobGraphInner as GraphBase>::NodeId, JobGraphError> {
use hashbrown::hash_map::Entry;
match self.items.entry(name) {
Entry::Occupied(item_entry) => {
let node_id = *item_entry.get();
let JobGraphNode::Item {
name: _,
source_job,
} = &mut self.graph[node_id]
else {
unreachable!("known to be an item");
};
if let Some(new_source_job) = new_source_job {
if let Some(source_job) = source_job {
return Err(JobGraphError::MultipleJobsCreateSameOutput {
output_item: item_entry.key().clone(),
existing_job: source_job.clone(),
new_job: new_source_job,
});
} else {
*source_job = Some(new_source_job);
}
}
Ok(node_id)
}
Entry::Vacant(item_entry) => {
let node_id = self.graph.add_node(JobGraphNode::Item {
name,
source_job: new_source_job,
});
new_nodes.insert(node_id);
item_entry.insert(node_id);
Ok(node_id)
}
}
}
pub fn try_add_jobs<I: IntoIterator<Item = DynJob>>(
&mut self,
jobs: I,
) -> Result<(), JobGraphError> {
use hashbrown::hash_map::Entry;
let jobs = jobs.into_iter();
struct RemoveNewNodesOnError<'a> {
this: &'a mut JobGraph,
new_nodes: HashSet<<JobGraphInner as GraphBase>::NodeId>,
}
impl Drop for RemoveNewNodesOnError<'_> {
fn drop(&mut self) {
for node in self.new_nodes.drain() {
self.this.graph.remove_node(node);
}
}
}
let mut remove_new_nodes_on_error = RemoveNewNodesOnError {
this: self,
new_nodes: HashSet::with_capacity_and_hasher(jobs.size_hint().0, Default::default()),
};
let new_nodes = &mut remove_new_nodes_on_error.new_nodes;
let this = &mut *remove_new_nodes_on_error.this;
for job in jobs {
let Entry::Vacant(job_entry) = this.jobs.entry(job.clone()) else {
continue;
};
let job_node_id = this
.graph
.add_node(JobGraphNode::Job(job_entry.key().clone()));
new_nodes.insert(job_node_id);
job_entry.insert(job_node_id);
for name in job.outputs() {
let item_node_id = this.try_add_item_node(name, Some(job.clone()), new_nodes)?;
this.graph.add_edge(job_node_id, item_node_id, ());
}
for name in job.inputs() {
let item_node_id = this.try_add_item_node(name, None, new_nodes)?;
this.graph.add_edge(item_node_id, job_node_id, ());
}
}
match toposort(&this.graph, Some(&mut this.space)) {
Ok(v) => {
this.topological_order = v;
// no need to remove any of the new nodes on drop since we didn't encounter any errors
remove_new_nodes_on_error.new_nodes.clear();
Ok(())
}
Err(_) => {
// there's at least one cycle, find one!
let cycle = kosaraju_scc(&this.graph)
.into_iter()
.find_map(|scc| {
if scc.len() <= 1 {
// can't be a cycle since our graph is bipartite --
// jobs only connect to items, never jobs to jobs or items to items
None
} else {
Some(scc)
}
})
.expect("we know there's a cycle");
let cycle_set = HashSet::from_iter(cycle.iter().copied());
let job = cycle
.into_iter()
.find_map(|node_id| {
if let JobGraphNode::Job(job) = &this.graph[node_id] {
Some(job.clone())
} else {
None
}
})
.expect("a job must be part of the cycle");
let output = job
.outputs()
.into_iter()
.find(|output| cycle_set.contains(&this.items[output]))
.expect("an output must be part of the cycle");
Err(JobGraphError::CycleError { job, output })
}
}
}
#[track_caller]
pub fn add_jobs<I: IntoIterator<Item = DynJob>>(&mut self, jobs: I) {
match self.try_add_jobs(jobs) {
Ok(()) => {}
Err(e) => panic!("error: {e}"),
}
}
pub fn to_unix_makefile(&self) -> String {
self.to_unix_makefile_with_internal_program_prefix(&[program_name_for_internal_jobs()])
}
pub fn to_unix_makefile_with_internal_program_prefix(
&self,
internal_program_prefix: &[Interned<str>],
) -> String {
let mut retval = String::new();
let mut needed_variables = BTreeSet::new();
for &node_id in &self.topological_order {
let JobGraphNode::Job(job) = &self.graph[node_id] else {
continue;
};
let outputs = job.outputs();
if outputs.is_empty() {
retval.push_str(":");
} else {
for output in job.outputs() {
match output {
JobItemName::Path { path } => {
write_str!(
retval,
"{} ",
EscapeForUnixMakefile::new(
&path,
UnixMakefileEscapeKind::NonRecipe,
&mut needed_variables
)
);
}
}
}
if outputs.len() == 1 {
retval.push_str(":");
} else {
retval.push_str("&:");
}
}
for input in job.inputs() {
match input {
JobItemName::Path { path } => {
write_str!(
retval,
" {}",
EscapeForUnixMakefile::new(
&path,
UnixMakefileEscapeKind::NonRecipe,
&mut needed_variables
)
);
}
}
}
retval.push_str("\n\t");
for (index, arg) in job
.command_line_with_internal_program_prefix(internal_program_prefix)
.into_iter()
.enumerate()
{
if index != 0 {
retval.push_str(" ");
}
write_str!(
retval,
"{}",
EscapeForUnixMakefile::new(
&arg,
UnixMakefileEscapeKind::RecipeWithShellEscaping,
&mut needed_variables
)
);
}
retval.push_str("\n\n");
}
if !needed_variables.is_empty() {
retval.insert_str(
0,
&String::from_iter(needed_variables.into_iter().map(|v| format!("{v}\n"))),
);
}
retval
}
pub fn to_unix_shell_script(&self) -> String {
self.to_unix_shell_script_with_internal_program_prefix(&[program_name_for_internal_jobs()])
}
pub fn to_unix_shell_script_with_internal_program_prefix(
&self,
internal_program_prefix: &[Interned<str>],
) -> String {
let mut retval = String::from(
"#!/bin/sh\n\
set -ex\n",
);
for &node_id in &self.topological_order {
let JobGraphNode::Job(job) = &self.graph[node_id] else {
continue;
};
for (index, arg) in job
.command_line_with_internal_program_prefix(internal_program_prefix)
.into_iter()
.enumerate()
{
if index != 0 {
retval.push_str(" ");
}
write_str!(retval, "{}", EscapeForUnixShell::new(&arg));
}
retval.push_str("\n");
}
retval
}
pub fn run(&self, params: &JobParams) -> eyre::Result<()> {
// use scope to auto-join threads on errors
thread::scope(|scope| {
struct WaitingJobState {
job_node_id: <JobGraphInner as GraphBase>::NodeId,
job: DynJob,
inputs: BTreeMap<JobItemName, OnceCell<JobItem>>,
}
let mut ready_jobs = VecDeque::new();
let mut item_name_to_waiting_jobs_map = HashMap::<_, Vec<_>>::default();
for &node_id in &self.topological_order {
let JobGraphNode::Job(job) = &self.graph[node_id] else {
continue;
};
let waiting_job = WaitingJobState {
job_node_id: node_id,
job: job.clone(),
inputs: job
.inputs()
.iter()
.map(|&name| (name, OnceCell::new()))
.collect(),
};
if waiting_job.inputs.is_empty() {
ready_jobs.push_back(waiting_job);
} else {
let waiting_job = Rc::new(waiting_job);
for &input_item in waiting_job.inputs.keys() {
item_name_to_waiting_jobs_map
.entry(input_item)
.or_default()
.push(waiting_job.clone());
}
}
}
struct RunningJob<'scope> {
job: DynJob,
thread: ScopedJoinHandle<'scope, eyre::Result<Vec<JobItem>>>,
}
let mut running_jobs = HashMap::default();
let (finished_jobs_sender, finished_jobs_receiver) = mpsc::channel();
loop {
while let Some(finished_job) = finished_jobs_receiver.try_recv().ok() {
let Some(RunningJob { job, thread }) = running_jobs.remove(&finished_job)
else {
unreachable!();
};
let output_items = thread.join().map_err(panic::resume_unwind)??;
assert!(
output_items.iter().map(JobItem::name).eq(job.outputs()),
"job's run() method returned the wrong output items:\n\
output items:\n\
{output_items:?}\n\
expected outputs:\n\
{:?}\n\
job:\n\
{job:?}",
job.outputs(),
);
for output_item in output_items {
for waiting_job in item_name_to_waiting_jobs_map
.remove(&output_item.name())
.unwrap_or_default()
{
let Ok(()) =
waiting_job.inputs[&output_item.name()].set(output_item.clone())
else {
unreachable!();
};
if let Some(waiting_job) = Rc::into_inner(waiting_job) {
ready_jobs.push_back(waiting_job);
}
}
}
}
if let Some(WaitingJobState {
job_node_id,
job,
inputs,
}) = ready_jobs.pop_front()
{
struct RunningJobInThread<'a> {
job_node_id: <JobGraphInner as GraphBase>::NodeId,
job: DynJob,
inputs: Vec<JobItem>,
params: &'a JobParams,
acquired_job: AcquiredJob,
finished_jobs_sender: mpsc::Sender<<JobGraphInner as GraphBase>::NodeId>,
}
impl RunningJobInThread<'_> {
fn run(mut self) -> eyre::Result<Vec<JobItem>> {
self.job
.run(&self.inputs, self.params, &mut self.acquired_job)
}
}
impl Drop for RunningJobInThread<'_> {
fn drop(&mut self) {
let _ = self.finished_jobs_sender.send(self.job_node_id);
}
}
let name = job.kind().name();
let running_job_in_thread = RunningJobInThread {
job_node_id,
job: job.clone(),
inputs: Vec::from_iter(
inputs
.into_values()
.map(|input| input.into_inner().expect("was set earlier")),
),
params,
acquired_job: AcquiredJob::acquire()?,
finished_jobs_sender: finished_jobs_sender.clone(),
};
running_jobs.insert(
job_node_id,
RunningJob {
job,
thread: thread::Builder::new()
.name(format!("job:{name}"))
.spawn_scoped(scope, move || running_job_in_thread.run())
.expect("failed to spawn thread for job"),
},
);
}
if running_jobs.is_empty() {
assert!(item_name_to_waiting_jobs_map.is_empty());
assert!(ready_jobs.is_empty());
return Ok(());
}
}
})
}
}
impl Extend<DynJob> for JobGraph {
#[track_caller]
fn extend<T: IntoIterator<Item = DynJob>>(&mut self, iter: T) {
self.add_jobs(iter);
}
}
impl FromIterator<DynJob> for JobGraph {
#[track_caller]
fn from_iter<T: IntoIterator<Item = DynJob>>(iter: T) -> Self {
let mut retval = Self::new();
retval.add_jobs(iter);
retval
}
}
impl Serialize for JobGraph {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut serializer = serializer.serialize_seq(Some(self.jobs.len()))?;
for &node_id in &self.topological_order {
let JobGraphNode::Job(job) = &self.graph[node_id] else {
continue;
};
serializer.serialize_element(job)?;
}
serializer.end()
}
}
impl<'de> Deserialize<'de> for JobGraph {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let jobs = Vec::<DynJob>::deserialize(deserializer)?;
let mut retval = JobGraph::new();
retval.try_add_jobs(jobs).map_err(D::Error::custom)?;
Ok(retval)
}
}

341
crates/fayalite/src/build/registry.rs Normal file
View file

@ -0,0 +1,341 @@
// SPDX-License-Identifier: LGPL-3.0-or-later
// See Notices.txt for copyright information
use crate::{
build::{DynJobKind, JobKind},
intern::Interned,
};
use std::{
borrow::Borrow,
cmp::Ordering,
collections::BTreeMap,
fmt,
sync::{Arc, OnceLock, RwLock, RwLockWriteGuard},
};
impl DynJobKind {
pub fn registry() -> JobKindRegistrySnapshot {
JobKindRegistrySnapshot(JobKindRegistry::get())
}
#[track_caller]
pub fn register(self) {
JobKindRegistry::register(JobKindRegistry::lock(), self);
}
}
#[derive(Copy, Clone, PartialEq, Eq)]
struct InternedStrCompareAsStr(Interned<str>);
impl fmt::Debug for InternedStrCompareAsStr {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
impl Ord for InternedStrCompareAsStr {
fn cmp(&self, other: &Self) -> Ordering {
str::cmp(&self.0, &other.0)
}
}
impl PartialOrd for InternedStrCompareAsStr {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl Borrow<str> for InternedStrCompareAsStr {
fn borrow(&self) -> &str {
&self.0
}
}
#[derive(Clone, Debug)]
struct JobKindRegistry {
job_kinds: BTreeMap<InternedStrCompareAsStr, DynJobKind>,
}
enum JobKindRegisterError {
SameName {
name: InternedStrCompareAsStr,
old_job_kind: DynJobKind,
new_job_kind: DynJobKind,
},
}
impl fmt::Display for JobKindRegisterError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::SameName {
name,
old_job_kind,
new_job_kind,
} => write!(
f,
"two different `JobKind` can't share the same name:\n\
{name:?}\n\
old job kind:\n\
{old_job_kind:?}\n\
new job kind:\n\
{new_job_kind:?}",
),
}
}
}
trait JobKindRegistryRegisterLock {
type Locked;
fn lock(self) -> Self::Locked;
fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry;
}
impl JobKindRegistryRegisterLock for &'static RwLock<Arc<JobKindRegistry>> {
type Locked = RwLockWriteGuard<'static, Arc<JobKindRegistry>>;
fn lock(self) -> Self::Locked {
self.write().expect("shouldn't be poisoned")
}
fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry {
Arc::make_mut(locked)
}
}
impl JobKindRegistryRegisterLock for &'_ mut JobKindRegistry {
type Locked = Self;
fn lock(self) -> Self::Locked {
self
}
fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry {
locked
}
}
impl JobKindRegistry {
fn lock() -> &'static RwLock<Arc<Self>> {
static REGISTRY: OnceLock<RwLock<Arc<JobKindRegistry>>> = OnceLock::new();
REGISTRY.get_or_init(Default::default)
}
fn try_register<L: JobKindRegistryRegisterLock>(
lock: L,
job_kind: DynJobKind,
) -> Result<(), JobKindRegisterError> {
use std::collections::btree_map::Entry;
let name = InternedStrCompareAsStr(job_kind.name());
// run user code only outside of lock
let mut locked = lock.lock();
let this = L::make_mut(&mut locked);
let result = match this.job_kinds.entry(name) {
Entry::Occupied(entry) => Err(JobKindRegisterError::SameName {
name,
old_job_kind: entry.get().clone(),
new_job_kind: job_kind,
}),
Entry::Vacant(entry) => {
entry.insert(job_kind);
Ok(())
}
};
drop(locked);
// outside of lock now, so we can test if it's the same DynJobKind
match result {
Err(JobKindRegisterError::SameName {
name: _,
old_job_kind,
new_job_kind,
}) if old_job_kind == new_job_kind => Ok(()),
result => result,
}
}
#[track_caller]
fn register<L: JobKindRegistryRegisterLock>(lock: L, job_kind: DynJobKind) {
match Self::try_register(lock, job_kind) {
Err(e) => panic!("{e}"),
Ok(()) => {}
}
}
fn get() -> Arc<Self> {
Self::lock().read().expect("shouldn't be poisoned").clone()
}
}
impl Default for JobKindRegistry {
fn default() -> Self {
let mut retval = Self {
job_kinds: BTreeMap::new(),
};
for job_kind in [] {
Self::register(&mut retval, job_kind);
}
retval
}
}
#[derive(Clone, Debug)]
pub struct JobKindRegistrySnapshot(Arc<JobKindRegistry>);
impl JobKindRegistrySnapshot {
pub fn get() -> Self {
JobKindRegistrySnapshot(JobKindRegistry::get())
}
pub fn get_by_name<'a>(&'a self, name: &str) -> Option<&'a DynJobKind> {
self.0.job_kinds.get(name)
}
pub fn iter_with_names(&self) -> JobKindRegistryIterWithNames<'_> {
JobKindRegistryIterWithNames(self.0.job_kinds.iter())
}
pub fn iter(&self) -> JobKindRegistryIter<'_> {
JobKindRegistryIter(self.0.job_kinds.values())
}
}
impl<'a> IntoIterator for &'a JobKindRegistrySnapshot {
type Item = &'a DynJobKind;
type IntoIter = JobKindRegistryIter<'a>;
fn into_iter(self) -> Self::IntoIter {
self.iter()
}
}
impl<'a> IntoIterator for &'a mut JobKindRegistrySnapshot {
type Item = &'a DynJobKind;
type IntoIter = JobKindRegistryIter<'a>;
fn into_iter(self) -> Self::IntoIter {
self.iter()
}
}
#[derive(Clone, Debug)]
pub struct JobKindRegistryIter<'a>(
std::collections::btree_map::Values<'a, InternedStrCompareAsStr, DynJobKind>,
);
impl<'a> Iterator for JobKindRegistryIter<'a> {
type Item = &'a DynJobKind;
fn next(&mut self) -> Option<Self::Item> {
self.0.next()
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.0.size_hint()
}
fn count(self) -> usize
where
Self: Sized,
{
self.0.count()
}
fn last(self) -> Option<Self::Item> {
self.0.last()
}
fn nth(&mut self, n: usize) -> Option<Self::Item> {
self.0.nth(n)
}
fn fold<B, F>(self, init: B, f: F) -> B
where
F: FnMut(B, Self::Item) -> B,
{
self.0.fold(init, f)
}
}
impl<'a> std::iter::FusedIterator for JobKindRegistryIter<'a> {}
impl<'a> ExactSizeIterator for JobKindRegistryIter<'a> {}
impl<'a> DoubleEndedIterator for JobKindRegistryIter<'a> {
fn next_back(&mut self) -> Option<Self::Item> {
self.0.next_back()
}
fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
self.0.nth_back(n)
}
fn rfold<B, F>(self, init: B, f: F) -> B
where
F: FnMut(B, Self::Item) -> B,
{
self.0.rfold(init, f)
}
}
#[derive(Clone, Debug)]
pub struct JobKindRegistryIterWithNames<'a>(
std::collections::btree_map::Iter<'a, InternedStrCompareAsStr, DynJobKind>,
);
impl<'a> Iterator for JobKindRegistryIterWithNames<'a> {
type Item = (Interned<str>, &'a DynJobKind);
fn next(&mut self) -> Option<Self::Item> {
self.0.next().map(|(name, job_kind)| (name.0, job_kind))
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.0.size_hint()
}
fn count(self) -> usize
where
Self: Sized,
{
self.0.count()
}
fn last(self) -> Option<Self::Item> {
self.0.last().map(|(name, job_kind)| (name.0, job_kind))
}
fn nth(&mut self, n: usize) -> Option<Self::Item> {
self.0.nth(n).map(|(name, job_kind)| (name.0, job_kind))
}
fn fold<B, F>(self, init: B, f: F) -> B
where
F: FnMut(B, Self::Item) -> B,
{
self.0
.map(|(name, job_kind)| (name.0, job_kind))
.fold(init, f)
}
}
impl<'a> std::iter::FusedIterator for JobKindRegistryIterWithNames<'a> {}
impl<'a> ExactSizeIterator for JobKindRegistryIterWithNames<'a> {}
impl<'a> DoubleEndedIterator for JobKindRegistryIterWithNames<'a> {
fn next_back(&mut self) -> Option<Self::Item> {
self.0
.next_back()
.map(|(name, job_kind)| (name.0, job_kind))
}
fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
self.0
.nth_back(n)
.map(|(name, job_kind)| (name.0, job_kind))
}
fn rfold<B, F>(self, init: B, f: F) -> B
where
F: FnMut(B, Self::Item) -> B,
{
self.0
.map(|(name, job_kind)| (name.0, job_kind))
.rfold(init, f)
}
}
#[track_caller]
pub fn register_job_kind<K: JobKind>(kind: K) {
DynJobKind::new(kind).register();
}

2
crates/fayalite/src/cli.rs
View file

@ -50,7 +50,7 @@ impl From<io::Error> for CliError {
pub trait RunPhase<Arg> {
type Output;
fn run(&self, arg: Arg) -> Result<Self::Output> {
self.run_with_job(arg, &mut AcquiredJob::acquire())
self.run_with_job(arg, &mut AcquiredJob::acquire()?)
}
fn run_with_job(&self, arg: Arg, acquired_job: &mut AcquiredJob) -> Result<Self::Output>;
}

43
crates/fayalite/src/firrtl.rs
View file

@ -7,6 +7,7 @@ use crate::{
DocStringAnnotation, DontTouchAnnotation, SVAttributeAnnotation,
},
array::Array,
build::JobArgs,
bundle::{Bundle, BundleField, BundleType},
clock::Clock,
enum_::{Enum, EnumType, EnumVariant},
@ -42,7 +43,7 @@ use crate::{
use bitvec::slice::BitSlice;
use clap::value_parser;
use num_traits::Signed;
use serde::Serialize;
use serde::{Deserialize, Serialize};
use std::{
cell::{Cell, RefCell},
cmp::Ordering,
@ -2749,14 +2750,23 @@ impl clap::builder::TypedValueParser for OptionSimplifyEnumsKindValueParser {
#[derive(Copy, Clone, PartialEq, Eq, Hash)]
pub struct ExportOptionsPrivate(());
#[derive(clap::Parser, Copy, Clone, PartialEq, Eq, Hash)]
impl ExportOptionsPrivate {
fn private_new() -> Self {
Self(())
}
}
#[derive(clap::Parser, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct ExportOptions {
#[clap(long = "no-simplify-memories", action = clap::ArgAction::SetFalse)]
#[serde(default = "ExportOptions::default_simplify_memories")]
pub simplify_memories: bool,
#[clap(long, value_parser = OptionSimplifyEnumsKindValueParser, default_value = "replace-with-bundle-of-uints")]
#[serde(default = "ExportOptions::default_simplify_enums")]
pub simplify_enums: std::option::Option<SimplifyEnumsKind>, // use std::option::Option instead of Option to avoid clap mis-parsing
#[doc(hidden)]
#[clap(skip = ExportOptionsPrivate(()))]
#[serde(skip, default = "ExportOptionsPrivate::private_new")]
/// `#[non_exhaustive]` except allowing struct update syntax
pub __private: ExportOptionsPrivate,
}
@ -2767,16 +2777,15 @@ impl fmt::Debug for ExportOptions {
}
}
impl ExportOptions {
pub fn to_args(&self) -> Vec<Interned<str>> {
impl JobArgs for ExportOptions {
fn to_args<Args: Extend<Interned<str>> + ?Sized>(&self, args: &mut Args) {
let Self {
simplify_memories,
simplify_enums,
__private: ExportOptionsPrivate(()),
} = self;
let mut retval = Vec::new();
if !*simplify_memories {
retval.push("--no-simplify-memories".intern());
} = *self;
if !simplify_memories {
args.extend(["--no-simplify-memories".intern()]);
}
let simplify_enums = simplify_enums.map(|v| {
clap::ValueEnum::to_possible_value(&v).expect("there are no skipped variants")
@ -2785,10 +2794,18 @@ impl ExportOptions {
None => OptionSimplifyEnumsKindValueParser::NONE_NAME,
Some(v) => v.get_name(),
};
retval.push(str::intern_owned(format!(
args.extend([str::intern_owned(format!(
"--simplify-enums={simplify_enums}"
)));
retval
))]);
}
}
impl ExportOptions {
fn default_simplify_memories() -> bool {
true
}
fn default_simplify_enums() -> Option<SimplifyEnumsKind> {
Some(SimplifyEnumsKind::ReplaceWithBundleOfUInts)
}
fn debug_fmt(
&self,
@ -2846,8 +2863,8 @@ impl ExportOptions {
impl Default for ExportOptions {
fn default() -> Self {
Self {
simplify_memories: true,
simplify_enums: Some(SimplifyEnumsKind::ReplaceWithBundleOfUInts),
simplify_memories: Self::default_simplify_memories(),
simplify_enums: Self::default_simplify_enums(),
__private: ExportOptionsPrivate(()),
}
}

7
crates/fayalite/src/module/transform/simplify_enums.rs
View file

@ -1,3 +1,5 @@
use serde::{Deserialize, Serialize};
// SPDX-License-Identifier: LGPL-3.0-or-later
// See Notices.txt for copyright information
use crate::{
@ -955,12 +957,15 @@ impl Folder for State {
}
}
#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, clap::ValueEnum)]
#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, clap::ValueEnum, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum SimplifyEnumsKind {
SimplifyToEnumsWithNoBody,
#[clap(name = "replace-with-bundle-of-uints")]
#[serde(rename = "replace-with-bundle-of-uints")]
ReplaceWithBundleOfUInts,
#[clap(name = "replace-with-uint")]
#[serde(rename = "replace-with-uint")]
ReplaceWithUInt,
}

7
crates/fayalite/src/util.rs
View file

@ -36,8 +36,11 @@ pub use scoped_ref::ScopedRef;
pub(crate) use misc::chain;
#[doc(inline)]
pub use misc::{
BitSliceWriteWithBase, DebugAsDisplay, DebugAsRawString, MakeMutSlice, RcWriter, interned_bit,
iter_eq_by, slice_range, try_slice_range,
BitSliceWriteWithBase, DebugAsDisplay, DebugAsRawString, MakeMutSlice, RcWriter,
SerdeJsonEscapeIf, SerdeJsonEscapeIfFormatter, SerdeJsonEscapeIfTest,
SerdeJsonEscapeIfTestResult, interned_bit, iter_eq_by, serialize_to_json_ascii,
serialize_to_json_ascii_pretty, serialize_to_json_ascii_pretty_with_indent, slice_range,
try_slice_range,
};
pub mod job_server;

186
crates/fayalite/src/util/job_server.rs
View file

@ -1,26 +1,36 @@
// SPDX-License-Identifier: LGPL-3.0-or-later
// See Notices.txt for copyright information
use ctor::ctor;
use jobslot::{Acquired, Client};
use ctor::{ctor, dtor};
use jobslot::Client;
use std::{
ffi::OsString,
mem,
io, mem,
num::NonZeroUsize,
sync::{Condvar, Mutex, Once, OnceLock},
thread::spawn,
sync::{Mutex, MutexGuard},
};
fn get_or_make_client() -> &'static Client {
#[ctor]
static CLIENT: OnceLock<Client> = unsafe {
match Client::from_env() {
Some(client) => OnceLock::from(client),
None => OnceLock::new(),
}
};
#[ctor]
static CLIENT: Mutex<Option<Option<Client>>> = unsafe { Mutex::new(Some(Client::from_env())) };
CLIENT.get_or_init(|| {
#[dtor]
fn drop_client() {
drop(
match CLIENT.lock() {
Ok(v) => v,
Err(e) => e.into_inner(),
}
.take(),
);
}
fn get_or_make_client() -> Client {
CLIENT
.lock()
.expect("shouldn't have panicked")
.as_mut()
.expect("shutting down")
.get_or_insert_with(|| {
let mut available_parallelism = None;
let mut args = std::env::args_os().skip(1);
while let Some(arg) = args.next() {
@ -52,141 +62,95 @@ fn get_or_make_client() -> &'static Client {
} else {
NonZeroUsize::new(1).unwrap()
};
Client::new_with_fifo(available_parallelism.get() - 1).expect("failed to create job server")
Client::new_with_fifo(available_parallelism.get() - 1)
.expect("failed to create job server")
})
.clone()
}
struct State {
obtained_count: usize,
waiting_count: usize,
available: Vec<Acquired>,
implicit_available: bool,
}
impl State {
fn total_available(&self) -> usize {
self.available.len() + self.implicit_available as usize
}
fn additional_waiting(&self) -> usize {
self.waiting_count.saturating_sub(self.total_available())
}
}
static STATE: Mutex<State> = Mutex::new(State {
obtained_count: 0,
waiting_count: 0,
available: Vec::new(),
implicit_available: true,
});
static COND_VAR: Condvar = Condvar::new();
#[derive(Debug)]
enum AcquiredJobInner {
FromJobServer(Acquired),
ImplicitJob,
}
#[derive(Debug)]
pub struct AcquiredJob {
job: AcquiredJobInner,
client: Client,
}
impl AcquiredJob {
fn start_acquire_thread() {
static STARTED_THREAD: Once = Once::new();
STARTED_THREAD.call_once(|| {
spawn(|| {
let mut acquired = None;
pub fn acquire() -> io::Result<Self> {
let client = get_or_make_client();
struct Waiting {}
impl Waiting {
fn done(self) -> MutexGuard<'static, State> {
mem::forget(self);
let mut state = STATE.lock().unwrap();
loop {
state = if state.additional_waiting() == 0 {
if acquired.is_some() {
drop(state);
drop(acquired.take()); // drop Acquired outside of lock
STATE.lock().unwrap()
} else {
COND_VAR.wait(state).unwrap()
}
} else if acquired.is_some() {
// allocate space before moving Acquired to ensure we
// drop Acquired outside of the lock on panic
state.available.reserve(1);
state.available.push(acquired.take().unwrap());
COND_VAR.notify_all();
state.waiting_count -= 1;
state
} else {
drop(state);
acquired = Some(
client
.acquire()
.expect("can't acquire token from job server"),
);
STATE.lock().unwrap()
};
}
});
});
}
fn acquire_inner(block: bool) -> Option<Self> {
Self::start_acquire_thread();
impl Drop for Waiting {
fn drop(&mut self) {
STATE.lock().unwrap().waiting_count -= 1;
}
}
let mut state = STATE.lock().unwrap();
loop {
if let Some(acquired) = state.available.pop() {
return Some(Self {
job: AcquiredJobInner::FromJobServer(acquired),
});
}
if state.implicit_available {
state.implicit_available = false;
return Some(Self {
job: AcquiredJobInner::ImplicitJob,
});
}
if !block {
return None;
if state.obtained_count == 0 && state.waiting_count == 0 {
state.obtained_count = 1; // get implicit token
return Ok(Self { client });
}
state.waiting_count += 1;
state = COND_VAR.wait(state).unwrap();
state.waiting_count -= 1;
}
}
pub fn try_acquire() -> Option<Self> {
Self::acquire_inner(false)
}
pub fn acquire() -> Self {
Self::acquire_inner(true).expect("failed to acquire token")
drop(state);
let waiting = Waiting {};
client.acquire_raw()?;
state = waiting.done();
state.obtained_count = state
.obtained_count
.checked_add(1)
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "obtained_count overflowed"))?;
drop(state);
Ok(Self { client })
}
pub fn run_command<R>(
&mut self,
cmd: std::process::Command,
f: impl FnOnce(&mut std::process::Command) -> std::io::Result<R>,
) -> std::io::Result<R> {
get_or_make_client().configure_make_and_run_with_fifo(cmd, f)
self.client.configure_make_and_run_with_fifo(cmd, f)
}
}
impl Drop for AcquiredJob {
fn drop(&mut self) {
let mut state = STATE.lock().unwrap();
match &self.job {
AcquiredJobInner::FromJobServer(_) => {
if state.waiting_count > state.available.len() + state.implicit_available as usize {
// allocate space before moving Acquired to ensure we
// drop Acquired outside of the lock on panic
state.available.reserve(1);
let AcquiredJobInner::FromJobServer(acquired) =
mem::replace(&mut self.job, AcquiredJobInner::ImplicitJob)
else {
unreachable!()
};
state.available.push(acquired);
COND_VAR.notify_all();
match &mut *state {
State {
obtained_count: 0, ..
} => unreachable!(),
State {
obtained_count: obtained_count @ 1,
waiting_count,
} => {
*obtained_count = 0; // drop implicit token
let any_waiting = *waiting_count != 0;
drop(state);
if any_waiting {
// we have the implicit token, but some other thread is trying to acquire a token,
// release the implicit token so they can acquire it.
let _ = self.client.release_raw(); // we're in drop, just ignore errors since we at least tried
}
}
AcquiredJobInner::ImplicitJob => {
state.implicit_available = true;
if state.waiting_count > state.available.len() {
COND_VAR.notify_all();
}
State { obtained_count, .. } => {
*obtained_count = obtained_count.saturating_sub(1);
drop(state);
let _ = self.client.release_raw(); // we're in drop, just ignore errors since we at least tried
}
}
}

321
crates/fayalite/src/util/misc.rs
View file

@ -5,6 +5,7 @@ use bitvec::{bits, order::Lsb0, slice::BitSlice, view::BitView};
use std::{
cell::Cell,
fmt::{self, Debug, Write},
io,
ops::{Bound, Range, RangeBounds},
rc::Rc,
sync::{Arc, OnceLock},
@ -243,3 +244,323 @@ pub fn try_slice_range<R: RangeBounds<usize>>(range: R, size: usize) -> Option<R
pub fn slice_range<R: RangeBounds<usize>>(range: R, size: usize) -> Range<usize> {
try_slice_range(range, size).expect("range out of bounds")
}
pub trait SerdeJsonEscapeIfTest {
fn char_needs_escape(&mut self, ch: char) -> serde_json::Result<bool>;
}
pub trait SerdeJsonEscapeIfTestResult {
fn to_result(self) -> serde_json::Result<bool>;
}
impl SerdeJsonEscapeIfTestResult for bool {
fn to_result(self) -> serde_json::Result<bool> {
Ok(self)
}
}
impl<E: Into<serde_json::Error>> SerdeJsonEscapeIfTestResult for Result<bool, E> {
fn to_result(self) -> serde_json::Result<bool> {
self.map_err(Into::into)
}
}
impl<T: ?Sized + FnMut(char) -> R, R: SerdeJsonEscapeIfTestResult> SerdeJsonEscapeIfTest for T {
fn char_needs_escape(&mut self, ch: char) -> serde_json::Result<bool> {
self(ch).to_result()
}
}
pub trait SerdeJsonEscapeIfFormatter: serde_json::ser::Formatter {
fn write_unicode_escape<W>(&mut self, writer: &mut W, ch: char) -> io::Result<()>
where
W: ?Sized + io::Write,
{
for utf16 in ch.encode_utf16(&mut [0; 2]) {
write!(writer, "\\u{utf16:04x}")?;
}
Ok(())
}
}
impl SerdeJsonEscapeIfFormatter for serde_json::ser::CompactFormatter {}
impl SerdeJsonEscapeIfFormatter for serde_json::ser::PrettyFormatter<'_> {}
pub struct SerdeJsonEscapeIf<Test, Base = serde_json::ser::CompactFormatter> {
pub base: Base,
pub test: Test,
}
impl<Test: SerdeJsonEscapeIfTest, Base: SerdeJsonEscapeIfFormatter> serde_json::ser::Formatter
for SerdeJsonEscapeIf<Test, Base>
{
fn write_null<W>(&mut self, writer: &mut W) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_null(writer)
}
fn write_bool<W>(&mut self, writer: &mut W, value: bool) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_bool(writer, value)
}
fn write_i8<W>(&mut self, writer: &mut W, value: i8) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_i8(writer, value)
}
fn write_i16<W>(&mut self, writer: &mut W, value: i16) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_i16(writer, value)
}
fn write_i32<W>(&mut self, writer: &mut W, value: i32) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_i32(writer, value)
}
fn write_i64<W>(&mut self, writer: &mut W, value: i64) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_i64(writer, value)
}
fn write_i128<W>(&mut self, writer: &mut W, value: i128) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_i128(writer, value)
}
fn write_u8<W>(&mut self, writer: &mut W, value: u8) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_u8(writer, value)
}
fn write_u16<W>(&mut self, writer: &mut W, value: u16) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_u16(writer, value)
}
fn write_u32<W>(&mut self, writer: &mut W, value: u32) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_u32(writer, value)
}
fn write_u64<W>(&mut self, writer: &mut W, value: u64) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_u64(writer, value)
}
fn write_u128<W>(&mut self, writer: &mut W, value: u128) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_u128(writer, value)
}
fn write_f32<W>(&mut self, writer: &mut W, value: f32) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_f32(writer, value)
}
fn write_f64<W>(&mut self, writer: &mut W, value: f64) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_f64(writer, value)
}
fn write_number_str<W>(&mut self, writer: &mut W, value: &str) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_number_str(writer, value)
}
fn begin_string<W>(&mut self, writer: &mut W) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.begin_string(writer)
}
fn end_string<W>(&mut self, writer: &mut W) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.end_string(writer)
}
fn write_string_fragment<W>(&mut self, writer: &mut W, mut fragment: &str) -> io::Result<()>
where
W: ?Sized + io::Write,
{
while let Some((next_escape_index, next_escape_char)) = fragment
.char_indices()
.find_map(|(index, ch)| match self.test.char_needs_escape(ch) {
Ok(false) => None,
Ok(true) => Some(Ok((index, ch))),
Err(e) => Some(Err(e)),
})
.transpose()?
{
let (no_escapes, rest) = fragment.split_at(next_escape_index);
fragment = &rest[next_escape_char.len_utf8()..];
self.base.write_string_fragment(writer, no_escapes)?;
self.base.write_unicode_escape(writer, next_escape_char)?;
}
self.base.write_string_fragment(writer, fragment)
}
fn write_char_escape<W>(
&mut self,
writer: &mut W,
char_escape: serde_json::ser::CharEscape,
) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_char_escape(writer, char_escape)
}
fn write_byte_array<W>(&mut self, writer: &mut W, value: &[u8]) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_byte_array(writer, value)
}
fn begin_array<W>(&mut self, writer: &mut W) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.begin_array(writer)
}
fn end_array<W>(&mut self, writer: &mut W) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.end_array(writer)
}
fn begin_array_value<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.begin_array_value(writer, first)
}
fn end_array_value<W>(&mut self, writer: &mut W) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.end_array_value(writer)
}
fn begin_object<W>(&mut self, writer: &mut W) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.begin_object(writer)
}
fn end_object<W>(&mut self, writer: &mut W) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.end_object(writer)
}
fn begin_object_key<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.begin_object_key(writer, first)
}
fn end_object_key<W>(&mut self, writer: &mut W) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.end_object_key(writer)
}
fn begin_object_value<W>(&mut self, writer: &mut W) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.begin_object_value(writer)
}
fn end_object_value<W>(&mut self, writer: &mut W) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.end_object_value(writer)
}
fn write_raw_fragment<W>(&mut self, writer: &mut W, fragment: &str) -> io::Result<()>
where
W: ?Sized + io::Write,
{
self.base.write_raw_fragment(writer, fragment)
}
}
fn serialize_to_json_ascii_helper<F: SerdeJsonEscapeIfFormatter, S: serde::Serialize + ?Sized>(
v: &S,
base: F,
) -> serde_json::Result<String> {
let mut retval = Vec::new();
v.serialize(&mut serde_json::ser::Serializer::with_formatter(
&mut retval,
SerdeJsonEscapeIf {
base,
test: |ch| ch < '\x20' || ch > '\x7F',
},
))?;
String::from_utf8(retval).map_err(|_| serde::ser::Error::custom("invalid UTF-8"))
}
pub fn serialize_to_json_ascii<T: serde::Serialize + ?Sized>(v: &T) -> serde_json::Result<String> {
serialize_to_json_ascii_helper(v, serde_json::ser::CompactFormatter)
}
pub fn serialize_to_json_ascii_pretty<T: serde::Serialize + ?Sized>(
v: &T,
) -> serde_json::Result<String> {
serialize_to_json_ascii_helper(v, serde_json::ser::PrettyFormatter::new())
}
pub fn serialize_to_json_ascii_pretty_with_indent<T: serde::Serialize + ?Sized>(
v: &T,
indent: &str,
) -> serde_json::Result<String> {
serialize_to_json_ascii_helper(
v,
serde_json::ser::PrettyFormatter::with_indent(indent.as_bytes()),
)
}