WIP refactoring to have JobKind be internal jobs

crates/fayalite/src/build/external.rs

@@ -2,7 +2,7 @@
 // See Notices.txt for copyright information
 
 use crate::{
-    build::{DynJob, EscapeForUnixShell, JobItem, JobItemName, JobKind},
+    build::{DynJob, JobItem, JobItemName, JobKind, graph::EscapeForUnixShell},
     intern::{Intern, Interned},
     util::{job_server::AcquiredJob, streaming_read_utf8::streaming_read_utf8},
 };
@@ -641,6 +641,7 @@ impl TemplatedExternalJobKind {
     }
 }
 
+#[cfg(todo)]
 impl JobKind for TemplatedExternalJobKind {
     type Job = TemplatedExternalJob;
 

crates/fayalite/src/build/graph.rs

@@ -0,0 +1,736 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// See Notices.txt for copyright information
+
+use crate::{
+    build::{DynJob, JobItem, JobItemName},
+    util::{HashMap, HashSet, job_server::AcquiredJob},
+};
+use petgraph::{
+    algo::{DfsSpace, kosaraju_scc, toposort},
+    graph::DiGraph,
+    visit::{GraphBase, Visitable},
+};
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use std::{
+    cell::OnceCell,
+    collections::{BTreeMap, BTreeSet, VecDeque},
+    fmt::{self, Write},
+    panic,
+    rc::Rc,
+    sync::mpsc,
+    thread::{self, ScopedJoinHandle},
+};
+
+macro_rules! write_str {
+    ($s:expr, $($rest:tt)*) => {
+        write!($s, $($rest)*).expect("String::write_fmt can't fail")
+    };
+}
+
+#[derive(Clone, Debug)]
+enum JobGraphNode {
+    Job(DynJob),
+    Item {
+        #[allow(dead_code, reason = "name used for debugging")]
+        name: JobItemName,
+        source_job: Option<DynJob>,
+    },
+}
+
+type JobGraphInner = DiGraph<JobGraphNode, ()>;
+
+#[derive(Clone, Default)]
+pub struct JobGraph {
+    jobs: HashMap<DynJob, <JobGraphInner as GraphBase>::NodeId>,
+    items: HashMap<JobItemName, <JobGraphInner as GraphBase>::NodeId>,
+    graph: JobGraphInner,
+    topological_order: Vec<<JobGraphInner as GraphBase>::NodeId>,
+    space: DfsSpace<<JobGraphInner as GraphBase>::NodeId, <JobGraphInner as Visitable>::Map>,
+}
+
+impl fmt::Debug for JobGraph {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let Self {
+            jobs: _,
+            items: _,
+            graph,
+            topological_order,
+            space: _,
+        } = self;
+        f.debug_struct("JobGraph")
+            .field("graph", graph)
+            .field("topological_order", topological_order)
+            .finish_non_exhaustive()
+    }
+}
+
+#[derive(Clone, Debug)]
+pub enum JobGraphError {
+    CycleError {
+        job: DynJob,
+        output: JobItemName,
+    },
+    MultipleJobsCreateSameOutput {
+        output_item: JobItemName,
+        existing_job: DynJob,
+        new_job: DynJob,
+    },
+}
+
+impl std::error::Error for JobGraphError {}
+
+impl fmt::Display for JobGraphError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::CycleError { job, output } => write!(
+                f,
+                "job can't be added to job graph because it would introduce a cyclic dependency through this job output:\n\
+                {output:?}\n\
+                job:\n{job:?}",
+            ),
+            JobGraphError::MultipleJobsCreateSameOutput {
+                output_item,
+                existing_job,
+                new_job,
+            } => write!(
+                f,
+                "job can't be added to job graph because the new job has an output that is also produced by an existing job.\n\
+                conflicting output:\n\
+                {output_item:?}\n\
+                existing job:\n\
+                {existing_job:?}\n\
+                new job:\n\
+                {new_job:?}",
+            ),
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+enum EscapeForUnixShellState {
+    DollarSingleQuote,
+    SingleQuote,
+    Unquoted,
+}
+
+#[derive(Clone)]
+pub struct EscapeForUnixShell<'a> {
+    state: EscapeForUnixShellState,
+    prefix: [u8; 3],
+    bytes: &'a [u8],
+}
+
+impl<'a> fmt::Debug for EscapeForUnixShell<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(self, f)
+    }
+}
+
+impl<'a> fmt::Display for EscapeForUnixShell<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        for c in self.clone() {
+            f.write_char(c)?;
+        }
+        Ok(())
+    }
+}
+
+impl<'a> EscapeForUnixShell<'a> {
+    pub fn new(s: &'a str) -> Self {
+        Self::from_bytes(s.as_bytes())
+    }
+    fn make_prefix(bytes: &[u8]) -> [u8; 3] {
+        let mut prefix = [0; 3];
+        prefix[..bytes.len()].copy_from_slice(bytes);
+        prefix
+    }
+    pub fn from_bytes(bytes: &'a [u8]) -> Self {
+        let mut needs_single_quote = bytes.is_empty();
+        for &b in bytes {
+            match b {
+                b'!' | b'\'' | b'\"' | b' ' => needs_single_quote = true,
+                0..0x20 | 0x7F.. => {
+                    return Self {
+                        state: EscapeForUnixShellState::DollarSingleQuote,
+                        prefix: Self::make_prefix(b"$'"),
+                        bytes,
+                    };
+                }
+                _ => {}
+            }
+        }
+        if needs_single_quote {
+            Self {
+                state: EscapeForUnixShellState::SingleQuote,
+                prefix: Self::make_prefix(b"'"),
+                bytes,
+            }
+        } else {
+            Self {
+                state: EscapeForUnixShellState::Unquoted,
+                prefix: Self::make_prefix(b""),
+                bytes,
+            }
+        }
+    }
+}
+
+impl Iterator for EscapeForUnixShell<'_> {
+    type Item = char;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match &mut self.prefix {
+            [0, 0, 0] => {}
+            [0, 0, v] | // find first
+            [0, v, _] | // non-zero byte
+            [v, _, _] => {
+                let retval = *v as char;
+                *v = 0;
+                return Some(retval);
+            }
+        }
+        let Some(&next_byte) = self.bytes.split_off_first() else {
+            return match self.state {
+                EscapeForUnixShellState::DollarSingleQuote
+                | EscapeForUnixShellState::SingleQuote => {
+                    self.state = EscapeForUnixShellState::Unquoted;
+                    Some('\'')
+                }
+                EscapeForUnixShellState::Unquoted => None,
+            };
+        };
+        match self.state {
+            EscapeForUnixShellState::DollarSingleQuote => match next_byte {
+                b'\'' | b'\\' => {
+                    self.prefix = Self::make_prefix(&[next_byte]);
+                    Some('\\')
+                }
+                b'\t' => {
+                    self.prefix = Self::make_prefix(b"t");
+                    Some('\\')
+                }
+                b'\n' => {
+                    self.prefix = Self::make_prefix(b"n");
+                    Some('\\')
+                }
+                b'\r' => {
+                    self.prefix = Self::make_prefix(b"r");
+                    Some('\\')
+                }
+                0x20..=0x7E => Some(next_byte as char),
+                _ => {
+                    self.prefix = [
+                        b'x',
+                        char::from_digit(next_byte as u32 >> 4, 0x10).expect("known to be in range")
+                            as u8,
+                        char::from_digit(next_byte as u32 & 0xF, 0x10)
+                            .expect("known to be in range") as u8,
+                    ];
+                    Some('\\')
+                }
+            },
+            EscapeForUnixShellState::SingleQuote => {
+                if next_byte == b'\'' {
+                    self.prefix = Self::make_prefix(b"\\''");
+                    Some('\'')
+                } else {
+                    Some(next_byte as char)
+                }
+            }
+            EscapeForUnixShellState::Unquoted => match next_byte {
+                b' ' | b'!' | b'"' | b'#' | b'$' | b'&' | b'\'' | b'(' | b')' | b'*' | b','
+                | b';' | b'<' | b'>' | b'?' | b'[' | b'\\' | b']' | b'^' | b'`' | b'{' | b'|'
+                | b'}' | b'~' => {
+                    self.prefix = Self::make_prefix(&[next_byte]);
+                    Some('\\')
+                }
+                _ => Some(next_byte as char),
+            },
+        }
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+#[non_exhaustive]
+pub enum UnixMakefileEscapeKind {
+    NonRecipe,
+    RecipeWithoutShellEscaping,
+    RecipeWithShellEscaping,
+}
+
+#[derive(Copy, Clone)]
+pub struct EscapeForUnixMakefile<'a> {
+    s: &'a str,
+    kind: UnixMakefileEscapeKind,
+}
+
+impl<'a> fmt::Debug for EscapeForUnixMakefile<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(self, f)
+    }
+}
+
+impl<'a> fmt::Display for EscapeForUnixMakefile<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.do_write(f, fmt::Write::write_str, fmt::Write::write_char, |_, _| {
+            Ok(())
+        })
+    }
+}
+
+impl<'a> EscapeForUnixMakefile<'a> {
+    fn do_write<S: ?Sized, E>(
+        &self,
+        state: &mut S,
+        write_str: impl Fn(&mut S, &str) -> Result<(), E>,
+        write_char: impl Fn(&mut S, char) -> Result<(), E>,
+        add_variable: impl Fn(&mut S, &'static str) -> Result<(), E>,
+    ) -> Result<(), E> {
+        let escape_recipe_char = |c| match c {
+            '$' => write_str(state, "$$"),
+            '\0'..='\x1F' | '\x7F' => {
+                panic!("can't escape a control character for Unix Makefile: {c:?}");
+            }
+            _ => write_char(state, c),
+        };
+        match self.kind {
+            UnixMakefileEscapeKind::NonRecipe => self.s.chars().try_for_each(|c| match c {
+                '=' => {
+                    add_variable(state, "EQUALS = =")?;
+                    write_str(state, "$(EQUALS)")
+                }
+                ';' => panic!("can't escape a semicolon (;) for Unix Makefile"),
+                '$' => write_str(state, "$$"),
+                '\\' | ' ' | '#' | ':' | '%' | '*' | '?' | '[' | ']' | '~' => {
+                    write_char(state, '\\')?;
+                    write_char(state, c)
+                }
+                '\0'..='\x1F' | '\x7F' => {
+                    panic!("can't escape a control character for Unix Makefile: {c:?}");
+                }
+                _ => write_char(state, c),
+            }),
+            UnixMakefileEscapeKind::RecipeWithoutShellEscaping => {
+                self.s.chars().try_for_each(escape_recipe_char)
+            }
+            UnixMakefileEscapeKind::RecipeWithShellEscaping => {
+                EscapeForUnixShell::new(self.s).try_for_each(escape_recipe_char)
+            }
+        }
+    }
+    pub fn new(
+        s: &'a str,
+        kind: UnixMakefileEscapeKind,
+        needed_variables: &mut BTreeSet<&'static str>,
+    ) -> Self {
+        let retval = Self { s, kind };
+        let Ok(()) = retval.do_write(
+            needed_variables,
+            |_, _| Ok(()),
+            |_, _| Ok(()),
+            |needed_variables, variable| -> Result<(), std::convert::Infallible> {
+                needed_variables.insert(variable);
+                Ok(())
+            },
+        );
+        retval
+    }
+}
+
+impl JobGraph {
+    pub fn new() -> Self {
+        Self::default()
+    }
+    fn try_add_item_node(
+        &mut self,
+        name: JobItemName,
+        new_source_job: Option<DynJob>,
+        new_nodes: &mut HashSet<<JobGraphInner as GraphBase>::NodeId>,
+    ) -> Result<<JobGraphInner as GraphBase>::NodeId, JobGraphError> {
+        use hashbrown::hash_map::Entry;
+        match self.items.entry(name) {
+            Entry::Occupied(item_entry) => {
+                let node_id = *item_entry.get();
+                let JobGraphNode::Item {
+                    name: _,
+                    source_job,
+                } = &mut self.graph[node_id]
+                else {
+                    unreachable!("known to be an item");
+                };
+                if let Some(new_source_job) = new_source_job {
+                    if let Some(source_job) = source_job {
+                        return Err(JobGraphError::MultipleJobsCreateSameOutput {
+                            output_item: item_entry.key().clone(),
+                            existing_job: source_job.clone(),
+                            new_job: new_source_job,
+                        });
+                    } else {
+                        *source_job = Some(new_source_job);
+                    }
+                }
+                Ok(node_id)
+            }
+            Entry::Vacant(item_entry) => {
+                let node_id = self.graph.add_node(JobGraphNode::Item {
+                    name,
+                    source_job: new_source_job,
+                });
+                new_nodes.insert(node_id);
+                item_entry.insert(node_id);
+                Ok(node_id)
+            }
+        }
+    }
+    pub fn try_add_jobs<I: IntoIterator<Item = DynJob>>(
+        &mut self,
+        jobs: I,
+    ) -> Result<(), JobGraphError> {
+        use hashbrown::hash_map::Entry;
+        let jobs = jobs.into_iter();
+        struct RemoveNewNodesOnError<'a> {
+            this: &'a mut JobGraph,
+            new_nodes: HashSet<<JobGraphInner as GraphBase>::NodeId>,
+        }
+        impl Drop for RemoveNewNodesOnError<'_> {
+            fn drop(&mut self) {
+                for node in self.new_nodes.drain() {
+                    self.this.graph.remove_node(node);
+                }
+            }
+        }
+        let mut remove_new_nodes_on_error = RemoveNewNodesOnError {
+            this: self,
+            new_nodes: HashSet::with_capacity_and_hasher(jobs.size_hint().0, Default::default()),
+        };
+        let new_nodes = &mut remove_new_nodes_on_error.new_nodes;
+        let this = &mut *remove_new_nodes_on_error.this;
+        let mut worklist = Vec::from_iter(jobs);
+        while let Some(job) = worklist.pop() {
+            let Entry::Vacant(job_entry) = this.jobs.entry(job.clone()) else {
+                continue;
+            };
+            let job_node_id = this
+                .graph
+                .add_node(JobGraphNode::Job(job_entry.key().clone()));
+            new_nodes.insert(job_node_id);
+            job_entry.insert(job_node_id);
+            for name in job.outputs() {
+                let item_node_id = this.try_add_item_node(name, Some(job.clone()), new_nodes)?;
+                this.graph.add_edge(job_node_id, item_node_id, ());
+            }
+            for (&name, direct_dependency) in job.inputs_and_direct_dependencies() {
+                worklist.extend(direct_dependency.clone());
+                let item_node_id = this.try_add_item_node(name, None, new_nodes)?;
+                this.graph.add_edge(item_node_id, job_node_id, ());
+            }
+        }
+        match toposort(&this.graph, Some(&mut this.space)) {
+            Ok(v) => {
+                this.topological_order = v;
+                // no need to remove any of the new nodes on drop since we didn't encounter any errors
+                remove_new_nodes_on_error.new_nodes.clear();
+                Ok(())
+            }
+            Err(_) => {
+                // there's at least one cycle, find one!
+                let cycle = kosaraju_scc(&this.graph)
+                    .into_iter()
+                    .find_map(|scc| {
+                        if scc.len() <= 1 {
+                            // can't be a cycle since our graph is bipartite --
+                            // jobs only connect to items, never jobs to jobs or items to items
+                            None
+                        } else {
+                            Some(scc)
+                        }
+                    })
+                    .expect("we know there's a cycle");
+                let cycle_set = HashSet::from_iter(cycle.iter().copied());
+                let job = cycle
+                    .into_iter()
+                    .find_map(|node_id| {
+                        if let JobGraphNode::Job(job) = &this.graph[node_id] {
+                            Some(job.clone())
+                        } else {
+                            None
+                        }
+                    })
+                    .expect("a job must be part of the cycle");
+                let output = job
+                    .outputs()
+                    .into_iter()
+                    .find(|output| cycle_set.contains(&this.items[output]))
+                    .expect("an output must be part of the cycle");
+                Err(JobGraphError::CycleError { job, output })
+            }
+        }
+    }
+    #[track_caller]
+    pub fn add_jobs<I: IntoIterator<Item = DynJob>>(&mut self, jobs: I) {
+        match self.try_add_jobs(jobs) {
+            Ok(()) => {}
+            Err(e) => panic!("error: {e}"),
+        }
+    }
+    pub fn to_unix_makefile(&self) -> String {
+        let mut retval = String::new();
+        let mut needed_variables = BTreeSet::new();
+        for &node_id in &self.topological_order {
+            let JobGraphNode::Job(job) = &self.graph[node_id] else {
+                continue;
+            };
+            for (index, output) in job.outputs().into_iter().enumerate() {
+                match output {
+                    JobItemName::Module { .. } => continue,
+                    JobItemName::File { path } => {
+                        if index != 0 {
+                            retval.push_str(" ");
+                        }
+                        write_str!(
+                            retval,
+                            "{}",
+                            EscapeForUnixMakefile::new(
+                                &path,
+                                UnixMakefileEscapeKind::NonRecipe,
+                                &mut needed_variables
+                            )
+                        );
+                    }
+                }
+            }
+            retval.push_str(":");
+            for input in job.inputs_and_direct_dependencies().keys() {
+                match input {
+                    JobItemName::Module { .. } => continue,
+                    JobItemName::File { path } => {
+                        write_str!(
+                            retval,
+                            " {}",
+                            EscapeForUnixMakefile::new(
+                                &path,
+                                UnixMakefileEscapeKind::NonRecipe,
+                                &mut needed_variables
+                            )
+                        );
+                    }
+                }
+            }
+            retval.push_str("\n\t");
+            for (index, arg) in job.to_command_line().into_iter().enumerate() {
+                if index != 0 {
+                    retval.push_str(" ");
+                }
+                write_str!(
+                    retval,
+                    "{}",
+                    EscapeForUnixMakefile::new(
+                        &arg,
+                        UnixMakefileEscapeKind::RecipeWithShellEscaping,
+                        &mut needed_variables
+                    )
+                );
+            }
+            retval.push_str("\n\n");
+        }
+        if !needed_variables.is_empty() {
+            retval.insert_str(
+                0,
+                &String::from_iter(needed_variables.into_iter().map(|v| format!("{v}\n"))),
+            );
+        }
+        retval
+    }
+    pub fn to_unix_shell_script(&self) -> String {
+        let mut retval = String::from(
+            "#!/bin/sh\n\
+            set -ex\n",
+        );
+        for &node_id in &self.topological_order {
+            let JobGraphNode::Job(job) = &self.graph[node_id] else {
+                continue;
+            };
+            for (index, arg) in job.to_command_line().into_iter().enumerate() {
+                if index != 0 {
+                    retval.push_str(" ");
+                }
+                write_str!(retval, "{}", EscapeForUnixShell::new(&arg));
+            }
+            retval.push_str("\n");
+        }
+        retval
+    }
+    pub fn run(&self) -> eyre::Result<()> {
+        // use scope to auto-join threads on errors
+        thread::scope(|scope| {
+            struct WaitingJobState {
+                job_node_id: <JobGraphInner as GraphBase>::NodeId,
+                job: DynJob,
+                inputs: BTreeMap<JobItemName, OnceCell<JobItem>>,
+            }
+            let mut ready_jobs = VecDeque::new();
+            let mut item_name_to_waiting_jobs_map = HashMap::<_, Vec<_>>::default();
+            for &node_id in &self.topological_order {
+                let JobGraphNode::Job(job) = &self.graph[node_id] else {
+                    continue;
+                };
+                let waiting_job = WaitingJobState {
+                    job_node_id: node_id,
+                    job: job.clone(),
+                    inputs: job
+                        .inputs_and_direct_dependencies()
+                        .keys()
+                        .map(|&name| (name, OnceCell::new()))
+                        .collect(),
+                };
+                if waiting_job.inputs.is_empty() {
+                    ready_jobs.push_back(waiting_job);
+                } else {
+                    let waiting_job = Rc::new(waiting_job);
+                    for &input_item in waiting_job.inputs.keys() {
+                        item_name_to_waiting_jobs_map
+                            .entry(input_item)
+                            .or_default()
+                            .push(waiting_job.clone());
+                    }
+                }
+            }
+            struct RunningJob<'scope> {
+                job: DynJob,
+                thread: ScopedJoinHandle<'scope, eyre::Result<Vec<JobItem>>>,
+            }
+            let mut running_jobs = HashMap::default();
+            let (finished_jobs_sender, finished_jobs_receiver) = mpsc::channel();
+            loop {
+                while let Some(finished_job) = finished_jobs_receiver.try_recv().ok() {
+                    let Some(RunningJob { job, thread }) = running_jobs.remove(&finished_job)
+                    else {
+                        unreachable!();
+                    };
+                    let output_items = thread.join().map_err(panic::resume_unwind)??;
+                    assert!(
+                        output_items.iter().map(JobItem::name).eq(job.outputs()),
+                        "job's run() method returned the wrong output items:\n\
+                        output items:\n\
+                        {output_items:?}\n\
+                        expected outputs:\n\
+                        {:?}\n\
+                        job:\n\
+                        {job:?}",
+                        job.outputs(),
+                    );
+                    for output_item in output_items {
+                        for waiting_job in item_name_to_waiting_jobs_map
+                            .remove(&output_item.name())
+                            .unwrap_or_default()
+                        {
+                            let Ok(()) =
+                                waiting_job.inputs[&output_item.name()].set(output_item.clone())
+                            else {
+                                unreachable!();
+                            };
+                            if let Some(waiting_job) = Rc::into_inner(waiting_job) {
+                                ready_jobs.push_back(waiting_job);
+                            }
+                        }
+                    }
+                }
+                if let Some(WaitingJobState {
+                    job_node_id,
+                    job,
+                    inputs,
+                }) = ready_jobs.pop_front()
+                {
+                    struct RunningJobInThread {
+                        job_node_id: <JobGraphInner as GraphBase>::NodeId,
+                        job: DynJob,
+                        inputs: Vec<JobItem>,
+                        acquired_job: AcquiredJob,
+                        finished_jobs_sender: mpsc::Sender<<JobGraphInner as GraphBase>::NodeId>,
+                    }
+                    impl RunningJobInThread {
+                        fn run(mut self) -> eyre::Result<Vec<JobItem>> {
+                            self.job.run(&self.inputs, &mut self.acquired_job)
+                        }
+                    }
+                    impl Drop for RunningJobInThread {
+                        fn drop(&mut self) {
+                            let _ = self.finished_jobs_sender.send(self.job_node_id);
+                        }
+                    }
+                    let name = job.debug_name();
+                    let running_job_in_thread = RunningJobInThread {
+                        job_node_id,
+                        job: job.clone(),
+                        inputs: Vec::from_iter(
+                            inputs
+                                .into_values()
+                                .map(|input| input.into_inner().expect("was set earlier")),
+                        ),
+                        acquired_job: AcquiredJob::acquire(),
+                        finished_jobs_sender: finished_jobs_sender.clone(),
+                    };
+                    running_jobs.insert(
+                        job_node_id,
+                        RunningJob {
+                            job,
+                            thread: thread::Builder::new()
+                                .name(name)
+                                .spawn_scoped(scope, move || running_job_in_thread.run())
+                                .expect("failed to spawn thread for job"),
+                        },
+                    );
+                }
+                if running_jobs.is_empty() {
+                    assert!(item_name_to_waiting_jobs_map.is_empty());
+                    assert!(ready_jobs.is_empty());
+                    return Ok(());
+                }
+            }
+        })
+    }
+}
+
+impl Extend<DynJob> for JobGraph {
+    #[track_caller]
+    fn extend<T: IntoIterator<Item = DynJob>>(&mut self, iter: T) {
+        self.add_jobs(iter);
+    }
+}
+
+impl FromIterator<DynJob> for JobGraph {
+    #[track_caller]
+    fn from_iter<T: IntoIterator<Item = DynJob>>(iter: T) -> Self {
+        let mut retval = Self::new();
+        retval.add_jobs(iter);
+        retval
+    }
+}
+
+impl Serialize for JobGraph {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut serializer = serializer.serialize_seq(Some(self.jobs.len()))?;
+        for &node_id in &self.topological_order {
+            let JobGraphNode::Job(job) = &self.graph[node_id] else {
+                continue;
+            };
+            serializer.serialize_element(job)?;
+        }
+        serializer.end()
+    }
+}
+
+impl<'de> Deserialize<'de> for JobGraph {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let jobs = Vec::<DynJob>::deserialize(deserializer)?;
+        let mut retval = JobGraph::new();
+        retval.try_add_jobs(jobs).map_err(D::Error::custom)?;
+        Ok(retval)
+    }
+}

crates/fayalite/src/build/registry.rs

@@ -0,0 +1,341 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// See Notices.txt for copyright information
+
+use crate::{
+    build::{DynJobKind, JobKind},
+    intern::Interned,
+};
+use std::{
+    borrow::Borrow,
+    cmp::Ordering,
+    collections::BTreeMap,
+    fmt,
+    sync::{Arc, OnceLock, RwLock, RwLockWriteGuard},
+};
+
+impl DynJobKind {
+    pub fn registry() -> JobKindRegistrySnapshot {
+        JobKindRegistrySnapshot(JobKindRegistry::get())
+    }
+    #[track_caller]
+    pub fn register(self) {
+        JobKindRegistry::register(JobKindRegistry::lock(), self);
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+struct InternedStrCompareAsStr(Interned<str>);
+
+impl fmt::Debug for InternedStrCompareAsStr {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl Ord for InternedStrCompareAsStr {
+    fn cmp(&self, other: &Self) -> Ordering {
+        str::cmp(&self.0, &other.0)
+    }
+}
+
+impl PartialOrd for InternedStrCompareAsStr {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Borrow<str> for InternedStrCompareAsStr {
+    fn borrow(&self) -> &str {
+        &self.0
+    }
+}
+
+#[derive(Clone, Debug)]
+struct JobKindRegistry {
+    job_kinds: BTreeMap<InternedStrCompareAsStr, DynJobKind>,
+}
+
+enum JobKindRegisterError {
+    SameName {
+        name: InternedStrCompareAsStr,
+        old_job_kind: DynJobKind,
+        new_job_kind: DynJobKind,
+    },
+}
+
+impl fmt::Display for JobKindRegisterError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::SameName {
+                name,
+                old_job_kind,
+                new_job_kind,
+            } => write!(
+                f,
+                "two different `JobKind` can't share the same name:\n\
+                {name:?}\n\
+                old job kind:\n\
+                {old_job_kind:?}\n\
+                new job kind:\n\
+                {new_job_kind:?}",
+            ),
+        }
+    }
+}
+
+trait JobKindRegistryRegisterLock {
+    type Locked;
+    fn lock(self) -> Self::Locked;
+    fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry;
+}
+
+impl JobKindRegistryRegisterLock for &'static RwLock<Arc<JobKindRegistry>> {
+    type Locked = RwLockWriteGuard<'static, Arc<JobKindRegistry>>;
+    fn lock(self) -> Self::Locked {
+        self.write().expect("shouldn't be poisoned")
+    }
+    fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry {
+        Arc::make_mut(locked)
+    }
+}
+
+impl JobKindRegistryRegisterLock for &'_ mut JobKindRegistry {
+    type Locked = Self;
+
+    fn lock(self) -> Self::Locked {
+        self
+    }
+
+    fn make_mut(locked: &mut Self::Locked) -> &mut JobKindRegistry {
+        locked
+    }
+}
+
+impl JobKindRegistry {
+    fn lock() -> &'static RwLock<Arc<Self>> {
+        static REGISTRY: OnceLock<RwLock<Arc<JobKindRegistry>>> = OnceLock::new();
+        REGISTRY.get_or_init(Default::default)
+    }
+    fn try_register<L: JobKindRegistryRegisterLock>(
+        lock: L,
+        job_kind: DynJobKind,
+    ) -> Result<(), JobKindRegisterError> {
+        use std::collections::btree_map::Entry;
+        let name = InternedStrCompareAsStr(job_kind.name());
+        // run user code only outside of lock
+        let mut locked = lock.lock();
+        let this = L::make_mut(&mut locked);
+        let result = match this.job_kinds.entry(name) {
+            Entry::Occupied(entry) => Err(JobKindRegisterError::SameName {
+                name,
+                old_job_kind: entry.get().clone(),
+                new_job_kind: job_kind,
+            }),
+            Entry::Vacant(entry) => {
+                entry.insert(job_kind);
+                Ok(())
+            }
+        };
+        drop(locked);
+        // outside of lock now, so we can test if it's the same DynJobKind
+        match result {
+            Err(JobKindRegisterError::SameName {
+                name: _,
+                old_job_kind,
+                new_job_kind,
+            }) if old_job_kind == new_job_kind => Ok(()),
+            result => result,
+        }
+    }
+    #[track_caller]
+    fn register<L: JobKindRegistryRegisterLock>(lock: L, job_kind: DynJobKind) {
+        match Self::try_register(lock, job_kind) {
+            Err(e) => panic!("{e}"),
+            Ok(()) => {}
+        }
+    }
+    fn get() -> Arc<Self> {
+        Self::lock().read().expect("shouldn't be poisoned").clone()
+    }
+}
+
+impl Default for JobKindRegistry {
+    fn default() -> Self {
+        let mut retval = Self {
+            job_kinds: BTreeMap::new(),
+        };
+        for job_kind in [] {
+            Self::register(&mut retval, job_kind);
+        }
+        retval
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct JobKindRegistrySnapshot(Arc<JobKindRegistry>);
+
+impl JobKindRegistrySnapshot {
+    pub fn get() -> Self {
+        JobKindRegistrySnapshot(JobKindRegistry::get())
+    }
+    pub fn get_by_name<'a>(&'a self, name: &str) -> Option<&'a DynJobKind> {
+        self.0.job_kinds.get(name)
+    }
+    pub fn iter_with_names(&self) -> JobKindRegistryIterWithNames<'_> {
+        JobKindRegistryIterWithNames(self.0.job_kinds.iter())
+    }
+    pub fn iter(&self) -> JobKindRegistryIter<'_> {
+        JobKindRegistryIter(self.0.job_kinds.values())
+    }
+}
+
+impl<'a> IntoIterator for &'a JobKindRegistrySnapshot {
+    type Item = &'a DynJobKind;
+    type IntoIter = JobKindRegistryIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.iter()
+    }
+}
+
+impl<'a> IntoIterator for &'a mut JobKindRegistrySnapshot {
+    type Item = &'a DynJobKind;
+    type IntoIter = JobKindRegistryIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.iter()
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct JobKindRegistryIter<'a>(
+    std::collections::btree_map::Values<'a, InternedStrCompareAsStr, DynJobKind>,
+);
+
+impl<'a> Iterator for JobKindRegistryIter<'a> {
+    type Item = &'a DynJobKind;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.0.next()
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.0.size_hint()
+    }
+
+    fn count(self) -> usize
+    where
+        Self: Sized,
+    {
+        self.0.count()
+    }
+
+    fn last(self) -> Option<Self::Item> {
+        self.0.last()
+    }
+
+    fn nth(&mut self, n: usize) -> Option<Self::Item> {
+        self.0.nth(n)
+    }
+
+    fn fold<B, F>(self, init: B, f: F) -> B
+    where
+        F: FnMut(B, Self::Item) -> B,
+    {
+        self.0.fold(init, f)
+    }
+}
+
+impl<'a> std::iter::FusedIterator for JobKindRegistryIter<'a> {}
+
+impl<'a> ExactSizeIterator for JobKindRegistryIter<'a> {}
+
+impl<'a> DoubleEndedIterator for JobKindRegistryIter<'a> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.0.next_back()
+    }
+
+    fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
+        self.0.nth_back(n)
+    }
+
+    fn rfold<B, F>(self, init: B, f: F) -> B
+    where
+        F: FnMut(B, Self::Item) -> B,
+    {
+        self.0.rfold(init, f)
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct JobKindRegistryIterWithNames<'a>(
+    std::collections::btree_map::Iter<'a, InternedStrCompareAsStr, DynJobKind>,
+);
+
+impl<'a> Iterator for JobKindRegistryIterWithNames<'a> {
+    type Item = (Interned<str>, &'a DynJobKind);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.0.next().map(|(name, job_kind)| (name.0, job_kind))
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.0.size_hint()
+    }
+
+    fn count(self) -> usize
+    where
+        Self: Sized,
+    {
+        self.0.count()
+    }
+
+    fn last(self) -> Option<Self::Item> {
+        self.0.last().map(|(name, job_kind)| (name.0, job_kind))
+    }
+
+    fn nth(&mut self, n: usize) -> Option<Self::Item> {
+        self.0.nth(n).map(|(name, job_kind)| (name.0, job_kind))
+    }
+
+    fn fold<B, F>(self, init: B, f: F) -> B
+    where
+        F: FnMut(B, Self::Item) -> B,
+    {
+        self.0
+            .map(|(name, job_kind)| (name.0, job_kind))
+            .fold(init, f)
+    }
+}
+
+impl<'a> std::iter::FusedIterator for JobKindRegistryIterWithNames<'a> {}
+
+impl<'a> ExactSizeIterator for JobKindRegistryIterWithNames<'a> {}
+
+impl<'a> DoubleEndedIterator for JobKindRegistryIterWithNames<'a> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.0
+            .next_back()
+            .map(|(name, job_kind)| (name.0, job_kind))
+    }
+
+    fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
+        self.0
+            .nth_back(n)
+            .map(|(name, job_kind)| (name.0, job_kind))
+    }
+
+    fn rfold<B, F>(self, init: B, f: F) -> B
+    where
+        F: FnMut(B, Self::Item) -> B,
+    {
+        self.0
+            .map(|(name, job_kind)| (name.0, job_kind))
+            .rfold(init, f)
+    }
+}
+
+#[track_caller]
+pub fn register_job_kind<K: JobKind>(kind: K) {
+    DynJobKind::new(kind).register();
+}

crates/fayalite/src/util.rs

@@ -36,8 +36,11 @@ pub use scoped_ref::ScopedRef;
 pub(crate) use misc::chain;
 #[doc(inline)]
 pub use misc::{
-    BitSliceWriteWithBase, DebugAsDisplay, DebugAsRawString, MakeMutSlice, RcWriter, interned_bit,
+    BitSliceWriteWithBase, DebugAsDisplay, DebugAsRawString, MakeMutSlice, RcWriter,
-    iter_eq_by, slice_range, try_slice_range,
+    SerdeJsonEscapeIf, SerdeJsonEscapeIfFormatter, SerdeJsonEscapeIfTest,
+    SerdeJsonEscapeIfTestResult, interned_bit, iter_eq_by, serialize_to_json_ascii,
+    serialize_to_json_ascii_pretty, serialize_to_json_ascii_pretty_with_indent, slice_range,
+    try_slice_range,
 };
 
 pub mod job_server;

crates/fayalite/src/util/misc.rs

@@ -5,6 +5,7 @@ use bitvec::{bits, order::Lsb0, slice::BitSlice, view::BitView};
 use std::{
     cell::Cell,
     fmt::{self, Debug, Write},
+    io,
     ops::{Bound, Range, RangeBounds},
     rc::Rc,
     sync::{Arc, OnceLock},
@@ -243,3 +244,323 @@ pub fn try_slice_range<R: RangeBounds<usize>>(range: R, size: usize) -> Option<R
 pub fn slice_range<R: RangeBounds<usize>>(range: R, size: usize) -> Range<usize> {
     try_slice_range(range, size).expect("range out of bounds")
 }
+
+pub trait SerdeJsonEscapeIfTest {
+    fn char_needs_escape(&mut self, ch: char) -> serde_json::Result<bool>;
+}
+
+pub trait SerdeJsonEscapeIfTestResult {
+    fn to_result(self) -> serde_json::Result<bool>;
+}
+
+impl SerdeJsonEscapeIfTestResult for bool {
+    fn to_result(self) -> serde_json::Result<bool> {
+        Ok(self)
+    }
+}
+
+impl<E: Into<serde_json::Error>> SerdeJsonEscapeIfTestResult for Result<bool, E> {
+    fn to_result(self) -> serde_json::Result<bool> {
+        self.map_err(Into::into)
+    }
+}
+
+impl<T: ?Sized + FnMut(char) -> R, R: SerdeJsonEscapeIfTestResult> SerdeJsonEscapeIfTest for T {
+    fn char_needs_escape(&mut self, ch: char) -> serde_json::Result<bool> {
+        self(ch).to_result()
+    }
+}
+
+pub trait SerdeJsonEscapeIfFormatter: serde_json::ser::Formatter {
+    fn write_unicode_escape<W>(&mut self, writer: &mut W, ch: char) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        for utf16 in ch.encode_utf16(&mut [0; 2]) {
+            write!(writer, "\\u{utf16:04x}")?;
+        }
+        Ok(())
+    }
+}
+
+impl SerdeJsonEscapeIfFormatter for serde_json::ser::CompactFormatter {}
+impl SerdeJsonEscapeIfFormatter for serde_json::ser::PrettyFormatter<'_> {}
+
+pub struct SerdeJsonEscapeIf<Test, Base = serde_json::ser::CompactFormatter> {
+    pub base: Base,
+    pub test: Test,
+}
+
+impl<Test: SerdeJsonEscapeIfTest, Base: SerdeJsonEscapeIfFormatter> serde_json::ser::Formatter
+    for SerdeJsonEscapeIf<Test, Base>
+{
+    fn write_null<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_null(writer)
+    }
+
+    fn write_bool<W>(&mut self, writer: &mut W, value: bool) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_bool(writer, value)
+    }
+
+    fn write_i8<W>(&mut self, writer: &mut W, value: i8) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_i8(writer, value)
+    }
+
+    fn write_i16<W>(&mut self, writer: &mut W, value: i16) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_i16(writer, value)
+    }
+
+    fn write_i32<W>(&mut self, writer: &mut W, value: i32) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_i32(writer, value)
+    }
+
+    fn write_i64<W>(&mut self, writer: &mut W, value: i64) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_i64(writer, value)
+    }
+
+    fn write_i128<W>(&mut self, writer: &mut W, value: i128) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_i128(writer, value)
+    }
+
+    fn write_u8<W>(&mut self, writer: &mut W, value: u8) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_u8(writer, value)
+    }
+
+    fn write_u16<W>(&mut self, writer: &mut W, value: u16) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_u16(writer, value)
+    }
+
+    fn write_u32<W>(&mut self, writer: &mut W, value: u32) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_u32(writer, value)
+    }
+
+    fn write_u64<W>(&mut self, writer: &mut W, value: u64) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_u64(writer, value)
+    }
+
+    fn write_u128<W>(&mut self, writer: &mut W, value: u128) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_u128(writer, value)
+    }
+
+    fn write_f32<W>(&mut self, writer: &mut W, value: f32) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_f32(writer, value)
+    }
+
+    fn write_f64<W>(&mut self, writer: &mut W, value: f64) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_f64(writer, value)
+    }
+
+    fn write_number_str<W>(&mut self, writer: &mut W, value: &str) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_number_str(writer, value)
+    }
+
+    fn begin_string<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_string(writer)
+    }
+
+    fn end_string<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_string(writer)
+    }
+
+    fn write_string_fragment<W>(&mut self, writer: &mut W, mut fragment: &str) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        while let Some((next_escape_index, next_escape_char)) = fragment
+            .char_indices()
+            .find_map(|(index, ch)| match self.test.char_needs_escape(ch) {
+                Ok(false) => None,
+                Ok(true) => Some(Ok((index, ch))),
+                Err(e) => Some(Err(e)),
+            })
+            .transpose()?
+        {
+            let (no_escapes, rest) = fragment.split_at(next_escape_index);
+            fragment = &rest[next_escape_char.len_utf8()..];
+            self.base.write_string_fragment(writer, no_escapes)?;
+            self.base.write_unicode_escape(writer, next_escape_char)?;
+        }
+        self.base.write_string_fragment(writer, fragment)
+    }
+
+    fn write_char_escape<W>(
+        &mut self,
+        writer: &mut W,
+        char_escape: serde_json::ser::CharEscape,
+    ) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_char_escape(writer, char_escape)
+    }
+
+    fn write_byte_array<W>(&mut self, writer: &mut W, value: &[u8]) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_byte_array(writer, value)
+    }
+
+    fn begin_array<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_array(writer)
+    }
+
+    fn end_array<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_array(writer)
+    }
+
+    fn begin_array_value<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_array_value(writer, first)
+    }
+
+    fn end_array_value<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_array_value(writer)
+    }
+
+    fn begin_object<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_object(writer)
+    }
+
+    fn end_object<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_object(writer)
+    }
+
+    fn begin_object_key<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_object_key(writer, first)
+    }
+
+    fn end_object_key<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_object_key(writer)
+    }
+
+    fn begin_object_value<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.begin_object_value(writer)
+    }
+
+    fn end_object_value<W>(&mut self, writer: &mut W) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.end_object_value(writer)
+    }
+
+    fn write_raw_fragment<W>(&mut self, writer: &mut W, fragment: &str) -> io::Result<()>
+    where
+        W: ?Sized + io::Write,
+    {
+        self.base.write_raw_fragment(writer, fragment)
+    }
+}
+
+fn serialize_to_json_ascii_helper<F: SerdeJsonEscapeIfFormatter, S: serde::Serialize + ?Sized>(
+    v: &S,
+    base: F,
+) -> serde_json::Result<String> {
+    let mut retval = Vec::new();
+    v.serialize(&mut serde_json::ser::Serializer::with_formatter(
+        &mut retval,
+        SerdeJsonEscapeIf {
+            base,
+            test: |ch| ch < '\x20' || ch > '\x7F',
+        },
+    ))?;
+    String::from_utf8(retval).map_err(|_| serde::ser::Error::custom("invalid UTF-8"))
+}
+
+pub fn serialize_to_json_ascii<T: serde::Serialize + ?Sized>(v: &T) -> serde_json::Result<String> {
+    serialize_to_json_ascii_helper(v, serde_json::ser::CompactFormatter)
+}
+
+pub fn serialize_to_json_ascii_pretty<T: serde::Serialize + ?Sized>(
+    v: &T,
+) -> serde_json::Result<String> {
+    serialize_to_json_ascii_helper(v, serde_json::ser::PrettyFormatter::new())
+}
+
+pub fn serialize_to_json_ascii_pretty_with_indent<T: serde::Serialize + ?Sized>(
+    v: &T,
+    indent: &str,
+) -> serde_json::Result<String> {
+    serialize_to_json_ascii_helper(
+        v,
+        serde_json::ser::PrettyFormatter::with_indent(indent.as_bytes()),
+    )
+}