WIP adding support for speculative loads/stores

crates/cpu/src/rename_execute_retire.rs

@@ -1,6 +1,9 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
 // See Notices.txt for copyright information
 
+//! Rename/Execute/Retire Control System
+//! [#8](https://git.libre-chip.org/libre-chip/grant-tracking/issues/8)
+
 use crate::{
     config::{
         CpuConfig, CpuConfig2PowOutRegNumWidth, CpuConfigFetchWidth, CpuConfigPRegNumWidth,
@@ -91,31 +94,48 @@ pub type RenamedMOp<C: PhantomConstGet<CpuConfig>> =
 #[hdl]
 pub type RenamedSrcRegUInt<C: PhantomConstGet<CpuConfig>> = UIntType<CpuConfigPRegNumWidth<C>>;
 
+/// Enqueues happen in program order, they are not re-ordered by out-of-order execution.
+/// the whole `MOpInstance` is sent again in [`UnitInputsReady`] so Units can just ignore all
+/// [`UnitEnqueue`] messages if they don't need to keep track of program order -- so, pure computation
+/// instructions.
+/// Loads/Stores need to keep track of program order to ensure they properly handle memory dependencies.
 #[hdl(no_static)]
-pub struct UnitStart<C: PhantomConstGet<CpuConfig>> {
+pub struct UnitEnqueue<C: PhantomConstGet<CpuConfig>> {
+    pub mop: MOpInstance<RenamedMOp<C>>,
+    pub config: C,
+}
+
+#[hdl(no_static)]
+pub struct UnitInputsReady<C: PhantomConstGet<CpuConfig>> {
+    /// the whole `MOpInstance` is sent again so Units can just ignore all [`UnitEnqueue`] messages if desired.
     pub mop: MOpInstance<RenamedMOp<C>>,
     pub src_values: Array<PRegValue, { COMMON_MOP_SRC_LEN }>,
     pub config: C,
 }
 
 #[hdl(no_static)]
-pub struct UnitFinishedSuccessfully<C: PhantomConstGet<CpuConfig>> {
+pub struct UnitOutputReady<C: PhantomConstGet<CpuConfig>> {
-    pub dest: UnitOutRegNum<C>,
+    pub id: MOpId,
     pub dest_value: PRegValue,
     pub predictor_op: NextPcPredictorOp<C>,
 }
 
 #[hdl(no_static)]
 pub struct UnitCausedCancel<C: PhantomConstGet<CpuConfig>> {
-    pub id: MOpId,
     pub start_at_pc: UInt<64>,
+    /// `true` if this instruction should be retired and then cause a cancel
+    /// (e.g. a branch ran successfully but the next pc was mispredicted so the following instructions
+    /// needs to be canceled).
+    /// `false` if this instruction should be canceled without retiring it
+    /// (e.g. a load ran before a store it should have run after so it needs to retry after the memory
+    /// is in the right state).
+    pub cancel_after_retire: Bool,
     pub config: C,
 }
 
 #[hdl(no_static)]
-pub struct UnitFinished<C: PhantomConstGet<CpuConfig>> {
+pub struct UnitFinishCauseCancel<C: PhantomConstGet<CpuConfig>> {
     pub id: MOpId,
-    pub finished_successfully: HdlOption<UnitFinishedSuccessfully<C>>,
     pub caused_cancel: HdlOption<UnitCausedCancel<C>>,
     pub config: C,
 }
@@ -132,15 +152,29 @@ pub struct UnitMOpCantCauseCancel<C: PhantomConstGet<CpuConfig>> {
     pub config: C,
 }
 
+/// Interface from the Rename/Execute/Retire control logic to a single Unit.
+///
+/// ## State diagram for a single &micro;Op in a Unit
+/// Notes:
+/// * The diagram ignores `cancel_all`.
+/// * Multiple state transitions can happen in a single clock cycle.
+/// * Any state marked "Can cause cancel", can immediately finish with [`Self::finish_cause_cancel`] where:
+///     * [`UnitCausedCancel::cancel_after_retire`] must be `false` unless there's a "Finish" edge from this state.
+///     * [`UnitFinishCauseCancel::caused_cancel`] must be `HdlSome` unless there's a "Finish" edge from this state.
+#[doc = simple_mermaid::mermaid!("rename_execute_retire/unit.mermaid")]
 #[hdl(no_static)]
 pub struct ExecuteToUnitInterface<C: PhantomConstGet<CpuConfig>> {
-    pub start: ReadyValid<UnitStart<C>>,
+    /// Enqueues happen in program order, they are not re-ordered by out-of-order execution.
+    pub enqueue: ReadyValid<UnitEnqueue<C>>,
+    pub inputs_ready: ReadyValid<UnitInputsReady<C>>,
     pub cancel_all: ReadyValid<()>,
     pub is_no_longer_speculative: ReadyValid<UnitMOpIsNoLongerSpeculative<C>>,
     #[hdl(flip)]
     pub cant_cause_cancel: ReadyValid<UnitMOpCantCauseCancel<C>>,
     #[hdl(flip)]
-    pub finished: ReadyValid<UnitFinished<C>>,
+    pub output_ready: ReadyValid<UnitOutputReady<C>>,
+    #[hdl(flip)]
+    pub finish_cause_cancel: ReadyValid<UnitFinishCauseCancel<C>>,
     pub config: C,
 }
 
@@ -331,63 +365,218 @@ impl<C: PhantomConstCpuConfig> RenameTable<C> {
     }
 }
 
-#[hdl(no_static)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, Default)]
-enum MOpExecutionProgressDebugState<C: PhantomConstGet<CpuConfig>> {
+enum MOpInUnitState {
-    NotStarted,
+    #[default]
-    Started,
+    NotYetEnqueued,
-    Finished(NextPcPredictorOp<C>),
+    InputsNotReadySpeculative {
-    Canceled,
+        can_cause_cancel: bool,
+    },
+    InputsReady {
+        speculative: bool,
+        can_cause_cancel: bool,
+    },
+    OutputReady {
+        speculative: bool,
+        can_cause_cancel: bool,
+    },
+    FinishedAndOrCausedCancel,
 }
 
+impl MOpInUnitState {
+    fn debug_str(self) -> &'static str {
+        match self {
+            Self::NotYetEnqueued => "NotYetEnqueued",
+            Self::InputsNotReadySpeculative { can_cause_cancel } => {
+                if can_cause_cancel {
+                    "INR_S_C"
+                } else {
+                    "INR_S"
+                }
+            }
+            Self::InputsReady {
+                speculative,
+                can_cause_cancel,
+            } => {
+                if speculative {
+                    if can_cause_cancel { "IR_S_C" } else { "IR_S" }
+                } else {
+                    if can_cause_cancel { "IR_C" } else { "IR" }
+                }
+            }
+            Self::OutputReady {
+                speculative,
+                can_cause_cancel,
+            } => {
+                if speculative {
+                    if can_cause_cancel { "OR_S_C" } else { "OR_S" }
+                } else {
+                    if can_cause_cancel { "OR_C" } else { "OR" }
+                }
+            }
+            Self::FinishedAndOrCausedCancel => "F_C",
+        }
+    }
+    #[must_use]
+    fn after_enqueue(self) -> Option<Self> {
+        match self {
+            Self::NotYetEnqueued => Some(Self::InputsNotReadySpeculative {
+                can_cause_cancel: true,
+            }),
+            _ => None,
+        }
+    }
+    #[must_use]
+    fn after_output_ready(self) -> Option<Self> {
+        match self {
+            Self::InputsReady {
+                speculative,
+                can_cause_cancel,
+            } => Some(Self::OutputReady {
+                speculative,
+                can_cause_cancel,
+            }),
+            _ => None,
+        }
+    }
+    #[must_use]
+    fn after_finish_cause_cancel(
+        self,
+        cancel_after_retire: bool,
+        cause_cancel: bool,
+    ) -> Option<Self> {
+        if cause_cancel && !cancel_after_retire {
+            match self {
+                Self::NotYetEnqueued => None,
+                Self::InputsNotReadySpeculative { can_cause_cancel }
+                | Self::InputsReady {
+                    speculative: _,
+                    can_cause_cancel,
+                }
+                | Self::OutputReady {
+                    speculative: _,
+                    can_cause_cancel,
+                } => can_cause_cancel.then_some(Self::FinishedAndOrCausedCancel),
+                Self::FinishedAndOrCausedCancel => todo!(),
+            }
+        } else {
+            assert!(cause_cancel == cancel_after_retire);
+            // see if we can eventually retire MOp
+            match self {
+                Self::OutputReady {
+                    speculative: _,
+                    can_cause_cancel,
+                } => {
+                    if cause_cancel && !can_cause_cancel {
+                        None
+                    } else {
+                        Some(Self::FinishedAndOrCausedCancel)
+                    }
+                }
+                Self::NotYetEnqueued
+                | Self::InputsNotReadySpeculative { .. }
+                | Self::InputsReady { .. }
+                | Self::FinishedAndOrCausedCancel => None,
+            }
+        }
+    }
+    #[must_use]
+    fn with_inputs_ready(self) -> Option<Self> {
+        match self {
+            Self::InputsNotReadySpeculative { can_cause_cancel } => Some(Self::InputsReady {
+                speculative: true,
+                can_cause_cancel,
+            }),
+            _ => None,
+        }
+    }
+    #[must_use]
+    fn without_speculative(self) -> Option<Self> {
+        match self {
+            Self::NotYetEnqueued => None,
+            Self::InputsNotReadySpeculative { .. } => None,
+            Self::InputsReady {
+                speculative,
+                can_cause_cancel,
+            } => speculative.then_some(Self::InputsReady {
+                speculative: false,
+                can_cause_cancel,
+            }),
+            Self::OutputReady {
+                speculative,
+                can_cause_cancel,
+            } => speculative.then_some(Self::OutputReady {
+                speculative: false,
+                can_cause_cancel,
+            }),
+            Self::FinishedAndOrCausedCancel => None,
+        }
+    }
+    #[must_use]
+    fn with_cant_cause_cancel(self) -> Option<Self> {
+        match self {
+            Self::NotYetEnqueued => None,
+            Self::InputsNotReadySpeculative { can_cause_cancel } => {
+                can_cause_cancel.then_some(Self::InputsNotReadySpeculative {
+                    can_cause_cancel: false,
+                })
+            }
+            Self::InputsReady {
+                speculative,
+                can_cause_cancel,
+            } => can_cause_cancel.then_some(Self::InputsReady {
+                speculative,
+                can_cause_cancel: false,
+            }),
+            Self::OutputReady {
+                speculative,
+                can_cause_cancel,
+            } => can_cause_cancel.then_some(Self::OutputReady {
+                speculative,
+                can_cause_cancel: false,
+            }),
+            Self::FinishedAndOrCausedCancel => None,
+        }
+    }
+}
+
+impl fmt::Debug for MOpInUnitState {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(self.debug_str())
+    }
+}
+
+#[hdl]
+type SimOnlyMOpInUnitState = SimOnly<MOpInUnitState>;
+
 #[hdl(no_static)]
 struct RobEntryDebugState<C: PhantomConstGet<CpuConfig>> {
     mop: MOpInstance<RenamedMOp<C>>,
+    mop_in_unit_state: SimOnlyMOpInUnitState,
     is_speculative: Bool,
-    sent_is_no_longer_speculative: Bool,
+    finished: HdlOption<NextPcPredictorOp<C>>,
-    can_cause_cancel: Bool,
-    progress: MOpExecutionProgressDebugState<C>,
     caused_cancel: HdlOption<UnitCausedCancel<C>>,
 }
 
 impl<C: PhantomConstCpuConfig> SimValueDefault for RobEntryDebugState<C> {
-    fn sim_value_default(self) -> SimValue<Self> {
-        zeroed(self)
-    }
-}
-
-#[derive(Clone, Debug)]
-enum MOpExecutionProgress<C: Type + PhantomConstGet<CpuConfig>> {
-    NotStarted,
-    Started,
-    Finished(SimValue<NextPcPredictorOp<C>>),
-    Canceled,
-}
-
-impl<C: Type + PhantomConstGet<CpuConfig>> MOpExecutionProgress<C> {
     #[hdl]
-    fn debug_state(&self, config: C) -> SimValue<MOpExecutionProgressDebugState<C>> {
+    fn sim_value_default(self) -> SimValue<Self> {
-        let ret_ty = MOpExecutionProgressDebugState[config];
+        let Self {
-        match self {
+            mop,
-            Self::NotStarted =>
+            mop_in_unit_state: _,
-            {
+            is_speculative: _,
-                #[hdl(sim)]
+            finished,
-                ret_ty.NotStarted()
+            caused_cancel,
-            }
+        } = self;
-            Self::Started =>
+        #[hdl(sim)]
-            {
+        Self {
-                #[hdl(sim)]
+            mop: zeroed(mop),
-                ret_ty.Started()
+            mop_in_unit_state: SimOnlyValue::default(),
-            }
+            is_speculative: false,
-            Self::Finished(v) =>
+            finished: #[hdl(sim)]
-            {
+            finished.HdlNone(),
-                #[hdl(sim)]
+            caused_cancel: #[hdl(sim)]
-                ret_ty.Finished(v)
+            caused_cancel.HdlNone(),
-            }
-            Self::Canceled =>
-            {
-                #[hdl(sim)]
-                ret_ty.Canceled()
-            }
         }
     }
 }
@@ -395,10 +584,9 @@ impl<C: Type + PhantomConstGet<CpuConfig>> MOpExecutionProgress<C> {
 #[derive(Debug)]
 struct RobEntry<C: PhantomConstCpuConfig> {
     mop: SimValue<MOpInstance<RenamedMOp<C>>>,
+    mop_in_unit_state: MOpInUnitState,
     is_speculative: bool,
-    sent_is_no_longer_speculative: bool,
+    finished: Option<SimValue<NextPcPredictorOp<C>>>,
-    can_cause_cancel: bool,
-    progress: MOpExecutionProgress<C>,
     caused_cancel: Option<SimValue<UnitCausedCancel<C>>>,
 }
 
@@ -406,10 +594,9 @@ impl<C: PhantomConstCpuConfig> RobEntry<C> {
     fn new(mop: SimValue<MOpInstance<RenamedMOp<C>>>) -> Self {
         Self {
             mop,
+            mop_in_unit_state: MOpInUnitState::NotYetEnqueued,
             is_speculative: true,
-            sent_is_no_longer_speculative: false,
+            finished: None,
-            can_cause_cancel: true,
-            progress: MOpExecutionProgress::NotStarted,
             caused_cancel: None,
         }
     }
@@ -428,6 +615,37 @@ impl<C: PhantomConstCpuConfig> RobEntry<C> {
     fn unit_out_reg_index(&self) -> usize {
         UnitOutRegNum::value_sim(&self.unit_out_reg())
     }
+    #[hdl]
+    fn debug_state(&self, config: C) -> SimValue<RobEntryDebugState<C>> {
+        let Self {
+            mop,
+            mop_in_unit_state,
+            is_speculative,
+            finished,
+            caused_cancel,
+        } = self;
+        let ret_ty = RobEntryDebugState[config];
+        #[hdl(sim)]
+        RobEntryDebugState::<C> {
+            mop,
+            mop_in_unit_state: SimOnlyValue::new(*mop_in_unit_state),
+            is_speculative,
+            finished: if let Some(finished) = finished {
+                #[hdl(sim)]
+                (ret_ty.finished).HdlSome(finished)
+            } else {
+                #[hdl(sim)]
+                (ret_ty.finished).HdlNone()
+            },
+            caused_cancel: if let Some(caused_cancel) = caused_cancel {
+                #[hdl(sim)]
+                (ret_ty.caused_cancel).HdlSome(caused_cancel)
+            } else {
+                #[hdl(sim)]
+                (ret_ty.caused_cancel).HdlNone()
+            },
+        }
+    }
 }
 
 #[hdl]
@@ -564,33 +782,8 @@ impl<C: PhantomConstCpuConfig> ReorderBuffer<C> {
             renamed: ty
                 .renamed
                 .from_iter_sim(
-                    zeroed(ty.renamed.element()),
+                    ty.renamed.element().sim_value_default(),
-                    self.renamed().map(|entry| {
+                    self.renamed().map(|v| v.debug_state(*config)),
-                        let RobEntry {
-                            mop,
-                            is_speculative,
-                            sent_is_no_longer_speculative,
-                            can_cause_cancel,
-                            progress,
-                            caused_cancel,
-                        } = entry;
-                        let caused_cancel_ty = HdlOption[UnitCausedCancel[self.config]];
-                        #[hdl(sim)]
-                        RobEntryDebugState::<_> {
-                            mop,
-                            is_speculative,
-                            sent_is_no_longer_speculative,
-                            can_cause_cancel,
-                            progress: progress.debug_state(self.config),
-                            caused_cancel: if let Some(caused_cancel) = caused_cancel {
-                                #[hdl(sim)]
-                                caused_cancel_ty.HdlSome(caused_cancel)
-                            } else {
-                                #[hdl(sim)]
-                                caused_cancel_ty.HdlNone()
-                            },
-                        }
-                    }),
                 )
                 .ok()
                 .expect("known to fit"),
@@ -793,14 +986,9 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
         for rob in self.rob.renamed() {
             let masked_id = rob.mop.id.as_int() as usize & mask;
             **retval[masked_id] = fmt::from_fn(|f| {
-                if rob.is_speculative {
+                f.write_str(rob.mop_in_unit_state.debug_str())?;
-                    f.write_str("(S)")?;
+                if rob.finished.is_some() {
-                }
+                    f.write_str("(finished)")?;
-                match rob.progress {
-                    MOpExecutionProgress::NotStarted => f.write_str("NotStarted")?,
-                    MOpExecutionProgress::Started => f.write_str("Started")?,
-                    MOpExecutionProgress::Finished(_) => f.write_str("Finished")?,
-                    MOpExecutionProgress::Canceled => f.write_str("Canceled")?,
                 }
                 if rob.caused_cancel.is_some() {
                     f.write_str("(caused cancel)")?;
@@ -1213,8 +1401,34 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
         Ok(())
     }
     #[hdl]
-    fn get_unit_start(&self, unit_index: usize) -> SimValue<HdlOption<UnitStart<C>>> {
+    fn get_unit_enqueue(&self, unit_index: usize) -> SimValue<HdlOption<UnitEnqueue<C>>> {
-        let ret_ty = HdlOption[UnitStart[self.config]];
+        let ret_ty = HdlOption[UnitEnqueue[self.config]];
+        if self.is_canceling() {
+            let retval = #[hdl(sim)]
+            ret_ty.HdlNone();
+            return retval; // separate variable to work around rust-analyzer parse error
+        }
+        for rob in self.rob.renamed() {
+            if rob.unit_index() == unit_index
+                && let Some(_) = rob.mop_in_unit_state.after_enqueue()
+            {
+                let retval = #[hdl(sim)]
+                ret_ty.HdlSome(
+                    #[hdl(sim)]
+                    UnitEnqueue::<_> {
+                        mop: &rob.mop,
+                        config: self.config,
+                    },
+                );
+                return retval;
+            }
+        }
+        #[hdl(sim)]
+        ret_ty.HdlNone()
+    }
+    #[hdl]
+    fn get_unit_inputs_ready(&self, unit_index: usize) -> SimValue<HdlOption<UnitInputsReady<C>>> {
+        let ret_ty = HdlOption[UnitInputsReady[self.config]];
         if self.is_canceling() {
             let retval = #[hdl(sim)]
             ret_ty.HdlNone();
@@ -1223,8 +1437,8 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
         let zero_reg = PRegNum[self.config].const_zero().into_sim_value();
         let zero_value = zeroed(PRegValue);
         for rob in self.rob.renamed() {
-            if let MOpExecutionProgress::NotStarted = rob.progress
+            if rob.unit_index() == unit_index
-                && rob.unit_index() == unit_index
+                && let Some(_) = rob.mop_in_unit_state.with_inputs_ready()
             {
                 let mut src_values: [_; COMMON_MOP_SRC_LEN] =
                     std::array::from_fn(|_| Some(zero_value.clone()));
@@ -1249,7 +1463,7 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
                     let retval = #[hdl(sim)]
                     ret_ty.HdlSome(
                         #[hdl(sim)]
-                        UnitStart::<_> {
+                        UnitInputsReady::<_> {
                             mop: &rob.mop,
                             src_values,
                             config: self.config,
@@ -1276,7 +1490,7 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
         for rob in self.rob.renamed() {
             if rob.unit_index() == unit_index
                 && !rob.is_speculative
-                && !rob.sent_is_no_longer_speculative
+                && let Some(_) = rob.mop_in_unit_state.without_speculative()
             {
                 let retval = #[hdl(sim)]
                 ret_ty.HdlSome(
@@ -1293,54 +1507,72 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
         ret_ty.HdlNone()
     }
     #[hdl]
-    fn unit_finished(&mut self, finished: SimValue<UnitFinished<C>>) {
+    fn unit_output_ready(&mut self, output_ready: SimValue<UnitOutputReady<C>>) {
         #[hdl(sim)]
-        let UnitFinished::<_> {
+        let UnitOutputReady::<_> {
             id,
-            finished_successfully,
+            dest_value,
-            caused_cancel: unit_caused_cancel,
+            predictor_op,
-            config: _,
+        } = output_ready;
-        } = finished;
         assert!(!self.is_canceling());
         let rob = self.rob.renamed_by_id_mut(&id);
         let unit_index = rob.unit_index();
         let out_reg_index = rob.unit_out_reg_index();
         let RobEntry {
             mop: _,
+            mop_in_unit_state,
             is_speculative: _,
-            sent_is_no_longer_speculative: _,
+            finished,
-            can_cause_cancel,
-            progress,
             caused_cancel,
         } = rob;
-        assert!(matches!(progress, MOpExecutionProgress::Started));
+        assert!(finished.is_none());
         assert!(caused_cancel.is_none());
+        let l1_reg = &mut self.l1_reg_file[unit_index][out_reg_index];
+        assert!(l1_reg.is_none());
+        *l1_reg = Some(dest_value);
+        *finished = Some(predictor_op);
+        *mop_in_unit_state = mop_in_unit_state
+            .after_output_ready()
+            .expect("should be valid state for output to become ready");
+    }
+    #[hdl]
+    fn unit_finish_cause_cancel(
+        &mut self,
+        finish_cause_cancel: SimValue<UnitFinishCauseCancel<C>>,
+    ) {
         #[hdl(sim)]
-        if let HdlSome(finished_successfully) = finished_successfully {
+        let UnitFinishCauseCancel::<_> {
-            #[hdl(sim)]
+            id,
-            let UnitFinishedSuccessfully::<_> {
+            caused_cancel: unit_caused_cancel,
-                dest: _,
+            config: _,
-                dest_value,
+        } = finish_cause_cancel;
-                predictor_op,
+        assert!(!self.is_canceling());
-            } = finished_successfully;
+        let RobEntry {
-            let l1_reg = &mut self.l1_reg_file[unit_index][out_reg_index];
+            mop: _,
-            assert!(l1_reg.is_none());
+            mop_in_unit_state,
-            *l1_reg = Some(dest_value);
+            is_speculative: _,
-            *progress = MOpExecutionProgress::Finished(predictor_op);
+            finished,
-        } else {
+            caused_cancel,
-            *progress = MOpExecutionProgress::Canceled;
+        } = self.rob.renamed_by_id_mut(&id);
-        }
+        assert!(caused_cancel.is_none());
+        let cancel_after_retire;
         #[hdl(sim)]
         if let HdlSome(unit_caused_cancel) = unit_caused_cancel {
-            assert!(
+            cancel_after_retire = *unit_caused_cancel.cancel_after_retire;
-                *can_cause_cancel,
-                "MOp {id:?} said it won't cause a cancel, then caused a cancel: {unit_caused_cancel:#?}"
-            );
             *caused_cancel = Some(unit_caused_cancel);
         } else {
-            assert!(
+            cancel_after_retire = false;
-                matches!(progress, MOpExecutionProgress::Finished(_)),
+        }
-                "MOp {id:?} must finish successfully and/or cause a cancel: progress={progress:?}",
+        if let Some(v) = mop_in_unit_state
+            .after_finish_cause_cancel(cancel_after_retire, caused_cancel.is_some())
+        {
+            *mop_in_unit_state = v
+        } else {
+            panic!(
+                "MOp {id:?} made an invalid attempt to finish/cause a cancel:\n\
+                    mop_in_unit_state={mop_in_unit_state:?}\n\
+                    finished={finished:?}\n\
+                    caused_cancel={caused_cancel:?}"
             );
         }
     }
@@ -1388,7 +1620,7 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
             rob,
             next_pc_canceling,
             unit_canceling: _,
-            l1_reg_file,
+            l1_reg_file: _,
             l2_reg_file_unit_index: _,
             config: _,
         } = self;
@@ -1414,7 +1646,7 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
         let mut retval = Vec::new();
         let mut prev_caused_cancel = false;
         for RobEntries {
-            unrenamed,
+            unrenamed: _,
             rename_table_updates: _,
             renamed_entries,
         } in &self.rob.entries
@@ -1435,13 +1667,15 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
                     return retval;
                 }
                 if let RobEntry {
-                    mop,
+                    mop: _,
-                    is_speculative,
+                    mop_in_unit_state: MOpInUnitState::FinishedAndOrCausedCancel,
-                    sent_is_no_longer_speculative,
+                    is_speculative: _,
-                    can_cause_cancel,
+                    finished: Some(renamed_op),
-                    progress: MOpExecutionProgress::Finished(renamed_op),
                     caused_cancel,
                 } = renamed
+                    && caused_cancel
+                        .as_ref()
+                        .is_none_or(|caused_cancel| *caused_cancel.cancel_after_retire)
                 {
                     prev_caused_cancel = caused_cancel.is_some();
                     #[hdl(sim)]
@@ -1520,15 +1754,15 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
             .for_each(|v| self.retire_rename_table.update(v, "retire_rename_table"));
         for RobEntry {
             mop: _,
+            mop_in_unit_state,
             is_speculative: _,
-            sent_is_no_longer_speculative: _,
+            finished: _,
-            can_cause_cancel: _,
-            progress,
             caused_cancel,
         } in renamed_entries
         {
-            assert!(matches!(progress, MOpExecutionProgress::Finished(_)));
+            assert_eq!(mop_in_unit_state, MOpInUnitState::FinishedAndOrCausedCancel);
             if let Some(caused_cancel) = caused_cancel {
+                assert!(*caused_cancel.cancel_after_retire);
                 self.start_cancel(caused_cancel);
                 return;
             }
@@ -1539,8 +1773,8 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
         assert!(!self.is_canceling());
         #[hdl(sim)]
         let UnitCausedCancel::<_> {
-            id: _,
             start_at_pc,
+            cancel_after_retire: _,
             config: _,
         } = caused_cancel;
         self.next_pc_canceling = Some(NextPcCancelingState::NeedSendCancel(start_at_pc.as_int()));
@@ -1552,13 +1786,19 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
             return;
         }
         for renamed in self.rob.renamed_mut() {
-            let can_cause_cancel = match renamed.progress {
+            let can_cause_cancel = match renamed.mop_in_unit_state {
-                MOpExecutionProgress::NotStarted => break,
+                MOpInUnitState::NotYetEnqueued => true,
-                MOpExecutionProgress::Started => renamed.can_cause_cancel,
+                MOpInUnitState::InputsNotReadySpeculative { can_cause_cancel } => can_cause_cancel,
-                MOpExecutionProgress::Finished(_) => renamed.caused_cancel.is_some(),
+                MOpInUnitState::InputsReady {
-                MOpExecutionProgress::Canceled => true,
+                    speculative: _,
+                    can_cause_cancel,
+                } => can_cause_cancel,
+                MOpInUnitState::OutputReady {
+                    speculative: _,
+                    can_cause_cancel,
+                } => can_cause_cancel,
+                MOpInUnitState::FinishedAndOrCausedCancel => renamed.caused_cancel.is_some(),
             };
-            renamed.can_cause_cancel = can_cause_cancel;
             renamed.is_speculative = false;
             if can_cause_cancel {
                 break;
@@ -1567,12 +1807,12 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
         let first_renamed = self.rob.renamed().next();
         if let Some(RobEntry {
             mop: _,
+            mop_in_unit_state: MOpInUnitState::FinishedAndOrCausedCancel,
             is_speculative: _,
-            sent_is_no_longer_speculative: _,
+            finished: _,
-            can_cause_cancel: _,
-            progress: MOpExecutionProgress::Canceled,
             caused_cancel: Some(caused_cancel),
         }) = first_renamed
+            && !*caused_cancel.cancel_after_retire
         {
             let caused_cancel = caused_cancel.clone();
             self.start_cancel(caused_cancel);
@@ -1615,14 +1855,18 @@ async fn rename_execute_retire_run(
         {
             #[hdl]
             let ExecuteToUnitInterface::<_> {
-                start,
+                enqueue,
+                inputs_ready,
                 cancel_all,
                 is_no_longer_speculative,
                 cant_cause_cancel,
-                finished,
+                output_ready,
+                finish_cause_cancel,
                 config: _,
             } = to_unit;
-            sim.write(start.data, state.get_unit_start(unit_index))
+            sim.write(enqueue.data, state.get_unit_enqueue(unit_index))
+                .await;
+            sim.write(inputs_ready.data, state.get_unit_inputs_ready(unit_index))
                 .await;
             sim.write(
                 cancel_all.data,
@@ -1641,7 +1885,8 @@ async fn rename_execute_retire_run(
             )
             .await;
             sim.write(cant_cause_cancel.ready, !is_canceling).await;
-            sim.write(finished.ready, !is_canceling).await;
+            sim.write(output_ready.ready, !is_canceling).await;
+            sim.write(finish_cause_cancel.ready, !is_canceling).await;
         }
         sim.wait_for_clock_edge(cd.clk).await;
         let from_post_decode_insns = sim.read_past(from_post_decode.insns, cd.clk).await;
@@ -1657,28 +1902,49 @@ async fn rename_execute_retire_run(
         {
             #[hdl]
             let ExecuteToUnitInterface::<_> {
-                start,
+                enqueue,
+                inputs_ready,
                 cancel_all,
                 is_no_longer_speculative,
                 cant_cause_cancel,
-                finished,
+                output_ready,
+                finish_cause_cancel,
                 config: _,
             } = to_unit;
-            if sim.read_past_bool(start.ready, cd.clk).await {
+            if sim.read_past_bool(enqueue.ready, cd.clk).await {
                 #[hdl(sim)]
-                if let HdlSome(start) = sim.read_past(start.data, cd.clk).await {
+                if let HdlSome(enqueue) = sim.read_past(enqueue.data, cd.clk).await {
                     assert!(!state.is_canceling());
                     let RobEntry {
                         mop: _,
+                        mop_in_unit_state,
                         is_speculative: _,
-                        sent_is_no_longer_speculative: _,
+                        finished,
-                        can_cause_cancel: _,
-                        progress,
                         caused_cancel,
-                    } = state.rob.renamed_by_id_mut(&start.mop.id);
+                    } = state.rob.renamed_by_id_mut(&enqueue.mop.id);
+                    assert!(finished.is_none());
                     assert!(caused_cancel.is_none());
-                    assert!(matches!(progress, MOpExecutionProgress::NotStarted));
+                    *mop_in_unit_state = mop_in_unit_state
-                    *progress = MOpExecutionProgress::Started;
+                        .after_enqueue()
+                        .expect("UnitEnqueue is known to be valid");
+                }
+            }
+            if sim.read_past_bool(inputs_ready.ready, cd.clk).await {
+                #[hdl(sim)]
+                if let HdlSome(inputs_ready) = sim.read_past(inputs_ready.data, cd.clk).await {
+                    assert!(!state.is_canceling());
+                    let RobEntry {
+                        mop: _,
+                        mop_in_unit_state,
+                        is_speculative: _,
+                        finished,
+                        caused_cancel,
+                    } = state.rob.renamed_by_id_mut(&inputs_ready.mop.id);
+                    assert!(finished.is_none());
+                    assert!(caused_cancel.is_none());
+                    *mop_in_unit_state = mop_in_unit_state
+                        .with_inputs_ready()
+                        .expect("UnitInputsReady is known to be valid");
                 }
             }
             if sim.read_past_bool(cancel_all.ready, cd.clk).await {
@@ -1698,21 +1964,18 @@ async fn rename_execute_retire_run(
                     sim.read_past(is_no_longer_speculative.data, cd.clk).await
                 {
                     assert!(!state.is_canceling());
-                    if let Some(RobEntry {
+                    let RobEntry {
                         mop: _,
-                        is_speculative,
+                        mop_in_unit_state,
-                        sent_is_no_longer_speculative,
+                        is_speculative: _,
-                        can_cause_cancel: _,
+                        finished,
-                        progress: _,
+                        caused_cancel,
-                        caused_cancel: _,
+                    } = state.rob.renamed_by_id_mut(&is_no_longer_speculative.id);
-                    }) = state
+                    assert!(finished.is_none());
-                        .rob
+                    assert!(caused_cancel.is_none());
-                        .try_renamed_by_id_mut(&is_no_longer_speculative.id)
+                    *mop_in_unit_state = mop_in_unit_state
-                    {
+                        .with_inputs_ready()
-                        assert!(!*is_speculative);
+                        .expect("UnitMOpIsNoLongerSpeculative is known to be valid");
-                        assert!(!*sent_is_no_longer_speculative);
-                        *sent_is_no_longer_speculative = true;
-                    }
                 }
             }
             if sim.read_past_bool(cant_cause_cancel.ready, cd.clk).await {
@@ -1725,21 +1988,30 @@ async fn rename_execute_retire_run(
                     assert!(!state.is_canceling());
                     let RobEntry {
                         mop: _,
+                        mop_in_unit_state,
                         is_speculative: _,
-                        sent_is_no_longer_speculative: _,
+                        finished,
-                        can_cause_cancel,
-                        progress,
                         caused_cancel,
                     } = state.rob.renamed_by_id_mut(&id);
+                    assert!(finished.is_none());
                     assert!(caused_cancel.is_none());
-                    assert!(!matches!(progress, MOpExecutionProgress::Canceled));
+                    *mop_in_unit_state = mop_in_unit_state
-                    *can_cause_cancel = false;
+                        .with_inputs_ready()
+                        .expect("UnitMOpCantCauseCancel should be valid");
                 }
             }
-            if sim.read_past_bool(finished.ready, cd.clk).await {
+            if sim.read_past_bool(output_ready.ready, cd.clk).await {
                 #[hdl(sim)]
-                if let HdlSome(finished) = sim.read_past(finished.data, cd.clk).await {
+                if let HdlSome(output_ready) = sim.read_past(output_ready.data, cd.clk).await {
-                    state.unit_finished(finished);
+                    state.unit_output_ready(output_ready);
+                }
+            }
+            if sim.read_past_bool(finish_cause_cancel.ready, cd.clk).await {
+                #[hdl(sim)]
+                if let HdlSome(finish_cause_cancel) =
+                    sim.read_past(finish_cause_cancel.data, cd.clk).await
+                {
+                    state.unit_finish_cause_cancel(finish_cause_cancel);
                 }
             }
         }
@@ -1819,17 +2091,25 @@ pub fn rename_execute_retire(config: PhantomConst<CpuConfig>) {
                     for unit_field in ExecuteToUnitInterfaces::unit_fields(to_units) {
                         #[hdl]
                         let ExecuteToUnitInterface::<_> {
-                            start,
+                            enqueue,
+                            inputs_ready,
                             cancel_all,
                             is_no_longer_speculative,
                             cant_cause_cancel,
-                            finished,
+                            output_ready,
+                            finish_cause_cancel,
                             config: _,
                         } = unit_field;
                         sim.write(
-                            start.data,
+                            enqueue.data,
                             #[hdl(sim)]
-                            (start.ty().data).HdlNone(),
+                            (enqueue.ty().data).HdlNone(),
+                        )
+                        .await;
+                        sim.write(
+                            inputs_ready.data,
+                            #[hdl(sim)]
+                            (inputs_ready.ty().data).HdlNone(),
                         )
                         .await;
                         sim.write(
@@ -1845,7 +2125,8 @@ pub fn rename_execute_retire(config: PhantomConst<CpuConfig>) {
                         )
                         .await;
                         sim.write(cant_cause_cancel.ready, false).await;
-                        sim.write(finished.ready, false).await;
+                        sim.write(output_ready.ready, false).await;
+                        sim.write(finish_cause_cancel.ready, false).await;
                     }
                 },
                 |sim, ()| {

crates/cpu/src/rename_execute_retire/unit.mermaid

@@ -0,0 +1,41 @@
+%% SPDX-License-Identifier: LGPL-3.0-or-later
+%% See Notices.txt for copyright information
+stateDiagram-v2
+    direction LR
+
+    state "Inputs not ready<br/>Speculative<br/>Can cause cancel" as inr_s_c
+    state "Inputs not ready<br/>Speculative" as inr_s
+    state "Inputs ready<br/>Speculative<br/>Can cause cancel" as ir_s_c
+    state "Inputs ready<br/>Speculative" as ir_s
+    state "Inputs ready<br/>Can cause cancel" as ir_c
+    state "Inputs ready" as ir
+    state "Output ready<br/>Speculative<br/>Can cause cancel" as or_s_c
+    state "Output ready<br/>Can cause cancel" as or_c
+    state "Output ready<br/>Speculative" as or_s
+    state "Output ready" as or
+
+    [*] --> inr_s_c: Enqueue
+
+    inr_s_c --> inr_s: Can't cause cancel
+    ir_s_c --> ir_s: Can't cause cancel
+    ir_c --> ir: Can't cause cancel
+    or_s_c --> or_s: Can't cause cancel
+    or_c --> or: Can't cause cancel
+
+    ir_s_c --> ir_c: No longer speculative
+    ir_s --> ir: No longer speculative
+    or_s_c --> or_c: No longer speculative
+    or_s --> or: No longer speculative
+
+    inr_s_c --> ir_s_c: Inputs ready
+    inr_s --> ir_s: Inputs ready
+
+    ir_s_c --> or_s_c: Output Ready
+    ir_c --> or_c: Output Ready
+    ir_s --> or_s: Output Ready
+    ir --> or: Output Ready
+
+    or_s_c --> [*]: Finish
+    or_c --> [*]: Finish
+    or_s --> [*]: Finish
+    or --> [*]: Finish

crates/cpu/tests/rename_execute_retire.rs

@@ -17,8 +17,8 @@ use cpu::{
     rename_execute_retire::{
         ExecuteToUnitInterface, MOpId, MOpInstance, NextPcPredictorOp, PostDecodeOutputInterface,
         RenamedMOp, RetireToNextPcInterface, RetireToNextPcInterfaceInner, UnitCausedCancel,
-        UnitFinished, UnitFinishedSuccessfully, UnitMOpCantCauseCancel,
+        UnitFinishCauseCancel, UnitInputsReady, UnitMOpCantCauseCancel,
-        UnitMOpIsNoLongerSpeculative, UnitStart, rename_execute_retire,
+        UnitMOpIsNoLongerSpeculative, UnitOutputReady, rename_execute_retire,
         to_unit_interfaces::ExecuteToUnitInterfaces,
     },
     unit::{UnitKind, UnitMOp},
@@ -1321,8 +1321,8 @@ trait MockExecutionStateTrait {
             Some(
                 #[hdl(sim)]
                 UnitCausedCancel::<C> {
-                    id,
                     start_at_pc: next_pc,
+                    cancel_after_retire: true,
                     config,
                 },
             )
@@ -1677,7 +1677,8 @@ struct MockUnitOpDebugState<C: PhantomConstGet<CpuConfig>> {
     src_values: Array<PRegValue, { COMMON_MOP_SRC_LEN }>,
     is_speculative: Bool,
     sent_cant_cause_cancel: Bool,
-    finished: HdlOption<UnitFinished<C>>,
+    output_ready: HdlOption<UnitOutputReady<C>>,
+    finish_cause_cancel: HdlOption<UnitFinishCauseCancel<C>>,
     config: C,
 }
 
@@ -1687,7 +1688,8 @@ struct MockUnitOp<C: PhantomConstCpuConfig> {
     src_values: [SimValue<PRegValue>; COMMON_MOP_SRC_LEN],
     is_speculative: bool,
     sent_cant_cause_cancel: bool,
-    finished: Option<SimValue<UnitFinished<C>>>,
+    output_ready: Option<SimValue<UnitOutputReady<C>>>,
+    finish_cause_cancel: Option<SimValue<UnitFinishCauseCancel<C>>>,
     config: C,
 }
 
@@ -1699,29 +1701,38 @@ impl<C: PhantomConstCpuConfig> MockUnitOp<C> {
             src_values,
             is_speculative,
             sent_cant_cause_cancel,
-            finished,
+            output_ready,
+            finish_cause_cancel,
             config,
         } = self;
-        let finished_ty = HdlOption[UnitFinished[*config]];
+        let output_ready_ty = HdlOption[UnitOutputReady[*config]];
+        let finish_cause_cancel_ty = HdlOption[UnitFinishCauseCancel[*config]];
         #[hdl(sim)]
         MockUnitOpDebugState::<_> {
             mop,
             src_values,
             is_speculative,
             sent_cant_cause_cancel,
-            finished: if let Some(finished) = finished {
+            output_ready: if let Some(output_ready) = output_ready {
                 #[hdl(sim)]
-                finished_ty.HdlSome(finished)
+                output_ready_ty.HdlSome(output_ready)
             } else {
                 #[hdl(sim)]
-                finished_ty.HdlNone()
+                output_ready_ty.HdlNone()
+            },
+            finish_cause_cancel: if let Some(finish_cause_cancel) = finish_cause_cancel {
+                #[hdl(sim)]
+                finish_cause_cancel_ty.HdlSome(finish_cause_cancel)
+            } else {
+                #[hdl(sim)]
+                finish_cause_cancel_ty.HdlNone()
             },
             config,
         }
     }
     #[hdl]
     fn try_run<E: MockExecutionStateTrait>(&mut self, execution_state: &mut E) {
-        if self.finished.is_some() {
+        if self.output_ready.is_some() || self.finish_cause_cancel.is_some() {
             return;
         }
         let (output, caused_cancel) = execution_state.run_mop(
@@ -1922,13 +1933,13 @@ impl<C: PhantomConstCpuConfig, E: MockExecutionStateTrait> MockUnitState<C, E> {
         self.ops.remove(&id);
     }
     #[hdl]
-    fn handle_start(&mut self, start: SimValue<UnitStart<C>>) {
+    fn handle_inputs_ready(&mut self, inputs_ready: SimValue<UnitInputsReady<C>>) {
         #[hdl(sim)]
-        let UnitStart::<_> {
+        let UnitInputsReady::<_> {
             mop,
             src_values,
             config: _,
-        } = start;
+        } = inputs_ready;
         assert_eq!(
             UnitNum::index_sim(&MOpTrait::dest_reg_sim_ref(&mop.mop).unit_num),
             Some(self.unit_index),
@@ -1968,23 +1979,6 @@ impl<C: PhantomConstCpuConfig, E: MockExecutionStateTrait> MockUnitState<C, E> {
     }
 }
 
-fn is_the_load_store_unit(config: PhantomConst<CpuConfig>, unit_index: usize) -> bool {
-    let load_store_unit_index = config
-        .get()
-        .units
-        .iter()
-        .position(|v| v.kind == UnitKind::LoadStore);
-    let load_store_last_unit_index = config
-        .get()
-        .units
-        .iter()
-        .rposition(|v| v.kind == UnitKind::LoadStore);
-    if load_store_unit_index != load_store_last_unit_index {
-        todo!("multiple LoadStore units aren't implemented");
-    }
-    Some(unit_index) == load_store_unit_index
-}
-
 fn is_the_l2_reg_file_unit(config: PhantomConst<CpuConfig>, unit_index: usize) -> bool {
     let first_unit_index = config
         .get()
@@ -2036,14 +2030,16 @@ fn mock_unit<#[hdl(skip)] MI: MakeInsns, #[hdl(skip)] E: MockExecutionStateTrait
                 async |mut sim| {
                     #[hdl]
                     let ExecuteToUnitInterface::<_> {
-                        start,
+                        enqueue,
+                        inputs_ready,
                         cancel_all,
                         is_no_longer_speculative,
                         cant_cause_cancel,
                         finished,
                         config: _,
                     } = from_execute;
-                    sim.write(start.ready, false).await;
+                    sim.write(enqueue.ready, false).await;
+                    sim.write(inputs_ready.ready, false).await;
                     sim.write(cancel_all.ready, false).await;
                     sim.write(is_no_longer_speculative.ready, false).await;
                     sim.write(
@@ -2101,14 +2097,16 @@ fn mock_unit<#[hdl(skip)] MI: MakeInsns, #[hdl(skip)] E: MockExecutionStateTrait
             {
                 #[hdl]
                 let ExecuteToUnitInterface::<_> {
-                    start,
+                    enqueue,
+                    inputs_ready,
                     cancel_all,
                     is_no_longer_speculative,
                     cant_cause_cancel,
                     finished,
                     config: _,
                 } = from_execute;
-                sim.write(start.ready, true).await;
+                sim.write(enqueue.ready, true).await;
+                sim.write(inputs_ready.ready, true).await;
                 sim.write(cancel_all.ready, true).await;
                 sim.write(is_no_longer_speculative.ready, true).await;
                 sim.write(cant_cause_cancel.data, state.peek_cant_cause_cancel())
@@ -2125,17 +2123,24 @@ fn mock_unit<#[hdl(skip)] MI: MakeInsns, #[hdl(skip)] E: MockExecutionStateTrait
             {
                 #[hdl]
                 let ExecuteToUnitInterface::<_> {
-                    start,
+                    enqueue,
+                    inputs_ready,
                     cancel_all,
                     is_no_longer_speculative,
                     cant_cause_cancel,
                     finished,
                     config: _,
                 } = from_execute;
-                if sim.read_past_bool(start.ready, cd.clk).await {
+                if sim.read_past_bool(enqueue.ready, cd.clk).await {
                     #[hdl(sim)]
-                    if let HdlSome(start) = sim.read_past(start.data, cd.clk).await {
+                    if let HdlSome(enqueue) = sim.read_past(enqueue.data, cd.clk).await {
-                        state.handle_start(start);
+                        state.handle_enqueue(enqueue);
+                    }
+                }
+                if sim.read_past_bool(inputs_ready.ready, cd.clk).await {
+                    #[hdl(sim)]
+                    if let HdlSome(inputs_ready) = sim.read_past(inputs_ready.data, cd.clk).await {
+                        state.handle_inputs_ready(inputs_ready);
                     }
                 }
                 if sim
@@ -2175,6 +2180,240 @@ fn mock_unit<#[hdl(skip)] MI: MakeInsns, #[hdl(skip)] E: MockExecutionStateTrait
     }
 }
 
+struct MockLoadStoreOp<C: PhantomConstCpuConfig> {
+    config: C,
+}
+
+#[hdl(no_static)]
+struct MockLoadStoreUnitDebugState<C: PhantomConstGet<CpuConfig>> {
+    config: C,
+}
+
+struct MockLoadStoreUnitState<C: PhantomConstCpuConfig> {
+    ops: VecDeque<MockLoadStoreOp<C>>,
+    config: C,
+    unit_index: usize,
+    load_store_state: MockLoadStoreState,
+}
+
+impl<C: PhantomConstCpuConfig> MockLoadStoreUnitState<C> {
+    fn new(config: C, unit_index: usize, load_store_state: MockLoadStoreState) -> Self {
+        Self {
+            ops: VecDeque::new(),
+            config,
+            unit_index,
+            load_store_state,
+        }
+    }
+}
+
+#[hdl_module(extern)]
+fn mock_load_store_unit<#[hdl(skip)] MI: MakeInsns>(
+    config: PhantomConst<CpuConfig>,
+    unit_index: usize,
+) {
+    assert!(
+        config
+            .get()
+            .units
+            .iter()
+            .enumerate()
+            .all(|(i, unit)| (unit_index == i) == (unit.kind == UnitKind::LoadStore)),
+        "unit index {unit_index} must be the load/store unit: {config:#?}",
+    );
+    #[hdl]
+    let cd: ClockDomain = m.input();
+    #[hdl]
+    let from_execute: ExecuteToUnitInterface<PhantomConst<CpuConfig>> =
+        m.input(ExecuteToUnitInterface[config]);
+    #[hdl]
+    let debug_state: MockLoadStoreUnitDebugState<PhantomConst<CpuConfig>> =
+        m.output(MockLoadStoreUnitDebugState[config]);
+    #[hdl]
+    let all_outputs_written: Bool = m.output();
+    m.register_clock_for_past(cd.clk);
+    m.extern_module_simulation_fn(
+        (
+            cd,
+            from_execute,
+            debug_state,
+            all_outputs_written,
+            config,
+            unit_index,
+        ),
+        async |args, mut sim| {
+            let (cd, from_execute, debug_state, all_outputs_written, config, unit_index) = args;
+            sim.resettable(
+                cd,
+                async |mut sim| {
+                    #[hdl]
+                    let ExecuteToUnitInterface::<_> {
+                        enqueue,
+                        inputs_ready,
+                        cancel_all,
+                        is_no_longer_speculative,
+                        cant_cause_cancel,
+                        output_ready,
+                        finish_cause_cancel,
+                        config: _,
+                    } = from_execute;
+                    sim.write(enqueue.ready, false).await;
+                    sim.write(inputs_ready.ready, false).await;
+                    sim.write(cancel_all.ready, false).await;
+                    sim.write(is_no_longer_speculative.ready, false).await;
+                    sim.write(
+                        cant_cause_cancel.data,
+                        #[hdl(sim)]
+                        (cant_cause_cancel.ty().data).HdlNone(),
+                    )
+                    .await;
+                    sim.write(
+                        output_ready.data,
+                        #[hdl(sim)]
+                        (output_ready.ty().data).HdlNone(),
+                    )
+                    .await;
+                    sim.write(
+                        finish_cause_cancel.data,
+                        #[hdl(sim)]
+                        (finish_cause_cancel.ty().data).HdlNone(),
+                    )
+                    .await;
+                    sim.write(
+                        debug_state,
+                        #[hdl(sim)]
+                        MockLoadStoreUnitDebugState::<_> { config },
+                    )
+                    .await;
+                },
+                |sim, ()| {
+                    run_fn(
+                        cd,
+                        from_execute,
+                        debug_state,
+                        all_outputs_written,
+                        MI::make_load_store_execution_state(),
+                        config,
+                        unit_index,
+                        sim,
+                    )
+                },
+            )
+            .await;
+        },
+    );
+    #[hdl]
+    async fn run_fn(
+        cd: Expr<ClockDomain>,
+        from_execute: Expr<ExecuteToUnitInterface<PhantomConst<CpuConfig>>>,
+        debug_state: Expr<MockLoadStoreUnitDebugState<PhantomConst<CpuConfig>>>,
+        all_outputs_written: Expr<Bool>,
+        load_store_state: MockLoadStoreState,
+        config: PhantomConst<CpuConfig>,
+        unit_index: usize,
+        mut sim: ExternModuleSimulationState,
+    ) {
+        let mut state = MockLoadStoreUnitState::new(config, unit_index, load_store_state);
+        loop {
+            {
+                #[hdl]
+                let ExecuteToUnitInterface::<_> {
+                    enqueue,
+                    inputs_ready,
+                    cancel_all,
+                    is_no_longer_speculative,
+                    cant_cause_cancel,
+                    output_ready,
+                    finish_cause_cancel,
+                    config: _,
+                } = from_execute;
+                sim.write(enqueue.ready, true).await;
+                sim.write(inputs_ready.ready, true).await;
+                sim.write(cancel_all.ready, true).await;
+                sim.write(is_no_longer_speculative.ready, true).await;
+                sim.write(cant_cause_cancel.data, state.peek_cant_cause_cancel())
+                    .await;
+                sim.write(output_ready.data, state.peek_output_ready())
+                    .await;
+                sim.write(finish_cause_cancel.data, state.peek_finish_cause_cancel())
+                    .await;
+            }
+            sim.write(debug_state, state.debug_state()).await;
+            sim.write(
+                all_outputs_written,
+                state.load_store_state.all_outputs_written(),
+            )
+            .await;
+            sim.wait_for_clock_edge(cd.clk).await;
+            {
+                #[hdl]
+                let ExecuteToUnitInterface::<_> {
+                    enqueue,
+                    inputs_ready,
+                    cancel_all,
+                    is_no_longer_speculative,
+                    cant_cause_cancel,
+                    output_ready,
+                    finish_cause_cancel,
+                    config: _,
+                } = from_execute;
+                if sim.read_past_bool(enqueue.ready, cd.clk).await {
+                    #[hdl(sim)]
+                    if let HdlSome(enqueue) = sim.read_past(enqueue.data, cd.clk).await {
+                        state.handle_enqueue(enqueue);
+                    }
+                }
+                if sim.read_past_bool(inputs_ready.ready, cd.clk).await {
+                    #[hdl(sim)]
+                    if let HdlSome(inputs_ready) = sim.read_past(inputs_ready.data, cd.clk).await {
+                        state.handle_inputs_ready(inputs_ready);
+                    }
+                }
+                if sim
+                    .read_past_bool(is_no_longer_speculative.ready, cd.clk)
+                    .await
+                {
+                    #[hdl(sim)]
+                    if let HdlSome(is_no_longer_speculative) =
+                        sim.read_past(is_no_longer_speculative.data, cd.clk).await
+                    {
+                        state.handle_mop_is_no_longer_speculative(is_no_longer_speculative);
+                    }
+                }
+                if sim.read_past_bool(cant_cause_cancel.ready, cd.clk).await {
+                    #[hdl(sim)]
+                    if let HdlSome(cant_cause_cancel) =
+                        sim.read_past(cant_cause_cancel.data, cd.clk).await
+                    {
+                        state.handle_mop_cant_cause_cancel(cant_cause_cancel);
+                    }
+                }
+                if sim.read_past_bool(output_ready.ready, cd.clk).await {
+                    #[hdl(sim)]
+                    if let HdlSome(output_ready) = sim.read_past(output_ready.data, cd.clk).await {
+                        state.handle_output_ready(output_ready);
+                    }
+                }
+                if sim.read_past_bool(finish_cause_cancel.ready, cd.clk).await {
+                    #[hdl(sim)]
+                    if let HdlSome(finish_cause_cancel) =
+                        sim.read_past(finish_cause_cancel.data, cd.clk).await
+                    {
+                        state.handle_finish_cause_cancel(finish_cause_cancel);
+                    }
+                }
+                if sim.read_past_bool(cancel_all.ready, cd.clk).await {
+                    #[hdl(sim)]
+                    if let HdlSome(cancel_all) = sim.read_past(cancel_all.data, cd.clk).await {
+                        let () = *cancel_all;
+                        state.cancel_all();
+                    }
+                }
+            }
+        }
+    }
+}
+
 #[hdl_module]
 fn rename_execute_retire_test_harness<#[hdl(skip)] MI: MakeInsns>(config: PhantomConst<CpuConfig>) {
     #[hdl]