WIP: adding load/store unit

crates/cpu/src/instruction.rs

@@ -19,6 +19,7 @@ use std::{
     borrow::Cow,
     fmt,
     marker::PhantomData,
+    num::NonZeroU64,
     ops::{ControlFlow, Range},
 };
 
@@ -2983,6 +2984,19 @@ pub enum LoadStoreWidth {
     Width64Bit,
 }
 
+impl LoadStoreWidth {
+    #[hdl]
+    pub fn access_byte_size_sim(this: &<Self as Type>::SimValue) -> NonZeroU64 {
+        #[hdl(sim)]
+        match this {
+            Self::Width8Bit => const { NonZeroU64::new(1).unwrap() },
+            Self::Width16Bit => const { NonZeroU64::new(2).unwrap() },
+            Self::Width32Bit => const { NonZeroU64::new(4).unwrap() },
+            Self::Width64Bit => const { NonZeroU64::new(8).unwrap() },
+        }
+    }
+}
+
 #[hdl(cmp_eq)]
 pub enum LoadStoreConversion {
     ZeroExt,

crates/cpu/src/unit.rs

@@ -8,6 +8,7 @@ use crate::{
         MOpVariantVisitOps, MOpVariantVisitor, MOpVisitVariants, RenamedMOp, mop_enum,
     },
     rename_execute_retire::ExecuteToUnitInterface,
+    unit::load_store::LoadStoreToDCacheInterface,
 };
 use fayalite::{
     bundle::{Bundle, BundleType},
@@ -18,6 +19,7 @@ use serde::{Deserialize, Serialize};
 use std::ops::ControlFlow;
 
 pub mod alu_branch;
+pub mod load_store;
 
 macro_rules! all_units {
     (
@@ -333,7 +335,7 @@ all_units! {
         #[create_dyn_unit_fn = |config, unit_index, filter| todo!()]
         #[extract(transformed_move_mop, transformed_move_mop_sim, transformed_move_mop_sim_ref, transformed_move_mop_sim_mut)]
         TransformedMove(TransformedMoveOp),
-        #[create_dyn_unit_fn = |config, unit_index, filter| todo!()]
+        #[create_dyn_unit_fn = |config, unit_index, filter| load_store::LoadStore::new(config, unit_index, filter).to_dyn()]
         #[extract(load_store_mop, load_store_mop_sim, load_store_mop_sim_ref, load_store_mop_sim_mut)]
         LoadStore(LoadStoreMOp<DestReg, SrcReg>),
     }
@@ -391,6 +393,7 @@ impl RenamedMOpFilter for () {
 pub struct UnitIO {
     pub cd: Option<Expr<ClockDomain>>,
     pub from_execute: Expr<ExecuteToUnitInterface<PhantomConst<CpuConfig>>>,
+    pub to_d_cache: Option<Expr<LoadStoreToDCacheInterface<PhantomConst<CpuConfig>>>>,
 }
 
 pub trait UnitTrait:

crates/cpu/src/unit/alu_branch.rs

@@ -1617,6 +1617,7 @@ impl UnitTrait for AluBranch {
         UnitIO {
             cd: None,
             from_execute: this.from_execute,
+            to_d_cache: None,
         }
     }
 

crates/cpu/src/unit/load_store.rs

@@ -0,0 +1,870 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// See Notices.txt for copyright information
+
+use crate::{
+    config::{CpuConfig, PhantomConstCpuConfig},
+    instruction::{COMMON_MOP_SRC_LEN, LoadStoreCommonMOp, LoadStoreMOp, LoadStoreWidth, PRegNum},
+    main_memory_and_io::{AddressRange, MemoryOperationErrorKind},
+    next_pc::{CallStackOp, SimValueDefault},
+    register::PRegValue,
+    rename_execute_retire::{
+        ExecuteToUnitInterface, GlobalState, MOpId, MOpInstance, NextPcPredictorOp, UnitEnqueue,
+        UnitFinishCauseCancel, UnitInputsReady, UnitMOpCantCauseCancel,
+        UnitMOpIsNoLongerSpeculative, UnitOutputReady,
+    },
+    unit::{DynUnit, DynUnitWrapper, RenamedMOpFilter, UnitIO, UnitKind, UnitMOp, UnitTrait},
+};
+use fayalite::{intern::Interned, prelude::*, ty::SimValueDebug, util::ready_valid::ReadyValid};
+use std::{collections::VecDeque, fmt};
+
+#[hdl]
+pub struct DCacheOpKindLoad {
+    /// if this is `false`, then if the address refers to a valid cache line, that cache line must have any dirty data
+    /// written back to memory and then be set to invalid before performing the load.
+    pub is_cacheable: Bool,
+}
+
+#[hdl]
+pub struct DCacheOpKindStore {
+    /// if this is `false`, then if the address refers to a valid cache line, that cache line must have any dirty data
+    /// written back to memory and then be set to invalid before performing the store.
+    pub is_cacheable: Bool,
+}
+
+#[hdl]
+pub enum DCacheOpKind {
+    Load(DCacheOpKindLoad),
+    Store(DCacheOpKindStore),
+}
+
+#[hdl]
+pub struct DCacheStart<C: PhantomConstGet<CpuConfig>> {
+    pub id: MOpId,
+    pub kind: DCacheOpKind,
+    /// unaligned addresses may cause a load/store to cross a cache-line boundary
+    pub address: UInt<64>,
+    pub data_and_mask: Array<HdlOption<UInt<8>>, 8>,
+    pub config: C,
+}
+
+#[hdl]
+pub enum DCacheFinishStatus {
+    /// The operation finished successfully
+    Success,
+    MemoryError(MemoryOperationErrorKind),
+}
+
+#[hdl]
+pub struct DCacheFinish<C: PhantomConstGet<CpuConfig>> {
+    pub status: DCacheFinishStatus,
+    pub data: Array<UInt<8>, 8>,
+    pub config: C,
+}
+
+/// This load must only look in the L1 cache (must not propagate to the L2/L3 cache or to memory), must not
+/// cause the timing of any earlier speculative operations any non-speculative operations to change because
+/// this load exists, and must not change any cache state that remains after all operations finish,
+/// including LRU state, random number generators, predictive prefetching state, etc.
+///
+/// Speculative loads that miss the L1 cache must return with [`DCacheFinishSpeculativeLoad::data`] set to
+/// [`HdlNone()`] instead of trying to propagate to the L2/L3 or memory.
+#[hdl]
+pub struct DCacheStartSpeculativeLoad<C: PhantomConstGet<CpuConfig>> {
+    pub id: MOpId,
+    /// unaligned addresses may cause a load to cross a cache-line boundary
+    pub address: UInt<64>,
+    pub mask: Array<Bool, 8>,
+    pub config: C,
+}
+
+#[hdl]
+pub struct DCacheFinishSpeculativeLoad<C: PhantomConstGet<CpuConfig>> {
+    pub data: HdlOption<Array<UInt<8>, 8>>,
+    pub config: C,
+}
+
+#[hdl]
+pub struct LoadStoreToDCacheInterface<C: PhantomConstGet<CpuConfig>> {
+    pub start: ReadyValid<DCacheStart<C>>,
+    #[hdl(flip)]
+    pub finish: ReadyValid<DCacheFinish<C>>,
+    /// it's valid for `start_speculative_load.ready` to be `false` at any time (or even always)
+    /// -- this doesn't block forward progress since speculative loads that don't start will eventually
+    /// convert to non-speculative loads and go through `start` instead.
+    pub start_speculative_load: ReadyValid<DCacheStartSpeculativeLoad<C>>,
+    #[hdl(flip)]
+    pub finish_speculative_load: ReadyValid<DCacheFinishSpeculativeLoad<C>>,
+    pub config: C,
+}
+
+#[hdl(custom_debug(sim))]
+struct OpDebugState {
+    v: SimOnly<String>,
+}
+
+impl SimValueDebug for OpDebugState {
+    #[hdl]
+    fn sim_value_debug(
+        value: &<Self as Type>::SimValue,
+        f: &mut fmt::Formatter<'_>,
+    ) -> fmt::Result {
+        f.write_str(&value.v)
+    }
+}
+
+impl SimValueDefault for OpDebugState {
+    #[hdl]
+    fn sim_value_default(self) -> SimValue<Self> {
+        thread_local! {
+            static VALUE: SimOnlyValue<String> = SimOnlyValue::new("None".into());
+        }
+        VALUE.with(|v| {
+            #[hdl(sim)]
+            Self { v }
+        })
+    }
+}
+
+#[derive(Debug)]
+struct Op<C: PhantomConstCpuConfig> {
+    id: SimValue<MOpId>,
+    pc: SimValue<UInt<64>>,
+    size_in_bytes: u8,
+    mop: SimValue<LoadStoreMOp<PRegNum<C>, PRegNum<C>>>,
+    /// earlier instructions can cause a cancel, so this instruction is still speculative.
+    /// not to be confused with this instruction being able to cause a cancel.
+    is_speculative: bool,
+    can_cause_cancel: bool,
+    src_values: Option<[SimValue<TraceAsString<PRegValue>>; COMMON_MOP_SRC_LEN]>,
+    dest_value: Option<SimValue<TraceAsString<PRegValue>>>,
+    sent_cant_cause_cancel: bool,
+    sent_output_ready: bool,
+    started_speculative_load: bool,
+    finished_speculative_load: bool,
+    started_load_store: bool,
+    config: C,
+}
+
+impl<C: PhantomConstCpuConfig> fmt::Display for Op<C> {
+    #[hdl]
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let Self {
+            id,
+            pc,
+            size_in_bytes,
+            mop,
+            is_speculative,
+            can_cause_cancel,
+            src_values,
+            dest_value,
+            sent_cant_cause_cancel,
+            sent_output_ready,
+            started_speculative_load,
+            finished_speculative_load,
+            started_load_store,
+            config: _,
+        } = self;
+        if *is_speculative {
+            f.write_str("(s)")?;
+        }
+        if *can_cause_cancel {
+            f.write_str("(ccc)")?;
+        }
+        if *sent_cant_cause_cancel {
+            f.write_str("(sccc)")?;
+        }
+        if *sent_output_ready {
+            f.write_str("(sor)")?;
+        }
+        if *started_speculative_load {
+            f.write_str("(ssl)")?;
+        }
+        if *finished_speculative_load {
+            f.write_str("(fsl)")?;
+        }
+        if *started_load_store {
+            f.write_str("(sls)")?;
+        }
+        write!(f, "id={id:?} pc={pc:?} sz={size_in_bytes}: {mop:?}")?;
+        if let Some(src_values) = src_values {
+            write!(f, "src_values={src_values:?}")?;
+        }
+        if let Some(dest_value) = dest_value {
+            write!(f, "dest_value={dest_value:?}")?;
+        }
+        Ok(())
+    }
+}
+
+struct AccessRangeUnknown;
+
+impl<C: PhantomConstCpuConfig> Op<C> {
+    fn new(
+        id: SimValue<MOpId>,
+        pc: SimValue<UInt<64>>,
+        size_in_bytes: u8,
+        mop: SimValue<LoadStoreMOp<PRegNum<C>, PRegNum<C>>>,
+        config: C,
+    ) -> Self {
+        Self {
+            id,
+            pc,
+            size_in_bytes,
+            mop,
+            is_speculative: true,
+            can_cause_cancel: true,
+            src_values: None,
+            dest_value: None,
+            sent_cant_cause_cancel: false,
+            sent_output_ready: false,
+            started_speculative_load: false,
+            finished_speculative_load: false,
+            started_load_store: false,
+            config,
+        }
+    }
+    #[hdl]
+    fn debug_state(&self) -> SimValue<OpDebugState> {
+        #[hdl(sim)]
+        OpDebugState {
+            v: SimOnlyValue::new(self.to_string()),
+        }
+    }
+    #[hdl]
+    fn access_range_helper<SrcCount: KnownSize>(
+        &self,
+        load_store_common: &SimValue<LoadStoreCommonMOp<PRegNum<C>, PRegNum<C>, SrcCount>>,
+    ) -> Result<AddressRange, AccessRangeUnknown> {
+        let Some([src0, ..]) = &self.src_values else {
+            return Err(AccessRangeUnknown);
+        };
+        let size = LoadStoreWidth::access_byte_size_sim(&load_store_common.width);
+        Ok(AddressRange::Limited {
+            start: std::num::Wrapping(src0.inner().int_fp.as_int()),
+            size,
+        })
+    }
+    #[hdl]
+    fn access_range(
+        &self,
+        override_load_range: Option<Result<AddressRange, AccessRangeUnknown>>,
+    ) -> Result<AddressRange, AccessRangeUnknown> {
+        #[hdl(sim)]
+        match &self.mop {
+            LoadStoreMOp::<_, _>::Load(mop) => {
+                if let Some(override_load_range) = override_load_range {
+                    return override_load_range;
+                }
+                self.access_range_helper(&mop.load_store_common)
+            }
+            LoadStoreMOp::<_, _>::Store(mop) => self.access_range_helper(&mop.load_store_common),
+        }
+    }
+}
+
+#[hdl(get(|c| c.rob_size.get().next_power_of_two()))]
+type OpsDebugLen<C: PhantomConstGet<CpuConfig>> = DynSize;
+
+#[hdl]
+type PhantomConstUnitIndex = PhantomConst<usize>;
+
+#[hdl(no_static)]
+pub struct LoadStoreDebugState<C: PhantomConstGet<CpuConfig>> {
+    global_state: GlobalState,
+    ops: ArrayType<TraceAsString<OpDebugState>, OpsDebugLen<C>>,
+    canceling: Bool,
+    config: C,
+    unit_index: PhantomConstUnitIndex,
+    filter: LoadStoreFilter,
+}
+
+#[derive(Debug)]
+struct LoadStoreState<C: PhantomConstCpuConfig> {
+    global_state: SimValue<GlobalState>,
+    ops: VecDeque<Op<C>>,
+    canceling: bool,
+    config: C,
+    unit_index: usize,
+    filter: SimValue<LoadStoreFilter>,
+}
+
+impl<C: PhantomConstCpuConfig> LoadStoreState<C> {
+    fn new(config: C, unit_index: usize, filter: SimValue<LoadStoreFilter>) -> Self {
+        Self {
+            global_state: GlobalState.sim_value_default(),
+            ops: VecDeque::new(),
+            canceling: false,
+            config,
+            unit_index,
+            filter,
+        }
+    }
+    #[hdl]
+    fn debug_state(&self) -> SimValue<LoadStoreDebugState<C>> {
+        let Self {
+            ref global_state,
+            ops: _,
+            canceling,
+            config,
+            unit_index,
+            ref filter,
+        } = *self;
+        let ty = LoadStoreDebugState[config];
+        let mut ops = vec![OpDebugState.sim_value_default().into_trace_as_string(); ty.ops.len()];
+        for op in &self.ops {
+            ops[op.id.as_int() as usize % ty.ops.len()] = op.debug_state().into_trace_as_string();
+        }
+        #[hdl(sim)]
+        LoadStoreDebugState::<_> {
+            global_state,
+            ops,
+            canceling,
+            config,
+            unit_index: PhantomConst::new_sized(unit_index),
+            filter,
+        }
+    }
+    #[track_caller]
+    fn op_by_id(&mut self, id: &<MOpId as Type>::SimValue) -> &mut Op<C> {
+        match self.ops.iter_mut().find(|op| *op.id == *id) {
+            Some(op) => op,
+            None => panic!("can't find load_store Op with id {id:?}"),
+        }
+    }
+    fn from_execute_enqueue_ready(&self) -> bool {
+        !self.canceling
+            && self.ops.len() < self.config.get().unit_max_in_flight(self.unit_index).get()
+    }
+    #[hdl]
+    fn from_execute_enqueue(&mut self, enqueue: SimValue<UnitEnqueue<C>>) {
+        #[hdl(sim)]
+        let UnitEnqueue::<_> { mop, config: _ } = enqueue;
+        #[hdl(sim)]
+        let MOpInstance::<_> {
+            fetch_block_id: _,
+            id,
+            pc,
+            predicted_next_pc: _,
+            size_in_bytes,
+            is_first_mop_in_insn: _,
+            is_last_mop_in_insn: _,
+            mop,
+        } = mop;
+        let mop = #[hdl(sim)]
+        match SimValue::into_value(mop).into_inner() {
+            UnitMOp::<_, _, _>::LoadStore(mop) => mop,
+            _ => unreachable!(),
+        };
+        self.ops.push_back(Op::new(
+            id,
+            pc,
+            size_in_bytes.cast_to_static::<UInt<8>>().as_int(),
+            mop,
+            self.config,
+        ));
+    }
+    #[hdl]
+    fn from_execute_inputs_ready(&mut self, inputs_ready: SimValue<UnitInputsReady<C>>) {
+        #[hdl(sim)]
+        let UnitInputsReady::<_> {
+            mop,
+            src_values,
+            config: _,
+        } = inputs_ready;
+        self.op_by_id(&mop.id).src_values = Some(SimValue::into_value(src_values));
+    }
+    #[hdl]
+    fn from_execute_is_no_longer_speculative(
+        &mut self,
+        is_no_longer_speculative: SimValue<UnitMOpIsNoLongerSpeculative<C>>,
+    ) {
+        #[hdl(sim)]
+        let UnitMOpIsNoLongerSpeculative::<_> { id, config: _ } = is_no_longer_speculative;
+        self.op_by_id(&id).is_speculative = false;
+    }
+    #[hdl]
+    fn from_execute_cant_cause_cancel(&self) -> Option<SimValue<UnitMOpCantCauseCancel<C>>> {
+        for op in &self.ops {
+            if op.sent_cant_cause_cancel || op.can_cause_cancel {
+                continue;
+            }
+            return Some(
+                #[hdl(sim)]
+                UnitMOpCantCauseCancel::<_> {
+                    id: op.id,
+                    config: self.config,
+                },
+            );
+        }
+        None
+    }
+    #[hdl]
+    fn from_execute_sent_cant_cause_cancel(
+        &mut self,
+        cant_cause_cancel: SimValue<UnitMOpCantCauseCancel<C>>,
+    ) {
+        #[hdl(sim)]
+        let UnitMOpCantCauseCancel::<_> { id, config: _ } = cant_cause_cancel;
+        self.op_by_id(&id).sent_cant_cause_cancel = true;
+    }
+    #[hdl]
+    fn from_execute_output_ready(&self) -> Option<SimValue<UnitOutputReady<C>>> {
+        for op in &self.ops {
+            if op.sent_output_ready {
+                continue;
+            }
+            if let Some(dest_value) = &op.dest_value {
+                return Some(
+                    #[hdl(sim)]
+                    UnitOutputReady::<_> {
+                        id: op.id,
+                        dest_value,
+                        predictor_op: #[hdl(sim)]
+                        NextPcPredictorOp::<_> {
+                            call_stack_op: #[hdl(sim)]
+                            CallStackOp.None(),
+                            cond_br_taken: #[hdl(sim)]
+                            HdlNone(),
+                            config: self.config,
+                        },
+                    },
+                );
+            }
+        }
+        None
+    }
+    #[hdl]
+    fn from_execute_sent_output_ready(&mut self, output_ready: SimValue<UnitOutputReady<C>>) {
+        #[hdl(sim)]
+        let UnitOutputReady::<_> {
+            id,
+            dest_value: _,
+            predictor_op: _,
+        } = output_ready;
+        self.op_by_id(&id).sent_output_ready = true;
+    }
+    #[hdl]
+    fn from_execute_finish_cause_cancel(&self) -> Option<SimValue<UnitFinishCauseCancel<C>>> {
+        todo!()
+    }
+    #[hdl]
+    fn from_execute_sent_finish_cause_cancel(
+        &mut self,
+        finish_cause_cancel: SimValue<UnitFinishCauseCancel<C>>,
+    ) {
+        let op = self
+            .ops
+            .pop_front_if(|front| front.id == finish_cause_cancel.id);
+        assert!(
+            op.is_some(),
+            "inconsistent state -- sent finish_cause_cancel but id doesn't match ops.front():\n{finish_cause_cancel:#?}\nfront={front:#?}",
+            front = self.ops.front(),
+        );
+    }
+    fn from_execute_cancel_all_ready(&self) -> bool {
+        true
+    }
+    #[hdl]
+    fn from_execute_cancel_all(&mut self) {
+        self.canceling = true;
+    }
+    #[hdl]
+    fn to_d_cache_start_data(&self) -> Option<SimValue<DCacheStart<C>>> {
+        todo!()
+    }
+    #[hdl]
+    fn to_d_cache_sent_start(&mut self, start: SimValue<DCacheStart<C>>) {
+        self.op_by_id(&start.id).started_load_store = true;
+    }
+    #[hdl]
+    fn to_d_cache_finish_ready(&self) -> bool {
+        true
+    }
+    #[hdl]
+    fn to_d_cache_finish(&mut self, finish: SimValue<DCacheFinish<C>>) {
+        todo!()
+    }
+    #[hdl]
+    fn to_d_cache_start_speculative_load_data(
+        &self,
+    ) -> Option<SimValue<DCacheStartSpeculativeLoad<C>>> {
+        todo!()
+    }
+    #[hdl]
+    fn to_d_cache_sent_start_speculative_load(
+        &mut self,
+        start_speculative_load: SimValue<DCacheStartSpeculativeLoad<C>>,
+    ) {
+        self.op_by_id(&start_speculative_load.id)
+            .started_speculative_load = true;
+    }
+    #[hdl]
+    fn to_d_cache_finish_speculative_load_ready(&self) -> bool {
+        true
+    }
+    #[hdl]
+    fn to_d_cache_finish_speculative_load(
+        &mut self,
+        finish_speculative_load: SimValue<DCacheFinishSpeculativeLoad<C>>,
+    ) {
+        todo!()
+    }
+    #[hdl]
+    fn step(&mut self) {
+        if self.canceling {
+            todo!()
+        }
+    }
+}
+
+#[hdl]
+async fn load_store_impl(
+    config: PhantomConst<CpuConfig>,
+    unit_index: usize,
+    filter: SimValue<LoadStoreFilter>,
+    cd: Expr<ClockDomain>,
+    from_execute: Expr<ExecuteToUnitInterface<PhantomConst<CpuConfig>>>,
+    to_d_cache: Expr<LoadStoreToDCacheInterface<PhantomConst<CpuConfig>>>,
+    debug_state: Expr<LoadStoreDebugState<PhantomConst<CpuConfig>>>,
+    mut sim: ExternModuleSimulationState,
+) {
+    let mut state = LoadStoreState::new(config, unit_index, filter);
+    loop {
+        {
+            #[hdl]
+            let ExecuteToUnitInterface::<_> {
+                global_state: _,
+                enqueue,
+                inputs_ready: _,
+                is_no_longer_speculative: _,
+                cant_cause_cancel,
+                output_ready,
+                finish_cause_cancel,
+                unit_outputs_ready: _,
+                cancel_all,
+                config: _,
+            } = from_execute;
+            sim.write(enqueue.ready, state.from_execute_enqueue_ready())
+                .await;
+            sim.write(cant_cause_cancel, state.from_execute_cant_cause_cancel())
+                .await;
+            sim.write(output_ready, state.from_execute_output_ready())
+                .await;
+            sim.write(
+                finish_cause_cancel,
+                state.from_execute_finish_cause_cancel(),
+            )
+            .await;
+            sim.write(cancel_all.ready, state.from_execute_cancel_all_ready())
+                .await;
+        }
+        {
+            #[hdl]
+            let LoadStoreToDCacheInterface::<_> {
+                start,
+                finish,
+                start_speculative_load,
+                finish_speculative_load,
+                config: _,
+            } = to_d_cache;
+            sim.write(start.data, state.to_d_cache_start_data()).await;
+            sim.write(finish.ready, state.to_d_cache_finish_ready())
+                .await;
+            sim.write(
+                start_speculative_load.data,
+                state.to_d_cache_start_speculative_load_data(),
+            )
+            .await;
+            sim.write(
+                finish_speculative_load.ready,
+                state.to_d_cache_finish_speculative_load_ready(),
+            )
+            .await;
+        }
+        sim.write(debug_state, state.debug_state()).await;
+        sim.wait_for_clock_edge(cd.clk).await;
+        {
+            #[hdl]
+            let ExecuteToUnitInterface::<_> {
+                global_state,
+                enqueue,
+                inputs_ready,
+                is_no_longer_speculative,
+                cant_cause_cancel,
+                output_ready,
+                finish_cause_cancel,
+                unit_outputs_ready,
+                cancel_all,
+                config: _,
+            } = from_execute;
+            state.global_state = sim.read_past(global_state, cd.clk).await;
+            if sim.read_past_bool(enqueue.ready, cd.clk).await {
+                #[hdl(sim)]
+                if let HdlSome(enqueue) = sim.read_past(enqueue.data, cd.clk).await {
+                    state.from_execute_enqueue(enqueue);
+                }
+            }
+            #[hdl(sim)]
+            if let HdlSome(inputs_ready) = sim.read_past(inputs_ready, cd.clk).await {
+                state.from_execute_inputs_ready(inputs_ready);
+            }
+            #[hdl(sim)]
+            if let HdlSome(is_no_longer_speculative) =
+                sim.read_past(is_no_longer_speculative, cd.clk).await
+            {
+                state.from_execute_is_no_longer_speculative(is_no_longer_speculative);
+            }
+            if sim.read_past_bool(unit_outputs_ready, cd.clk).await {
+                #[hdl(sim)]
+                if let HdlSome(cant_cause_cancel) = sim.read_past(cant_cause_cancel, cd.clk).await {
+                    state.from_execute_sent_cant_cause_cancel(cant_cause_cancel);
+                }
+                #[hdl(sim)]
+                if let HdlSome(output_ready) = sim.read_past(output_ready, cd.clk).await {
+                    state.from_execute_sent_output_ready(output_ready);
+                }
+                #[hdl(sim)]
+                if let HdlSome(finish_cause_cancel) =
+                    sim.read_past(finish_cause_cancel, cd.clk).await
+                {
+                    state.from_execute_sent_finish_cause_cancel(finish_cause_cancel);
+                }
+            }
+            if sim.read_past_bool(cancel_all.ready, cd.clk).await {
+                #[hdl(sim)]
+                if let HdlSome(cancel_all) = sim.read_past(cancel_all.data, cd.clk).await {
+                    #[hdl]
+                    let () = cancel_all;
+                    state.from_execute_cancel_all();
+                }
+            }
+        }
+        {
+            #[hdl]
+            let LoadStoreToDCacheInterface::<_> {
+                start,
+                finish,
+                start_speculative_load,
+                finish_speculative_load,
+                config: _,
+            } = to_d_cache;
+            if sim.read_past_bool(start.ready, cd.clk).await {
+                #[hdl(sim)]
+                if let HdlSome(start) = sim.read_past(start.data, cd.clk).await {
+                    state.to_d_cache_sent_start(start);
+                }
+            }
+            if sim.read_past_bool(finish.ready, cd.clk).await {
+                #[hdl(sim)]
+                if let HdlSome(finish) = sim.read_past(finish.data, cd.clk).await {
+                    state.to_d_cache_finish(finish);
+                }
+            }
+            if sim
+                .read_past_bool(start_speculative_load.ready, cd.clk)
+                .await
+            {
+                #[hdl(sim)]
+                if let HdlSome(start_speculative_load) =
+                    sim.read_past(start_speculative_load.data, cd.clk).await
+                {
+                    state.to_d_cache_sent_start_speculative_load(start_speculative_load);
+                }
+            }
+            if sim
+                .read_past_bool(finish_speculative_load.ready, cd.clk)
+                .await
+            {
+                #[hdl(sim)]
+                if let HdlSome(finish_speculative_load) =
+                    sim.read_past(finish_speculative_load.data, cd.clk).await
+                {
+                    state.to_d_cache_finish_speculative_load(finish_speculative_load);
+                }
+            }
+        }
+        state.step();
+    }
+}
+
+#[hdl]
+struct LoadStoreFilter {
+    load: Bool,
+    store: Bool,
+}
+
+impl LoadStoreFilter {
+    #[hdl]
+    fn new(
+        from_execute: ExecuteToUnitInterface<PhantomConst<CpuConfig>>,
+        filter: &mut impl RenamedMOpFilter,
+    ) -> SimValue<Self> {
+        let LoadStoreMOp { Load, Store } = from_execute
+            .inputs_ready
+            .HdlSome
+            .mop
+            .mop
+            .inner_ty()
+            .LoadStore;
+        #[hdl(sim)]
+        Self {
+            load: filter.should_include_ty(Load),
+            store: filter.should_include_ty(Store),
+        }
+    }
+}
+
+#[hdl_module(extern)]
+pub fn load_store(
+    config: PhantomConst<CpuConfig>,
+    unit_index: usize,
+    filter: &mut impl RenamedMOpFilter,
+) {
+    #[hdl]
+    let cd: ClockDomain = m.input();
+
+    #[hdl]
+    let from_execute: ExecuteToUnitInterface<PhantomConst<CpuConfig>> =
+        m.input(ExecuteToUnitInterface[config]);
+
+    #[hdl]
+    let to_d_cache: LoadStoreToDCacheInterface<PhantomConst<CpuConfig>> =
+        m.output(LoadStoreToDCacheInterface[config]);
+
+    #[hdl]
+    let debug_state: LoadStoreDebugState<PhantomConst<CpuConfig>> =
+        m.output(LoadStoreDebugState[config]);
+
+    let filter = LoadStoreFilter::new(from_execute.ty(), filter).to_expr();
+
+    assert_eq!(config.get().units[unit_index].kind, UnitKind::LoadStore);
+
+    m.register_clock_for_past(cd.clk);
+
+    m.extern_module_simulation_fn(
+        (
+            config,
+            unit_index,
+            filter,
+            cd,
+            from_execute,
+            to_d_cache,
+            debug_state,
+        ),
+        async |(config, unit_index, filter, cd, from_execute, to_d_cache, debug_state), mut sim| {
+            let filter = filter.into_sim_value();
+            sim.resettable(
+                cd,
+                async |mut sim| {
+                    #[hdl]
+                    let ExecuteToUnitInterface::<_> {
+                        global_state: _,
+                        enqueue,
+                        inputs_ready: _,
+                        is_no_longer_speculative: _,
+                        cant_cause_cancel,
+                        output_ready,
+                        finish_cause_cancel,
+                        unit_outputs_ready: _,
+                        cancel_all,
+                        config: _,
+                    } = from_execute;
+                    sim.write(enqueue.ready, false).await;
+                    sim.write(cant_cause_cancel, cant_cause_cancel.ty().HdlNone())
+                        .await;
+                    sim.write(output_ready, output_ready.ty().HdlNone()).await;
+                    sim.write(finish_cause_cancel, finish_cause_cancel.ty().HdlNone())
+                        .await;
+                    sim.write(cancel_all.ready, false).await;
+                    #[hdl]
+                    let LoadStoreToDCacheInterface::<_> {
+                        start,
+                        finish,
+                        start_speculative_load,
+                        finish_speculative_load,
+                        config: _,
+                    } = to_d_cache;
+                    sim.write(start.data, start.ty().data.HdlNone()).await;
+                    sim.write(finish.ready, false).await;
+                    sim.write(
+                        debug_state,
+                        LoadStoreState::new(config, unit_index, filter.clone()).debug_state(),
+                    )
+                    .await;
+                    sim.write(
+                        start_speculative_load.data,
+                        start_speculative_load.ty().data.HdlNone(),
+                    )
+                    .await;
+                    sim.write(finish_speculative_load.ready, false).await;
+                    sim.write(
+                        debug_state,
+                        LoadStoreState::new(config, unit_index, filter.clone()).debug_state(),
+                    )
+                    .await;
+                },
+                async |sim, ()| {
+                    load_store_impl(
+                        config,
+                        unit_index,
+                        filter.clone(),
+                        cd,
+                        from_execute,
+                        to_d_cache,
+                        debug_state,
+                        sim,
+                    )
+                    .await
+                },
+            )
+            .await
+        },
+    );
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct LoadStore {
+    config: PhantomConst<CpuConfig>,
+    module: Interned<Module<load_store>>,
+}
+
+impl LoadStore {
+    pub fn new(
+        config: PhantomConst<CpuConfig>,
+        unit_index: usize,
+        filter: &mut impl RenamedMOpFilter,
+    ) -> Self {
+        Self {
+            config,
+            module: load_store(config, unit_index, filter),
+        }
+    }
+}
+
+impl UnitTrait for LoadStore {
+    type Type = load_store;
+
+    fn ty(&self) -> Self::Type {
+        self.module.io_ty()
+    }
+
+    fn unit_kind(&self) -> UnitKind {
+        UnitKind::LoadStore
+    }
+
+    fn module(&self) -> Interned<Module<Self::Type>> {
+        self.module
+    }
+
+    fn io(&self, this: Expr<Self::Type>) -> UnitIO {
+        UnitIO {
+            cd: Some(this.cd),
+            from_execute: this.from_execute,
+            to_d_cache: Some(this.to_d_cache),
+        }
+    }
+
+    fn to_dyn(&self) -> DynUnit {
+        DynUnitWrapper(*self).to_dyn()
+    }
+}

crates/cpu/tests/units_formal.rs

@@ -94,13 +94,17 @@ fn formal_harness(
         let UnitIO {
             cd: unit_cd,
             from_execute: unit_from_execute,
+            to_d_cache,
         } = dyn_unit.io(unit);
+        if let Some(unit_cd) = unit_cd {
+            connect(unit_cd, cd);
+        }
         connect(
             unit_from_execute,
             ExecuteToUnitInterfaces::unit_fields(decode_and_run.to_units)[unit_index],
         );
-        if let Some(unit_cd) = unit_cd {
-            connect(unit_cd, cd);
+        if let Some(to_d_cache) = to_d_cache {
+            unimplemented!("to_d_cache for {unit:?}: {to_d_cache:?}");
         }
     }
     hdl_assert(cd.clk, !decode_and_run.error, "");