diff --git a/crates/cpu/src/instruction.rs b/crates/cpu/src/instruction.rs index 336da53..bf4be7b 100644 --- a/crates/cpu/src/instruction.rs +++ b/crates/cpu/src/instruction.rs @@ -19,6 +19,7 @@ use std::{ borrow::Cow, fmt, marker::PhantomData, + num::NonZeroU64, ops::{ControlFlow, Range}, }; @@ -2983,6 +2984,19 @@ pub enum LoadStoreWidth { Width64Bit, } +impl LoadStoreWidth { + #[hdl] + pub fn access_byte_size_sim(this: &::SimValue) -> NonZeroU64 { + #[hdl(sim)] + match this { + Self::Width8Bit => const { NonZeroU64::new(1).unwrap() }, + Self::Width16Bit => const { NonZeroU64::new(2).unwrap() }, + Self::Width32Bit => const { NonZeroU64::new(4).unwrap() }, + Self::Width64Bit => const { NonZeroU64::new(8).unwrap() }, + } + } +} + #[hdl(cmp_eq)] pub enum LoadStoreConversion { ZeroExt, diff --git a/crates/cpu/src/main_memory_and_io.rs b/crates/cpu/src/main_memory_and_io.rs index 5a1a853..795bf84 100644 --- a/crates/cpu/src/main_memory_and_io.rs +++ b/crates/cpu/src/main_memory_and_io.rs @@ -127,8 +127,47 @@ impl AddressRange { }, } } + pub const fn is_overlapping(self, other: Self) -> bool { + self.contains(other.start().0) || other.contains(self.start().0) + } } +const _: () = { + const N: u64 = 4; + const fn is_overlapping_reference(range1: AddressRange, range2: AddressRange) -> bool { + // we just need to check values < N since all starts/ends are < N + let mut v = 0; + while v < N { + if range1.contains(v) && range2.contains(v) { + return true; + } + v += 1; + } + false + } + let mut start1 = 0; + while start1 < N { + let mut end1 = 0; + while end1 < N { + let mut start2 = 0; + while start2 < N { + let mut end2 = 0; + while end2 < N { + let range1 = AddressRange::from_wrapping_range_inclusive(start1..=end1); + let range2 = AddressRange::from_wrapping_range_inclusive(start2..=end2); + assert!( + range1.is_overlapping(range2) == is_overlapping_reference(range1, range2) + ); + end2 += 1; + } + start2 += 1; + } + end1 += 1; + } + start1 += 1; + } +}; + #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, serde::Serialize, serde::Deserialize)] #[non_exhaustive] pub struct MemoryInterfaceConfig { diff --git a/crates/cpu/src/unit.rs b/crates/cpu/src/unit.rs index ce024a4..4864125 100644 --- a/crates/cpu/src/unit.rs +++ b/crates/cpu/src/unit.rs @@ -8,6 +8,7 @@ use crate::{ MOpVariantVisitOps, MOpVariantVisitor, MOpVisitVariants, RenamedMOp, mop_enum, }, rename_execute_retire::ExecuteToUnitInterface, + unit::load_store::LoadStoreToDCacheInterface, }; use fayalite::{ bundle::{Bundle, BundleType}, @@ -18,6 +19,7 @@ use serde::{Deserialize, Serialize}; use std::ops::ControlFlow; pub mod alu_branch; +pub mod load_store; macro_rules! all_units { ( @@ -333,7 +335,7 @@ all_units! { #[create_dyn_unit_fn = |config, unit_index, filter| todo!()] #[extract(transformed_move_mop, transformed_move_mop_sim, transformed_move_mop_sim_ref, transformed_move_mop_sim_mut)] TransformedMove(TransformedMoveOp), - #[create_dyn_unit_fn = |config, unit_index, filter| todo!()] + #[create_dyn_unit_fn = |config, unit_index, filter| load_store::LoadStore::new(config, unit_index, filter).to_dyn()] #[extract(load_store_mop, load_store_mop_sim, load_store_mop_sim_ref, load_store_mop_sim_mut)] LoadStore(LoadStoreMOp), } @@ -387,6 +389,13 @@ impl RenamedMOpFilter for () { } } +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +pub struct UnitIO { + pub cd: Option>, + pub from_execute: Expr>>, + pub to_d_cache: Option>>>, +} + pub trait UnitTrait: 'static + Send + Sync + std::fmt::Debug + fayalite::intern::SupportsPtrEqWithTypeId { @@ -395,11 +404,7 @@ pub trait UnitTrait: fn ty(&self) -> Self::Type; fn unit_kind(&self) -> UnitKind; fn module(&self) -> Interned>; - fn cd(&self, this: Expr) -> Option>; - fn from_execute( - &self, - this: Expr, - ) -> Expr>>; + fn io(&self, this: Expr) -> UnitIO; fn to_dyn(&self) -> DynUnit; } @@ -434,15 +439,8 @@ impl UnitTrait for DynUnit { self.unit.module() } - fn cd(&self, this: Expr) -> Option> { - self.unit.cd(this) - } - - fn from_execute( - &self, - this: Expr, - ) -> Expr>> { - self.unit.from_execute(this) + fn io(&self, this: Expr) -> UnitIO { + self.unit.io(this) } fn to_dyn(&self) -> DynUnit { @@ -468,15 +466,8 @@ impl UnitTrait for DynUnitWrapper) -> Option> { - self.0.cd(Expr::from_bundle(this)) - } - - fn from_execute( - &self, - this: Expr, - ) -> Expr>> { - self.0.from_execute(Expr::from_bundle(this)) + fn io(&self, this: Expr) -> UnitIO { + self.0.io(Expr::from_bundle(this)) } fn to_dyn(&self) -> DynUnit { diff --git a/crates/cpu/src/unit/alu_branch.rs b/crates/cpu/src/unit/alu_branch.rs index 9ad3f4b..4af688a 100644 --- a/crates/cpu/src/unit/alu_branch.rs +++ b/crates/cpu/src/unit/alu_branch.rs @@ -18,7 +18,7 @@ use crate::{ ExecuteToUnitInterface, GlobalState, NextPcPredictorOp, RenamedMOp, UnitCausedCancel, UnitFinishCauseCancel, UnitInputsReady, UnitOutputReady, }, - unit::{DynUnit, DynUnitWrapper, RenamedMOpFilter, UnitKind, UnitTrait}, + unit::{DynUnit, DynUnitWrapper, RenamedMOpFilter, UnitIO, UnitKind, UnitTrait}, }; use fayalite::{ expr::CastToImpl, @@ -1613,15 +1613,12 @@ impl UnitTrait for AluBranch { self.module } - fn cd(&self, _this: Expr) -> Option> { - None - } - - fn from_execute( - &self, - this: Expr, - ) -> Expr>> { - this.from_execute + fn io(&self, this: Expr) -> UnitIO { + UnitIO { + cd: None, + from_execute: this.from_execute, + to_d_cache: None, + } } fn to_dyn(&self) -> DynUnit { diff --git a/crates/cpu/src/unit/load_store.rs b/crates/cpu/src/unit/load_store.rs new file mode 100644 index 0000000..858763b --- /dev/null +++ b/crates/cpu/src/unit/load_store.rs @@ -0,0 +1,1328 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// See Notices.txt for copyright information + +use crate::{ + config::{CpuConfig, PhantomConstCpuConfig}, + instruction::{ + COMMON_MOP_SRC_LEN, LoadMOp, LoadStoreCommonMOp, LoadStoreConversion, LoadStoreMOp, + LoadStoreWidth, PRegNum, StoreMOp, + }, + main_memory_and_io::{AddressRange, MemoryOperationErrorKind}, + next_pc::{CallStackOp, SimValueDefault}, + register::{PRegFlags, PRegValue}, + rename_execute_retire::{ + ExecuteToUnitInterface, GlobalState, MOpId, MOpInstance, NextPcPredictorOp, UnitEnqueue, + UnitFinishCauseCancel, UnitInputsReady, UnitMOpCantCauseCancel, + UnitMOpIsNoLongerSpeculative, UnitOutputReady, + }, + unit::{DynUnit, DynUnitWrapper, RenamedMOpFilter, UnitIO, UnitKind, UnitMOp, UnitTrait}, +}; +use fayalite::{intern::Interned, prelude::*, ty::SimValueDebug, util::ready_valid::ReadyValid}; +use std::{collections::VecDeque, fmt}; + +#[hdl] +pub struct DCacheOpKindLoad { + /// if this is `false`, then if the address refers to a valid cache line, that cache line must have any dirty data + /// written back to memory and then be set to invalid before performing the load. + pub is_cacheable: Bool, +} + +#[hdl] +pub struct DCacheOpKindStore { + /// if this is `false`, then if the address refers to a valid cache line, that cache line must have any dirty data + /// written back to memory and then be set to invalid before performing the store. + pub is_cacheable: Bool, +} + +#[hdl] +pub enum DCacheOpKind { + Load(DCacheOpKindLoad), + Store(DCacheOpKindStore), +} + +#[hdl] +pub struct DCacheStart> { + pub id: MOpId, + pub kind: DCacheOpKind, + /// unaligned addresses may cause a load/store to cross a cache-line boundary + pub address: UInt<64>, + pub data_and_mask: Array>, 8>, + pub config: C, +} + +#[hdl] +pub enum DCacheFinishStatus { + /// The operation finished successfully + Success, + MemoryError(MemoryOperationErrorKind), +} + +#[hdl] +pub struct DCacheFinish> { + pub id: MOpId, + pub status: DCacheFinishStatus, + pub data: Array, 8>, + pub config: C, +} + +/// This load must only look in the L1 cache (must not propagate to the L2/L3 cache or to memory), must not +/// cause the timing of any earlier speculative operations or any non-speculative operations to change +/// because this load exists, and must not change any cache state that remains after all operations finish, +/// including LRU state, random number generators, predictive prefetching state, etc. +/// +/// Speculative loads that miss the L1 cache must return with [`DCacheFinishSpeculativeLoad::data`] set to +/// [`HdlNone()`] instead of trying to propagate to the L2/L3 or memory. +#[hdl] +pub struct DCacheStartSpeculativeLoad> { + pub id: MOpId, + /// unaligned addresses may cause a load to cross a cache-line boundary + pub address: UInt<64>, + pub mask: Array, + pub config: C, +} + +#[hdl] +pub struct DCacheFinishSpeculativeLoad> { + pub id: MOpId, + pub data: HdlOption, 8>>, + pub config: C, +} + +/// This CPU core's D-cache must report to the [load/store unit](load_store()) whenever: +/// * A cache line is removed from the D-cache by this CPU core (e.g. the cache ran out of space or the +/// cache line was intentionally removed by some instruction). +/// * Or a cache line is removed and/or written by something other than this CPU core (so other cores +/// and I/O devices that write to memory). +/// +/// That allows the [load/store unit](load_store()) to restart speculative loads as necessary to +/// maintain sequential consistency. +#[hdl] +pub struct DCacheForeignWriteToCacheLine> { + /// The address of the cache line that was written. + pub cache_line_address: UInt<64>, + /// Must be set such that for all bytes that were written and/or removed from the D-cache, that + /// `cache_line_address` must equal `cache_line_address_mask & byte_address`. + /// + /// e.g. if a 64-byte cache line at `0x1000` was removed, then `cache_line_address` is `0x1000` + /// and `cache_line_address_mask` is `0xFFFF_FFFF_FFFF_FFC0`. + pub cache_line_address_mask: UInt<64>, + pub config: C, +} + +#[hdl] +pub struct LoadStoreToDCacheInterface> { + /// Operations started through `start` must maintain sequential consistency -- so earlier operations + /// can't appear to have happened after later operations. + pub start: ReadyValid>, + /// operations must finish in the same order they started. + #[hdl(flip)] + pub finish: ReadyValid>, + /// It's valid for `start_speculative_load.ready` to be `false` at any time (or even always) + /// -- this doesn't block forward progress since speculative loads that don't start will eventually + /// convert to non-speculative loads and go through `start` instead. + /// + /// Speculative loads don't need to maintain any order relative to non-speculative operations, except + /// that speculative loads started after non-speculative operations have finished must be ordered after + /// those non-speculative operations (that should be trivial to implement in the D-cache). + /// + /// The [load/store unit](load_store()) will handle restarting speculative loads as necessary to + /// maintain sequential consistency when there is a `foreign_write_to_cache_line` that conflicts with + /// that load. + pub start_speculative_load: ReadyValid>, + /// Operations must finish in the same order they started. + #[hdl(flip)] + pub finish_speculative_load: ReadyValid>, + #[hdl(flip)] + pub foreign_write_to_cache_line: HdlOption>, + pub config: C, +} + +#[hdl(custom_debug(sim))] +struct OpDebugState { + v: SimOnly, +} + +impl SimValueDebug for OpDebugState { + #[hdl] + fn sim_value_debug( + value: &::SimValue, + f: &mut fmt::Formatter<'_>, + ) -> fmt::Result { + f.write_str(&value.v) + } +} + +impl SimValueDefault for OpDebugState { + #[hdl] + fn sim_value_default(self) -> SimValue { + thread_local! { + static VALUE: SimOnlyValue = SimOnlyValue::new("None".into()); + } + VALUE.with(|v| { + #[hdl(sim)] + Self { v } + }) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum OpState { + InputNotReady, + StartedSpeculativeLoad, + FinishedSpeculativeLoad, + StartedLoadStore, + FinishedLoadStore, +} + +impl OpState { + fn as_str(self) -> &'static str { + match self { + Self::InputNotReady => "inr", + Self::StartedSpeculativeLoad => "ssl", + Self::FinishedSpeculativeLoad => "fsl", + Self::StartedLoadStore => "sls", + Self::FinishedLoadStore => "fls", + } + } +} + +impl fmt::Display for OpState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug)] +struct Op { + id: SimValue, + pc: SimValue>, + size_in_bytes: u8, + mop: SimValue, PRegNum>>, + /// earlier instructions can cause a cancel, so this instruction is still speculative. + /// not to be confused with this instruction being able to cause a cancel. + is_speculative: bool, + src_values: Option<[SimValue>; COMMON_MOP_SRC_LEN]>, + dest_value: Option>>, + sent_cant_cause_cancel: bool, + sent_output_ready: bool, + state: OpState, +} + +impl fmt::Display for Op { + #[hdl] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + id, + pc, + size_in_bytes, + mop, + is_speculative, + src_values, + dest_value, + sent_cant_cause_cancel, + sent_output_ready, + state, + } = self; + if *is_speculative { + f.write_str("(s)")?; + } + if *sent_cant_cause_cancel { + f.write_str("(sccc)")?; + } + if *sent_output_ready { + f.write_str("(sor)")?; + } + write!( + f, + "({state}) id={id:?} pc={pc:?} sz={size_in_bytes}: {mop:?}" + )?; + if let Some(src_values) = src_values { + write!(f, "src_values={src_values:?}")?; + } + if let Some(dest_value) = dest_value { + write!(f, "dest_value={dest_value:?}")?; + } + Ok(()) + } +} + +struct AccessRangeUnknown; + +impl Op { + fn new( + id: SimValue, + pc: SimValue>, + size_in_bytes: u8, + mop: SimValue, PRegNum>>, + ) -> Self { + Self { + id, + pc, + size_in_bytes, + mop, + is_speculative: true, + src_values: None, + dest_value: None, + sent_cant_cause_cancel: false, + sent_output_ready: false, + state: OpState::InputNotReady, + } + } + #[hdl] + fn debug_state(&self) -> SimValue { + #[hdl(sim)] + OpDebugState { + v: SimOnlyValue::new(self.to_string()), + } + } + #[hdl] + fn access_range_helper( + &self, + load_store_common: &SimValue, PRegNum, SrcCount>>, + ) -> Result { + let Some([src0, ..]) = &self.src_values else { + return Err(AccessRangeUnknown); + }; + let size = LoadStoreWidth::access_byte_size_sim(&load_store_common.width); + Ok(AddressRange::Limited { + start: std::num::Wrapping(src0.inner().int_fp.as_int()), + size, + }) + } + #[hdl] + fn access_range(&self) -> Result { + #[hdl(sim)] + match &self.mop { + LoadStoreMOp::<_, _>::Load(mop) => self.access_range_helper(&mop.load_store_common), + LoadStoreMOp::<_, _>::Store(mop) => self.access_range_helper(&mop.load_store_common), + } + } + fn can_cause_cancel(&self) -> bool { + if self.is_speculative { + return true; + } + match self.state { + OpState::InputNotReady + | OpState::StartedSpeculativeLoad + | OpState::StartedLoadStore => true, + OpState::FinishedSpeculativeLoad | OpState::FinishedLoadStore => { + self.dest_value.is_none() + } + } + } +} + +#[hdl(get(|c| c.rob_size.get().next_power_of_two()))] +type OpsDebugLen> = DynSize; + +#[hdl] +type PhantomConstUnitIndex = PhantomConst; + +#[hdl(no_static)] +pub struct LoadStoreDebugState> { + global_state: GlobalState, + ops: ArrayType, OpsDebugLen>, + canceling: Bool, + config: C, + unit_index: PhantomConstUnitIndex, + filter: LoadStoreFilter, +} + +#[derive(Debug)] +struct LoadStoreState { + global_state: SimValue, + ops: VecDeque>, + d_cache_start: Option>>, + d_cache_start_speculative_load: Option>>, + canceling: bool, + config: C, + unit_index: usize, + filter: SimValue, +} + +impl LoadStoreState { + fn new(config: C, unit_index: usize, filter: SimValue) -> Self { + Self { + global_state: GlobalState.sim_value_default(), + ops: VecDeque::new(), + d_cache_start: None, + d_cache_start_speculative_load: None, + canceling: false, + config, + unit_index, + filter, + } + } + #[hdl] + fn debug_state(&self) -> SimValue> { + let Self { + ref global_state, + ops: _, + // not included in debug state since it's identical to `to_d_cache.start.data` + d_cache_start: _, + // not included in debug state since it's identical to `to_d_cache.start_speculative_load.data` + d_cache_start_speculative_load: _, + canceling, + config, + unit_index, + ref filter, + } = *self; + let ty = LoadStoreDebugState[config]; + let mut ops = vec![OpDebugState.sim_value_default().into_trace_as_string(); ty.ops.len()]; + for op in &self.ops { + ops[op.id.as_int() as usize % ty.ops.len()] = op.debug_state().into_trace_as_string(); + } + #[hdl(sim)] + LoadStoreDebugState::<_> { + global_state, + ops, + canceling, + config, + unit_index: PhantomConst::new_sized(unit_index), + filter, + } + } + #[track_caller] + fn op_by_id(&mut self, id: &::SimValue) -> &mut Op { + match self.ops.iter_mut().find(|op| *op.id == *id) { + Some(op) => op, + None => panic!("can't find load_store Op with id {id:?}"), + } + } + fn from_execute_enqueue_ready(&self) -> bool { + !self.canceling + && self.ops.len() < self.config.get().unit_max_in_flight(self.unit_index).get() + } + #[hdl] + fn from_execute_enqueue(&mut self, enqueue: SimValue>) { + #[hdl(sim)] + let UnitEnqueue::<_> { mop, config: _ } = enqueue; + #[hdl(sim)] + let MOpInstance::<_> { + fetch_block_id: _, + id, + pc, + predicted_next_pc: _, + size_in_bytes, + is_first_mop_in_insn: _, + is_last_mop_in_insn: _, + mop, + } = mop; + let mop = #[hdl(sim)] + match SimValue::into_value(mop).into_inner() { + UnitMOp::<_, _, _>::LoadStore(mop) => mop, + _ => unreachable!(), + }; + self.ops.push_back(Op::new( + id, + pc, + size_in_bytes.cast_to_static::>().as_int(), + mop, + )); + } + #[hdl] + fn from_execute_inputs_ready(&mut self, inputs_ready: SimValue>) { + #[hdl(sim)] + let UnitInputsReady::<_> { + mop, + src_values, + config: _, + } = inputs_ready; + self.op_by_id(&mop.id).src_values = Some(SimValue::into_value(src_values)); + } + #[hdl] + fn from_execute_is_no_longer_speculative( + &mut self, + is_no_longer_speculative: SimValue>, + ) { + #[hdl(sim)] + let UnitMOpIsNoLongerSpeculative::<_> { id, config: _ } = is_no_longer_speculative; + self.op_by_id(&id).is_speculative = false; + } + #[hdl] + fn from_execute_cant_cause_cancel(&self) -> Option>> { + if self.canceling { + return None; + } + for op in &self.ops { + if op.sent_cant_cause_cancel || op.can_cause_cancel() { + continue; + } + return Some( + #[hdl(sim)] + UnitMOpCantCauseCancel::<_> { + id: op.id, + config: self.config, + }, + ); + } + None + } + #[hdl] + fn from_execute_sent_cant_cause_cancel( + &mut self, + cant_cause_cancel: SimValue>, + ) { + #[hdl(sim)] + let UnitMOpCantCauseCancel::<_> { id, config: _ } = cant_cause_cancel; + self.op_by_id(&id).sent_cant_cause_cancel = true; + } + #[hdl] + fn from_execute_output_ready(&self) -> Option>> { + if self.canceling { + return None; + } + for op in &self.ops { + if op.sent_output_ready { + continue; + } + if let Some(dest_value) = &op.dest_value { + return Some( + #[hdl(sim)] + UnitOutputReady::<_> { + id: op.id, + dest_value, + predictor_op: #[hdl(sim)] + NextPcPredictorOp::<_> { + call_stack_op: #[hdl(sim)] + CallStackOp.None(), + cond_br_taken: #[hdl(sim)] + HdlNone(), + config: self.config, + }, + }, + ); + } + } + None + } + #[hdl] + fn from_execute_sent_output_ready(&mut self, output_ready: SimValue>) { + #[hdl(sim)] + let UnitOutputReady::<_> { + id, + dest_value: _, + predictor_op: _, + } = output_ready; + self.op_by_id(&id).sent_output_ready = true; + } + #[hdl] + fn from_execute_finish_cause_cancel(&self) -> Option>> { + if self.canceling { + return None; + } + let Op { + id, + pc: _, + size_in_bytes: _, + mop: _, + is_speculative, + src_values: _, + dest_value, + sent_cant_cause_cancel: _, + sent_output_ready, + state, + } = self.ops.front()?; + if *is_speculative { + return None; + } + let retval_ty = UnitFinishCauseCancel[self.config]; + match state { + OpState::InputNotReady => None, + OpState::StartedSpeculativeLoad => None, + OpState::FinishedSpeculativeLoad => { + if *sent_output_ready { + // TODO: handle when some other CPU stores to the same cache line before the speculative + // load sends UnitFinishCauseCancel -- that needs to make the load cause a cancel so it + // can be redone with the correct data. That enforces sequential consistency. + Some( + #[hdl(sim)] + UnitFinishCauseCancel::<_> { + id, + caused_cancel: #[hdl(sim)] + (retval_ty.caused_cancel).HdlNone(), + config: self.config, + }, + ) + } else { + None + } + } + OpState::StartedLoadStore => None, + OpState::FinishedLoadStore => { + if dest_value.is_none() { + todo!("load/store caused error -- need to translate to a cancel"); + } + if *sent_output_ready { + Some( + #[hdl(sim)] + UnitFinishCauseCancel::<_> { + id, + caused_cancel: #[hdl(sim)] + (retval_ty.caused_cancel).HdlNone(), + config: self.config, + }, + ) + } else { + None + } + } + } + } + #[hdl] + fn from_execute_sent_finish_cause_cancel( + &mut self, + finish_cause_cancel: SimValue>, + ) { + let op = self + .ops + .pop_front_if(|front| front.id == finish_cause_cancel.id); + assert!( + op.is_some(), + "inconsistent state -- sent finish_cause_cancel but id doesn't match ops.front():\n{finish_cause_cancel:#?}\nfront={front:#?}", + front = self.ops.front(), + ); + } + fn from_execute_cancel_all_ready(&self) -> bool { + true + } + #[hdl] + fn from_execute_cancel_all(&mut self) { + self.canceling = true; + } + #[hdl] + fn to_d_cache_sent_start(&mut self, start: SimValue>) { + self.op_by_id(&start.id).state = OpState::StartedLoadStore; + } + #[hdl] + fn to_d_cache_finish_ready(&self) -> bool { + true + } + #[hdl] + fn to_d_cache_finish(&mut self, finish: SimValue>) { + #[hdl(sim)] + let DCacheFinish::<_> { + id, + status, + data, + config: _, + } = finish; + let canceling = self.canceling; + let op = self.op_by_id(&id); + op.state = OpState::FinishedLoadStore; + if canceling { + return; + } + #[hdl(sim)] + match status { + DCacheFinishStatus::Success => { + op.dest_value = #[hdl(sim)] + match &op.mop { + LoadStoreMOp::<_, _>::Load(mop) => Some(Self::process_load_data( + mop, + &SimValue::into_value(data).map(|v| v.as_int()), + )), + LoadStoreMOp::<_, _>::Store(_) => { + Some(PRegValue::zeroed_sim().into_trace_as_string()) + } + }; + } + DCacheFinishStatus::MemoryError(error) => { + todo!("{error:?}") + } + } + } + #[hdl] + fn to_d_cache_sent_start_speculative_load( + &mut self, + start_speculative_load: SimValue>, + ) { + self.op_by_id(&start_speculative_load.id).state = OpState::StartedSpeculativeLoad; + } + #[hdl] + fn to_d_cache_finish_speculative_load_ready(&self) -> bool { + true + } + #[hdl] + fn to_d_cache_finish_speculative_load( + &mut self, + finish_speculative_load: SimValue>, + ) { + #[hdl(sim)] + let DCacheFinishSpeculativeLoad::<_> { + id, + data, + config: _, + } = finish_speculative_load; + let canceling = self.canceling; + let op = self.op_by_id(&id); + op.state = OpState::FinishedSpeculativeLoad; + if canceling { + return; + } + #[hdl(sim)] + if let HdlSome(data) = data { + op.dest_value = #[hdl(sim)] + match &op.mop { + LoadStoreMOp::<_, _>::Load(mop) => Some(Self::process_load_data( + mop, + &SimValue::into_value(data).map(|v| v.as_int()), + )), + LoadStoreMOp::<_, _>::Store(_) => { + unreachable!() + } + }; + } + } + #[hdl] + fn to_d_cache_foreign_write_to_cache_line( + &mut self, + foreign_write_to_cache_line: SimValue>, + ) { + #[hdl(sim)] + let DCacheForeignWriteToCacheLine::<_> { + cache_line_address, + cache_line_address_mask, + config: _, + } = foreign_write_to_cache_line; + todo!( + "foreign_write_to_cache_line:\n\ + cache_line_address={cache_line_address}, cache_line_address_mask={cache_line_address_mask}" + ); + } + #[hdl] + fn canceling_step(&mut self) { + self.d_cache_start = None; + self.d_cache_start_speculative_load = None; + let finished_canceling = |op: &mut Op<_>| match op.state { + OpState::InputNotReady => true, + OpState::StartedSpeculativeLoad => false, + OpState::FinishedSpeculativeLoad => true, + OpState::StartedLoadStore => false, + OpState::FinishedLoadStore => true, + }; + while let Some(_) = self.ops.pop_front_if(finished_canceling) {} + if self.ops.is_empty() { + self.canceling = false; + } + } + fn is_cacheable_load(mop: &SimValue, PRegNum>>) -> bool { + // TODO: implement un-cacheable loads + true + } + fn is_cacheable_store(mop: &SimValue, PRegNum>>) -> bool { + // TODO: implement un-cacheable stores + true + } + #[hdl] + fn process_load_data( + mop: &SimValue, PRegNum>>, + data: &[u8; 8], + ) -> SimValue> { + #[hdl(sim)] + let LoadMOp::<_, _> { load_store_common } = mop; + #[hdl(sim)] + let LoadStoreCommonMOp::<_, _, _> { + common: _, + width, + conversion, + } = load_store_common; + macro_rules! cvt { + ($ty:ident) => { + $ty::from_le_bytes(std::array::from_fn(|i| data[i])) as u64 + }; + } + let int_fp = #[hdl(sim)] + match conversion { + LoadStoreConversion::ZeroExt => + { + #[hdl(sim)] + match width { + LoadStoreWidth::Width8Bit => cvt!(u8), + LoadStoreWidth::Width16Bit => cvt!(u16), + LoadStoreWidth::Width32Bit => cvt!(u32), + LoadStoreWidth::Width64Bit => cvt!(u64), + } + } + LoadStoreConversion::SignExt => + { + #[hdl(sim)] + match width { + LoadStoreWidth::Width8Bit => cvt!(i8), + LoadStoreWidth::Width16Bit => cvt!(i16), + LoadStoreWidth::Width32Bit => cvt!(i32), + LoadStoreWidth::Width64Bit => cvt!(i64), + } + } + }; + let retval = #[hdl(sim)] + PRegValue { + int_fp, + flags: PRegFlags::zeroed_sim(), + }; + retval.into_trace_as_string() + } + #[hdl] + fn process_store_data( + mop: &SimValue, PRegNum>>, + data: u64, + ) -> [Option; 8] { + #[hdl(sim)] + let StoreMOp::<_, _> { load_store_common } = mop; + #[hdl(sim)] + let LoadStoreCommonMOp::<_, _, _> { + common: _, + width, + conversion, + } = load_store_common; + macro_rules! cvt { + ($ty:ident) => {{ + let data = (data as $ty).to_le_bytes(); + std::array::from_fn(|i| data.get(i).copied()) + }}; + } + #[hdl(sim)] + match conversion { + LoadStoreConversion::ZeroExt => + { + #[hdl(sim)] + match width { + LoadStoreWidth::Width8Bit => cvt!(u8), + LoadStoreWidth::Width16Bit => cvt!(u16), + LoadStoreWidth::Width32Bit => cvt!(u32), + LoadStoreWidth::Width64Bit => cvt!(u64), + } + } + LoadStoreConversion::SignExt => + { + #[hdl(sim)] + match width { + LoadStoreWidth::Width8Bit => cvt!(i8), + LoadStoreWidth::Width16Bit => cvt!(i16), + LoadStoreWidth::Width32Bit => cvt!(i32), + LoadStoreWidth::Width64Bit => cvt!(i64), + } + } + } + } + #[hdl] + fn step(&mut self) { + if self.canceling { + return self.canceling_step(); + } + self.d_cache_start = None; + self.d_cache_start_speculative_load = None; + let store_queue_capacity = 4; // TODO: make configurable + struct StoreQueueEntry { + access_range: AddressRange, + data: [u8; 8], + } + let mut store_queue = Vec::::with_capacity(store_queue_capacity); + for op in &mut self.ops { + let access_range = op.access_range(); + let Op { + ref id, + pc: _, + size_in_bytes: _, + ref mop, + is_speculative, + ref src_values, + ref mut dest_value, + sent_cant_cause_cancel: _, + sent_output_ready: _, + ref mut state, + } = *op; + #[hdl(sim)] + match mop { + LoadStoreMOp::<_, _>::Load(mop) => { + let Some([src0, ..]) = src_values else { + continue; + }; + let Ok(access_range) = access_range else { + continue; + }; + let is_cacheable = Self::is_cacheable_load(mop); + let mut blocking_store = false; + for StoreQueueEntry { + access_range: store_range, + data, + } in &store_queue + { + if access_range == *store_range { + // matching load and store -- speculatively forward data + if let OpState::InputNotReady = state + && is_cacheable + { + *state = OpState::FinishedSpeculativeLoad; + *dest_value = Some(Self::process_load_data(mop, data)); + } + blocking_store = true; + break; + } + if store_range.is_overlapping(access_range) { + blocking_store = true; + break; + } + } + let address = src0.inner().int_fp.as_int(); + let size = LoadStoreWidth::access_byte_size_sim(&mop.load_store_common.width); + let mask = std::array::from_fn(|i| i < size.get() as usize); + match state { + OpState::InputNotReady + if is_speculative && !blocking_store && is_cacheable => + { + self.d_cache_start_speculative_load.get_or_insert_with(|| { + #[hdl(sim)] + DCacheStartSpeculativeLoad::<_> { + id, + address, + mask, + config: self.config, + } + }); + } + OpState::InputNotReady | OpState::FinishedSpeculativeLoad => { + if !is_speculative && dest_value.is_none() { + self.d_cache_start.get_or_insert_with(|| { + #[hdl(sim)] + DCacheStart::<_> { + id, + kind: #[hdl(sim)] + DCacheOpKind.Load( + #[hdl(sim)] + DCacheOpKindLoad { is_cacheable }, + ), + address, + data_and_mask: mask.map(|v| v.then_some(0)), + config: self.config, + } + }); + } + } + OpState::StartedSpeculativeLoad => {} + OpState::StartedLoadStore => {} + OpState::FinishedLoadStore => {} + } + } + LoadStoreMOp::<_, _>::Store(mop) => { + let Op { + ref id, + pc: _, + size_in_bytes: _, + mop: _, + is_speculative, + ref src_values, + ref dest_value, + sent_cant_cause_cancel: _, + sent_output_ready: _, + state, + } = *op; + let Some([src0, src1, ..]) = src_values else { + break; + }; + let Ok(access_range) = access_range else { + continue; + }; + let is_cacheable = Self::is_cacheable_store(mop); + let address = src0.inner().int_fp.as_int(); + let data_and_mask = Self::process_store_data(mop, src1.inner().int_fp.as_int()); + if dest_value.is_none() { + if store_queue.len() >= store_queue_capacity { + break; + } + store_queue.push(StoreQueueEntry { + access_range, + data: data_and_mask.map(|v| v.unwrap_or(0)), + }); + } + match state { + OpState::InputNotReady => { + if !is_speculative { + self.d_cache_start.get_or_insert_with(|| { + #[hdl(sim)] + DCacheStart::<_> { + id, + kind: #[hdl(sim)] + DCacheOpKind.Store( + #[hdl(sim)] + DCacheOpKindStore { is_cacheable }, + ), + address, + data_and_mask, + config: self.config, + } + }); + } + } + OpState::StartedSpeculativeLoad | OpState::FinishedSpeculativeLoad => { + unreachable!() + } + OpState::StartedLoadStore => {} + OpState::FinishedLoadStore => {} + } + } + } + } + } +} + +#[hdl] +async fn load_store_impl( + config: PhantomConst, + unit_index: usize, + filter: SimValue, + cd: Expr, + from_execute: Expr>>, + to_d_cache: Expr>>, + debug_state: Expr>>, + mut sim: ExternModuleSimulationState, +) { + let mut state = LoadStoreState::new(config, unit_index, filter); + loop { + { + #[hdl] + let ExecuteToUnitInterface::<_> { + global_state: _, + enqueue, + inputs_ready: _, + is_no_longer_speculative: _, + cant_cause_cancel, + output_ready, + finish_cause_cancel, + unit_outputs_ready: _, + cancel_all, + config: _, + } = from_execute; + sim.write(enqueue.ready, state.from_execute_enqueue_ready()) + .await; + sim.write(cant_cause_cancel, state.from_execute_cant_cause_cancel()) + .await; + sim.write(output_ready, state.from_execute_output_ready()) + .await; + sim.write( + finish_cause_cancel, + state.from_execute_finish_cause_cancel(), + ) + .await; + sim.write(cancel_all.ready, state.from_execute_cancel_all_ready()) + .await; + } + { + #[hdl] + let LoadStoreToDCacheInterface::<_> { + start, + finish, + start_speculative_load, + finish_speculative_load, + foreign_write_to_cache_line: _, + config: _, + } = to_d_cache; + sim.write(start.data, &state.d_cache_start).await; + sim.write(finish.ready, state.to_d_cache_finish_ready()) + .await; + sim.write( + start_speculative_load.data, + &state.d_cache_start_speculative_load, + ) + .await; + sim.write( + finish_speculative_load.ready, + state.to_d_cache_finish_speculative_load_ready(), + ) + .await; + } + sim.write(debug_state, state.debug_state()).await; + sim.wait_for_clock_edge(cd.clk).await; + { + #[hdl] + let ExecuteToUnitInterface::<_> { + global_state, + enqueue, + inputs_ready, + is_no_longer_speculative, + cant_cause_cancel, + output_ready, + finish_cause_cancel, + unit_outputs_ready, + cancel_all, + config: _, + } = from_execute; + state.global_state = sim.read_past(global_state, cd.clk).await; + if sim.read_past_bool(enqueue.ready, cd.clk).await { + #[hdl(sim)] + if let HdlSome(enqueue) = sim.read_past(enqueue.data, cd.clk).await { + state.from_execute_enqueue(enqueue); + } + } + #[hdl(sim)] + if let HdlSome(inputs_ready) = sim.read_past(inputs_ready, cd.clk).await { + state.from_execute_inputs_ready(inputs_ready); + } + #[hdl(sim)] + if let HdlSome(is_no_longer_speculative) = + sim.read_past(is_no_longer_speculative, cd.clk).await + { + state.from_execute_is_no_longer_speculative(is_no_longer_speculative); + } + if sim.read_past_bool(unit_outputs_ready, cd.clk).await { + #[hdl(sim)] + if let HdlSome(cant_cause_cancel) = sim.read_past(cant_cause_cancel, cd.clk).await { + state.from_execute_sent_cant_cause_cancel(cant_cause_cancel); + } + #[hdl(sim)] + if let HdlSome(output_ready) = sim.read_past(output_ready, cd.clk).await { + state.from_execute_sent_output_ready(output_ready); + } + #[hdl(sim)] + if let HdlSome(finish_cause_cancel) = + sim.read_past(finish_cause_cancel, cd.clk).await + { + state.from_execute_sent_finish_cause_cancel(finish_cause_cancel); + } + } + if sim.read_past_bool(cancel_all.ready, cd.clk).await { + #[hdl(sim)] + if let HdlSome(cancel_all) = sim.read_past(cancel_all.data, cd.clk).await { + #[hdl] + let () = cancel_all; + state.from_execute_cancel_all(); + } + } + } + { + #[hdl] + let LoadStoreToDCacheInterface::<_> { + start, + finish, + start_speculative_load, + finish_speculative_load, + foreign_write_to_cache_line, + config: _, + } = to_d_cache; + if sim.read_past_bool(start.ready, cd.clk).await { + #[hdl(sim)] + if let HdlSome(start) = sim.read_past(start.data, cd.clk).await { + state.to_d_cache_sent_start(start); + } + } + if sim.read_past_bool(finish.ready, cd.clk).await { + #[hdl(sim)] + if let HdlSome(finish) = sim.read_past(finish.data, cd.clk).await { + state.to_d_cache_finish(finish); + } + } + if sim + .read_past_bool(start_speculative_load.ready, cd.clk) + .await + { + #[hdl(sim)] + if let HdlSome(start_speculative_load) = + sim.read_past(start_speculative_load.data, cd.clk).await + { + state.to_d_cache_sent_start_speculative_load(start_speculative_load); + } + } + if sim + .read_past_bool(finish_speculative_load.ready, cd.clk) + .await + { + #[hdl(sim)] + if let HdlSome(finish_speculative_load) = + sim.read_past(finish_speculative_load.data, cd.clk).await + { + state.to_d_cache_finish_speculative_load(finish_speculative_load); + } + } + #[hdl(sim)] + if let HdlSome(foreign_write_to_cache_line) = + sim.read_past(foreign_write_to_cache_line, cd.clk).await + { + state.to_d_cache_foreign_write_to_cache_line(foreign_write_to_cache_line); + } + } + state.step(); + } +} + +#[hdl] +struct LoadStoreFilter { + load: Bool, + store: Bool, +} + +impl LoadStoreFilter { + #[hdl] + fn new( + from_execute: ExecuteToUnitInterface>, + filter: &mut impl RenamedMOpFilter, + ) -> SimValue { + let LoadStoreMOp { Load, Store } = from_execute + .inputs_ready + .HdlSome + .mop + .mop + .inner_ty() + .LoadStore; + #[hdl(sim)] + Self { + load: filter.should_include_ty(Load), + store: filter.should_include_ty(Store), + } + } +} + +#[hdl_module(extern)] +pub fn load_store( + config: PhantomConst, + unit_index: usize, + filter: &mut impl RenamedMOpFilter, +) { + #[hdl] + let cd: ClockDomain = m.input(); + + #[hdl] + let from_execute: ExecuteToUnitInterface> = + m.input(ExecuteToUnitInterface[config]); + + #[hdl] + let to_d_cache: LoadStoreToDCacheInterface> = + m.output(LoadStoreToDCacheInterface[config]); + + #[hdl] + let debug_state: LoadStoreDebugState> = + m.output(LoadStoreDebugState[config]); + + let filter = LoadStoreFilter::new(from_execute.ty(), filter).to_expr(); + + assert_eq!(config.get().units[unit_index].kind, UnitKind::LoadStore); + + m.register_clock_for_past(cd.clk); + + m.extern_module_simulation_fn( + ( + config, + unit_index, + filter, + cd, + from_execute, + to_d_cache, + debug_state, + ), + async |(config, unit_index, filter, cd, from_execute, to_d_cache, debug_state), mut sim| { + let filter = filter.into_sim_value(); + sim.resettable( + cd, + async |mut sim| { + #[hdl] + let ExecuteToUnitInterface::<_> { + global_state: _, + enqueue, + inputs_ready: _, + is_no_longer_speculative: _, + cant_cause_cancel, + output_ready, + finish_cause_cancel, + unit_outputs_ready: _, + cancel_all, + config: _, + } = from_execute; + sim.write(enqueue.ready, false).await; + sim.write(cant_cause_cancel, cant_cause_cancel.ty().HdlNone()) + .await; + sim.write(output_ready, output_ready.ty().HdlNone()).await; + sim.write(finish_cause_cancel, finish_cause_cancel.ty().HdlNone()) + .await; + sim.write(cancel_all.ready, false).await; + #[hdl] + let LoadStoreToDCacheInterface::<_> { + start, + finish, + start_speculative_load, + finish_speculative_load, + foreign_write_to_cache_line: _, + config: _, + } = to_d_cache; + sim.write(start.data, start.ty().data.HdlNone()).await; + sim.write(finish.ready, false).await; + sim.write( + debug_state, + LoadStoreState::new(config, unit_index, filter.clone()).debug_state(), + ) + .await; + sim.write( + start_speculative_load.data, + start_speculative_load.ty().data.HdlNone(), + ) + .await; + sim.write(finish_speculative_load.ready, false).await; + sim.write( + debug_state, + LoadStoreState::new(config, unit_index, filter.clone()).debug_state(), + ) + .await; + }, + async |sim, ()| { + load_store_impl( + config, + unit_index, + filter.clone(), + cd, + from_execute, + to_d_cache, + debug_state, + sim, + ) + .await + }, + ) + .await + }, + ); +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct LoadStore { + config: PhantomConst, + module: Interned>, +} + +impl LoadStore { + pub fn new( + config: PhantomConst, + unit_index: usize, + filter: &mut impl RenamedMOpFilter, + ) -> Self { + Self { + config, + module: load_store(config, unit_index, filter), + } + } +} + +impl UnitTrait for LoadStore { + type Type = load_store; + + fn ty(&self) -> Self::Type { + self.module.io_ty() + } + + fn unit_kind(&self) -> UnitKind { + UnitKind::LoadStore + } + + fn module(&self) -> Interned> { + self.module + } + + fn io(&self, this: Expr) -> UnitIO { + UnitIO { + cd: Some(this.cd), + from_execute: this.from_execute, + to_d_cache: Some(this.to_d_cache), + } + } + + fn to_dyn(&self) -> DynUnit { + DynUnitWrapper(*self).to_dyn() + } +} diff --git a/crates/cpu/tests/units_formal.rs b/crates/cpu/tests/units_formal.rs index 47ad415..2a818b8 100644 --- a/crates/cpu/tests/units_formal.rs +++ b/crates/cpu/tests/units_formal.rs @@ -14,7 +14,7 @@ use cpu::{ DecodeAndRunSingleInsnInput, DecodeAndRunSingleInsnOutput, DecodeOneInsnInput, DecodeOneInsnMaxMOpCount, decode_and_run_single_insn, }, - unit::{RenamedMOpFilter, UnitKind, UnitMOp, UnitTrait}, + unit::{RenamedMOpFilter, UnitIO, UnitKind, UnitMOp, UnitTrait}, util::array_vec::ArrayVec, }; use fayalite::{ @@ -91,12 +91,20 @@ fn formal_harness( dyn_unit.module(), SourceLocation::caller(), ); + let UnitIO { + cd: unit_cd, + from_execute: unit_from_execute, + to_d_cache, + } = dyn_unit.io(unit); + if let Some(unit_cd) = unit_cd { + connect(unit_cd, cd); + } connect( - dyn_unit.from_execute(unit), + unit_from_execute, ExecuteToUnitInterfaces::unit_fields(decode_and_run.to_units)[unit_index], ); - if let Some(unit_cd) = dyn_unit.cd(unit) { - connect(unit_cd, cd); + if let Some(to_d_cache) = to_d_cache { + unimplemented!("to_d_cache for {unit:?}: {to_d_cache:?}"); } } hdl_assert(cd.clk, !decode_and_run.error, "");