diff --git a/crates/cpu/src/instruction.rs b/crates/cpu/src/instruction.rs index bf4be7b..336da53 100644 --- a/crates/cpu/src/instruction.rs +++ b/crates/cpu/src/instruction.rs @@ -19,7 +19,6 @@ use std::{ borrow::Cow, fmt, marker::PhantomData, - num::NonZeroU64, ops::{ControlFlow, Range}, }; @@ -2984,19 +2983,6 @@ pub enum LoadStoreWidth { Width64Bit, } -impl LoadStoreWidth { - #[hdl] - pub fn access_byte_size_sim(this: &::SimValue) -> NonZeroU64 { - #[hdl(sim)] - match this { - Self::Width8Bit => const { NonZeroU64::new(1).unwrap() }, - Self::Width16Bit => const { NonZeroU64::new(2).unwrap() }, - Self::Width32Bit => const { NonZeroU64::new(4).unwrap() }, - Self::Width64Bit => const { NonZeroU64::new(8).unwrap() }, - } - } -} - #[hdl(cmp_eq)] pub enum LoadStoreConversion { ZeroExt, diff --git a/crates/cpu/src/main_memory_and_io.rs b/crates/cpu/src/main_memory_and_io.rs index 795bf84..5a1a853 100644 --- a/crates/cpu/src/main_memory_and_io.rs +++ b/crates/cpu/src/main_memory_and_io.rs @@ -127,47 +127,8 @@ impl AddressRange { }, } } - pub const fn is_overlapping(self, other: Self) -> bool { - self.contains(other.start().0) || other.contains(self.start().0) - } } -const _: () = { - const N: u64 = 4; - const fn is_overlapping_reference(range1: AddressRange, range2: AddressRange) -> bool { - // we just need to check values < N since all starts/ends are < N - let mut v = 0; - while v < N { - if range1.contains(v) && range2.contains(v) { - return true; - } - v += 1; - } - false - } - let mut start1 = 0; - while start1 < N { - let mut end1 = 0; - while end1 < N { - let mut start2 = 0; - while start2 < N { - let mut end2 = 0; - while end2 < N { - let range1 = AddressRange::from_wrapping_range_inclusive(start1..=end1); - let range2 = AddressRange::from_wrapping_range_inclusive(start2..=end2); - assert!( - range1.is_overlapping(range2) == is_overlapping_reference(range1, range2) - ); - end2 += 1; - } - start2 += 1; - } - end1 += 1; - } - start1 += 1; - } -}; - #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, serde::Serialize, serde::Deserialize)] #[non_exhaustive] pub struct MemoryInterfaceConfig { diff --git a/crates/cpu/src/unit.rs b/crates/cpu/src/unit.rs index 4864125..ce024a4 100644 --- a/crates/cpu/src/unit.rs +++ b/crates/cpu/src/unit.rs @@ -8,7 +8,6 @@ use crate::{ MOpVariantVisitOps, MOpVariantVisitor, MOpVisitVariants, RenamedMOp, mop_enum, }, rename_execute_retire::ExecuteToUnitInterface, - unit::load_store::LoadStoreToDCacheInterface, }; use fayalite::{ bundle::{Bundle, BundleType}, @@ -19,7 +18,6 @@ use serde::{Deserialize, Serialize}; use std::ops::ControlFlow; pub mod alu_branch; -pub mod load_store; macro_rules! all_units { ( @@ -335,7 +333,7 @@ all_units! { #[create_dyn_unit_fn = |config, unit_index, filter| todo!()] #[extract(transformed_move_mop, transformed_move_mop_sim, transformed_move_mop_sim_ref, transformed_move_mop_sim_mut)] TransformedMove(TransformedMoveOp), - #[create_dyn_unit_fn = |config, unit_index, filter| load_store::LoadStore::new(config, unit_index, filter).to_dyn()] + #[create_dyn_unit_fn = |config, unit_index, filter| todo!()] #[extract(load_store_mop, load_store_mop_sim, load_store_mop_sim_ref, load_store_mop_sim_mut)] LoadStore(LoadStoreMOp), } @@ -389,13 +387,6 @@ impl RenamedMOpFilter for () { } } -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] -pub struct UnitIO { - pub cd: Option>, - pub from_execute: Expr>>, - pub to_d_cache: Option>>>, -} - pub trait UnitTrait: 'static + Send + Sync + std::fmt::Debug + fayalite::intern::SupportsPtrEqWithTypeId { @@ -404,7 +395,11 @@ pub trait UnitTrait: fn ty(&self) -> Self::Type; fn unit_kind(&self) -> UnitKind; fn module(&self) -> Interned>; - fn io(&self, this: Expr) -> UnitIO; + fn cd(&self, this: Expr) -> Option>; + fn from_execute( + &self, + this: Expr, + ) -> Expr>>; fn to_dyn(&self) -> DynUnit; } @@ -439,8 +434,15 @@ impl UnitTrait for DynUnit { self.unit.module() } - fn io(&self, this: Expr) -> UnitIO { - self.unit.io(this) + fn cd(&self, this: Expr) -> Option> { + self.unit.cd(this) + } + + fn from_execute( + &self, + this: Expr, + ) -> Expr>> { + self.unit.from_execute(this) } fn to_dyn(&self) -> DynUnit { @@ -466,8 +468,15 @@ impl UnitTrait for DynUnitWrapper) -> UnitIO { - self.0.io(Expr::from_bundle(this)) + fn cd(&self, this: Expr) -> Option> { + self.0.cd(Expr::from_bundle(this)) + } + + fn from_execute( + &self, + this: Expr, + ) -> Expr>> { + self.0.from_execute(Expr::from_bundle(this)) } fn to_dyn(&self) -> DynUnit { diff --git a/crates/cpu/src/unit/alu_branch.rs b/crates/cpu/src/unit/alu_branch.rs index 4af688a..9ad3f4b 100644 --- a/crates/cpu/src/unit/alu_branch.rs +++ b/crates/cpu/src/unit/alu_branch.rs @@ -18,7 +18,7 @@ use crate::{ ExecuteToUnitInterface, GlobalState, NextPcPredictorOp, RenamedMOp, UnitCausedCancel, UnitFinishCauseCancel, UnitInputsReady, UnitOutputReady, }, - unit::{DynUnit, DynUnitWrapper, RenamedMOpFilter, UnitIO, UnitKind, UnitTrait}, + unit::{DynUnit, DynUnitWrapper, RenamedMOpFilter, UnitKind, UnitTrait}, }; use fayalite::{ expr::CastToImpl, @@ -1613,12 +1613,15 @@ impl UnitTrait for AluBranch { self.module } - fn io(&self, this: Expr) -> UnitIO { - UnitIO { - cd: None, - from_execute: this.from_execute, - to_d_cache: None, - } + fn cd(&self, _this: Expr) -> Option> { + None + } + + fn from_execute( + &self, + this: Expr, + ) -> Expr>> { + this.from_execute } fn to_dyn(&self) -> DynUnit { diff --git a/crates/cpu/src/unit/load_store.rs b/crates/cpu/src/unit/load_store.rs deleted file mode 100644 index 858763b..0000000 --- a/crates/cpu/src/unit/load_store.rs +++ /dev/null @@ -1,1328 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-or-later -// See Notices.txt for copyright information - -use crate::{ - config::{CpuConfig, PhantomConstCpuConfig}, - instruction::{ - COMMON_MOP_SRC_LEN, LoadMOp, LoadStoreCommonMOp, LoadStoreConversion, LoadStoreMOp, - LoadStoreWidth, PRegNum, StoreMOp, - }, - main_memory_and_io::{AddressRange, MemoryOperationErrorKind}, - next_pc::{CallStackOp, SimValueDefault}, - register::{PRegFlags, PRegValue}, - rename_execute_retire::{ - ExecuteToUnitInterface, GlobalState, MOpId, MOpInstance, NextPcPredictorOp, UnitEnqueue, - UnitFinishCauseCancel, UnitInputsReady, UnitMOpCantCauseCancel, - UnitMOpIsNoLongerSpeculative, UnitOutputReady, - }, - unit::{DynUnit, DynUnitWrapper, RenamedMOpFilter, UnitIO, UnitKind, UnitMOp, UnitTrait}, -}; -use fayalite::{intern::Interned, prelude::*, ty::SimValueDebug, util::ready_valid::ReadyValid}; -use std::{collections::VecDeque, fmt}; - -#[hdl] -pub struct DCacheOpKindLoad { - /// if this is `false`, then if the address refers to a valid cache line, that cache line must have any dirty data - /// written back to memory and then be set to invalid before performing the load. - pub is_cacheable: Bool, -} - -#[hdl] -pub struct DCacheOpKindStore { - /// if this is `false`, then if the address refers to a valid cache line, that cache line must have any dirty data - /// written back to memory and then be set to invalid before performing the store. - pub is_cacheable: Bool, -} - -#[hdl] -pub enum DCacheOpKind { - Load(DCacheOpKindLoad), - Store(DCacheOpKindStore), -} - -#[hdl] -pub struct DCacheStart> { - pub id: MOpId, - pub kind: DCacheOpKind, - /// unaligned addresses may cause a load/store to cross a cache-line boundary - pub address: UInt<64>, - pub data_and_mask: Array>, 8>, - pub config: C, -} - -#[hdl] -pub enum DCacheFinishStatus { - /// The operation finished successfully - Success, - MemoryError(MemoryOperationErrorKind), -} - -#[hdl] -pub struct DCacheFinish> { - pub id: MOpId, - pub status: DCacheFinishStatus, - pub data: Array, 8>, - pub config: C, -} - -/// This load must only look in the L1 cache (must not propagate to the L2/L3 cache or to memory), must not -/// cause the timing of any earlier speculative operations or any non-speculative operations to change -/// because this load exists, and must not change any cache state that remains after all operations finish, -/// including LRU state, random number generators, predictive prefetching state, etc. -/// -/// Speculative loads that miss the L1 cache must return with [`DCacheFinishSpeculativeLoad::data`] set to -/// [`HdlNone()`] instead of trying to propagate to the L2/L3 or memory. -#[hdl] -pub struct DCacheStartSpeculativeLoad> { - pub id: MOpId, - /// unaligned addresses may cause a load to cross a cache-line boundary - pub address: UInt<64>, - pub mask: Array, - pub config: C, -} - -#[hdl] -pub struct DCacheFinishSpeculativeLoad> { - pub id: MOpId, - pub data: HdlOption, 8>>, - pub config: C, -} - -/// This CPU core's D-cache must report to the [load/store unit](load_store()) whenever: -/// * A cache line is removed from the D-cache by this CPU core (e.g. the cache ran out of space or the -/// cache line was intentionally removed by some instruction). -/// * Or a cache line is removed and/or written by something other than this CPU core (so other cores -/// and I/O devices that write to memory). -/// -/// That allows the [load/store unit](load_store()) to restart speculative loads as necessary to -/// maintain sequential consistency. -#[hdl] -pub struct DCacheForeignWriteToCacheLine> { - /// The address of the cache line that was written. - pub cache_line_address: UInt<64>, - /// Must be set such that for all bytes that were written and/or removed from the D-cache, that - /// `cache_line_address` must equal `cache_line_address_mask & byte_address`. - /// - /// e.g. if a 64-byte cache line at `0x1000` was removed, then `cache_line_address` is `0x1000` - /// and `cache_line_address_mask` is `0xFFFF_FFFF_FFFF_FFC0`. - pub cache_line_address_mask: UInt<64>, - pub config: C, -} - -#[hdl] -pub struct LoadStoreToDCacheInterface> { - /// Operations started through `start` must maintain sequential consistency -- so earlier operations - /// can't appear to have happened after later operations. - pub start: ReadyValid>, - /// operations must finish in the same order they started. - #[hdl(flip)] - pub finish: ReadyValid>, - /// It's valid for `start_speculative_load.ready` to be `false` at any time (or even always) - /// -- this doesn't block forward progress since speculative loads that don't start will eventually - /// convert to non-speculative loads and go through `start` instead. - /// - /// Speculative loads don't need to maintain any order relative to non-speculative operations, except - /// that speculative loads started after non-speculative operations have finished must be ordered after - /// those non-speculative operations (that should be trivial to implement in the D-cache). - /// - /// The [load/store unit](load_store()) will handle restarting speculative loads as necessary to - /// maintain sequential consistency when there is a `foreign_write_to_cache_line` that conflicts with - /// that load. - pub start_speculative_load: ReadyValid>, - /// Operations must finish in the same order they started. - #[hdl(flip)] - pub finish_speculative_load: ReadyValid>, - #[hdl(flip)] - pub foreign_write_to_cache_line: HdlOption>, - pub config: C, -} - -#[hdl(custom_debug(sim))] -struct OpDebugState { - v: SimOnly, -} - -impl SimValueDebug for OpDebugState { - #[hdl] - fn sim_value_debug( - value: &::SimValue, - f: &mut fmt::Formatter<'_>, - ) -> fmt::Result { - f.write_str(&value.v) - } -} - -impl SimValueDefault for OpDebugState { - #[hdl] - fn sim_value_default(self) -> SimValue { - thread_local! { - static VALUE: SimOnlyValue = SimOnlyValue::new("None".into()); - } - VALUE.with(|v| { - #[hdl(sim)] - Self { v } - }) - } -} - -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -enum OpState { - InputNotReady, - StartedSpeculativeLoad, - FinishedSpeculativeLoad, - StartedLoadStore, - FinishedLoadStore, -} - -impl OpState { - fn as_str(self) -> &'static str { - match self { - Self::InputNotReady => "inr", - Self::StartedSpeculativeLoad => "ssl", - Self::FinishedSpeculativeLoad => "fsl", - Self::StartedLoadStore => "sls", - Self::FinishedLoadStore => "fls", - } - } -} - -impl fmt::Display for OpState { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -#[derive(Debug)] -struct Op { - id: SimValue, - pc: SimValue>, - size_in_bytes: u8, - mop: SimValue, PRegNum>>, - /// earlier instructions can cause a cancel, so this instruction is still speculative. - /// not to be confused with this instruction being able to cause a cancel. - is_speculative: bool, - src_values: Option<[SimValue>; COMMON_MOP_SRC_LEN]>, - dest_value: Option>>, - sent_cant_cause_cancel: bool, - sent_output_ready: bool, - state: OpState, -} - -impl fmt::Display for Op { - #[hdl] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Self { - id, - pc, - size_in_bytes, - mop, - is_speculative, - src_values, - dest_value, - sent_cant_cause_cancel, - sent_output_ready, - state, - } = self; - if *is_speculative { - f.write_str("(s)")?; - } - if *sent_cant_cause_cancel { - f.write_str("(sccc)")?; - } - if *sent_output_ready { - f.write_str("(sor)")?; - } - write!( - f, - "({state}) id={id:?} pc={pc:?} sz={size_in_bytes}: {mop:?}" - )?; - if let Some(src_values) = src_values { - write!(f, "src_values={src_values:?}")?; - } - if let Some(dest_value) = dest_value { - write!(f, "dest_value={dest_value:?}")?; - } - Ok(()) - } -} - -struct AccessRangeUnknown; - -impl Op { - fn new( - id: SimValue, - pc: SimValue>, - size_in_bytes: u8, - mop: SimValue, PRegNum>>, - ) -> Self { - Self { - id, - pc, - size_in_bytes, - mop, - is_speculative: true, - src_values: None, - dest_value: None, - sent_cant_cause_cancel: false, - sent_output_ready: false, - state: OpState::InputNotReady, - } - } - #[hdl] - fn debug_state(&self) -> SimValue { - #[hdl(sim)] - OpDebugState { - v: SimOnlyValue::new(self.to_string()), - } - } - #[hdl] - fn access_range_helper( - &self, - load_store_common: &SimValue, PRegNum, SrcCount>>, - ) -> Result { - let Some([src0, ..]) = &self.src_values else { - return Err(AccessRangeUnknown); - }; - let size = LoadStoreWidth::access_byte_size_sim(&load_store_common.width); - Ok(AddressRange::Limited { - start: std::num::Wrapping(src0.inner().int_fp.as_int()), - size, - }) - } - #[hdl] - fn access_range(&self) -> Result { - #[hdl(sim)] - match &self.mop { - LoadStoreMOp::<_, _>::Load(mop) => self.access_range_helper(&mop.load_store_common), - LoadStoreMOp::<_, _>::Store(mop) => self.access_range_helper(&mop.load_store_common), - } - } - fn can_cause_cancel(&self) -> bool { - if self.is_speculative { - return true; - } - match self.state { - OpState::InputNotReady - | OpState::StartedSpeculativeLoad - | OpState::StartedLoadStore => true, - OpState::FinishedSpeculativeLoad | OpState::FinishedLoadStore => { - self.dest_value.is_none() - } - } - } -} - -#[hdl(get(|c| c.rob_size.get().next_power_of_two()))] -type OpsDebugLen> = DynSize; - -#[hdl] -type PhantomConstUnitIndex = PhantomConst; - -#[hdl(no_static)] -pub struct LoadStoreDebugState> { - global_state: GlobalState, - ops: ArrayType, OpsDebugLen>, - canceling: Bool, - config: C, - unit_index: PhantomConstUnitIndex, - filter: LoadStoreFilter, -} - -#[derive(Debug)] -struct LoadStoreState { - global_state: SimValue, - ops: VecDeque>, - d_cache_start: Option>>, - d_cache_start_speculative_load: Option>>, - canceling: bool, - config: C, - unit_index: usize, - filter: SimValue, -} - -impl LoadStoreState { - fn new(config: C, unit_index: usize, filter: SimValue) -> Self { - Self { - global_state: GlobalState.sim_value_default(), - ops: VecDeque::new(), - d_cache_start: None, - d_cache_start_speculative_load: None, - canceling: false, - config, - unit_index, - filter, - } - } - #[hdl] - fn debug_state(&self) -> SimValue> { - let Self { - ref global_state, - ops: _, - // not included in debug state since it's identical to `to_d_cache.start.data` - d_cache_start: _, - // not included in debug state since it's identical to `to_d_cache.start_speculative_load.data` - d_cache_start_speculative_load: _, - canceling, - config, - unit_index, - ref filter, - } = *self; - let ty = LoadStoreDebugState[config]; - let mut ops = vec![OpDebugState.sim_value_default().into_trace_as_string(); ty.ops.len()]; - for op in &self.ops { - ops[op.id.as_int() as usize % ty.ops.len()] = op.debug_state().into_trace_as_string(); - } - #[hdl(sim)] - LoadStoreDebugState::<_> { - global_state, - ops, - canceling, - config, - unit_index: PhantomConst::new_sized(unit_index), - filter, - } - } - #[track_caller] - fn op_by_id(&mut self, id: &::SimValue) -> &mut Op { - match self.ops.iter_mut().find(|op| *op.id == *id) { - Some(op) => op, - None => panic!("can't find load_store Op with id {id:?}"), - } - } - fn from_execute_enqueue_ready(&self) -> bool { - !self.canceling - && self.ops.len() < self.config.get().unit_max_in_flight(self.unit_index).get() - } - #[hdl] - fn from_execute_enqueue(&mut self, enqueue: SimValue>) { - #[hdl(sim)] - let UnitEnqueue::<_> { mop, config: _ } = enqueue; - #[hdl(sim)] - let MOpInstance::<_> { - fetch_block_id: _, - id, - pc, - predicted_next_pc: _, - size_in_bytes, - is_first_mop_in_insn: _, - is_last_mop_in_insn: _, - mop, - } = mop; - let mop = #[hdl(sim)] - match SimValue::into_value(mop).into_inner() { - UnitMOp::<_, _, _>::LoadStore(mop) => mop, - _ => unreachable!(), - }; - self.ops.push_back(Op::new( - id, - pc, - size_in_bytes.cast_to_static::>().as_int(), - mop, - )); - } - #[hdl] - fn from_execute_inputs_ready(&mut self, inputs_ready: SimValue>) { - #[hdl(sim)] - let UnitInputsReady::<_> { - mop, - src_values, - config: _, - } = inputs_ready; - self.op_by_id(&mop.id).src_values = Some(SimValue::into_value(src_values)); - } - #[hdl] - fn from_execute_is_no_longer_speculative( - &mut self, - is_no_longer_speculative: SimValue>, - ) { - #[hdl(sim)] - let UnitMOpIsNoLongerSpeculative::<_> { id, config: _ } = is_no_longer_speculative; - self.op_by_id(&id).is_speculative = false; - } - #[hdl] - fn from_execute_cant_cause_cancel(&self) -> Option>> { - if self.canceling { - return None; - } - for op in &self.ops { - if op.sent_cant_cause_cancel || op.can_cause_cancel() { - continue; - } - return Some( - #[hdl(sim)] - UnitMOpCantCauseCancel::<_> { - id: op.id, - config: self.config, - }, - ); - } - None - } - #[hdl] - fn from_execute_sent_cant_cause_cancel( - &mut self, - cant_cause_cancel: SimValue>, - ) { - #[hdl(sim)] - let UnitMOpCantCauseCancel::<_> { id, config: _ } = cant_cause_cancel; - self.op_by_id(&id).sent_cant_cause_cancel = true; - } - #[hdl] - fn from_execute_output_ready(&self) -> Option>> { - if self.canceling { - return None; - } - for op in &self.ops { - if op.sent_output_ready { - continue; - } - if let Some(dest_value) = &op.dest_value { - return Some( - #[hdl(sim)] - UnitOutputReady::<_> { - id: op.id, - dest_value, - predictor_op: #[hdl(sim)] - NextPcPredictorOp::<_> { - call_stack_op: #[hdl(sim)] - CallStackOp.None(), - cond_br_taken: #[hdl(sim)] - HdlNone(), - config: self.config, - }, - }, - ); - } - } - None - } - #[hdl] - fn from_execute_sent_output_ready(&mut self, output_ready: SimValue>) { - #[hdl(sim)] - let UnitOutputReady::<_> { - id, - dest_value: _, - predictor_op: _, - } = output_ready; - self.op_by_id(&id).sent_output_ready = true; - } - #[hdl] - fn from_execute_finish_cause_cancel(&self) -> Option>> { - if self.canceling { - return None; - } - let Op { - id, - pc: _, - size_in_bytes: _, - mop: _, - is_speculative, - src_values: _, - dest_value, - sent_cant_cause_cancel: _, - sent_output_ready, - state, - } = self.ops.front()?; - if *is_speculative { - return None; - } - let retval_ty = UnitFinishCauseCancel[self.config]; - match state { - OpState::InputNotReady => None, - OpState::StartedSpeculativeLoad => None, - OpState::FinishedSpeculativeLoad => { - if *sent_output_ready { - // TODO: handle when some other CPU stores to the same cache line before the speculative - // load sends UnitFinishCauseCancel -- that needs to make the load cause a cancel so it - // can be redone with the correct data. That enforces sequential consistency. - Some( - #[hdl(sim)] - UnitFinishCauseCancel::<_> { - id, - caused_cancel: #[hdl(sim)] - (retval_ty.caused_cancel).HdlNone(), - config: self.config, - }, - ) - } else { - None - } - } - OpState::StartedLoadStore => None, - OpState::FinishedLoadStore => { - if dest_value.is_none() { - todo!("load/store caused error -- need to translate to a cancel"); - } - if *sent_output_ready { - Some( - #[hdl(sim)] - UnitFinishCauseCancel::<_> { - id, - caused_cancel: #[hdl(sim)] - (retval_ty.caused_cancel).HdlNone(), - config: self.config, - }, - ) - } else { - None - } - } - } - } - #[hdl] - fn from_execute_sent_finish_cause_cancel( - &mut self, - finish_cause_cancel: SimValue>, - ) { - let op = self - .ops - .pop_front_if(|front| front.id == finish_cause_cancel.id); - assert!( - op.is_some(), - "inconsistent state -- sent finish_cause_cancel but id doesn't match ops.front():\n{finish_cause_cancel:#?}\nfront={front:#?}", - front = self.ops.front(), - ); - } - fn from_execute_cancel_all_ready(&self) -> bool { - true - } - #[hdl] - fn from_execute_cancel_all(&mut self) { - self.canceling = true; - } - #[hdl] - fn to_d_cache_sent_start(&mut self, start: SimValue>) { - self.op_by_id(&start.id).state = OpState::StartedLoadStore; - } - #[hdl] - fn to_d_cache_finish_ready(&self) -> bool { - true - } - #[hdl] - fn to_d_cache_finish(&mut self, finish: SimValue>) { - #[hdl(sim)] - let DCacheFinish::<_> { - id, - status, - data, - config: _, - } = finish; - let canceling = self.canceling; - let op = self.op_by_id(&id); - op.state = OpState::FinishedLoadStore; - if canceling { - return; - } - #[hdl(sim)] - match status { - DCacheFinishStatus::Success => { - op.dest_value = #[hdl(sim)] - match &op.mop { - LoadStoreMOp::<_, _>::Load(mop) => Some(Self::process_load_data( - mop, - &SimValue::into_value(data).map(|v| v.as_int()), - )), - LoadStoreMOp::<_, _>::Store(_) => { - Some(PRegValue::zeroed_sim().into_trace_as_string()) - } - }; - } - DCacheFinishStatus::MemoryError(error) => { - todo!("{error:?}") - } - } - } - #[hdl] - fn to_d_cache_sent_start_speculative_load( - &mut self, - start_speculative_load: SimValue>, - ) { - self.op_by_id(&start_speculative_load.id).state = OpState::StartedSpeculativeLoad; - } - #[hdl] - fn to_d_cache_finish_speculative_load_ready(&self) -> bool { - true - } - #[hdl] - fn to_d_cache_finish_speculative_load( - &mut self, - finish_speculative_load: SimValue>, - ) { - #[hdl(sim)] - let DCacheFinishSpeculativeLoad::<_> { - id, - data, - config: _, - } = finish_speculative_load; - let canceling = self.canceling; - let op = self.op_by_id(&id); - op.state = OpState::FinishedSpeculativeLoad; - if canceling { - return; - } - #[hdl(sim)] - if let HdlSome(data) = data { - op.dest_value = #[hdl(sim)] - match &op.mop { - LoadStoreMOp::<_, _>::Load(mop) => Some(Self::process_load_data( - mop, - &SimValue::into_value(data).map(|v| v.as_int()), - )), - LoadStoreMOp::<_, _>::Store(_) => { - unreachable!() - } - }; - } - } - #[hdl] - fn to_d_cache_foreign_write_to_cache_line( - &mut self, - foreign_write_to_cache_line: SimValue>, - ) { - #[hdl(sim)] - let DCacheForeignWriteToCacheLine::<_> { - cache_line_address, - cache_line_address_mask, - config: _, - } = foreign_write_to_cache_line; - todo!( - "foreign_write_to_cache_line:\n\ - cache_line_address={cache_line_address}, cache_line_address_mask={cache_line_address_mask}" - ); - } - #[hdl] - fn canceling_step(&mut self) { - self.d_cache_start = None; - self.d_cache_start_speculative_load = None; - let finished_canceling = |op: &mut Op<_>| match op.state { - OpState::InputNotReady => true, - OpState::StartedSpeculativeLoad => false, - OpState::FinishedSpeculativeLoad => true, - OpState::StartedLoadStore => false, - OpState::FinishedLoadStore => true, - }; - while let Some(_) = self.ops.pop_front_if(finished_canceling) {} - if self.ops.is_empty() { - self.canceling = false; - } - } - fn is_cacheable_load(mop: &SimValue, PRegNum>>) -> bool { - // TODO: implement un-cacheable loads - true - } - fn is_cacheable_store(mop: &SimValue, PRegNum>>) -> bool { - // TODO: implement un-cacheable stores - true - } - #[hdl] - fn process_load_data( - mop: &SimValue, PRegNum>>, - data: &[u8; 8], - ) -> SimValue> { - #[hdl(sim)] - let LoadMOp::<_, _> { load_store_common } = mop; - #[hdl(sim)] - let LoadStoreCommonMOp::<_, _, _> { - common: _, - width, - conversion, - } = load_store_common; - macro_rules! cvt { - ($ty:ident) => { - $ty::from_le_bytes(std::array::from_fn(|i| data[i])) as u64 - }; - } - let int_fp = #[hdl(sim)] - match conversion { - LoadStoreConversion::ZeroExt => - { - #[hdl(sim)] - match width { - LoadStoreWidth::Width8Bit => cvt!(u8), - LoadStoreWidth::Width16Bit => cvt!(u16), - LoadStoreWidth::Width32Bit => cvt!(u32), - LoadStoreWidth::Width64Bit => cvt!(u64), - } - } - LoadStoreConversion::SignExt => - { - #[hdl(sim)] - match width { - LoadStoreWidth::Width8Bit => cvt!(i8), - LoadStoreWidth::Width16Bit => cvt!(i16), - LoadStoreWidth::Width32Bit => cvt!(i32), - LoadStoreWidth::Width64Bit => cvt!(i64), - } - } - }; - let retval = #[hdl(sim)] - PRegValue { - int_fp, - flags: PRegFlags::zeroed_sim(), - }; - retval.into_trace_as_string() - } - #[hdl] - fn process_store_data( - mop: &SimValue, PRegNum>>, - data: u64, - ) -> [Option; 8] { - #[hdl(sim)] - let StoreMOp::<_, _> { load_store_common } = mop; - #[hdl(sim)] - let LoadStoreCommonMOp::<_, _, _> { - common: _, - width, - conversion, - } = load_store_common; - macro_rules! cvt { - ($ty:ident) => {{ - let data = (data as $ty).to_le_bytes(); - std::array::from_fn(|i| data.get(i).copied()) - }}; - } - #[hdl(sim)] - match conversion { - LoadStoreConversion::ZeroExt => - { - #[hdl(sim)] - match width { - LoadStoreWidth::Width8Bit => cvt!(u8), - LoadStoreWidth::Width16Bit => cvt!(u16), - LoadStoreWidth::Width32Bit => cvt!(u32), - LoadStoreWidth::Width64Bit => cvt!(u64), - } - } - LoadStoreConversion::SignExt => - { - #[hdl(sim)] - match width { - LoadStoreWidth::Width8Bit => cvt!(i8), - LoadStoreWidth::Width16Bit => cvt!(i16), - LoadStoreWidth::Width32Bit => cvt!(i32), - LoadStoreWidth::Width64Bit => cvt!(i64), - } - } - } - } - #[hdl] - fn step(&mut self) { - if self.canceling { - return self.canceling_step(); - } - self.d_cache_start = None; - self.d_cache_start_speculative_load = None; - let store_queue_capacity = 4; // TODO: make configurable - struct StoreQueueEntry { - access_range: AddressRange, - data: [u8; 8], - } - let mut store_queue = Vec::::with_capacity(store_queue_capacity); - for op in &mut self.ops { - let access_range = op.access_range(); - let Op { - ref id, - pc: _, - size_in_bytes: _, - ref mop, - is_speculative, - ref src_values, - ref mut dest_value, - sent_cant_cause_cancel: _, - sent_output_ready: _, - ref mut state, - } = *op; - #[hdl(sim)] - match mop { - LoadStoreMOp::<_, _>::Load(mop) => { - let Some([src0, ..]) = src_values else { - continue; - }; - let Ok(access_range) = access_range else { - continue; - }; - let is_cacheable = Self::is_cacheable_load(mop); - let mut blocking_store = false; - for StoreQueueEntry { - access_range: store_range, - data, - } in &store_queue - { - if access_range == *store_range { - // matching load and store -- speculatively forward data - if let OpState::InputNotReady = state - && is_cacheable - { - *state = OpState::FinishedSpeculativeLoad; - *dest_value = Some(Self::process_load_data(mop, data)); - } - blocking_store = true; - break; - } - if store_range.is_overlapping(access_range) { - blocking_store = true; - break; - } - } - let address = src0.inner().int_fp.as_int(); - let size = LoadStoreWidth::access_byte_size_sim(&mop.load_store_common.width); - let mask = std::array::from_fn(|i| i < size.get() as usize); - match state { - OpState::InputNotReady - if is_speculative && !blocking_store && is_cacheable => - { - self.d_cache_start_speculative_load.get_or_insert_with(|| { - #[hdl(sim)] - DCacheStartSpeculativeLoad::<_> { - id, - address, - mask, - config: self.config, - } - }); - } - OpState::InputNotReady | OpState::FinishedSpeculativeLoad => { - if !is_speculative && dest_value.is_none() { - self.d_cache_start.get_or_insert_with(|| { - #[hdl(sim)] - DCacheStart::<_> { - id, - kind: #[hdl(sim)] - DCacheOpKind.Load( - #[hdl(sim)] - DCacheOpKindLoad { is_cacheable }, - ), - address, - data_and_mask: mask.map(|v| v.then_some(0)), - config: self.config, - } - }); - } - } - OpState::StartedSpeculativeLoad => {} - OpState::StartedLoadStore => {} - OpState::FinishedLoadStore => {} - } - } - LoadStoreMOp::<_, _>::Store(mop) => { - let Op { - ref id, - pc: _, - size_in_bytes: _, - mop: _, - is_speculative, - ref src_values, - ref dest_value, - sent_cant_cause_cancel: _, - sent_output_ready: _, - state, - } = *op; - let Some([src0, src1, ..]) = src_values else { - break; - }; - let Ok(access_range) = access_range else { - continue; - }; - let is_cacheable = Self::is_cacheable_store(mop); - let address = src0.inner().int_fp.as_int(); - let data_and_mask = Self::process_store_data(mop, src1.inner().int_fp.as_int()); - if dest_value.is_none() { - if store_queue.len() >= store_queue_capacity { - break; - } - store_queue.push(StoreQueueEntry { - access_range, - data: data_and_mask.map(|v| v.unwrap_or(0)), - }); - } - match state { - OpState::InputNotReady => { - if !is_speculative { - self.d_cache_start.get_or_insert_with(|| { - #[hdl(sim)] - DCacheStart::<_> { - id, - kind: #[hdl(sim)] - DCacheOpKind.Store( - #[hdl(sim)] - DCacheOpKindStore { is_cacheable }, - ), - address, - data_and_mask, - config: self.config, - } - }); - } - } - OpState::StartedSpeculativeLoad | OpState::FinishedSpeculativeLoad => { - unreachable!() - } - OpState::StartedLoadStore => {} - OpState::FinishedLoadStore => {} - } - } - } - } - } -} - -#[hdl] -async fn load_store_impl( - config: PhantomConst, - unit_index: usize, - filter: SimValue, - cd: Expr, - from_execute: Expr>>, - to_d_cache: Expr>>, - debug_state: Expr>>, - mut sim: ExternModuleSimulationState, -) { - let mut state = LoadStoreState::new(config, unit_index, filter); - loop { - { - #[hdl] - let ExecuteToUnitInterface::<_> { - global_state: _, - enqueue, - inputs_ready: _, - is_no_longer_speculative: _, - cant_cause_cancel, - output_ready, - finish_cause_cancel, - unit_outputs_ready: _, - cancel_all, - config: _, - } = from_execute; - sim.write(enqueue.ready, state.from_execute_enqueue_ready()) - .await; - sim.write(cant_cause_cancel, state.from_execute_cant_cause_cancel()) - .await; - sim.write(output_ready, state.from_execute_output_ready()) - .await; - sim.write( - finish_cause_cancel, - state.from_execute_finish_cause_cancel(), - ) - .await; - sim.write(cancel_all.ready, state.from_execute_cancel_all_ready()) - .await; - } - { - #[hdl] - let LoadStoreToDCacheInterface::<_> { - start, - finish, - start_speculative_load, - finish_speculative_load, - foreign_write_to_cache_line: _, - config: _, - } = to_d_cache; - sim.write(start.data, &state.d_cache_start).await; - sim.write(finish.ready, state.to_d_cache_finish_ready()) - .await; - sim.write( - start_speculative_load.data, - &state.d_cache_start_speculative_load, - ) - .await; - sim.write( - finish_speculative_load.ready, - state.to_d_cache_finish_speculative_load_ready(), - ) - .await; - } - sim.write(debug_state, state.debug_state()).await; - sim.wait_for_clock_edge(cd.clk).await; - { - #[hdl] - let ExecuteToUnitInterface::<_> { - global_state, - enqueue, - inputs_ready, - is_no_longer_speculative, - cant_cause_cancel, - output_ready, - finish_cause_cancel, - unit_outputs_ready, - cancel_all, - config: _, - } = from_execute; - state.global_state = sim.read_past(global_state, cd.clk).await; - if sim.read_past_bool(enqueue.ready, cd.clk).await { - #[hdl(sim)] - if let HdlSome(enqueue) = sim.read_past(enqueue.data, cd.clk).await { - state.from_execute_enqueue(enqueue); - } - } - #[hdl(sim)] - if let HdlSome(inputs_ready) = sim.read_past(inputs_ready, cd.clk).await { - state.from_execute_inputs_ready(inputs_ready); - } - #[hdl(sim)] - if let HdlSome(is_no_longer_speculative) = - sim.read_past(is_no_longer_speculative, cd.clk).await - { - state.from_execute_is_no_longer_speculative(is_no_longer_speculative); - } - if sim.read_past_bool(unit_outputs_ready, cd.clk).await { - #[hdl(sim)] - if let HdlSome(cant_cause_cancel) = sim.read_past(cant_cause_cancel, cd.clk).await { - state.from_execute_sent_cant_cause_cancel(cant_cause_cancel); - } - #[hdl(sim)] - if let HdlSome(output_ready) = sim.read_past(output_ready, cd.clk).await { - state.from_execute_sent_output_ready(output_ready); - } - #[hdl(sim)] - if let HdlSome(finish_cause_cancel) = - sim.read_past(finish_cause_cancel, cd.clk).await - { - state.from_execute_sent_finish_cause_cancel(finish_cause_cancel); - } - } - if sim.read_past_bool(cancel_all.ready, cd.clk).await { - #[hdl(sim)] - if let HdlSome(cancel_all) = sim.read_past(cancel_all.data, cd.clk).await { - #[hdl] - let () = cancel_all; - state.from_execute_cancel_all(); - } - } - } - { - #[hdl] - let LoadStoreToDCacheInterface::<_> { - start, - finish, - start_speculative_load, - finish_speculative_load, - foreign_write_to_cache_line, - config: _, - } = to_d_cache; - if sim.read_past_bool(start.ready, cd.clk).await { - #[hdl(sim)] - if let HdlSome(start) = sim.read_past(start.data, cd.clk).await { - state.to_d_cache_sent_start(start); - } - } - if sim.read_past_bool(finish.ready, cd.clk).await { - #[hdl(sim)] - if let HdlSome(finish) = sim.read_past(finish.data, cd.clk).await { - state.to_d_cache_finish(finish); - } - } - if sim - .read_past_bool(start_speculative_load.ready, cd.clk) - .await - { - #[hdl(sim)] - if let HdlSome(start_speculative_load) = - sim.read_past(start_speculative_load.data, cd.clk).await - { - state.to_d_cache_sent_start_speculative_load(start_speculative_load); - } - } - if sim - .read_past_bool(finish_speculative_load.ready, cd.clk) - .await - { - #[hdl(sim)] - if let HdlSome(finish_speculative_load) = - sim.read_past(finish_speculative_load.data, cd.clk).await - { - state.to_d_cache_finish_speculative_load(finish_speculative_load); - } - } - #[hdl(sim)] - if let HdlSome(foreign_write_to_cache_line) = - sim.read_past(foreign_write_to_cache_line, cd.clk).await - { - state.to_d_cache_foreign_write_to_cache_line(foreign_write_to_cache_line); - } - } - state.step(); - } -} - -#[hdl] -struct LoadStoreFilter { - load: Bool, - store: Bool, -} - -impl LoadStoreFilter { - #[hdl] - fn new( - from_execute: ExecuteToUnitInterface>, - filter: &mut impl RenamedMOpFilter, - ) -> SimValue { - let LoadStoreMOp { Load, Store } = from_execute - .inputs_ready - .HdlSome - .mop - .mop - .inner_ty() - .LoadStore; - #[hdl(sim)] - Self { - load: filter.should_include_ty(Load), - store: filter.should_include_ty(Store), - } - } -} - -#[hdl_module(extern)] -pub fn load_store( - config: PhantomConst, - unit_index: usize, - filter: &mut impl RenamedMOpFilter, -) { - #[hdl] - let cd: ClockDomain = m.input(); - - #[hdl] - let from_execute: ExecuteToUnitInterface> = - m.input(ExecuteToUnitInterface[config]); - - #[hdl] - let to_d_cache: LoadStoreToDCacheInterface> = - m.output(LoadStoreToDCacheInterface[config]); - - #[hdl] - let debug_state: LoadStoreDebugState> = - m.output(LoadStoreDebugState[config]); - - let filter = LoadStoreFilter::new(from_execute.ty(), filter).to_expr(); - - assert_eq!(config.get().units[unit_index].kind, UnitKind::LoadStore); - - m.register_clock_for_past(cd.clk); - - m.extern_module_simulation_fn( - ( - config, - unit_index, - filter, - cd, - from_execute, - to_d_cache, - debug_state, - ), - async |(config, unit_index, filter, cd, from_execute, to_d_cache, debug_state), mut sim| { - let filter = filter.into_sim_value(); - sim.resettable( - cd, - async |mut sim| { - #[hdl] - let ExecuteToUnitInterface::<_> { - global_state: _, - enqueue, - inputs_ready: _, - is_no_longer_speculative: _, - cant_cause_cancel, - output_ready, - finish_cause_cancel, - unit_outputs_ready: _, - cancel_all, - config: _, - } = from_execute; - sim.write(enqueue.ready, false).await; - sim.write(cant_cause_cancel, cant_cause_cancel.ty().HdlNone()) - .await; - sim.write(output_ready, output_ready.ty().HdlNone()).await; - sim.write(finish_cause_cancel, finish_cause_cancel.ty().HdlNone()) - .await; - sim.write(cancel_all.ready, false).await; - #[hdl] - let LoadStoreToDCacheInterface::<_> { - start, - finish, - start_speculative_load, - finish_speculative_load, - foreign_write_to_cache_line: _, - config: _, - } = to_d_cache; - sim.write(start.data, start.ty().data.HdlNone()).await; - sim.write(finish.ready, false).await; - sim.write( - debug_state, - LoadStoreState::new(config, unit_index, filter.clone()).debug_state(), - ) - .await; - sim.write( - start_speculative_load.data, - start_speculative_load.ty().data.HdlNone(), - ) - .await; - sim.write(finish_speculative_load.ready, false).await; - sim.write( - debug_state, - LoadStoreState::new(config, unit_index, filter.clone()).debug_state(), - ) - .await; - }, - async |sim, ()| { - load_store_impl( - config, - unit_index, - filter.clone(), - cd, - from_execute, - to_d_cache, - debug_state, - sim, - ) - .await - }, - ) - .await - }, - ); -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct LoadStore { - config: PhantomConst, - module: Interned>, -} - -impl LoadStore { - pub fn new( - config: PhantomConst, - unit_index: usize, - filter: &mut impl RenamedMOpFilter, - ) -> Self { - Self { - config, - module: load_store(config, unit_index, filter), - } - } -} - -impl UnitTrait for LoadStore { - type Type = load_store; - - fn ty(&self) -> Self::Type { - self.module.io_ty() - } - - fn unit_kind(&self) -> UnitKind { - UnitKind::LoadStore - } - - fn module(&self) -> Interned> { - self.module - } - - fn io(&self, this: Expr) -> UnitIO { - UnitIO { - cd: Some(this.cd), - from_execute: this.from_execute, - to_d_cache: Some(this.to_d_cache), - } - } - - fn to_dyn(&self) -> DynUnit { - DynUnitWrapper(*self).to_dyn() - } -} diff --git a/crates/cpu/tests/units_formal.rs b/crates/cpu/tests/units_formal.rs index 2a818b8..47ad415 100644 --- a/crates/cpu/tests/units_formal.rs +++ b/crates/cpu/tests/units_formal.rs @@ -14,7 +14,7 @@ use cpu::{ DecodeAndRunSingleInsnInput, DecodeAndRunSingleInsnOutput, DecodeOneInsnInput, DecodeOneInsnMaxMOpCount, decode_and_run_single_insn, }, - unit::{RenamedMOpFilter, UnitIO, UnitKind, UnitMOp, UnitTrait}, + unit::{RenamedMOpFilter, UnitKind, UnitMOp, UnitTrait}, util::array_vec::ArrayVec, }; use fayalite::{ @@ -91,20 +91,12 @@ fn formal_harness( dyn_unit.module(), SourceLocation::caller(), ); - let UnitIO { - cd: unit_cd, - from_execute: unit_from_execute, - to_d_cache, - } = dyn_unit.io(unit); - if let Some(unit_cd) = unit_cd { - connect(unit_cd, cd); - } connect( - unit_from_execute, + dyn_unit.from_execute(unit), ExecuteToUnitInterfaces::unit_fields(decode_and_run.to_units)[unit_index], ); - if let Some(to_d_cache) = to_d_cache { - unimplemented!("to_d_cache for {unit:?}: {to_d_cache:?}"); + if let Some(unit_cd) = dyn_unit.cd(unit) { + connect(unit_cd, cd); } } hdl_assert(cd.clk, !decode_and_run.error, "");