rename_execute_retire: add reference counting for L1 registers

crates/cpu/src/rename_execute_retire.rs

@@ -716,39 +716,15 @@ impl<C: PhantomConstCpuConfig> RenameExecuteRetireState<C> {
     }
     #[hdl]
     fn find_free_unit_out_reg(&self, unit_index: usize) -> Option<usize> {
-        // TODO: replace searching through instructions and rename tables with tracking when regs are free
-        let mut allocated_regs = vec![false; 1 << self.config.get().out_reg_num_width];
-        for renamed in self.rob.renamed() {
-            if renamed.unit_index == unit_index
-                && let Some(unit_out_reg_index) = renamed.unit_out_reg_index()
-            {
-                allocated_regs[unit_out_reg_index] = true;
-            }
-            MOpTrait::for_each_src_reg_sim_ref(renamed.mop.mop.inner(), &mut |src_reg, _index| {
-                #[hdl(sim)]
-                let PRegNum::<_> {
-                    unit_num,
-                    unit_out_reg,
-                } = src_reg;
-                if Some(unit_index) == UnitNum::index_sim(&unit_num) {
-                    allocated_regs[UnitOutRegNum::value_sim(&unit_out_reg)] = true;
-                }
-            });
-        }
-        for entry in self
-            .rename_table
-            .entries()
+        let rob_l1_reg_ref_counts = &self.rob.l1_reg_ref_counts()[unit_index];
+        let rename_table_l1_ref_counts = &self.rename_table.l1_ref_counts()[unit_index];
+        let retire_rename_table_l1_ref_counts =
+            &self.retire_rename_table.l1_ref_counts()[unit_index];
+        rob_l1_reg_ref_counts
             .iter()
-            .chain(self.retire_rename_table.entries().iter())
-        {
-            #[hdl(sim)]
-            if let HdlSome(l1) = &entry.inner().l1 {
-                if Some(unit_index) == UnitNum::index_sim(&l1.unit_num) {
-                    allocated_regs[UnitOutRegNum::value_sim(&l1.unit_out_reg)] = true;
-                }
-            }
-        }
-        allocated_regs.iter().position(|v| !v)
+            .zip(rename_table_l1_ref_counts)
+            .zip(retire_rename_table_l1_ref_counts)
+            .position(|((a, b), c)| *a == 0 && *b == 0 && *c == 0)
     }
     #[hdl]
     fn find_free_l2_reg(&self) -> Option<usize> {

crates/cpu/src/rename_execute_retire/rename_table.rs

@@ -2,12 +2,11 @@
 // See Notices.txt for copyright information
 
 use crate::{
-    config::{CpuConfig, PhantomConstCpuConfig},
+    config::{CpuConfig, CpuConfig2PowOutRegNumWidth, CpuConfigUnitCount, PhantomConstCpuConfig},
     instruction::{L2RegNum, MOpRegNum, PRegNum, UnitNum, UnitOutRegNum},
     rename_execute_retire::L2RegFileLen,
 };
 use fayalite::{int::UIntInRangeInclusiveType, prelude::*};
-use std::collections::BTreeSet;
 
 #[hdl(no_static)]
 pub(crate) struct RenameTableEntry<C: PhantomConstGet<CpuConfig>> {
@@ -35,6 +34,13 @@ type MOpRegCount<C: PhantomConstGet<CpuConfig>> = DynSize;
 #[hdl(no_static)]
 pub(crate) struct RenameTableDebugState<C: PhantomConstGet<CpuConfig>> {
     entries: ArrayType<TraceAsString<RenameTableEntry<C>>, MOpRegCount<C>>,
+    l1_ref_counts: ArrayType<
+        ArrayType<
+            UIntInRangeInclusiveType<ConstUsize<0>, MOpRegCount<C>>,
+            CpuConfig2PowOutRegNumWidth<C>,
+        >,
+        CpuConfigUnitCount<C>,
+    >,
     l2_ref_counts:
         ArrayType<UIntInRangeInclusiveType<ConstUsize<0>, MOpRegCount<C>>, L2RegFileLen<C>>,
     config: C,
@@ -43,6 +49,7 @@ pub(crate) struct RenameTableDebugState<C: PhantomConstGet<CpuConfig>> {
 #[derive(Debug)]
 pub(crate) struct RenameTable<C: PhantomConstCpuConfig> {
     entries: Box<[SimValue<TraceAsString<RenameTableEntry<C>>>; 1 << MOpRegNum::WIDTH]>,
+    l1_ref_counts: Box<[Box<[usize]>]>,
     l2_ref_counts: Box<[usize]>,
     config: C,
 }
@@ -52,16 +59,19 @@ impl<C: PhantomConstCpuConfig> Clone for RenameTable<C> {
         Self {
             entries: self.entries.clone(),
             config: self.config.clone(),
+            l1_ref_counts: self.l1_ref_counts.clone(),
             l2_ref_counts: self.l2_ref_counts.clone(),
         }
     }
     fn clone_from(&mut self, source: &Self) {
         let Self {
             entries,
+            l1_ref_counts,
             l2_ref_counts,
             config,
         } = self;
         entries.clone_from(&source.entries);
+        l1_ref_counts.clone_from(&source.l1_ref_counts);
         l2_ref_counts.clone_from(&source.l2_ref_counts);
         *config = source.config;
     }
@@ -95,6 +105,11 @@ impl<C: PhantomConstCpuConfig> RenameTable<C> {
             .expect("size is known to match");
         Self {
             entries,
+            l1_ref_counts: vec![
+                vec![0; CpuConfig2PowOutRegNumWidth[config]].into_boxed_slice();
+                CpuConfigUnitCount[config]
+            ]
+            .into_boxed_slice(),
             l2_ref_counts: vec![0; L2RegFileLen[config]].into_boxed_slice(),
             config,
         }
@@ -108,6 +123,7 @@ impl<C: PhantomConstCpuConfig> RenameTable<C> {
     pub(crate) fn to_debug_state(&self) -> SimValue<RenameTableDebugState<C>> {
         let Self {
             entries,
+            l1_ref_counts,
             l2_ref_counts,
             config,
         } = self;
@@ -115,11 +131,32 @@ impl<C: PhantomConstCpuConfig> RenameTable<C> {
         #[hdl(sim)]
         RenameTableDebugState::<_> {
             entries: entries.to_sim_value_with_type(ty.entries),
+            l1_ref_counts: l1_ref_counts.to_sim_value_with_type(ty.l1_ref_counts),
             l2_ref_counts: l2_ref_counts.to_sim_value_with_type(ty.l2_ref_counts),
             config,
         }
     }
     #[hdl]
+    pub(crate) fn l1_ref_counts(&self) -> &[Box<[usize]>] {
+        let mut expected = vec![
+            vec![0usize; CpuConfig2PowOutRegNumWidth[self.config]]
+                .into_boxed_slice();
+            CpuConfigUnitCount[self.config]
+        ];
+        for entry in self.entries.iter() {
+            #[hdl(sim)]
+            let RenameTableEntry::<_> { l1, l2: _ } = entry.inner();
+            #[hdl(sim)]
+            if let HdlSome(l1) = l1 {
+                if let Some(unit_index) = UnitNum::index_sim(&l1.unit_num) {
+                    expected[unit_index][UnitOutRegNum::value_sim(&l1.unit_out_reg)] += 1;
+                }
+            }
+        }
+        assert_eq!(*expected, *self.l1_ref_counts);
+        &self.l1_ref_counts
+    }
+    #[hdl]
     pub(crate) fn l2_ref_counts(&self) -> &[usize] {
         let mut expected = vec![0usize; L2RegNum.l2_reg_count()];
         for entry in self.entries.iter() {
@@ -139,7 +176,17 @@ impl<C: PhantomConstCpuConfig> RenameTable<C> {
             |entry: &mut SimValue<TraceAsString<RenameTableEntry<C>>>,
              new: SimValue<TraceAsString<RenameTableEntry<C>>>| {
                 #[hdl(sim)]
-                let RenameTableEntry::<_> { l1: _, l2 } = entry.inner();
+                let RenameTableEntry::<_> { l1, l2 } = entry.inner();
+                #[hdl(sim)]
+                if let HdlSome(l1) = l1 {
+                    if let Some(unit_index) = UnitNum::index_sim(&l1.unit_num) {
+                        let ref_count = &mut self.l1_ref_counts[unit_index]
+                            [UnitOutRegNum::value_sim(&l1.unit_out_reg)];
+                        *ref_count = ref_count.checked_sub(1).unwrap_or_else(|| {
+                            unreachable!("{rename_table_name}: l1 ref count went negative: {l1:?}")
+                        });
+                    }
+                }
                 #[hdl(sim)]
                 if let HdlSome(l2) = l2 {
                     let ref_count = &mut self.l2_ref_counts[L2RegNum::value_sim(l2)];
@@ -148,15 +195,27 @@ impl<C: PhantomConstCpuConfig> RenameTable<C> {
                     });
                 }
                 #[hdl(sim)]
-                let RenameTableEntry::<_> { l1: _, l2 } = new.inner();
+                let RenameTableEntry::<_> { l1, l2 } = new.inner();
+                #[hdl(sim)]
+                if let HdlSome(l1) = l1 {
+                    if let Some(unit_index) = UnitNum::index_sim(&l1.unit_num) {
+                        let ref_count = &mut self.l1_ref_counts[unit_index]
+                            [UnitOutRegNum::value_sim(&l1.unit_out_reg)];
+                        *ref_count += 1;
+                        assert!(
+                            *ref_count <= 1 << MOpRegNum::WIDTH,
+                            "{rename_table_name}: l1 ref count overflowed: {l1:?}",
+                        );
+                    }
+                }
                 #[hdl(sim)]
                 if let HdlSome(l2) = l2 {
                     let ref_count = &mut self.l2_ref_counts[L2RegNum::value_sim(l2)];
                     *ref_count += 1;
                     assert!(
-                        *ref_count <= L2RegNum.l2_reg_count(),
+                        *ref_count <= 1 << MOpRegNum::WIDTH,
                         "{rename_table_name}: l2 ref count overflowed: {l2:?}",
-                    )
+                    );
                 }
                 *entry = new;
             };
@@ -244,15 +303,9 @@ impl<C: PhantomConstCpuConfig> RenameTable<C> {
     }
     #[hdl]
     pub(crate) fn used_unit_out_reg_count(&self, unit_index: usize) -> usize {
-        let mut seen = BTreeSet::new();
-        for entry in self.entries.iter() {
-            #[hdl(sim)]
-            if let HdlSome(v) = &entry.inner().l1 {
-                if UnitNum::index_sim(&v.unit_num) == Some(unit_index) {
-                    seen.insert(UnitOutRegNum::value_sim(&v.unit_out_reg));
-                }
-            }
-        }
-        seen.len()
+        self.l1_ref_counts()[unit_index]
+            .iter()
+            .filter(|ref_count| **ref_count != 0)
+            .count()
     }
 }

crates/cpu/src/rename_execute_retire/reorder_buffer.rs

@@ -2,8 +2,14 @@
 // See Notices.txt for copyright information
 
 use crate::{
-    config::{CpuConfig, CpuConfigRobSize, CpuConfigUnitCount, PhantomConstCpuConfig},
-    instruction::{L2RegNum, L2RegisterFileMOp, MOp, MOpTrait, PRegNum, UnitNum, UnitOutRegNum},
+    config::{
+        CpuConfig, CpuConfig2PowOutRegNumWidth, CpuConfigRobSize, CpuConfigUnitCount,
+        PhantomConstCpuConfig,
+    },
+    instruction::{
+        COMMON_MOP_SRC_LEN, L2RegNum, L2RegisterFileMOp, MOp, MOpTrait, PRegNum, UnitNum,
+        UnitOutRegNum,
+    },
     next_pc::SimValueDefault,
     rename_execute_retire::{
         L2RegFileLen, MOpId, MOpInUnitState, MOpInstance, NextPcPredictorOp, RenamedMOp,
@@ -215,12 +221,22 @@ impl<C: PhantomConstCpuConfig> RobEntries<C> {
     }
 }
 
+#[hdl(get(|c| c.rob_size.get() * (1 + COMMON_MOP_SRC_LEN)))]
+type L1RegMaxRefCount<C: PhantomConstGet<CpuConfig>> = DynSize;
+
 #[hdl(no_static)]
 pub(crate) struct ReorderBufferDebugState<C: PhantomConstGet<CpuConfig>> {
     next_renamed_mop_id: MOpId,
     entries: ArrayVec<RobEntriesDebugState, CpuConfigRobSize<C>>,
     incomplete_back_entry: HdlOption<RobEntriesDebugState>,
     renamed: ArrayVec<RobEntryDebugState<C>, CpuConfigRobSize<C>>,
+    l1_reg_ref_counts: ArrayType<
+        ArrayType<
+            UIntInRangeInclusiveType<ConstUsize<0>, L1RegMaxRefCount<C>>,
+            CpuConfig2PowOutRegNumWidth<C>,
+        >,
+        CpuConfigUnitCount<C>,
+    >,
     l2_reg_ref_counts:
         ArrayType<UIntInRangeInclusiveType<ConstUsize<0>, CpuConfigRobSize<C>>, L2RegFileLen<C>>,
     config: C,
@@ -234,6 +250,7 @@ impl<C: PhantomConstCpuConfig> SimValueDefault for ReorderBufferDebugState<C> {
             entries,
             incomplete_back_entry,
             renamed,
+            l1_reg_ref_counts,
             l2_reg_ref_counts,
             config,
         } = self;
@@ -243,6 +260,7 @@ impl<C: PhantomConstCpuConfig> SimValueDefault for ReorderBufferDebugState<C> {
             entries: entries.sim_value_default(),
             incomplete_back_entry: incomplete_back_entry.sim_value_default(),
             renamed: renamed.sim_value_default(),
+            l1_reg_ref_counts: zeroed(l1_reg_ref_counts),
             l2_reg_ref_counts: zeroed(l2_reg_ref_counts),
             config,
         }
@@ -254,6 +272,7 @@ pub(crate) struct ReorderBuffer<C: PhantomConstCpuConfig> {
     next_renamed_mop_id: SimValue<MOpId>,
     entries: VecDeque<RobEntries<C>>,
     incomplete_back_entry: Option<RobEntries<C>>,
+    l1_reg_ref_counts: Box<[Box<[usize]>]>,
     l2_reg_ref_counts: Box<[usize]>,
     config: C,
 }
@@ -264,23 +283,45 @@ impl<C: PhantomConstCpuConfig> ReorderBuffer<C> {
             next_renamed_mop_id: MOpId.zero().into_sim_value(),
             entries: VecDeque::new(),
             incomplete_back_entry: None,
+            l1_reg_ref_counts: vec![
+                vec![0; CpuConfig2PowOutRegNumWidth[config]].into_boxed_slice();
+                CpuConfigUnitCount[config]
+            ]
+            .into_boxed_slice(),
             l2_reg_ref_counts: vec![0; L2RegNum.l2_reg_count()].into_boxed_slice(),
             config,
         }
     }
     #[hdl]
+    pub(crate) fn l1_reg_ref_counts(&self) -> &[Box<[usize]>] {
+        let mut expected = vec![
+            vec![0usize; CpuConfig2PowOutRegNumWidth[self.config]]
+                .into_boxed_slice();
+            CpuConfigUnitCount[self.config]
+        ];
+        for entry in self.renamed() {
+            entry.for_each_reg(
+                (),
+                |(), l1| {
+                    if let Some(unit_index) = UnitNum::index_sim(&l1.unit_num) {
+                        expected[unit_index][UnitOutRegNum::value_sim(&l1.unit_out_reg)] += 1
+                    }
+                },
+                |(), _l2| {},
+            );
+        }
+        assert_eq!(*expected, *self.l1_reg_ref_counts);
+        &self.l1_reg_ref_counts
+    }
+    #[hdl]
     pub(crate) fn l2_reg_ref_counts(&self) -> &[usize] {
         let mut expected = vec![0usize; L2RegNum.l2_reg_count()];
         for entry in self.renamed() {
-            #[hdl(sim)]
-            if let RenamedMOp::<_>::TransformedMove(mop) = entry.mop.mop.inner() {
-                let l2_reg = #[hdl(sim)]
-                match mop {
-                    L2RegisterFileMOp::<_, _>::ReadL2Reg(v) => &v.common.imm,
-                    L2RegisterFileMOp::<_, _>::WriteL2Reg(v) => &v.common.imm,
-                };
-                expected[L2RegNum::value_sim(l2_reg)] += 1;
-            }
+            entry.for_each_reg(
+                (),
+                |(), _l1| {},
+                |(), l2| expected[L2RegNum::value_sim(l2)] += 1,
+            );
         }
         assert_eq!(*expected, *self.l2_reg_ref_counts);
         &self.l2_reg_ref_counts
@@ -291,6 +332,7 @@ impl<C: PhantomConstCpuConfig> ReorderBuffer<C> {
             next_renamed_mop_id,
             entries,
             incomplete_back_entry,
+            l1_reg_ref_counts,
             l2_reg_ref_counts,
             config,
         } = self;
@@ -314,6 +356,7 @@ impl<C: PhantomConstCpuConfig> ReorderBuffer<C> {
                 )
                 .ok()
                 .expect("known to fit"),
+            l1_reg_ref_counts: l1_reg_ref_counts.to_sim_value_with_type(ty.l1_reg_ref_counts),
             l2_reg_ref_counts: l2_reg_ref_counts.to_sim_value_with_type(ty.l2_reg_ref_counts),
             config,
         }
@@ -358,6 +401,7 @@ impl<C: PhantomConstCpuConfig> ReorderBuffer<C> {
             next_renamed_mop_id: _,
             entries,
             incomplete_back_entry,
+            l1_reg_ref_counts: _,
             l2_reg_ref_counts: _,
             config: _,
         } = self;
@@ -372,6 +416,7 @@ impl<C: PhantomConstCpuConfig> ReorderBuffer<C> {
             next_renamed_mop_id: _,
             entries,
             incomplete_back_entry,
+            l1_reg_ref_counts: _,
             l2_reg_ref_counts: _,
             config: _,
         } = self;
@@ -385,6 +430,7 @@ impl<C: PhantomConstCpuConfig> ReorderBuffer<C> {
             next_renamed_mop_id: _,
             entries,
             incomplete_back_entry,
+            l1_reg_ref_counts: _,
             l2_reg_ref_counts: _,
             config: _,
         } = self;
@@ -416,7 +462,12 @@ impl<C: PhantomConstCpuConfig> ReorderBuffer<C> {
         println!("renamed_push_back_with_new_id: {:?}", renamed.mop);
         renamed.for_each_reg(
             (),
-            |(), _l1| {},
+            |(), l1| {
+                if let Some(unit_index) = UnitNum::index_sim(&l1.unit_num) {
+                    self.l1_reg_ref_counts[unit_index]
+                        [UnitOutRegNum::value_sim(&l1.unit_out_reg)] += 1;
+                }
+            },
             |(), l2| self.l2_reg_ref_counts[L2RegNum::value_sim(l2)] += 1,
         );
         let renamed_entries = &mut self
@@ -449,11 +500,15 @@ impl<C: PhantomConstCpuConfig> ReorderBuffer<C> {
             next_renamed_mop_id: _,
             entries,
             incomplete_back_entry,
+            l1_reg_ref_counts,
             l2_reg_ref_counts,
             config: _,
         } = self;
         entries.clear();
         l2_reg_ref_counts.fill(0);
+        for i in l1_reg_ref_counts {
+            i.fill(0);
+        }
         *incomplete_back_entry = None;
     }
     pub(crate) fn unrenamed_back_append_rename_table_update(
@@ -487,7 +542,15 @@ impl<C: PhantomConstCpuConfig> ReorderBuffer<C> {
         for entry in &renamed_entries {
             entry.for_each_reg(
                 (),
-                |(), _l1| {},
+                |(), l1| {
+                    if let Some(unit_index) = UnitNum::index_sim(&l1.unit_num) {
+                        let ref_count = &mut self.l1_reg_ref_counts[unit_index]
+                            [UnitOutRegNum::value_sim(&l1.unit_out_reg)];
+                        *ref_count = ref_count.checked_sub(1).unwrap_or_else(|| {
+                            unreachable!("ReorderBuffer: l1 ref count went negative: {l1:?}")
+                        });
+                    }
+                },
                 |(), l2| {
                     let ref_count = &mut self.l2_reg_ref_counts[L2RegNum::value_sim(l2)];
                     *ref_count = ref_count.checked_sub(1).unwrap_or_else(|| {