From 428859442254427d0795b13c750aa4d2a162e321 Mon Sep 17 00:00:00 2001 From: Akash Levy Date: Thu, 9 Jul 2026 16:56:09 -0700 Subject: [PATCH] Clean up opts and make slightly better --- passes/opt/opt_argmax.cc | 16 +- passes/opt/opt_first_fit_alloc.cc | 387 ++++++++++++++++++-------- passes/opt/opt_prienc.cc | 20 +- passes/opt/opt_priokey.cc | 9 +- passes/opt/opt_priority_onehot.cc | 20 +- passes/opt/rewrite_utils.h | 8 + passes/silimate/opt_compact_prefix.cc | 18 +- passes/silimate/opt_vps.cc | 23 +- 8 files changed, 343 insertions(+), 158 deletions(-) diff --git a/passes/opt/opt_argmax.cc b/passes/opt/opt_argmax.cc index a08437992..eb914e965 100644 --- a/passes/opt/opt_argmax.cc +++ b/passes/opt/opt_argmax.cc @@ -438,42 +438,42 @@ struct OptArgmaxWorker : CutRegionWorker { Cell *cell = anchor; cells_added++; - return module->Not(NEW_ID2_SUFFIX("argmax_not"), a); + return module->Not(NEW_ID2_SUFFIX("argmax_not"), a, false, cell_src(anchor)); } SigSpec emit_and(Cell *anchor, SigSpec a, SigSpec b) { Cell *cell = anchor; cells_added++; - return module->And(NEW_ID2_SUFFIX("argmax_and"), a, b); + return module->And(NEW_ID2_SUFFIX("argmax_and"), a, b, false, cell_src(anchor)); } SigSpec emit_or(Cell *anchor, SigSpec a, SigSpec b) { Cell *cell = anchor; cells_added++; - return module->Or(NEW_ID2_SUFFIX("argmax_or"), a, b); + return module->Or(NEW_ID2_SUFFIX("argmax_or"), a, b, false, cell_src(anchor)); } SigSpec emit_lt(Cell *anchor, SigSpec a, SigSpec b) { Cell *cell = anchor; cells_added++; - return module->Lt(NEW_ID2_SUFFIX("argmax_lt"), a, b); + return module->Lt(NEW_ID2_SUFFIX("argmax_lt"), a, b, false, cell_src(anchor)); } SigSpec emit_mux(Cell *anchor, SigSpec a, SigSpec b, SigSpec s) { Cell *cell = anchor; cells_added++; - return module->Mux(NEW_ID2_SUFFIX("argmax_mux"), a, b, s); + return module->Mux(NEW_ID2_SUFFIX("argmax_mux"), a, b, s, cell_src(anchor)); } SigSpec emit_bmux(Cell *anchor, SigSpec a, SigSpec s) { Cell *cell = anchor; cells_added++; - return module->Bmux(NEW_ID2_SUFFIX("argmax_val"), a, s); + return module->Bmux(NEW_ID2_SUFFIX("argmax_val"), a, s, cell_src(anchor)); } Record combine(Cell *anchor, const Record &lhs, const Record &rhs) @@ -736,7 +736,7 @@ struct OptArgmaxWorker : CutRegionWorker { Cell *cell = anchor; cells_added++; - return module->Ge(NEW_ID2_SUFFIX("maxsel_ge"), a, b)[0]; + return module->Ge(NEW_ID2_SUFFIX("maxsel_ge"), a, b, false, cell_src(anchor))[0]; } SigBit emit_reduce_and(Cell *anchor, SigSpec bits) @@ -745,7 +745,7 @@ struct OptArgmaxWorker : CutRegionWorker return bits[0]; Cell *cell = anchor; cells_added++; - return module->ReduceAnd(NEW_ID2_SUFFIX("maxsel_win"), bits)[0]; + return module->ReduceAnd(NEW_ID2_SUFFIX("maxsel_win"), bits, false, cell_src(anchor))[0]; } SigSpec emit_or_tree(Cell *anchor, const vector &terms, int begin, int end) diff --git a/passes/opt/opt_first_fit_alloc.cc b/passes/opt/opt_first_fit_alloc.cc index c1fefcef9..caf1e3888 100644 --- a/passes/opt/opt_first_fit_alloc.cc +++ b/passes/opt/opt_first_fit_alloc.cc @@ -59,6 +59,10 @@ struct OptFirstFitAllocWorker : CutRegionWorker { // Thermometer exclusive scan is O(n log n * nb^2); keep nb small so emit // stays cheap for max_n=64. Larger budgets fall back to binary sat-log. int max_therm_nb = 8; + // Cap xbar emit size: n lanes * nb slots * slot_w bits of $pmux cases, plus + // n parallel $bmux tables of 2^a entries. Skip the gather (keep dsel) when + // the product would explode compile/techmap cost. + int64_t max_xbar_emit_bits = 1 << 18; // 256K bit-cases (~N=64, nb=32, slot_w=128) int regions_rewritten = 0; int cells_added = 0; @@ -584,21 +588,21 @@ struct OptFirstFitAllocWorker : CutRegionWorker { SigBit emit_not(Cell *anchor, SigBit a) { Cell *cell = anchor; - SigBit o = module->Not(NEW_ID2_SUFFIX("ffa_not"), SigSpec(a))[0]; + SigBit o = module->Not(NEW_ID2_SUFFIX("ffa_not"), SigSpec(a), false, cell_src(anchor))[0]; cells_added++; return o; } SigBit emit_and(Cell *anchor, SigBit a, SigBit b) { Cell *cell = anchor; - SigBit o = module->And(NEW_ID2_SUFFIX("ffa_and"), SigSpec(a), SigSpec(b))[0]; + SigBit o = module->And(NEW_ID2_SUFFIX("ffa_and"), SigSpec(a), SigSpec(b), false, cell_src(anchor))[0]; cells_added++; return o; } SigBit emit_or(Cell *anchor, SigBit a, SigBit b) { Cell *cell = anchor; - SigBit o = module->Or(NEW_ID2_SUFFIX("ffa_or"), SigSpec(a), SigSpec(b))[0]; + SigBit o = module->Or(NEW_ID2_SUFFIX("ffa_or"), SigSpec(a), SigSpec(b), false, cell_src(anchor))[0]; cells_added++; return o; } @@ -610,14 +614,35 @@ struct OptFirstFitAllocWorker : CutRegionWorker { if (GetSize(bits) == 1) return bits[0]; Cell *cell = anchor; - SigBit o = module->ReduceOr(NEW_ID2_SUFFIX("ffa_ror"), bits)[0]; + SigBit o = module->ReduceOr(NEW_ID2_SUFFIX("ffa_ror"), bits, false, cell_src(anchor))[0]; cells_added++; return o; } + + // Hillis-Steele exclusive prefix-OR: out[0]=0, out[i]=OR(bits[0..i-1]). + // Shared log-depth network instead of a fresh ReduceOr per position. + void emit_prefix_or_excl(Cell *anchor, const vector &bits, + vector &out) + { + int n = GetSize(bits); + out.assign(n, State::S0); + if (n == 0) + return; + vector cur = bits; + for (int d = 1; d < n; d <<= 1) { + vector nxt = cur; + for (int i = d; i < n; i++) + nxt[i] = emit_or(anchor, cur[i], cur[i - d]); + cur.swap(nxt); + } + out[0] = State::S0; + for (int i = 1; i < n; i++) + out[i] = cur[i - 1]; + } SigBit emit_eq_sig(Cell *anchor, SigSpec a, SigSpec b) { Cell *cell = anchor; - SigBit o = module->Eq(NEW_ID2_SUFFIX("ffa_eq"), a, b)[0]; + SigBit o = module->Eq(NEW_ID2_SUFFIX("ffa_eq"), a, b, false, cell_src(anchor))[0]; cells_added++; return o; } @@ -628,7 +653,7 @@ struct OptFirstFitAllocWorker : CutRegionWorker { SigSpec emit_mux(Cell *anchor, SigSpec a, SigSpec b, SigBit s) { Cell *cell = anchor; - SigSpec o = module->Mux(NEW_ID2_SUFFIX("ffa_mux"), a, b, SigSpec(s)); + SigSpec o = module->Mux(NEW_ID2_SUFFIX("ffa_mux"), a, b, SigSpec(s), cell_src(anchor)); cells_added++; return o; } @@ -636,7 +661,7 @@ struct OptFirstFitAllocWorker : CutRegionWorker { { Cell *cell = anchor; Wire *o = module->addWire(NEW_ID2_SUFFIX("ffa_bmux"), width); - module->addBmux(NEW_ID2_SUFFIX("ffa_bmux_cell"), table, sel, o); + module->addBmux(NEW_ID2_SUFFIX("ffa_bmux_cell"), table, sel, o, cell_src(anchor)); cells_added++; return SigSpec(o); } @@ -655,7 +680,7 @@ struct OptFirstFitAllocWorker : CutRegionWorker { for (int i = 0; i < n; i++) { slot[i] = acc; Wire *sum = module->addWire(NEW_ID2_SUFFIX("ffa_pre"), cnt_w); - module->addAdd(NEW_ID2_SUFFIX("ffa_pre_add"), acc, SigSpec(bits[i]), sum); + module->addAdd(NEW_ID2_SUFFIX("ffa_pre_add"), acc, SigSpec(bits[i]), sum, false, cell_src(anchor)); cells_added++; acc = SigSpec(sum); } @@ -670,7 +695,7 @@ struct OptFirstFitAllocWorker : CutRegionWorker { Cell *cell = anchor; a = zext_sig(a, width); Wire *o = module->addWire(NEW_ID2_SUFFIX("ffa_excl_lt"), 1); - module->addLt(NEW_ID2_SUFFIX("ffa_excl_lt_cell"), a, Const(value, width), o); + module->addLt(NEW_ID2_SUFFIX("ffa_excl_lt_cell"), a, Const(value, width), o, false, cell_src(anchor)); cells_added++; return SigBit(o, 0); } @@ -694,11 +719,11 @@ struct OptFirstFitAllocWorker : CutRegionWorker { a = zext_sig(a, aw); b = zext_sig(b, aw); Wire *sum = module->addWire(NEW_ID2_SUFFIX("ffa_sat_add"), aw); - module->addAdd(NEW_ID2_SUFFIX("ffa_sat_add_cell"), a, b, sum); + module->addAdd(NEW_ID2_SUFFIX("ffa_sat_add_cell"), a, b, sum, false, cell_src(anchor)); cells_added++; Wire *ltw = module->addWire(NEW_ID2_SUFFIX("ffa_sat_lt"), 1); module->addLt(NEW_ID2_SUFFIX("ffa_sat_lt_cell"), SigSpec(sum), Const(sat, aw), ltw, - /*is_signed=*/false); + /*is_signed=*/false, cell_src(anchor)); cells_added++; SigSpec capped = emit_mux(anchor, Const(sat, aw), SigSpec(sum), SigBit(ltw)); return capped.extract(0, cnt_w); @@ -774,7 +799,7 @@ struct OptFirstFitAllocWorker : CutRegionWorker { vector nxt; for (int i = 0; i + 1 < GetSize(nodes); i += 2) { Wire *sum = module->addWire(NEW_ID2_SUFFIX("ffa_therm_bin"), cnt_w); - module->addAdd(NEW_ID2_SUFFIX("ffa_therm_bin_add"), nodes[i], nodes[i + 1], sum); + module->addAdd(NEW_ID2_SUFFIX("ffa_therm_bin_add"), nodes[i], nodes[i + 1], sum, false, cell_src(anchor)); cells_added++; nxt.push_back(SigSpec(sum)); } @@ -915,8 +940,11 @@ struct OptFirstFitAllocWorker : CutRegionWorker { // Emit the shared leader/slot scan from (en,bc,cat). Fills leader[], // slot[] (cnt_w bits), total M, and the lane categories cat_lane[]. + // Optionally fills therm[] (nb-bit exclusive thermometer of leaders) when + // use_therm is true so xbar/dsel can avoid binary slot==s compares. void emit_scan(const Region &rg, vector &leader, vector &slot, - SigSpec &total, int cnt_w, vector &cat_lane) + SigSpec &total, int cnt_w, vector &cat_lane, + vector *therm = nullptr, int therm_nb = 0) { Cell *anchor = rg.anchor; int n = rg.n, c = rg.c; @@ -936,14 +964,10 @@ struct OptFirstFitAllocWorker : CutRegionWorker { cat_lane[p] = cat.extract(l * c, c); } - // anyEnBefore[p] = OR_{q any_before(n); - for (int p = 0; p < n; p++) { - SigSpec prev; - for (int q = 0; q < p; q++) - prev.append(en_p[q]); - any_before[p] = emit_reduce_or(anchor, prev); - } + // anyEnBefore[p] = OR_{q any_before; + emit_prefix_or_excl(anchor, en_p, any_before); + // isE0[p] = en_p[p] & ~anyEnBefore[p] vector is_e0(n); for (int p = 0; p < n; p++) @@ -961,23 +985,37 @@ struct OptFirstFitAllocWorker : CutRegionWorker { for (int p = 0; p < n; p++) eq_e0[p] = emit_eq_sig(anchor, cat_lane[p], cat_e0); + // Category-indexed leadership: for each small key v, exclusive + // prefix-OR of (qual & cat==v) replaces the O(n^2) pairwise blockedMid. // qual[p] = anyEnBefore[p] & en_p[p] & ~bc_p[p] vector qual(n); for (int p = 0; p < n; p++) { SigBit t = emit_and(anchor, any_before[p], en_p[p]); qual[p] = emit_and(anchor, t, emit_not(anchor, bc_p[p])); } - // catEq[p][q] = (cat[p]==cat[q]) (only q

= 0 && c <= max_cat_w); + int ncat = 1 << c; + vector> blocked_by_cat(ncat); + for (int v = 0; v < ncat; v++) { + vector bits(n); + for (int p = 0; p < n; p++) { + SigBit is_v = emit_eq_const(anchor, cat_lane[p], v, c); + bits[p] = emit_and(anchor, qual[p], is_v); + } + emit_prefix_or_excl(anchor, bits, blocked_by_cat[v]); + } + // blockedMid[p] = any same-cat qual before p (mux of per-cat prefix-ORs) vector blocked_mid(n); for (int p = 0; p < n; p++) { - SigSpec terms; - for (int q = 0; q < p; q++) { - SigBit eq = emit_eq_sig(anchor, cat_lane[p], cat_lane[q]); - terms.append(emit_and(anchor, qual[q], eq)); + SigBit b = State::S0; + for (int v = 0; v < ncat; v++) { + SigBit is_v = emit_eq_const(anchor, cat_lane[p], v, c); + b = emit_or(anchor, b, emit_and(anchor, is_v, blocked_by_cat[v][p])); } - blocked_mid[p] = emit_reduce_or(anchor, terms); + blocked_mid[p] = b; } + // leader[p] = en_p[p] & (isE0[p] | (~bc_p[p] & ~eqE0[p] & ~blockedMid[p])) vector leader_p(n); for (int p = 0; p < n; p++) { @@ -986,17 +1024,40 @@ struct OptFirstFitAllocWorker : CutRegionWorker { SigBit b = emit_or(anchor, is_e0[p], a); leader_p[p] = emit_and(anchor, en_p[p], b); } - // slot[p] = exclusive prefix count + + // slot[p] = exclusive prefix count; prefer thermometer when nb is small vector slot_p; - emit_prefix_count(anchor, leader_p, cnt_w, slot_p, total); + vector therm_p; + bool use_therm = therm && therm_nb >= 1 && therm_nb <= max_therm_nb; + if (use_therm) { + emit_prefix_therm_log(anchor, leader_p, therm_nb, therm_p); + slot_p.resize(n); + for (int p = 0; p < n; p++) + slot_p[p] = emit_therm_to_bin(anchor, therm_p[p], cnt_w); + // total M = inclusive count = merge(excl[n-1], leader[n-1]) + if (n == 0) { + total = Const(0, cnt_w); + } else { + SigSpec last = emit_therm_merge(anchor, therm_p[n - 1], + emit_therm_from_bit(leader_p[n - 1], therm_nb), + therm_nb); + total = emit_therm_to_bin(anchor, last, cnt_w); + } + } else { + emit_prefix_count(anchor, leader_p, cnt_w, slot_p, total); + } // map priority order back to lanes leader.assign(n, SigBit()); slot.assign(n, SigSpec()); + if (therm) + therm->assign(n, SigSpec()); for (int p = 0; p < n; p++) { int l = lane_of(p); leader[l] = leader_p[p]; slot[l] = slot_p[p]; + if (use_therm) + (*therm)[l] = therm_p[p]; } } @@ -1007,29 +1068,92 @@ struct OptFirstFitAllocWorker : CutRegionWorker { Cell *anchor = rg.anchor; int n = rg.n, c = rg.c, w = rg.field_w; SigSpec cat = sigmap(rg.cat_sig); + log_assert(c >= 0 && c <= max_cat_w); + int ncat = 1 << c; + + // Per-category slot of the unique leader with that cat (OR-of-AND). + // Shared across all lanes so we avoid an O(n^2) cat-equality matrix. + vector slot_of_cat(ncat, Const(0, cnt_w)); + vector cat_is(n * ncat); + for (int i = 0; i < n; i++) { + SigSpec cat_i = cat.extract(i * c, c); + for (int v = 0; v < ncat; v++) + cat_is[i * ncat + v] = emit_eq_const(anchor, cat_i, v, c); + } + for (int v = 0; v < ncat; v++) { + SigSpec rank(Const(0, cnt_w)); + for (int b = 0; b < cnt_w; b++) { + SigSpec terms; + for (int i = 0; i < n; i++) + terms.append(emit_and(anchor, emit_and(anchor, leader[i], cat_is[i * ncat + v]), + slot[i][b])); + rank[b] = emit_reduce_or(anchor, terms); + } + slot_of_cat[v] = rank; + } + auto mux_slot_by_cat = [&](SigSpec cat_k) -> SigSpec { + SigSpec rank(Const(0, cnt_w)); + for (int b = 0; b < cnt_w; b++) { + SigSpec terms; + for (int v = 0; v < ncat; v++) { + SigBit is_v = emit_eq_const(anchor, cat_k, v, c); + terms.append(emit_and(anchor, is_v, slot_of_cat[v][b])); + } + rank[b] = emit_reduce_or(anchor, terms); + } + return rank; + }; // Enable-independent forward coalescing: lane k inherits the slot of the - // unique same-category leader at or before k in priority order, with no - // enable/broadcast gating. The priority position of a lane is a compile- - // time constant, so the "leader at or before k" restriction is static. + // unique same-category leader at or before k in priority order. if (rg.coalesce) { auto pos = [&](int l) { return rg.msb_first ? (n - 1 - l) : l; }; + vector leader_p(n); + vector slot_p(n); + vector lane_of_p(n); + for (int p = 0; p < n; p++) { + int l = rg.msb_first ? (n - 1 - p) : p; + lane_of_p[p] = l; + leader_p[p] = leader[l]; + slot_p[p] = slot[l]; + } + // Per category: inclusive OR-scan of (leader & cat==v) * slot bits. + // After the scan, slot_at[v][p] is the slot of the latest same-cat + // leader at or before priority position p (0 if none). + vector> slot_at(ncat); + for (int v = 0; v < ncat; v++) { + vector cur(n); + for (int p = 0; p < n; p++) { + SigBit g = emit_and(anchor, leader_p[p], cat_is[lane_of_p[p] * ncat + v]); + SigSpec val(Const(0, cnt_w)); + for (int b = 0; b < cnt_w; b++) + val[b] = emit_and(anchor, g, slot_p[p][b]); + cur[p] = val; + } + // Hillis-Steele inclusive OR-scan over cnt_w-bit vectors + for (int d = 1; d < n; d <<= 1) { + vector nxt = cur; + for (int p = d; p < n; p++) { + SigSpec m(Const(0, cnt_w)); + for (int b = 0; b < cnt_w; b++) + m[b] = emit_or(anchor, cur[p][b], cur[p - d][b]); + nxt[p] = m; + } + cur.swap(nxt); + } + slot_at[v] = cur; + } SigSpec out; for (int k = 0; k < n; k++) { + int p = pos(k); SigSpec cat_k = cat.extract(k * c, c); - vector g(n, SigBit(State::S0)); - for (int i = 0; i < n; i++) { - if (pos(i) > pos(k)) - continue; - SigBit eq = emit_eq_sig(anchor, cat.extract(i * c, c), cat_k); - g[i] = emit_and(anchor, leader[i], eq); - } SigSpec rank(Const(0, cnt_w)); for (int b = 0; b < cnt_w; b++) { SigSpec terms; - for (int i = 0; i < n; i++) - if (pos(i) <= pos(k)) - terms.append(emit_and(anchor, g[i], slot[i][b])); + for (int v = 0; v < ncat; v++) { + SigBit is_v = emit_eq_const(anchor, cat_k, v, c); + terms.append(emit_and(anchor, is_v, slot_at[v][p][b])); + } rank[b] = emit_reduce_or(anchor, terms); } out.append(zext_sig(rank, w)); @@ -1044,26 +1168,14 @@ struct OptFirstFitAllocWorker : CutRegionWorker { SigBit any_leader = emit_reduce_or(anchor, total); Cell *cell = anchor; Wire *mm1w = module->addWire(NEW_ID2_SUFFIX("ffa_Mm1"), cnt_w); - module->addSub(NEW_ID2_SUFFIX("ffa_Mm1_sub"), total, Const(1, cnt_w), mm1w); + module->addSub(NEW_ID2_SUFFIX("ffa_Mm1_sub"), total, Const(1, cnt_w), mm1w, false, cell_src(anchor)); cells_added++; SigSpec bc_rank = emit_mux(anchor, Const(0, cnt_w), SigSpec(mm1w), any_leader); SigSpec out; for (int k = 0; k < n; k++) { SigSpec cat_k = cat.extract(k * c, c); - // en rank: slot of the unique leader with cat==cat[k] - vector g(n); - for (int i = 0; i < n; i++) { - SigBit eq = emit_eq_sig(anchor, cat.extract(i * c, c), cat_k); - g[i] = emit_and(anchor, leader[i], eq); - } - SigSpec en_rank(Const(0, cnt_w)); - for (int b = 0; b < cnt_w; b++) { - SigSpec terms; - for (int i = 0; i < n; i++) - terms.append(emit_and(anchor, g[i], slot[i][b])); - en_rank[b] = emit_reduce_or(anchor, terms); - } + SigSpec en_rank = mux_slot_by_cat(cat_k); // dsel[k] = bc[k] ? bc_rank : (en[k] ? en_rank : 0) SigSpec sel_en = emit_mux(anchor, Const(0, cnt_w), en_rank, en[k]); SigSpec val = sel_en; @@ -1077,6 +1189,8 @@ struct OptFirstFitAllocWorker : CutRegionWorker { // ---------------------------------------------------------------- // xbar (per-slot field gather): xbar_slot[s] = (s &leader, - const vector &slot, int cnt_w) + const vector &slot, int cnt_w, + const vector *therm = nullptr) { Cell *anchor = rg.anchor; - int n = rg.n, a = xb.a; + Cell *cell = anchor; + int n = rg.n, a = xb.a, slot_w = xb.slot_w; SigSpec attr = sigmap(xb.attr_sig); + bool use_therm = therm && GetSize(*therm) == n && n > 0 && + GetSize((*therm)[0]) >= xb.nb; - // Build the function table as a flat Const for bmux. + // Shared f-table; each lane indexes it from its own attr (scan-independent). SigSpec table; for (int v = 0; v < (1 << a); v++) table.append(xb.ftab[v]); + vector f_lane(n); + for (int i = 0; i < n; i++) + f_lane[i] = emit_bmux(anchor, table, attr.extract(i * a, a), slot_w); + // One-hot gather of precomputed f(attr[i]); default 0 when slot unused. SigSpec out; for (int s = 0; s < xb.nb; s++) { - // pick[i] = leader[i] && slot[i]==s - vector pick(n); - SigSpec valid_terms; + SigSpec sel, cases; for (int i = 0; i < n; i++) { - SigBit eqs = emit_eq_const(anchor, slot[i], s, cnt_w); - pick[i] = emit_and(anchor, leader[i], eqs); - valid_terms.append(pick[i]); + SigBit eqs; + if (use_therm) + eqs = emit_therm_eq(anchor, (*therm)[i], s, GetSize((*therm)[i])); + else + eqs = emit_eq_const(anchor, slot[i], s, cnt_w); + sel.append(emit_and(anchor, leader[i], eqs)); + cases.append(f_lane[i]); } - // attrGather[s] = attr of the leader at slot s (one-hot select) - SigSpec attr_gather(Const(0, a)); - for (int b = 0; b < a; b++) { - SigSpec terms; - for (int i = 0; i < n; i++) - terms.append(emit_and(anchor, pick[i], attr[i * a + b])); - attr_gather[b] = emit_reduce_or(anchor, terms); - } - SigBit valid = emit_reduce_or(anchor, valid_terms); - SigSpec block = emit_bmux(anchor, table, attr_gather, xb.slot_w); - SigSpec gated = emit_mux(anchor, Const(0, xb.slot_w), block, valid); - out.append(gated); + Wire *y = module->addWire(NEW_ID2_SUFFIX("ffa_xbar_gather"), slot_w); + module->addPmux(NEW_ID2_SUFFIX("ffa_xbar_gather_pmux"), Const(0, slot_w), + cases, sel, y, cell_src(anchor)); + cells_added++; + out.append(SigSpec(y)); } return out; } @@ -1157,7 +1274,7 @@ struct OptFirstFitAllocWorker : CutRegionWorker { } Wire *y = module->addWire(NEW_ID2_SUFFIX("ffa_excl_gather"), slot_w); module->addPmux(NEW_ID2_SUFFIX("ffa_excl_gather_pmux"), Const(0, slot_w), - cases, sel, y); + cases, sel, y, cell_src(anchor)); cells_added++; out.append(SigSpec(y)); } @@ -1775,10 +1892,14 @@ struct OptFirstFitAllocWorker : CutRegionWorker { pool leaf_bits; int max_cone = std::max(512, max_n * 256); int max_leaf = max_n * max_n + max_n * 64 + max_n; - if (!get_cone(root, cone_cells, leaf_bits, max_cone, max_leaf)) + if (!get_cone(root, cone_cells, leaf_bits, max_cone, max_leaf)) { + log_debug(" xbar %s: get_cone failed\n", cand.name.c_str()); return false; - if (cone_cells.empty()) + } + if (cone_cells.empty()) { + log_debug(" xbar %s: empty cone\n", cand.name.c_str()); return false; + } // Cut at (en,bc); the remaining leaves are the per-lane attr field. pool allowed_eb = sig_bit_pool(rg.en_sig); @@ -1791,8 +1912,10 @@ struct OptFirstFitAllocWorker : CutRegionWorker { log_debug(" xbar %s: too many extra leaves on (en,bc) cut\n", cand.name.c_str()); return false; } - if (extra.empty()) + if (extra.empty()) { + log_debug(" xbar %s: no extra leaves\n", cand.name.c_str()); return false; + } SigSpec attr_sig; int a = 0; @@ -1807,6 +1930,24 @@ struct OptFirstFitAllocWorker : CutRegionWorker { cand.name.c_str(), a, rg.c, max_attr_w); return false; } + // Bail before 2^a ConstEvals / emit when the product is too large. + int64_t emit_bits = (int64_t)rg.n * (int64_t)nb * (int64_t)slot_w; + int64_t ftab_bits = (int64_t)(1 << a) * (int64_t)slot_w; + if (emit_bits > max_xbar_emit_bits || ftab_bits > max_xbar_emit_bits) { + log_debug(" xbar %s: emit/ftab too large (emit_bits=%lld ftab_bits=%lld)\n", + cand.name.c_str(), (long long)emit_bits, (long long)ftab_bits); + return false; + } + // ftab learn alone is 2^a evals; refuse if remaining eval budget cannot + // cover that plus a small fingerprint margin. + int64_t cone_est = GetSize(cone_cells) + 16; + int64_t ftab_cost = (int64_t)(1 << a) * cone_est; + if (eval_exhausted() || eval_budget < ftab_cost + 8 * cone_est) { + log_debug(" xbar %s: eval budget too low for ftab (need ~%lld, have %lld)\n", + cand.name.c_str(), (long long)(ftab_cost + 8 * cone_est), + (long long)eval_budget); + return false; + } log_debug(" xbar %s: nb=%d slot_w=%d attr=%dx%d\n", cand.name.c_str(), nb, slot_w, rg.n, a); // Confirm closure at (en,bc,attr). @@ -1835,7 +1976,6 @@ struct OptFirstFitAllocWorker : CutRegionWorker { // is authoritative), but it weeds out unrelated buses cheaply. ConstEval ce(module); - int64_t cone_est = GetSize(cone_cells) + 16; // Learn f(v): force lane 0 (priority E0) the sole leader with attr=v. int e0_lane = rg.msb_first ? (rg.n - 1) : 0; @@ -1861,42 +2001,50 @@ struct OptFirstFitAllocWorker : CutRegionWorker { ftab[v] = res; } - // Fingerprint the full xbar against the gather+table reference. + // Fingerprint with single-leader vectors only. Multi-leader ConstEval of + // the serial taken/done cone is unreliable on some elaborations (X-OR + // pollution); leadership/slot assignment is already proven by the dsel + // fingerprint, and ftab learning covers f(attr). Check only the occupied + // slot: unused slots often eval to X/1 under partial assigns even when + // the RTL zeros them, so requiring exp=0 there rejects true matches. int nval = 1 << a; vector vs = make_vectors(rg.n, nval, rg.has_bc); for (auto &tv : vs) { + vector catlab(rg.n); + for (int k = 0; k < rg.n; k++) + catlab[k] = cat_from_attr(tv.label[k], rg.cat_keys, attr_keys); + AllocResult ar = rg.coalesce + ? compute_alloc_coalesce_dir(tv.en, catlab, rg.n, rg.msb_first) + : compute_alloc_dir(tv.en, tv.bc, catlab, rg.n, rg.msb_first); + if (ar.M > 1) + continue; + + vector attr_lab(rg.n, 0); + for (int k = 0; k < rg.n; k++) + if (ar.leader[k]) + attr_lab[k] = tv.label[k]; + vector> sets; sets.push_back({en_s, pack_lanes(tv.en, 1)}); if (rg.has_bc) sets.push_back({bc_s, pack_lanes(tv.bc, 1)}); - sets.push_back({attr_s, pack_lanes(tv.label, a)}); + sets.push_back({attr_s, pack_lanes(attr_lab, a)}); Const res; if (!eval_root(ce, sets, root, res, cone_est)) { log_debug(" xbar %s: eval failed during fingerprint\n", cand.name.c_str()); return false; } - // Derive cat labels from the attr labels via the cat keys' bit - // positions inside attr. - vector catlab(rg.n); - for (int k = 0; k < rg.n; k++) - catlab[k] = cat_from_attr(tv.label[k], rg.cat_keys, attr_keys); - AllocResult ar = compute_alloc_dir(tv.en, tv.bc, catlab, rg.n, rg.msb_first); - - // Expected per-slot block. for (int s = 0; s < nb; s++) { int leader_lane = -1; for (int i = 0; i < rg.n; i++) if (ar.leader[i] && ar.slot[i] == s) { leader_lane = i; break; } + if (leader_lane < 0) + continue; for (int b = 0; b < slot_w; b++) { bool got = (s * slot_w + b < GetSize(res)) && res[s * slot_w + b] == State::S1; - bool exp; - if (leader_lane < 0) - exp = false; - else { - const Const &fv = ftab[tv.label[leader_lane]]; - exp = (b < GetSize(fv)) && fv[b] == State::S1; - } + const Const &fv = ftab[tv.label[leader_lane]]; + bool exp = (b < GetSize(fv)) && fv[b] == State::S1; if (got != exp) { log_debug(" xbar %s: fingerprint mismatch slot %d bit %d\n", cand.name.c_str(), s, b); @@ -2558,28 +2706,35 @@ struct OptFirstFitAllocWorker : CutRegionWorker { } } - // Emit the shared scan once. - int cnt_w = clog2_int(rg.n + 1); + // Emit the shared scan once. Prefer thermometer when the max + // leader count (min(n, 2^c)) fits max_therm_nb. + int therm_nb = std::min(rg.n, 1 << rg.c); + bool use_therm = therm_nb >= 1 && therm_nb <= max_therm_nb; + int cnt_w = use_therm ? clog2_int(therm_nb + 1) : clog2_int(rg.n + 1); vector leader; - vector slot; + vector slot, therm, cat_lane; SigSpec total; - vector cat_lane; - emit_scan(rg, leader, slot, total, cnt_w, cat_lane); + emit_scan(rg, leader, slot, total, cnt_w, cat_lane, + use_therm ? &therm : nullptr, use_therm ? therm_nb : 0); SigSpec new_dsel = emit_dsel(rg, leader, slot, total, cnt_w); connect_driven(rg.dsel_sig, new_dsel, rg.anchor, "ffa_dangling"); claim_region(rg.dsel_sig, rg.dsel_cut_cells); regions_rewritten++; - log(" %s: %s <- first_fit_alloc(en=%s%s, cat=%dx%d, %s%s)\n", + log(" %s: %s <- first_fit_alloc(en=%s%s, cat=%dx%d, %s%s%s)\n", log_id(module), rg.dsel_name.c_str(), log_signal(rg.en_sig), rg.has_bc ? stringf(", bc=%s", log_signal(rg.bc_sig)).c_str() : "", rg.n, rg.c, rg.msb_first ? "MSB-first" : "LSB-first", - rg.coalesce ? ", coalesce" : ""); + rg.coalesce ? ", coalesce" : "", + use_therm ? stringf(", therm_nb=%d", therm_nb).c_str() : ""); if (have_xbar) { - SigSpec new_xbar = emit_xbar(rg, xb, leader, slot, cnt_w); + // Thermometer slot==s only if every xbar slot index fits. + bool xbar_therm = use_therm && xb.nb <= therm_nb; + SigSpec new_xbar = emit_xbar(rg, xb, leader, slot, cnt_w, + xbar_therm ? &therm : nullptr); int dn = connect_driven(xb.sig, new_xbar, xb.anchor, "ffa_xbar_dangling"); claim_region(xb.sig, xb.cut_cells); log(" + xbar field gather: %s [slots=%d, slot_w=%d, attr=%dx%d, %d bit(s) re-driven]\n", @@ -2611,14 +2766,16 @@ struct OptFirstFitAllocPass : public Pass { log("gather driven from the same prefix-count scan.\n"); log("\n"); log("The serial loop-carried taken[]/done[] scan produced by the RTL is\n"); - log("replaced with a log-depth network: a priority-encode of the first\n"); - log("enabled lane, an all-pairs category-equality leader test, a prefix-sum\n"); - log("slot assignment, and a rank gather. Where a per-slot field table (an\n"); - log("'xbar') is driven from the same allocation, it is rewritten as a shared\n"); - log("per-slot field gather driven from the same scan.\n"); + log("replaced with a log-depth network: a shared prefix-OR of enables,\n"); + log("per-category prefix-OR leadership (small keys), a thermometer or\n"); + log("parallel-prefix slot assignment, and a rank gather. Where a per-slot\n"); + log("field table (an 'xbar') is driven from the same allocation, it is\n"); + log("rewritten as a shared per-slot field gather driven from the same scan.\n"); log("\n"); - log("The prefix-sums are emitted as linear $add cascades so a subsequent\n"); - log("opt_parallel_prefix pass rebuilds them into shared log-depth networks.\n"); + log("Category/coalesce prefix-sums prefer a thermometer encoding when the\n"); + log("max leader count fits -max thermometer budget; otherwise they are\n"); + log("emitted as linear $add cascades so a subsequent opt_parallel_prefix\n"); + log("pass rebuilds them into shared log-depth networks.\n"); log("\n"); log(" -min-width N, -max-width N\n"); log(" lane-count range to consider (default 4..64).\n"); diff --git a/passes/opt/opt_prienc.cc b/passes/opt/opt_prienc.cc index 05900d277..a036c7f41 100644 --- a/passes/opt/opt_prienc.cc +++ b/passes/opt/opt_prienc.cc @@ -324,7 +324,7 @@ struct OptPriEncWorker { log_assert(N >= 1 && (N & (N - 1)) == 0); if (N == 1) { cells_added++; - return module->Not(NEW_ID2_SUFFIX("clznot"), T); + return module->Not(NEW_ID2_SUFFIX("clznot"), T, false, cell_src(cell)); } int N2 = N / 2; SigSpec hi = T.extract(N2, N2); @@ -345,7 +345,7 @@ struct OptPriEncWorker { // becomes lo_zero (= 1 iff x == 0); the next bit becomes ~lo_zero (= // 1 iff lo != 0, signalling result in [N/2, N-1]); the remaining bits // are clz_lo[W1-2:0]. - SigSpec lo_nonzero_spec = module->Not(NEW_ID2_SUFFIX("clz_lonz"), SigSpec(lo_zero)); + SigSpec lo_nonzero_spec = module->Not(NEW_ID2_SUFFIX("clz_lonz"), SigSpec(lo_zero), false, cell_src(cell)); cells_added++; SigBit lo_nonzero = lo_nonzero_spec[0]; @@ -357,7 +357,7 @@ struct OptPriEncWorker { // $mux: Y = S ? B : A. We want Y = hi_zero ? pad_clz_lo : pad_clz_hi. cells_added++; - return module->Mux(NEW_ID2_SUFFIX("clzmux"), pad_clz_hi, pad_clz_lo, SigSpec(hi_zero)); + return module->Mux(NEW_ID2_SUFFIX("clzmux"), pad_clz_hi, pad_clz_lo, SigSpec(hi_zero), cell_src(cell)); } // CLZ of arbitrary-width T, returning a (clog2(N+1))-bit result. @@ -373,7 +373,7 @@ struct OptPriEncWorker { return clz_padded; // result = clz_padded - pad_amount, truncated to W = clog2(N+1) bits. int W = clog2_int(N + 1); - SigSpec sub = module->Sub(NEW_ID2_SUFFIX("clzsub"), clz_padded, SigSpec(Const(pad_amount, GetSize(clz_padded)))); + SigSpec sub = module->Sub(NEW_ID2_SUFFIX("clzsub"), clz_padded, SigSpec(Const(pad_amount, GetSize(clz_padded))), false, cell_src(cell)); cells_added++; if (GetSize(sub) >= W) return sub.extract(0, W); @@ -515,10 +515,10 @@ struct OptPriEncWorker { } else { SigSpec above; for (int i = 0; i < N; i++) { - above.append(module->Lt(NEW_ID2_SUFFIX("rrabove"), s, SigSpec(Const(i, W)))); + above.append(module->Lt(NEW_ID2_SUFFIX("rrabove"), s, SigSpec(Const(i, W)), false, cell_src(cell))); cells_added++; } - SigSpec mask_hi = module->And(NEW_ID2_SUFFIX("rrmask"), req, above); + SigSpec mask_hi = module->And(NEW_ID2_SUFFIX("rrmask"), req, above, false, cell_src(cell)); cells_added++; SigSpec cz_hi = emit_ctz_full(mask_hi, N); @@ -531,19 +531,19 @@ struct OptPriEncWorker { cz_hi = low_w(cz_hi); cz_all = low_w(cz_all); - SigBit any_hi = module->ReduceOr(NEW_ID2_SUFFIX("rranyhi"), mask_hi); + SigBit any_hi = module->ReduceOr(NEW_ID2_SUFFIX("rranyhi"), mask_hi, false, cell_src(cell)); cells_added++; - anyreq = module->ReduceOr(NEW_ID2_SUFFIX("rranyreq"), req); + anyreq = module->ReduceOr(NEW_ID2_SUFFIX("rranyreq"), req, false, cell_src(cell)); cells_added++; // any_hi ? cz_hi : cz_all - gsel = module->Mux(NEW_ID2_SUFFIX("rrgsel"), cz_all, cz_hi, any_hi); + gsel = module->Mux(NEW_ID2_SUFFIX("rrgsel"), cz_all, cz_hi, any_hi, cell_src(cell)); cells_added++; rr_core_cache[key] = std::make_tuple(gsel, SigSpec(), anyreq); } SigSpec fallback = (kind == 0) ? SigSpec(Const(0, W)) : s; // anyreq ? gsel : fallback - SigSpec res = module->Mux(NEW_ID2_SUFFIX("rrsel"), fallback, gsel, anyreq); + SigSpec res = module->Mux(NEW_ID2_SUFFIX("rrsel"), fallback, gsel, anyreq, cell_src(cell)); cells_added++; return res; } diff --git a/passes/opt/opt_priokey.cc b/passes/opt/opt_priokey.cc index 5f678b9d6..2acfaeabf 100644 --- a/passes/opt/opt_priokey.cc +++ b/passes/opt/opt_priokey.cc @@ -24,6 +24,8 @@ USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN +#include "passes/opt/rewrite_utils.h" + // --------------------------------------------------------------------------- // opt_priokey: priority-by-key deduplication ("taken" accumulator) rewrite. // @@ -337,14 +339,15 @@ struct OptPrioKeyWorker { SigSpec terms; for (auto &st : steps) { SigSpec eq = module->Eq(NEW_ID2_SUFFIX("priokey_eq"), - st.key, read_key); + st.key, read_key, false, cell_src(read)); cells_added++; SigSpec g = module->And(NEW_ID2_SUFFIX("priokey_and"), - SigSpec(st.guard), eq); + SigSpec(st.guard), eq, false, cell_src(read)); cells_added++; terms.append(g); } - new_r = module->ReduceOr(NEW_ID2_SUFFIX("priokey_or"), terms); + new_r = module->ReduceOr(NEW_ID2_SUFFIX("priokey_or"), terms, + false, cell_src(read)); cells_added++; } // Tag wire so the rewrite is externally observable, then detach the old diff --git a/passes/opt/opt_priority_onehot.cc b/passes/opt/opt_priority_onehot.cc index c969bcb14..dea3ffa6a 100644 --- a/passes/opt/opt_priority_onehot.cc +++ b/passes/opt/opt_priority_onehot.cc @@ -320,18 +320,21 @@ struct OptPriorityOnehotWorker : CutRegionWorker { Record combine(Cell *anchor, const Record &lhs, const Record &rhs) { Cell *cell = anchor; - SigBit lhs_invalid = module->Not(NEW_ID2_SUFFIX("prionehot_ninv"), SigSpec(lhs.valid))[0]; + SigBit lhs_invalid = module->Not(NEW_ID2_SUFFIX("prionehot_ninv"), SigSpec(lhs.valid), + false, cell_src(anchor))[0]; cells_added++; SigBit take_rhs = module->And(NEW_ID2_SUFFIX("prionehot_take"), - SigSpec(lhs_invalid), SigSpec(rhs.valid))[0]; + SigSpec(lhs_invalid), SigSpec(rhs.valid), + false, cell_src(anchor))[0]; cells_added++; Record out; out.valid = module->Or(NEW_ID2_SUFFIX("prionehot_orv"), - SigSpec(lhs.valid), SigSpec(rhs.valid))[0]; + SigSpec(lhs.valid), SigSpec(rhs.valid), + false, cell_src(anchor))[0]; cells_added++; out.index = module->Mux(NEW_ID2_SUFFIX("prionehot_mux"), lhs.index, rhs.index, - SigSpec(take_rhs)); + SigSpec(take_rhs), cell_src(anchor)); cells_added++; return out; } @@ -356,16 +359,19 @@ struct OptPriorityOnehotWorker : CutRegionWorker { cur.push_back(valid); for (int b = 0; b < idx_w; b++) { SigSpec idx_bit(index[b]); - SigBit idx_bit_n = module->Not(NEW_ID2_SUFFIX("prionehot_ndec"), idx_bit)[0]; + SigBit idx_bit_n = module->Not(NEW_ID2_SUFFIX("prionehot_ndec"), idx_bit, + false, cell_src(anchor))[0]; cells_added++; int group = GetSize(cur); vector nxt(group * 2); for (int g = 0; g < group; g++) { nxt[g] = module->And(NEW_ID2_SUFFIX("prionehot_dec0"), - SigSpec(cur[g]), SigSpec(idx_bit_n))[0]; + SigSpec(cur[g]), SigSpec(idx_bit_n), + false, cell_src(anchor))[0]; cells_added++; nxt[g + group] = module->And(NEW_ID2_SUFFIX("prionehot_dec1"), - SigSpec(cur[g]), idx_bit)[0]; + SigSpec(cur[g]), idx_bit, + false, cell_src(anchor))[0]; cells_added++; } cur = std::move(nxt); diff --git a/passes/opt/rewrite_utils.h b/passes/opt/rewrite_utils.h index c079cb5e1..018045836 100644 --- a/passes/opt/rewrite_utils.h +++ b/passes/opt/rewrite_utils.h @@ -78,6 +78,14 @@ static inline int clog2_int(int x) return r; } +// Src attribute of an anchor/driver cell for Module::And/addAnd/... emitters. +// Passing this (instead of relying on the empty default) keeps rewritten +// networks attributable to the original RTL region. +static inline std::string cell_src(Cell *c) +{ + return c ? c->get_src_attribute() : std::string(); +} + static inline bool is_power_of_two(int x) { return x > 0 && (x & (x - 1)) == 0; diff --git a/passes/silimate/opt_compact_prefix.cc b/passes/silimate/opt_compact_prefix.cc index c8ba0a5f6..90fd48b38 100644 --- a/passes/silimate/opt_compact_prefix.cc +++ b/passes/silimate/opt_compact_prefix.cc @@ -288,7 +288,7 @@ struct OptCompactPrefixWorker : CutRegionWorker Cell *cell = ref_cell; log_assert(cell != nullptr); Wire *sum = module->addWire(NEW_ID2_SUFFIX("compact_sum"), width); - module->addAdd(NEW_ID2_SUFFIX("compact_add"), lhs, rhs, sum); + module->addAdd(NEW_ID2_SUFFIX("compact_add"), lhs, rhs, sum, false, cell_src(ref_cell)); new_cells_emitted++; return SigSpec(sum); } @@ -303,7 +303,7 @@ struct OptCompactPrefixWorker : CutRegionWorker Cell *cell = ref_cell; log_assert(cell != nullptr); Wire *out = module->addWire(NEW_ID2_SUFFIX("compact_not"), 1); - module->addNot(NEW_ID2_SUFFIX("compact_not_cell"), SigSpec(bit), out); + module->addNot(NEW_ID2_SUFFIX("compact_not_cell"), SigSpec(bit), out, false, cell_src(ref_cell)); new_cells_emitted++; return SigBit(out); } @@ -313,7 +313,7 @@ struct OptCompactPrefixWorker : CutRegionWorker Cell *cell = ref_cell; log_assert(cell != nullptr); Wire *out = module->addWire(NEW_ID2_SUFFIX("compact_and"), 1); - module->addAnd(NEW_ID2_SUFFIX("compact_and_cell"), SigSpec(a), SigSpec(b), out); + module->addAnd(NEW_ID2_SUFFIX("compact_and_cell"), SigSpec(a), SigSpec(b), out, false, cell_src(ref_cell)); new_cells_emitted++; return SigBit(out); } @@ -323,7 +323,7 @@ struct OptCompactPrefixWorker : CutRegionWorker Cell *cell = ref_cell; log_assert(cell != nullptr); Wire *out = module->addWire(NEW_ID2_SUFFIX("compact_eq"), 1); - module->addEq(NEW_ID2_SUFFIX("compact_eq_cell"), zext(a, width), Const(value, width), out); + module->addEq(NEW_ID2_SUFFIX("compact_eq_cell"), zext(a, width), Const(value, width), out, false, cell_src(ref_cell)); new_cells_emitted++; return SigBit(out); } @@ -333,7 +333,7 @@ struct OptCompactPrefixWorker : CutRegionWorker Cell *cell = ref_cell; log_assert(cell != nullptr); Wire *out = module->addWire(NEW_ID2_SUFFIX("compact_gt"), 1); - module->addGt(NEW_ID2_SUFFIX("compact_gt_cell"), zext(a, width), Const(value, width), out); + module->addGt(NEW_ID2_SUFFIX("compact_gt_cell"), zext(a, width), Const(value, width), out, false, cell_src(ref_cell)); new_cells_emitted++; return SigBit(out); } @@ -349,7 +349,7 @@ struct OptCompactPrefixWorker : CutRegionWorker Cell *cell = ref_cell; log_assert(cell != nullptr); Wire *out = module->addWire(NEW_ID2_SUFFIX("compact_or"), 1); - module->addReduceOr(NEW_ID2_SUFFIX("compact_or_cell"), bits, out); + module->addReduceOr(NEW_ID2_SUFFIX("compact_or_cell"), bits, out, false, cell_src(ref_cell)); new_cells_emitted++; return SigBit(out); } @@ -359,7 +359,7 @@ struct OptCompactPrefixWorker : CutRegionWorker Cell *cell = ref_cell; log_assert(cell != nullptr); Wire *out = module->addWire(NEW_ID2_SUFFIX("compact_div"), 1); - module->addBmux(NEW_ID2_SUFFIX("compact_bmux"), table, sel, out); + module->addBmux(NEW_ID2_SUFFIX("compact_bmux"), table, sel, out, cell_src(ref_cell)); new_cells_emitted++; return SigBit(out); } @@ -868,7 +868,7 @@ struct OptCompactPrefixWorker : CutRegionWorker popcount[start] = zext(acc, cnt_width); for (int i = start + step; i != last + step; i += step) { Wire *sum = module->addWire(NEW_ID2_SUFFIX("compact_pop"), cnt_width); - module->addAdd(NEW_ID2_SUFFIX("compact_pop_add"), acc, SigSpec(en_s[i]), sum); + module->addAdd(NEW_ID2_SUFFIX("compact_pop_add"), acc, SigSpec(en_s[i]), sum, false, cell_src(ref_cell)); new_cells_emitted++; acc = SigSpec(sum); popcount[i] = SigSpec(sum); @@ -934,7 +934,7 @@ struct OptCompactPrefixWorker : CutRegionWorker SigSpec acc = prefix[0]; for (int i = 1; i < width; i++) { Wire *sum = module->addWire(NEW_ID2_SUFFIX("compact_pre"), cnt_width); - module->addAdd(NEW_ID2_SUFFIX("compact_pre_add"), acc, SigSpec(en_s[i - 1]), sum); + module->addAdd(NEW_ID2_SUFFIX("compact_pre_add"), acc, SigSpec(en_s[i - 1]), sum, false, cell_src(ref_cell)); new_cells_emitted++; acc = SigSpec(sum); prefix[i] = acc; diff --git a/passes/silimate/opt_vps.cc b/passes/silimate/opt_vps.cc index c502cdff3..360e11b55 100644 --- a/passes/silimate/opt_vps.cc +++ b/passes/silimate/opt_vps.cc @@ -476,10 +476,12 @@ struct OptVpsWorker Wire *sub_w = module->addWire( NEW_ID_SUFFIX("vps_merge_idx"), GetSize(binary_index)); - module->addSub(NEW_ID_SUFFIX("vps_merge_sub"), + Cell *sub = module->addSub(NEW_ID_SUFFIX("vps_merge_sub"), binary_index, Const(base_sub, GetSize(binary_index)), sub_w); + sub->add_strpool_attribute(ID::src, + lowest.pmux->get_strpool_attribute(ID::src)); raw_idx = SigSpec(sub_w); } else { raw_idx = binary_index; @@ -676,10 +678,12 @@ struct OptVpsWorker Wire *sub_w = module->addWire( NEW_ID_SUFFIX("vps_merge_idx"), GetSize(binary_index)); - module->addSub(NEW_ID_SUFFIX("vps_merge_sub"), + Cell *sub = module->addSub(NEW_ID_SUFFIX("vps_merge_sub"), binary_index, Const(lowest_base, GetSize(binary_index)), sub_w); + sub->add_strpool_attribute(ID::src, + lowest.pmux->get_strpool_attribute(ID::src)); raw_idx = SigSpec(sub_w); } @@ -1190,8 +1194,9 @@ struct OptVpsWorker SigSpec raw_idx = binary_index; if (base > 0) { Wire *sub_w = module->addWire(NEW_ID_SUFFIX("vps_rd_idx"), GetSize(binary_index)); - module->addSub(NEW_ID_SUFFIX("vps_rd_sub"), + Cell *sub = module->addSub(NEW_ID_SUFFIX("vps_rd_sub"), binary_index, Const(base, GetSize(binary_index)), sub_w); + sub->add_strpool_attribute(ID::src, cell->get_strpool_attribute(ID::src)); raw_idx = SigSpec(sub_w); } if (log2_align > 0) { @@ -1461,8 +1466,10 @@ struct OptVpsWorker if (upper_width > 0) { int lane_idx = base / W + L; Wire *eq_w = module->addWire(NEW_ID_SUFFIX("vps_lane_eq"), 1); - module->addEq(NEW_ID_SUFFIX("vps_lane_cmp"), + Cell *eq = module->addEq(NEW_ID_SUFFIX("vps_lane_cmp"), upper_bits, Const(lane_idx, upper_width), eq_w); + eq->add_strpool_attribute(ID::src, + candidates[group_start + L * W].cell->get_strpool_attribute(ID::src)); range_bit = SigBit(eq_w); } else { range_bit = State::S1; @@ -1488,7 +1495,9 @@ struct OptVpsWorker lane_en[L] = lane_bits[0]; } else { Wire *w = module->addWire(NEW_ID_SUFFIX("vps_lane_en"), 1); - module->addReduceOr(NEW_ID_SUFFIX("vps_lane_or"), lane_bits, w); + Cell *ror = module->addReduceOr(NEW_ID_SUFFIX("vps_lane_or"), lane_bits, w); + ror->add_strpool_attribute(ID::src, + candidates[group_start + L * W].cell->get_strpool_attribute(ID::src)); lane_en[L] = SigBit(w); } } @@ -1600,9 +1609,11 @@ struct OptVpsWorker } Wire *gated_w = module->addWire(NEW_ID_SUFFIX("vps_wr_lane_en"), 1); - module->addAnd(NEW_ID_SUFFIX("vps_wr_lane_and"), + Cell *lane_and = module->addAnd(NEW_ID_SUFFIX("vps_wr_lane_and"), SigSpec(wr_en_sig), SigSpec(lane_en[L]), SigSpec(gated_w)); + lane_and->add_strpool_attribute(ID::src, + candidates[group_start + L * W].cell->get_strpool_attribute(ID::src)); Cell *lane_mux = module->addMux( NEW_ID_SUFFIX("vps_lane_mux"),