From 00e48706df95ae8afe423e3b7d1edbee2740835a Mon Sep 17 00:00:00 2001 From: Akash Levy Date: Mon, 6 Jul 2026 12:56:38 -0700 Subject: [PATCH 1/2] opt: recognize three QoR logic-depth patterns Extend two existing opt passes and add one new pass to collapse serial/dynamic-index structures that were leaving high logic depth: - opt_first_fit_alloc: recognize the "coalesce-matrix" first-fit allocator variant (same_cat[i][k] coalescing gated on the leader's enable, driven from a raw input enable). Rewrite both the lane_slot allocation and the xbar field gather from one shared log-depth scan. - opt_prienc: detect round-robin / rotated-priority scans (req scanned from idx_last downward with wraparound) and rewrite the depth-N idx--/req[idx] mux chain to rotate -> log-depth priority-encode -> unrotate. - opt_priokey (new): recognize priority-by-key one-hot accumulators and replace each dynamic taken[key] read ($shiftx/$bmux) with the equivalent pairwise-key-compare reduction, dropping the wide dynamic indexing. Supports -strict for full-key-range formal validation. Each includes self-contained tests (equiv_opt / sat -prove-asserts, mux-bound and negative cases) in tests/opt/. Co-authored-by: Cursor --- passes/opt/CMakeLists.txt | 4 + passes/opt/opt_first_fit_alloc.cc | 130 ++++++++- passes/opt/opt_prienc.cc | 264 ++++++++++++++++- passes/opt/opt_priokey.cc | 464 ++++++++++++++++++++++++++++++ tests/opt/opt_first_fit_alloc.ys | 167 +++++++++++ tests/opt/opt_prienc.ys | 126 ++++++++ tests/opt/opt_priokey.ys | 240 ++++++++++++++++ 7 files changed, 1381 insertions(+), 14 deletions(-) create mode 100644 passes/opt/opt_priokey.cc create mode 100644 tests/opt/opt_priokey.ys diff --git a/passes/opt/CMakeLists.txt b/passes/opt/CMakeLists.txt index 8e7d866bd..a651c400d 100644 --- a/passes/opt/CMakeLists.txt +++ b/passes/opt/CMakeLists.txt @@ -108,6 +108,10 @@ yosys_pass(opt_first_fit_alloc opt_first_fit_alloc.cc ) +yosys_pass(opt_priokey + opt_priokey.cc +) + pmgen_command(peepopt peepopt_shiftmul_right.pmg peepopt_shiftmul_left.pmg diff --git a/passes/opt/opt_first_fit_alloc.cc b/passes/opt/opt_first_fit_alloc.cc index a2e2af74b..f8a51d766 100644 --- a/passes/opt/opt_first_fit_alloc.cc +++ b/passes/opt/opt_first_fit_alloc.cc @@ -165,6 +165,56 @@ struct OptFirstFitAllocWorker : CutRegionWorker { return r; } + // ---------------------------------------------------------------- + // Reference semantics of the "coalesce matrix" allocator variant. + // + // Leadership and slot assignment are identical to the greedy first-fit + // above, but the per-lane rank does NOT depend on the lane's own enable: + // every lane k (enabled or not) inherits the slot of the first leader at + // or before k (in priority order) whose category matches cat[k]. This + // models RTL that precomputes a per-leader "same_cat[i][k]" mask (gated + // only on the leader's enable) and forward-coalesces into lane k without + // re-checking en[k]. There is no broadcast lane in this variant. + // ---------------------------------------------------------------- + AllocResult compute_alloc_coalesce(const vector &en, const vector &cat, + int n) const + { + AllocResult r = compute_alloc(en, vector(n, 0), cat, n); + for (int k = 0; k < n; k++) { + r.dsel[k] = 0; + for (int i = 0; i <= k; i++) + if (r.leader[i] && cat[i] == cat[k]) { + r.dsel[k] = r.slot[i]; + break; + } + } + return r; + } + + AllocResult compute_alloc_coalesce_dir(const vector &en, const vector &cat, + int n, bool msb_first) const + { + if (!msb_first) + return compute_alloc_coalesce(en, cat, n); + vector er(n), cr(n); + for (int i = 0; i < n; i++) { + er[i] = en[n - 1 - i]; + cr[i] = cat[n - 1 - i]; + } + AllocResult rr = compute_alloc_coalesce(er, cr, n); + AllocResult r; + r.dsel.assign(n, 0); + r.leader.assign(n, 0); + r.slot.assign(n, 0); + r.M = rr.M; + for (int i = 0; i < n; i++) { + r.dsel[i] = rr.dsel[n - 1 - i]; + r.leader[i] = rr.leader[n - 1 - i]; + r.slot[i] = rr.slot[n - 1 - i]; + } + return r; + } + // ---------------------------------------------------------------- // Test vectors. `nval` is the number of distinct label values (2^c for // the category, 2^a for the xbar attribute). The vectors deliberately @@ -412,6 +462,10 @@ struct OptFirstFitAllocWorker : CutRegionWorker { int field_w = 0; SigSpec en_sig, bc_sig, cat_sig; bool has_bc = false; + // Enable-independent forward coalescing: lanes inherit the slot of the + // first same-category leader at or before them in priority order, + // regardless of their own enable (the "same_cat matrix" RTL shape). + bool coalesce = false; int c = 0; bool msb_first = false; Cell *anchor = nullptr; @@ -594,9 +648,39 @@ struct OptFirstFitAllocWorker : CutRegionWorker { { Cell *anchor = rg.anchor; int n = rg.n, c = rg.c, w = rg.field_w; + SigSpec cat = sigmap(rg.cat_sig); + + // Enable-independent forward coalescing: lane k inherits the slot of the + // unique same-category leader at or before k in priority order, with no + // enable/broadcast gating. The priority position of a lane is a compile- + // time constant, so the "leader at or before k" restriction is static. + if (rg.coalesce) { + auto pos = [&](int l) { return rg.msb_first ? (n - 1 - l) : l; }; + SigSpec out; + for (int k = 0; k < n; k++) { + SigSpec cat_k = cat.extract(k * c, c); + vector g(n, SigBit(State::S0)); + for (int i = 0; i < n; i++) { + if (pos(i) > pos(k)) + continue; + SigBit eq = emit_eq_sig(anchor, cat.extract(i * c, c), cat_k); + g[i] = emit_and(anchor, leader[i], eq); + } + SigSpec rank(Const(0, cnt_w)); + for (int b = 0; b < cnt_w; b++) { + SigSpec terms; + for (int i = 0; i < n; i++) + if (pos(i) <= pos(k)) + terms.append(emit_and(anchor, g[i], slot[i][b])); + rank[b] = emit_reduce_or(anchor, terms); + } + out.append(zext_sig(rank, w)); + } + return out; + } + SigSpec en = sigmap(rg.en_sig); SigSpec bc = rg.has_bc ? sigmap(rg.bc_sig) : SigSpec(); - SigSpec cat = sigmap(rg.cat_sig); // bc rank: (M>=1) ? M-1 : 0 SigBit any_leader = emit_reduce_or(anchor, total); @@ -707,15 +791,22 @@ struct OptFirstFitAllocWorker : CutRegionWorker { internal_bits.insert(bit); pool seen; - auto all_internal = [&](const SigSpec &s) { + // Accept a bus bit if it is a cone-internal (computed) signal or a + // primary input / undriven bit. The enable/broadcast lanes are usually + // computed signals (e.g. valid & format), but some RTL drives the scan + // straight from a top-level request port (e.g. lane_en), so input buses + // must be admissible too. Inputs sort shallowest (depth 0) below, so they + // survive the candidate cap ahead of the deep intermediate nets. + auto all_internal_or_input = [&](const SigSpec &s) { for (auto bit : s) - if (!bit.wire || !internal_bits.count(bit)) + if (!bit.wire || (!internal_bits.count(bit) && + bit_to_driver.at(bit, nullptr) != nullptr)) return false; return true; }; auto add = [&](const SigSpec &sig, const std::string &nm) { SigSpec s = sigmap(sig); - if (GetSize(s) != n || !sig_bus_ok(s) || !all_internal(s)) + if (GetSize(s) != n || !sig_bus_ok(s) || !all_internal_or_input(s)) return; if (!seen.insert(s).second) return; @@ -868,17 +959,28 @@ struct OptFirstFitAllocWorker : CutRegionWorker { bool fpm = fingerprint_dsel(ce, root_sig, n, field_w, en_bus.sig, bc_bus ? bc_bus->sig : SigSpec(), bc_bus != nullptr, cat_sig, c, msb_first, cone_est); - log_debug(" en=%s bc=%s cat=%dx%d %s: fingerprint %s\n", en_bus.name.c_str(), + // Standard first-fit failed: try the enable-independent + // forward-coalescing variant (no broadcast lane). + bool coalesce = false; + if (!fpm && bc_bus == nullptr) { + fpm = fingerprint_dsel(ce, root_sig, n, field_w, en_bus.sig, + SigSpec(), false, cat_sig, c, msb_first, + cone_est, /*coalesce=*/true); + coalesce = fpm; + } + log_debug(" en=%s bc=%s cat=%dx%d %s: fingerprint %s%s\n", en_bus.name.c_str(), bc_bus ? bc_bus->name.c_str() : "-", n, c, - msb_first ? "MSB" : "LSB", fpm ? "MATCH" : "no"); + msb_first ? "MSB" : "LSB", fpm ? "MATCH" : "no", + coalesce ? " (coalesce)" : ""); if (fpm) { out.dsel_sig = root_sig; out.dsel_name = root_name; out.n = n; out.field_w = field_w; out.en_sig = en_bus.sig; - out.bc_sig = bc_bus ? bc_bus->sig : SigSpec(); - out.has_bc = (bc_bus != nullptr); + out.bc_sig = (!coalesce && bc_bus) ? bc_bus->sig : SigSpec(); + out.has_bc = (!coalesce && bc_bus != nullptr); + out.coalesce = coalesce; out.cat_sig = cat_sig; out.c = c; out.msb_first = msb_first; @@ -901,7 +1003,8 @@ struct OptFirstFitAllocWorker : CutRegionWorker { // Any eval failure or single mismatch rejects the candidate. bool fingerprint_dsel(ConstEval &ce, const SigSpec &root, int n, int field_w, const SigSpec &en_sig, const SigSpec &bc_sig, bool has_bc, - const SigSpec &cat_sig, int c, bool msb_first, int64_t cone_est) + const SigSpec &cat_sig, int c, bool msb_first, int64_t cone_est, + bool coalesce = false) { int nval = 1 << c; vector vs = make_vectors(n, nval, has_bc); @@ -920,7 +1023,9 @@ struct OptFirstFitAllocWorker : CutRegionWorker { if (!eval_root(ce, sets, root, res, cone_est)) return false; - AllocResult ar = compute_alloc_dir(tv.en, tv.bc, tv.label, n, msb_first); + AllocResult ar = coalesce + ? compute_alloc_coalesce_dir(tv.en, tv.label, n, msb_first) + : compute_alloc_dir(tv.en, tv.bc, tv.label, n, msb_first); for (int k = 0; k < n; k++) { int got = lane_val(res, k, field_w); int exp = ar.dsel[k] & ((1 << field_w) - 1); @@ -1239,11 +1344,12 @@ struct OptFirstFitAllocWorker : CutRegionWorker { claim_region(rg.dsel_sig, rg.dsel_cut_cells); regions_rewritten++; - log(" %s: %s <- first_fit_alloc(en=%s%s, cat=%dx%d, %s)\n", + log(" %s: %s <- first_fit_alloc(en=%s%s, cat=%dx%d, %s%s)\n", log_id(module), rg.dsel_name.c_str(), log_signal(rg.en_sig), rg.has_bc ? stringf(", bc=%s", log_signal(rg.bc_sig)).c_str() : "", - rg.n, rg.c, rg.msb_first ? "MSB-first" : "LSB-first"); + rg.n, rg.c, rg.msb_first ? "MSB-first" : "LSB-first", + rg.coalesce ? ", coalesce" : ""); if (have_xbar) { SigSpec new_xbar = emit_xbar(rg, xb, leader, slot, cnt_w); diff --git a/passes/opt/opt_prienc.cc b/passes/opt/opt_prienc.cc index acb7b78f4..e7c4a966c 100644 --- a/passes/opt/opt_prienc.cc +++ b/passes/opt/opt_prienc.cc @@ -82,6 +82,7 @@ struct OptPriEncWorker { // Configuration. bool detect_clz = true; bool detect_ctz = true; + bool detect_rr = true; int max_input_width = 256; int min_input_width = 4; @@ -390,6 +391,171 @@ struct OptPriEncWorker { return full; } + // ------------------------------------------------------------------ + // Round-robin (rotated priority) detection + rewrite. + // + // A round-robin arbiter grants the first set request bit scanning + // *upward* (increasing index, wrapping) starting just after a stored + // pointer `s` (= idx_last): + // + // grant = anyreq ? (first set bit at index > s, else first set + // bit overall) : 0 + // idx_next = anyreq ? grant : s + // + // RTL usually spells this as a DEPTH-iteration loop that walks `idx` + // downward from idx_last with wraparound and keeps the last hit, which + // elaborates into a serial mux/shift chain of depth ~DEPTH. The rewrite + // below is log-depth: + // + // above[i] = (i > s) (per-bit threshold mask) + // mask_hi = req & above + // grant = anyreq ? (|mask_hi ? ctz(mask_hi) : ctz(req)) : 0 + // + // where ctz() reuses the log-depth CTZ network. For power-of-2 DEPTH the + // rewrite is fully combinationally equivalent for every pointer value; + // for non-power-of-2 DEPTH it is equivalent for every *reachable* pointer + // (idx_last only ever holds a valid index in [0,DEPTH)), which is the + // range the fingerprint checks. Detection therefore sweeps s over + // [0,DEPTH) only. + // ------------------------------------------------------------------ + + // kind: 0 = grant, 1 = idx_next. + int rr_expected(const Const& reqv, int s, int N, int W, int kind) { + auto bits = reqv.to_bits(); + int lo_all = -1, lo_hi = -1; + for (int i = 0; i < N; i++) { + bool set = (i < (int)bits.size() && bits[i] == State::S1); + if (!set) continue; + if (lo_all < 0) lo_all = i; + if (i > s && lo_hi < 0) lo_hi = i; + } + bool anyreq = (lo_all >= 0); + int gsel = (lo_hi >= 0) ? lo_hi : (lo_all >= 0 ? lo_all : 0); + int val = kind == 0 ? (anyreq ? gsel : 0) + : (anyreq ? gsel : s); + return val & ((W >= 31) ? -1 : ((1 << W) - 1)); + } + + // Returns matched kind (0 grant, 1 idx_next), or -1 for no match. + int fingerprint_rr(SigSpec req_sig, SigSpec start_sig, SigSpec S_sig, + int N, int W) { + ConstEval ce(module); + bool ok0 = true, ok1 = true; + auto deck = gen_test_vectors(N); + int checks = 0; + for (auto& rv : deck) { + for (int s = 0; s < N; s++) { + ce.push(); + ce.set(req_sig, rv); + ce.set(start_sig, Const(s, W)); + SigSpec out = S_sig, undef; + bool ok = ce.eval(out, undef); + ce.pop(); + if (!ok || !out.is_fully_const()) return -1; + int ov = out.as_const().as_int(); + if (ok0 && ov != rr_expected(rv, s, N, W, 0)) ok0 = false; + if (ok1 && ov != rr_expected(rv, s, N, W, 1)) ok1 = false; + checks++; + if (!ok0 && !ok1) return -1; + } + } + if (checks < 2 * N) return -1; + if (ok0) return 0; + if (ok1) return 1; + return -1; + } + + // Emit the log-depth round-robin network. Shared subexpressions across + // the grant / idx_next pair for the same (req, start) inputs are cached. + dict, std::tuple> rr_core_cache; + + SigSpec emit_rr(Wire* req_wire, Wire* start_wire, int N, int W, int kind) { + SigSpec req = sigmap(SigSpec(req_wire)); + SigSpec s = sigmap(SigSpec(start_wire)); + + SigSpec gsel; + SigBit anyreq; + auto key = std::make_pair(req_wire, start_wire); + auto it = rr_core_cache.find(key); + if (it != rr_core_cache.end()) { + SigSpec cached_gsel; + std::tie(cached_gsel, std::ignore, anyreq) = it->second; + gsel = cached_gsel; + } else { + SigSpec above; + for (int i = 0; i < N; i++) { + above.append(module->Lt(NEW_ID2_SUFFIX("rrabove"), s, SigSpec(Const(i, W)))); + cells_added++; + } + SigSpec mask_hi = module->And(NEW_ID2_SUFFIX("rrmask"), req, above); + cells_added++; + + SigSpec cz_hi = emit_ctz_full(mask_hi, N); + SigSpec cz_all = emit_ctz_full(req, N); + auto low_w = [&](SigSpec x) { + if (GetSize(x) > W) return x.extract(0, W); + while (GetSize(x) < W) x.append(SigSpec(State::S0)); + return x; + }; + cz_hi = low_w(cz_hi); + cz_all = low_w(cz_all); + + SigBit any_hi = module->ReduceOr(NEW_ID2_SUFFIX("rranyhi"), mask_hi); + cells_added++; + anyreq = module->ReduceOr(NEW_ID2_SUFFIX("rranyreq"), req); + cells_added++; + // any_hi ? cz_hi : cz_all + gsel = module->Mux(NEW_ID2_SUFFIX("rrgsel"), cz_all, cz_hi, any_hi); + cells_added++; + rr_core_cache[key] = std::make_tuple(gsel, SigSpec(), anyreq); + } + + SigSpec fallback = (kind == 0) ? SigSpec(Const(0, W)) : s; + // anyreq ? gsel : fallback + SigSpec res = module->Mux(NEW_ID2_SUFFIX("rrsel"), fallback, gsel, anyreq); + cells_added++; + return res; + } + + // Generalisation of cone_depends_only_on_T to a set of allowed leaf bits. + bool cone_depends_only_on_set(SigSpec S_sig, const pool& allowed) { + pool visited; + std::queue worklist; + for (auto bit : sigmap(S_sig)) { + if (!bit.wire) continue; + if (visited.insert(bit).second) worklist.push(bit); + } + while (!worklist.empty()) { + SigBit bit = worklist.front(); + worklist.pop(); + if (allowed.count(bit)) continue; + if (input_port_bits.count(bit)) return false; + auto it = bit_to_driver.find(bit); + if (it == bit_to_driver.end()) return false; + Cell* drv = it->second; + if (sequential_cells.count(drv)) return false; + for (auto& conn : drv->connections()) { + if (!drv->input(conn.first)) continue; + for (auto in_bit : sigmap(conn.second)) { + if (!in_bit.wire) continue; + if (visited.insert(in_bit).second) worklist.push(in_bit); + } + } + } + return true; + } + + struct RRRewrite { + Wire* S_wire; + Wire* req_wire; + Wire* start_wire; + int N; + int W; + int kind; + Cell* sole_driver; + IdString out_port; + }; + struct Rewrite { Wire* S_wire; Wire* T_wire; @@ -584,6 +750,74 @@ struct OptPriEncWorker { } } + // Stage 3: round-robin (rotated priority) detection. Reuses the same + // candidate cones; an output S is grant/idx_next of a round-robin + // arbiter over a wide request bus `req` and a same-width-as-S pointer + // `start`, both bottoming out the cone. + vector rr_rewrites; + if (detect_rr) { + const int max_pairs = 64; + for (auto& cand : candidates) { + if (claimed_outputs.count(cand.S_wire)) continue; + if (claimed_drivers.count(cand.sole_driver)) continue; + + int W = cand.S_wire->width; + if (W < 2 || W > max_W) continue; + SigSpec S_sig = sigmap(SigSpec(cand.S_wire)); + + vector req_cands, start_cands; + for (Wire* w : wires_snapshot) { + if (w == cand.S_wire) continue; + bool all_in = true; + for (auto bit : sigmap(SigSpec(w))) + if (!cand.cone_bits.count(bit)) { all_in = false; break; } + if (!all_in) continue; + int wn = w->width; + if (wn >= min_input_width && wn <= max_input_width && + clog2_int(wn) == W) + req_cands.push_back(w); + if (wn == W) + start_cands.push_back(w); + } + std::sort(req_cands.begin(), req_cands.end(), + [](Wire* a, Wire* b) { return a->width > b->width; }); + + int pairs = 0; + bool matched = false; + for (Wire* req_wire : req_cands) { + if (matched) break; + int N = req_wire->width; + SigSpec req_sig = sigmap(SigSpec(req_wire)); + pool req_bits; + for (auto bit : req_sig) + if (bit.wire) req_bits.insert(bit); + for (Wire* start_wire : start_cands) { + if (start_wire == req_wire) continue; + if (++pairs > max_pairs) break; + SigSpec start_sig = sigmap(SigSpec(start_wire)); + pool allowed = req_bits; + for (auto bit : start_sig) + if (bit.wire) allowed.insert(bit); + if (!cone_depends_only_on_set(S_sig, allowed)) continue; + + int kind = fingerprint_rr(req_sig, start_sig, S_sig, N, W); + if (kind < 0) continue; + + log(" %s: %s <- round_robin_%s(req=%s, start=%s) [N=%d, W=%d]\n", + log_id(module), log_id(cand.S_wire), + kind == 0 ? "grant" : "next", + log_id(req_wire), log_id(start_wire), N, W); + rr_rewrites.push_back({cand.S_wire, req_wire, start_wire, N, W, + kind, cand.sole_driver, cand.out_port}); + claimed_outputs.insert(cand.S_wire); + claimed_drivers.insert(cand.sole_driver); + matched = true; + break; + } + } + } + } + // Apply rewrites. We collected first to avoid the index growing stale // while we add new cells/wires. for (auto& r : rewrites) { @@ -595,6 +829,14 @@ struct OptPriEncWorker { module->connect(SigSpec(r.S_wire), new_S); regions_rewritten++; } + for (auto& r : rr_rewrites) { + cell = r.sole_driver; + SigSpec new_S = emit_rr(r.req_wire, r.start_wire, r.N, r.W, r.kind); + Wire* dangling = module->addWire(NEW_ID2_SUFFIX("dangling"), r.W); + r.sole_driver->setPort(r.out_port, dangling); + module->connect(SigSpec(r.S_wire), new_S); + regions_rewritten++; + } } }; @@ -624,11 +866,23 @@ struct OptPriEncPass : public Pass { log(" ctz_full : symmetric to clz_full from the LSB side.\n"); log(" ctz_short : symmetric to clz_short from the LSB side.\n"); log("\n"); + log("In addition, the pass detects round-robin (rotated priority)\n"); + log("arbiters: grant / idx_next = first set request bit scanning upward\n"); + log("(wrapping) from just after a stored pointer idx_last. RTL typically\n"); + log("spells this as a DEPTH-iteration idx-- loop over req[idx], which\n"); + log("elaborates into a serial chain; it is replaced with a log-depth\n"); + log("threshold-mask + CTZ network. For power-of-2 DEPTH the rewrite is\n"); + log("equivalent for every pointer value; for other widths it is\n"); + log("equivalent for every reachable pointer (idx_last in [0,DEPTH)).\n"); + log("\n"); log(" -clz\n"); - log(" detect CLZ patterns only.\n"); + log(" detect CLZ patterns only (also disables round-robin).\n"); log("\n"); log(" -ctz\n"); - log(" detect CTZ patterns only.\n"); + log(" detect CTZ patterns only (also disables round-robin).\n"); + log("\n"); + log(" -no-rr\n"); + log(" disable round-robin / rotated-priority detection.\n"); log("\n"); log(" -max-width N\n"); log(" maximum input bus width to consider (default 64).\n"); @@ -648,6 +902,7 @@ struct OptPriEncPass : public Pass { bool only_clz = false; bool only_ctz = false; + bool no_rr = false; int max_width = 64; int min_width = 4; @@ -655,6 +910,7 @@ struct OptPriEncPass : public Pass { for (argidx = 1; argidx < args.size(); argidx++) { if (args[argidx] == "-clz") { only_clz = true; continue; } if (args[argidx] == "-ctz") { only_ctz = true; continue; } + if (args[argidx] == "-no-rr") { no_rr = true; continue; } if (args[argidx] == "-max-width" && argidx + 1 < args.size()) { max_width = std::stoi(args[++argidx]); continue; } @@ -664,6 +920,9 @@ struct OptPriEncPass : public Pass { break; } extra_args(args, argidx, design); + // -clz / -ctz select a single leading/trailing variant and disable + // round-robin detection unless the user re-enables it explicitly. + if (only_clz || only_ctz) no_rr = true; int total_regions = 0; int total_cells_added = 0; @@ -671,6 +930,7 @@ struct OptPriEncPass : public Pass { OptPriEncWorker worker(module); worker.detect_clz = !only_ctz; worker.detect_ctz = !only_clz; + worker.detect_rr = !no_rr; worker.max_input_width = max_width; worker.min_input_width = min_width; worker.run(); diff --git a/passes/opt/opt_priokey.cc b/passes/opt/opt_priokey.cc new file mode 100644 index 000000000..f505c1e7e --- /dev/null +++ b/passes/opt/opt_priokey.cc @@ -0,0 +1,464 @@ +/* + * yosys -- Yosys Open SYnthesis Suite + * + * Copyright (C) 2026 Akash Levy + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + */ + +#include "kernel/yosys.h" +#include "kernel/sigtools.h" +#include "kernel/consteval.h" + +USING_YOSYS_NAMESPACE +PRIVATE_NAMESPACE_BEGIN + +// --------------------------------------------------------------------------- +// opt_priokey: priority-by-key deduplication ("taken" accumulator) rewrite. +// +// RTL that resolves conflicts between several sources that each carry a small +// key is often written as a serial scan over a wide one-hot "set" accumulator +// indexed by the key: +// +// taken = '0; +// for (i = 0; i < P; i++) +// if (act[i] && !taken[key[i]]) begin // this source wins its key +// taken[key[i]] = 1'b1; // claim the key +// ...use win[i]... +// end +// +// This elaborates into a serial chain of dynamic-index reads (taken[key[i]] = +// $shiftx into an S-bit vector) and dynamic-index writes (taken | 1< bit_to_driver; + + int max_slots = 1 << 14; // maximum accumulator width S + int max_chain = 256; // maximum number of sources P + int fp_trials = 256; // ConstEval validation vectors + bool strict = false; // validate over the full key range, not [0,S) + + int regions_rewritten = 0; + int cells_added = 0; + + OptPrioKeyWorker(Module *m) : module(m), sigmap(m) { build_index(); } + + void build_index() { + for (auto c : module->cells()) + for (auto &conn : c->connections()) + if (c->output(conn.first)) + for (auto bit : sigmap(conn.second)) + if (bit.wire) bit_to_driver[bit] = c; + } + + Cell *sole_driver(const SigSpec &s) { + SigSpec ss = sigmap(s); + Cell *d = nullptr; + for (auto bit : ss) { + if (!bit.wire) return nullptr; + auto it = bit_to_driver.find(bit); + if (it == bit_to_driver.end()) return nullptr; + if (d && d != it->second) return nullptr; + d = it->second; + } + return d; + } + + bool is_all_zero(const SigSpec &s) { + for (auto bit : s) + if (bit != SigBit(State::S0)) return false; + return true; + } + + bool is_const_one(const SigSpec &s) { + SigSpec ss = sigmap(s); + if (!ss.is_fully_const()) return false; + return ss.as_const().as_int() == 1; + } + + // Recover the index bus `key` from a one-hot set-mask (1 << key), spelled + // as $shl(1, key) or $shift(1, -key). + SigSpec decode_onehot_key(const SigSpec &mask) { + Cell *d = sole_driver(mask); + if (!d) return SigSpec(); + if ((d->type == ID($shl) || d->type == ID($sshl)) && + is_const_one(d->getPort(ID::A))) + return sigmap(d->getPort(ID::B)); + if (d->type == ID($shift) && is_const_one(d->getPort(ID::A))) { + Cell *neg = sole_driver(d->getPort(ID::B)); + if (neg && neg->type == ID($neg)) + return sigmap(neg->getPort(ID::A)); + } + return SigSpec(); + } + + // A set-arm is `taken_prev` with one bit (1 << key) OR'ed in. After opt it + // appears either as the raw one-hot (prev was 0) or as an $or whose two + // operands are the one-hot mask and the (masked) previous state. Return the + // key bus of the one-hot mask. + SigSpec extract_key_from_setarm(const SigSpec &arm) { + SigSpec k = decode_onehot_key(arm); + if (GetSize(k)) return k; + Cell *d = sole_driver(arm); + if (d && d->type == ID($or)) { + k = decode_onehot_key(d->getPort(ID::A)); + if (GetSize(k)) return k; + k = decode_onehot_key(d->getPort(ID::B)); + if (GetSize(k)) return k; + } + return SigSpec(); + } + + // One guarded "set key" applied to the accumulator (guard may be const-1 + // for an unconditional set). + struct SetStep { + SigBit guard; + SigSpec key; + }; + + // Cache of trace results: chained reads share accumulator prefixes (read j + // indexes the state produced by sets 0..j-1), so without memoization the + // same chain is re-walked once per read -> O(P^2 * S). The cache makes the + // total walk O(P * S). + dict>> acc_memo; + + // Walk an accumulator value back to constant zero, collecting the guarded + // key-sets that produced it. Returns false (leaving `steps` unchanged) if it + // is not a pure set-only accumulator (mux/or of prev with a one-hot key) + // rooted at 0. + bool trace_acc(SigSpec acc, vector &steps, int depth) { + acc = sigmap(acc); + auto mit = acc_memo.find(acc); + if (mit != acc_memo.end()) { + if (mit->second.first) + steps.insert(steps.end(), mit->second.second.begin(), + mit->second.second.end()); + return mit->second.first; + } + int start = GetSize(steps); + bool ok = trace_acc_uncached(acc, steps, depth); + if (ok) { + acc_memo[acc] = {true, vector(steps.begin() + start, + steps.end())}; + } else { + steps.resize(start); // discard any partial trace + acc_memo[acc] = {false, {}}; + } + return ok; + } + + bool trace_acc_uncached(SigSpec acc, vector &steps, int depth) { + if (is_all_zero(acc)) + return true; + if (depth > max_chain) + return false; + Cell *d = sole_driver(acc); + if (!d) + return false; + if (d->type == ID($mux)) { + SigSpec s = sigmap(d->getPort(ID::S)); + if (GetSize(s) != 1) + return false; + if (!trace_acc(d->getPort(ID::A), steps, depth + 1)) + return false; + SigSpec key = extract_key_from_setarm(d->getPort(ID::B)); + if (GetSize(key) == 0) + return false; + steps.push_back(SetStep{s[0], key}); + return true; + } + if (d->type == ID($or)) { + SigSpec a = d->getPort(ID::A), b = d->getPort(ID::B); + SigSpec ka = decode_onehot_key(a), kb = decode_onehot_key(b); + if (GetSize(ka) && trace_acc(b, steps, depth + 1)) { + steps.push_back(SetStep{SigBit(State::S1), ka}); + return true; + } + if (GetSize(kb) && trace_acc(a, steps, depth + 1)) { + steps.push_back(SetStep{SigBit(State::S1), kb}); + return true; + } + } + return false; + } + + // A dynamic read of the accumulator: the 1-bit read cell, the accumulator + // value it indexes (with any out-of-range x-padding stripped) and the index. + struct Read { + Cell *cell; + SigSpec acc; + SigSpec key; + }; + + // Recognize a 1-bit dynamic read of a vector: either $shiftx(A=acc, B=key) + // or $bmux(A={x.., acc}, S=key). Returns false otherwise. + bool match_read(Cell *c, Read &r) { + if (c->type == ID($shiftx)) { + if (GetSize(c->getPort(ID::Y)) != 1) + return false; + r.cell = c; + r.acc = sigmap(c->getPort(ID::A)); + r.key = sigmap(c->getPort(ID::B)); + return GetSize(r.acc) >= 2; + } + if (c->type == ID($bmux)) { + if (c->getParam(ID::WIDTH).as_int() != 1) + return false; + SigSpec a = sigmap(c->getPort(ID::A)); + // Strip the high x-padding to recover the real accumulator bits. + int w = 0; + while (w < GetSize(a) && a[w] != SigBit(State::Sx)) + w++; + if (w < 2) + return false; + r.cell = c; + r.acc = a.extract(0, w); + r.key = sigmap(c->getPort(ID::S)); + return true; + } + return false; + } + + // Prove read == OR over steps of ( guard & key == read_key ) by ConstEval + // fingerprinting over the reachable key range [0,S). Guards and key buses + // (which are disjoint sel slices) are driven as free inputs. + bool validate_read(const Read &rd, const vector &steps, int S) { + Cell *read = rd.cell; + SigSpec read_key = rd.key; + // In strict mode sweep the full key range so the rewrite is proven for + // every value the index bus can take (out-of-range reads included); + // otherwise sweep only the reachable slots [0,S). + int kw = GetSize(read_key); + uint64_t range = (uint64_t)S; + if (strict) { + int cap = kw < 30 ? kw : 30; + range = 1ULL << cap; + } + ConstEval ce(module); + uint64_t lfsr = 0x9e3779b97f4a7c15ULL ^ (uintptr_t)read; + auto rnd = [&]() { + lfsr ^= lfsr << 13; lfsr ^= lfsr >> 7; lfsr ^= lfsr << 17; + return lfsr; + }; + for (int t = 0; t < fp_trials; t++) { + ce.push(); + int rk = (int)(rnd() % range); + ce.set(read_key, Const(rk, GetSize(read_key))); + vector kv(GetSize(steps)); + vector gv(GetSize(steps)); + for (int i = 0; i < GetSize(steps); i++) { + kv[i] = (int)(rnd() % range); + ce.set(steps[i].key, Const(kv[i], GetSize(steps[i].key))); + if (steps[i].guard == State::S1) { + gv[i] = 1; + } else { + gv[i] = (int)(rnd() & 1); + ce.set(SigSpec(steps[i].guard), Const(gv[i], 1)); + } + } + SigSpec out(read->getPort(ID::Y)); + SigSpec undef; + bool ok = ce.eval(out, undef) && out.is_fully_const(); + int actual = ok ? (out.as_const().as_int() & 1) : -1; + int expect = 0; + for (int i = 0; i < GetSize(steps); i++) + if (gv[i] && kv[i] == rk) { expect = 1; break; } + ce.pop(); + if (!ok || actual != expect) + return false; + } + return true; + } + + void rewrite_read(const Read &rd, const vector &steps) { + Cell *read = rd.cell; + cell = read; + SigSpec read_key = rd.key; + SigSpec new_r; + if (steps.empty()) { + new_r = SigSpec(State::S0); + } else { + SigSpec terms; + for (auto &st : steps) { + SigSpec eq = module->Eq(NEW_ID2_SUFFIX("priokey_eq"), + st.key, read_key); + cells_added++; + SigSpec g = module->And(NEW_ID2_SUFFIX("priokey_and"), + SigSpec(st.guard), eq); + cells_added++; + terms.append(g); + } + new_r = module->ReduceOr(NEW_ID2_SUFFIX("priokey_or"), terms); + cells_added++; + } + // Tag wire so the rewrite is externally observable, then detach the old + // dynamic read and drive its consumers from the reduction. + Wire *tag = module->addWire(NEW_ID2_SUFFIX("priokey_read"), 1); + module->connect(SigSpec(tag), new_r); + SigSpec old_y = sigmap(read->getPort(ID::Y)); + Wire *dangling = module->addWire(NEW_ID2_SUFFIX("priokey_dangling"), + GetSize(read->getPort(ID::Y))); + read->setPort(ID::Y, dangling); + module->connect(old_y, SigSpec(tag)); + regions_rewritten++; + } + + void run() { + vector reads; + for (auto c : module->cells()) { + Read r; + if (!match_read(c, r)) + continue; + if (GetSize(r.acc) > max_slots) + continue; + reads.push_back(r); + } + + int max_sources = 0; + int accum_width = 0; + vector zero_reads; // read of the all-zero head (== 0) + for (auto &rd : reads) { + int S = GetSize(rd.acc); + vector steps; + if (!trace_acc(rd.acc, steps, 0)) + continue; + // The all-zero head read is only rewritten (to 0) once we know the + // pattern is really present in this module. + if (steps.empty()) { + zero_reads.push_back(rd); + continue; + } + if (!validate_read(rd, steps, S)) + continue; + rewrite_read(rd, steps); + max_sources = std::max(max_sources, GetSize(steps) + 1); + accum_width = S; + } + if (regions_rewritten) + for (auto &rd : zero_reads) + rewrite_read(rd, {}); + + if (regions_rewritten) + log(" %s: priority-by-key dedup, up to %d source(s), " + "%d-slot accumulator\n", + log_id(module), max_sources, accum_width); + } +}; + +struct OptPrioKeyPass : public Pass { + OptPrioKeyPass() : Pass("opt_priokey", + "detect and rewrite priority-by-key deduplication accumulators") {} + + void help() override { + // |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---| + log("\n"); + log(" opt_priokey [options] [selection]\n"); + log("\n"); + log("This pass detects a serial 'set accumulator' that resolves conflicts\n"); + log("between several sources that each carry a small key:\n"); + log("\n"); + log(" taken = '0;\n"); + log(" for (i = 0; i < P; i++)\n"); + log(" if (act[i] && !taken[key[i]]) begin taken[key[i]] = 1; ... end\n"); + log("\n"); + log("Such RTL elaborates into a chain of dynamic-index reads/writes into a\n"); + log("wide one-hot vector ($shiftx / $shift), whose depth grows with both the\n"); + log("number of sources and the accumulator width. Each dynamic read\n"); + log("taken[key[j]] is provably equal to\n"); + log("\n"); + log(" OR over i args, RTLIL::Design *design) override { + log_header(design, "Executing OPT_PRIOKEY pass (priority-by-key dedup).\n"); + + int max_slots = 1 << 14; + int max_sources = 256; + bool strict = false; + size_t argidx; + for (argidx = 1; argidx < args.size(); argidx++) { + if (args[argidx] == "-max-slots" && argidx + 1 < args.size()) { + max_slots = std::stoi(args[++argidx]); continue; + } + if (args[argidx] == "-max-sources" && argidx + 1 < args.size()) { + max_sources = std::stoi(args[++argidx]); continue; + } + if (args[argidx] == "-strict") { + strict = true; continue; + } + break; + } + extra_args(args, argidx, design); + + int total_regions = 0, total_cells = 0; + for (auto module : design->selected_modules()) { + OptPrioKeyWorker worker(module); + worker.max_slots = max_slots; + worker.max_chain = max_sources; + worker.strict = strict; + worker.run(); + total_regions += worker.regions_rewritten; + total_cells += worker.cells_added; + } + + log("Rewrote %d dynamic key-read(s); emitted %d new cell(s).\n", + total_regions, total_cells); + + if (total_regions) + Yosys::run_pass("clean -purge"); + } +} OptPrioKeyPass; + +PRIVATE_NAMESPACE_END diff --git a/tests/opt/opt_first_fit_alloc.ys b/tests/opt/opt_first_fit_alloc.ys index 07101da6a..a4bec64b0 100644 --- a/tests/opt/opt_first_fit_alloc.ys +++ b/tests/opt/opt_first_fit_alloc.ys @@ -590,3 +590,170 @@ design -load postopt select -assert-min 1 w:*ffa_* design -reset log -pop + +# ============================================================================ +# Group G: coalesce-matrix variant (precomputed same_cat[i][k], raw-input en) +# ============================================================================ +# +# Some RTL precomputes a per-leader "same_cat[i][k]" mask (gated ONLY on the +# leader's enable) and forward-coalesces the leader's slot into lane k without +# re-checking en[k]. Disabled lanes after a same-category leader therefore +# inherit that leader's slot (rather than 0). The pass detects this as the +# enable-independent forward-coalescing variant. These modules also drive the +# scan straight from a top-level request port (lane_en), exercising the +# primary-input enable/broadcast candidate path. + +# G1: coalesce-matrix dsel-only, raw-input enable, N=8 (equiv). +log -header "G1: coalesce-matrix allocator, raw-input enable, N=8 (equiv)" +log -push +design -reset +read_verilog -sv < no rewrite" +log -push +design -reset +read_verilog -sv <= N) ? (si - N) : si; + rr_dut #(N,W) u1(.req(req),.s(s),.grant(g1),.idx_next(n1)); + rr_ref #(N,W) u2(.req(req),.s(s),.grant(g2),.idx_next(n2)); + always_comb begin + assert (g1 == g2); + assert (n1 == n2); + end +endmodule +EOF +hierarchy -top tb +flatten +chformal -lower +opt -full +sat -verify -prove-asserts -show-ports tb +design -reset +log -pop + +# RR3: negative -- a downward-scanning arbiter (opposite rotation) is a +# different function and must not be rewritten as round-robin. +log -header "RR3: downward-scan arbiter -> no round-robin rewrite" +log -push +design -reset +read_verilog -sv < no rewrite" +log -push +design -reset +read_verilog -sv < no rewrite" +log -push +design -reset +read_verilog -sv < no rewrite" +log -push +design -reset +read_verilog -sv < Date: Mon, 6 Jul 2026 13:28:54 -0700 Subject: [PATCH 2/2] opt_prienc: give each round-robin req candidate its own fingerprint budget The max_pairs budget was a single running counter shared across all req_wire iterations, so once a start-candidate-heavy first req size exhausted it, every later req size broke on its first start candidate and was silently skipped. Reset the budget per req_wire so all req sizes get a fair chance. (Completeness only; fingerprint_rr still validates every match, so this never affected correctness.) Co-authored-by: Cursor --- passes/opt/opt_prienc.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/passes/opt/opt_prienc.cc b/passes/opt/opt_prienc.cc index e7c4a966c..7231db45e 100644 --- a/passes/opt/opt_prienc.cc +++ b/passes/opt/opt_prienc.cc @@ -782,7 +782,6 @@ struct OptPriEncWorker { std::sort(req_cands.begin(), req_cands.end(), [](Wire* a, Wire* b) { return a->width > b->width; }); - int pairs = 0; bool matched = false; for (Wire* req_wire : req_cands) { if (matched) break; @@ -791,6 +790,10 @@ struct OptPriEncWorker { pool req_bits; for (auto bit : req_sig) if (bit.wire) req_bits.insert(bit); + // Per-req_wire fingerprint budget: a start-candidate-heavy + // first req size must not exhaust a shared budget and starve + // later (narrower) req sizes. + int pairs = 0; for (Wire* start_wire : start_cands) { if (start_wire == req_wire) continue; if (++pairs > max_pairs) break;