diff --git a/passes/opt/cut_region.h b/passes/opt/cut_region.h new file mode 100644 index 000000000..52ce38719 --- /dev/null +++ b/passes/opt/cut_region.h @@ -0,0 +1,855 @@ +/* + * yosys -- Yosys Open SYnthesis Suite + * + * Copyright (C) 2026 Silimate Inc. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +// Shared cut-region matching infrastructure for the functional rewrite +// passes (opt_argmax, opt_priority_onehot, opt_compact_prefix). These passes +// find combinational regions between "cut" signals (module ports, FF data +// pins, or internal buses), verify their function by ConstEval +// fingerprinting, and replace the region while leaving surrounding logic +// untouched. +// +// This header is designed to be included INSIDE each pass's private +// namespace (after PRIVATE_NAMESPACE_BEGIN), so the shared code has a single +// source without introducing link-level coupling between the passes. +// +// All graph walks and fingerprint evaluations are charged against +// per-module work budgets so that adversarial netlist shapes (deep shared +// cones with hundreds of same-width candidate buses) degrade into skipped +// candidates instead of multi-minute runtimes. + +struct CutRegionWorker +{ + struct RootCand { + SigSpec sig; + std::string name; + }; + + struct BusCand { + SigSpec sig; + std::string name; + int entries = 0; + int elem_width = 0; + bool is_const = false; + }; + + Module *module; + SigMap sigmap; + dict bit_to_driver; + pool input_port_bits; + pool claimed_bits; + std::string last_cut_fail; + + // Work budgets, decremented as the search runs. Walk steps count cells + // visited by cone/cut traversals; eval steps approximate ConstEval cost + // as (test vectors x cone cells); attempts count cut-closure trials + // (each one also carries pool/queue setup overhead, so the count is + // bounded separately from the step total). When a budget runs out the + // remaining candidates in the module are skipped (matching is + // best-effort). + int64_t walk_budget = 20000000; + int64_t eval_budget = 20000000; + int64_t attempt_budget = 65536; + + bool walk_exhausted() const { return walk_budget <= 0 || attempt_budget <= 0; } + bool eval_exhausted() const { return eval_budget <= 0; } + void charge_walk(int64_t n) { walk_budget -= n; } + void charge_eval(int64_t n) { eval_budget -= n; } + + // One visible note per module when a budget runs out, so QoR changes + // caused by truncated candidate searches are diagnosable from the log + // (the pass options can then raise the budget for that design). + bool budget_noted = false; + void note_budget(const char *pass_name, int skipped_roots) + { + if (budget_noted) + return; + if (!walk_exhausted() && !eval_exhausted()) + return; + budget_noted = true; + const char *which = attempt_budget <= 0 ? "attempt budget" : + walk_budget <= 0 ? "walk budget" : "eval budget"; + log("Note: %s search %s exhausted in module %s; %d remaining root candidate(s) skipped. " + "Use the pass budget options to raise the limit if QoR matters more than runtime here.\n", + pass_name, which, log_id(module), skipped_roots); + } + + CutRegionWorker(Module *module) : module(module), sigmap(module) + { + build_indexes(); + } + + bool is_sequential(Cell *c) + { + return c->type.in( + ID($ff), ID($dff), ID($dffe), ID($adff), ID($adffe), + ID($sdff), ID($sdffe), ID($sdffce), ID($dffsr), ID($dffsre), + ID($_DFF_P_), ID($_DFF_N_), + ID($_DFFE_PP_), ID($_DFFE_PN_), ID($_DFFE_NP_), ID($_DFFE_NN_), + ID($_DFF_PP0_), ID($_DFF_PP1_), ID($_DFF_PN0_), ID($_DFF_PN1_), + ID($_DFF_NP0_), ID($_DFF_NP1_), ID($_DFF_NN0_), ID($_DFF_NN1_), + ID($dlatch), ID($adlatch), ID($dlatchsr), + ID($mem), ID($mem_v2), ID($meminit), ID($meminit_v2), + ID($memrd), ID($memrd_v2), ID($memwr), ID($memwr_v2), + ID($fsm), + ID($assert), ID($assume), ID($cover), ID($live), ID($fair), + ID($print), ID($check), + ID($anyconst), ID($anyseq), ID($allconst), ID($allseq), + ID($initstate)); + } + + void build_indexes() + { + for (auto c : module->cells()) { + if (is_sequential(c)) + continue; + for (auto &conn : c->connections()) { + if (!c->output(conn.first)) + continue; + for (auto bit : sigmap(conn.second)) { + if (!bit.wire) + continue; + auto it = bit_to_driver.find(bit); + if (it == bit_to_driver.end()) + bit_to_driver[bit] = c; + else if (it->second != c) + it->second = nullptr; + } + } + } + + for (auto w : module->wires()) { + if (!w->port_input) + continue; + for (auto bit : sigmap(SigSpec(w))) + if (bit.wire) + input_port_bits.insert(bit); + } + } + + // Combinational fanin cone of `from`. Leaves are port-input bits or bits + // driven by sequential cells / undriven. Returns false if size limits + // are exceeded. `cell_order` records the cells in BFS discovery order + // (closest to `from` first). + bool get_cone(SigSpec from, pool &cone_cells, pool &leaf_bits, + int max_cone_cells, int max_leaf_bits, vector *cell_order = nullptr) + { + pool visited; + std::queue worklist; + for (auto bit : sigmap(from)) { + if (!bit.wire) + continue; + if (visited.insert(bit).second) + worklist.push(bit); + } + + while (!worklist.empty()) { + SigBit bit = worklist.front(); + worklist.pop(); + + if (input_port_bits.count(bit)) { + leaf_bits.insert(bit); + if (GetSize(leaf_bits) > max_leaf_bits) + return false; + continue; + } + + Cell *drv = bit_to_driver.at(bit, nullptr); + if (drv == nullptr) { + leaf_bits.insert(bit); + if (GetSize(leaf_bits) > max_leaf_bits) + return false; + continue; + } + + if (!cone_cells.insert(drv).second) + continue; + charge_walk(1); + if (GetSize(cone_cells) > max_cone_cells) + return false; + if (cell_order != nullptr) + cell_order->push_back(drv); + + for (auto &conn : drv->connections()) { + if (!drv->input(conn.first)) + continue; + for (auto in_bit : sigmap(conn.second)) { + if (!in_bit.wire) + continue; + if (visited.insert(in_bit).second) + worklist.push(in_bit); + } + } + } + + return true; + } + + // Walk the cone of `root`, cutting it at the bits in `allowed`. Returns + // true iff the cut cone closes (no other primary input / undriven bit is + // reached). `hit_bits`, when given, collects the allowed bits the cone + // actually uses. `forced_bits`, when given, is the subset of allowed + // bits the fingerprint will force: no forced bit may be driven by a cell + // inside the cut cone, since ConstEval caches whole cell outputs and + // evaluating such a driver would conflict with the forced values (when + // `forced_bits` is null, all of `allowed` is treated as forced). + bool cut_cone_walk(const SigSpec &root, const pool &allowed, int max_cells, + pool *hit_bits = nullptr, pool *cells_out = nullptr, + const pool *forced_bits = nullptr, + const pool *full_leaves = nullptr, + const pool *full_cells = nullptr) + { + attempt_budget--; + // Walk-free fast path: when no allowed bit has a combinational + // driver, no cut can shadow any cone leaf, so the cut closes iff it + // covers every leaf of the full cone (and the cut cone is the full + // cone). This answers the dominant class of failing candidates + // (port/FF-level bus pairs) in a handful of hash lookups. + if (full_leaves != nullptr && full_cells != nullptr) { + bool allowed_all_leaf = true; + for (auto bit : allowed) + if (bit_to_driver.at(bit, nullptr) != nullptr) { + allowed_all_leaf = false; + break; + } + if (allowed_all_leaf) { + for (auto leaf : *full_leaves) + if (!allowed.count(leaf)) { + last_cut_fail = stringf("leaf %s", log_signal(leaf)); + return false; + } + if (GetSize(*full_cells) > max_cells) { + last_cut_fail = "size limit"; + return false; + } + if (hit_bits != nullptr) + *hit_bits = *full_leaves; + if (cells_out != nullptr) + *cells_out = *full_cells; + return true; + } + } + + pool visited; + pool cells_seen; + std::queue worklist; + for (auto bit : sigmap(root)) { + if (!bit.wire) + continue; + if (visited.insert(bit).second) + worklist.push(bit); + } + + while (!worklist.empty()) { + SigBit bit = worklist.front(); + worklist.pop(); + + if (allowed.count(bit)) { + if (hit_bits != nullptr) + hit_bits->insert(bit); + continue; + } + + Cell *drv = bit_to_driver.at(bit, nullptr); + if (drv == nullptr) { + last_cut_fail = stringf("leaf %s", log_signal(bit)); + return false; + } + + if (!cells_seen.insert(drv).second) + continue; + charge_walk(1); + if (GetSize(cells_seen) > max_cells || walk_exhausted()) { + last_cut_fail = "size limit"; + return false; + } + + for (auto &conn : drv->connections()) { + if (!drv->input(conn.first)) + continue; + for (auto in_bit : sigmap(conn.second)) { + if (!in_bit.wire) + continue; + if (visited.insert(in_bit).second) + worklist.push(in_bit); + } + } + } + + const pool &check = (forced_bits != nullptr) ? *forced_bits : + (hit_bits != nullptr) ? *hit_bits : allowed; + for (auto bit : check) { + Cell *drv = bit_to_driver.at(bit, nullptr); + if (drv != nullptr && cells_seen.count(drv)) { + last_cut_fail = stringf("forced bit %s driven inside cone", log_signal(bit)); + return false; + } + } + + if (cells_out != nullptr) + *cells_out = cells_seen; + return true; + } + + // Walk the cone of `root` cut at `allowed`, collecting up to `max_extra` + // remaining boundary bits (inputs the cut does not cover) instead of + // failing on them. Aborts early once the limit is crossed, since callers + // only probe small uncovered sets. + bool cut_cone_extra_leaves(const SigSpec &root, const pool &allowed, int max_cells, + pool &extra_leaves, int max_extra) + { + attempt_budget--; + pool visited; + pool cells_seen; + std::queue worklist; + for (auto bit : sigmap(root)) { + if (!bit.wire) + continue; + if (visited.insert(bit).second) + worklist.push(bit); + } + + while (!worklist.empty()) { + SigBit bit = worklist.front(); + worklist.pop(); + + if (allowed.count(bit)) + continue; + + Cell *drv = bit_to_driver.at(bit, nullptr); + if (drv == nullptr) { + extra_leaves.insert(bit); + if (GetSize(extra_leaves) > max_extra) + return false; + continue; + } + + if (!cells_seen.insert(drv).second) + continue; + charge_walk(1); + if (GetSize(cells_seen) > max_cells || walk_exhausted()) + return false; + + for (auto &conn : drv->connections()) { + if (!drv->input(conn.first)) + continue; + for (auto in_bit : sigmap(conn.second)) { + if (!in_bit.wire) + continue; + if (visited.insert(in_bit).second) + worklist.push(in_bit); + } + } + } + + return true; + } + + bool sig_fully_driven(const SigSpec &sig) + { + for (auto bit : sigmap(sig)) { + if (!bit.wire) + return false; + if (input_port_bits.count(bit)) + return false; + if (bit_to_driver.at(bit, nullptr) == nullptr) + return false; + } + return true; + } + + // Collect the bus bits into `seen_bits`, rejecting constant or repeated + // bits (fingerprints drive each bus bit independently). + bool sig_bits_unique(const SigSpec &sig, pool &seen_bits) + { + for (auto bit : sigmap(sig)) + if (!bit.wire || !seen_bits.insert(bit).second) + return false; + return true; + } + + // Cut buses only need to consist of wire bits; FF outputs (cone leaves) + // are valid region boundaries even though they have no comb driver. + bool sig_bus_ok(const SigSpec &sig) + { + for (auto bit : sigmap(sig)) + if (!bit.wire) + return false; + return true; + } + + pool sig_bit_pool(const SigSpec &sig) + { + pool bits; + for (auto bit : sigmap(sig)) + if (bit.wire) + bits.insert(bit); + return bits; + } + + // Depth of each cone cell measured from the cone leaves (cells reading + // only leaf/port bits have depth 1). Used to order candidate cut buses + // so signals produced by shallow pre-logic are tried first. + dict compute_cone_depths(const pool &cone_cells) + { + dict depth; + dict> succs; + dict npreds; + std::queue ready; + + for (auto c : cone_cells) { + pool preds; + for (auto &conn : c->connections()) { + if (!c->input(conn.first)) + continue; + for (auto bit : sigmap(conn.second)) { + if (!bit.wire) + continue; + Cell *drv = bit_to_driver.at(bit, nullptr); + if (drv != nullptr && drv != c && cone_cells.count(drv)) + preds.insert(drv); + } + } + npreds[c] = GetSize(preds); + for (auto p : preds) + succs[p].push_back(c); + if (preds.empty()) { + depth[c] = 1; + ready.push(c); + } + } + + while (!ready.empty()) { + Cell *c = ready.front(); + ready.pop(); + for (auto s : succs.at(c, vector())) { + if (depth.at(s, 0) < depth.at(c) + 1) + depth[s] = depth.at(c) + 1; + if (--npreds.at(s) == 0) + ready.push(s); + } + } + + return depth; + } + + bool find_anchor_driver(const SigSpec &out_sig, Cell *&anchor) + { + for (auto bit : sigmap(out_sig)) { + Cell *drv = bit_to_driver.at(bit, nullptr); + if (drv != nullptr) { + anchor = drv; + return true; + } + } + return false; + } + + // Detach the existing drivers of `out_sig` (bit-precise: other bits of + // shared driver outputs keep their connections). The caller then drives + // `out_sig` from the replacement logic; the disconnected cells become + // dead and are removed by the trailing 'clean -purge'. + void disconnect_root(const SigSpec &out_sig, Cell *anchor, const char *dangling_suffix) + { + pool target_bits; + for (auto bit : sigmap(out_sig)) + if (bit.wire) + target_bits.insert(bit); + + pool seen_cells; + for (auto target : target_bits) { + Cell *drv = bit_to_driver.at(target, nullptr); + if (drv == nullptr || seen_cells.count(drv)) + continue; + seen_cells.insert(drv); + + for (auto &conn : drv->connections()) { + if (!drv->output(conn.first)) + continue; + + SigSpec orig = conn.second; + SigSpec replacement = orig; + bool changed = false; + Cell *cell = drv; + Wire *dangling = module->addWire(NEW_ID2_SUFFIX(dangling_suffix), GetSize(orig)); + for (int i = 0; i < GetSize(orig); i++) { + if (target_bits.count(sigmap(orig[i]))) { + replacement[i] = SigBit(dangling, i); + changed = true; + } + } + if (changed) + drv->setPort(conn.first, replacement); + } + } + (void)anchor; + } + + // Claim the root and every signal produced inside the matched region, so + // functionally identical sub-roots (e.g. the data input of the region's + // final mux) are not rewritten again. + void claim_region(const SigSpec &root_sig, const pool &cut_cells) + { + for (auto bit : sigmap(root_sig)) + if (bit.wire) + claimed_bits.insert(bit); + for (auto c : cut_cells) + for (auto &conn : c->connections()) { + if (!c->output(conn.first)) + continue; + for (auto bit : sigmap(conn.second)) + if (bit.wire) + claimed_bits.insert(bit); + } + } + + bool root_claimed(const SigSpec &root_sig) + { + for (auto bit : sigmap(root_sig)) + if (bit.wire && claimed_bits.count(bit)) + return true; + return false; + } + + // Parse a split name of the form "base[index]" (Verific lowers packed + // multi-dimensional ports, nets and array FFs into per-lane wires named + // this way). + bool parse_indexed_port_name(Wire *wire, std::string &base, int &index) + { + std::string name = wire->name.str(); + size_t rbrack = name.size(); + if (rbrack == 0 || name[rbrack - 1] != ']') + return false; + size_t lbrack = name.rfind('['); + if (lbrack == std::string::npos || lbrack + 1 >= rbrack - 1) + return false; + for (size_t i = lbrack + 1; i < rbrack - 1; i++) + if (!isdigit(name[i])) + return false; + base = name.substr(0, lbrack); + index = atoi(name.substr(lbrack + 1, rbrack - lbrack - 2).c_str()); + return true; + } + + // Group per-lane split wires into contiguous, equal-width buses. The run + // may start at any base index; the resulting sig is the ascending-index + // concatenation, so lane k is sig[k*elem_width ...] = the (base+k)-th + // wire. + vector collect_split_buses(const vector &wires) + { + std::map>> groups; + for (auto w : wires) { + std::string base; + int index = -1; + if (parse_indexed_port_name(w, base, index)) + groups[base].push_back({index, w}); + } + + vector buses; + for (auto &it : groups) { + auto entries = it.second; + std::sort(entries.begin(), entries.end(), + [](const std::pair &a, const std::pair &b) { + return a.first < b.first; + }); + if (entries.empty()) + continue; + bool contiguous = true; + int base_index = entries.front().first; + int elem_width = GetSize(entries.front().second); + for (int i = 0; i < GetSize(entries); i++) { + if (entries[i].first != base_index + i || + GetSize(entries[i].second) != elem_width) { + contiguous = false; + break; + } + } + if (!contiguous) + continue; + + SigSpec sig; + for (auto &entry : entries) + sig.append(SigSpec(entry.second)); + buses.push_back({sig, it.first, GetSize(entries), elem_width}); + } + + return buses; + } + + // All bits produced inside the cone or appearing as its leaves; the + // universe wire-run and split-bus collection draws candidates from. + pool cone_sig_bit_pool(const pool &cone_cells, const pool &leaf_bits) + { + pool bits = leaf_bits; + for (auto c : cone_cells) + for (auto &conn : c->connections()) + if (c->output(conn.first)) + for (auto bit : sigmap(conn.second)) + if (bit.wire) + bits.insert(bit); + return bits; + } + + // Maximal contiguous in-cone wire-bit runs, longest first (real region + // buses are wide; incidental wires are short and must not exhaust the + // cap). Constant edge bits (e.g. the never-written top bit of a [W:0] + // vector) are trimmed instead of rejecting the whole wire. + vector collect_wire_run_buses(const pool &cone_sig_bits, int cap) + { + vector wire_runs; + for (auto wb : module->wires()) { + if (GetSize(wb) < 2) + continue; + SigSpec sig = sigmap(SigSpec(wb)); + int run_start = -1; + for (int i = 0; i <= GetSize(sig); i++) { + bool ok = i < GetSize(sig) && sig[i].wire && cone_sig_bits.count(sig[i]); + if (ok && run_start < 0) + run_start = i; + if (!ok && run_start >= 0) { + int run_len = i - run_start; + if (run_len >= 2) { + SigSpec run = sig.extract(run_start, run_len); + std::string name = (run_len == GetSize(wb)) + ? wb->name.str() + : stringf("%s[%d+:%d]", wb->name.str().c_str(), run_start, run_len); + wire_runs.push_back({run, name}); + } + run_start = -1; + } + } + } + std::stable_sort(wire_runs.begin(), wire_runs.end(), + [](const BusCand &a, const BusCand &b) { + return GetSize(a.sig) > GetSize(b.sig); + }); + if (GetSize(wire_runs) > cap) + wire_runs.resize(cap); + return wire_runs; + } + + // Split-wire buses whose lanes touch the cone. + vector collect_cone_split_buses(const pool &cone_sig_bits) + { + vector cone_wires; + for (auto wb : module->wires()) { + bool touches = false; + for (auto bit : sigmap(SigSpec(wb))) + if (bit.wire && cone_sig_bits.count(bit)) { + touches = true; + break; + } + if (touches) + cone_wires.push_back(wb); + } + return collect_split_buses(cone_wires); + } + + // Per-seed cone cache: the seed sweep is the dominant fixed cost in + // FF-heavy modules and is shared by every matching mode of a pass. + struct SeedCone { + pool cells; + vector order; + bool valid = false; + }; + dict> seed_cone_cache; + + std::shared_ptr seed_cone(const SigSpec &seed, int max_cone_cells, int max_leaf_bits) + { + auto it = seed_cone_cache.find(seed); + if (it != seed_cone_cache.end()) + return it->second; + auto sc = std::make_shared(); + pool leaf_bits; + sc->valid = !walk_exhausted() && + get_cone(seed, sc->cells, leaf_bits, max_cone_cells, max_leaf_bits, &sc->order); + seed_cone_cache[seed] = sc; + return sc; + } + + // Collect candidate root signals. Module output ports and FF data inputs + // are seeds; internal signals inside seed cones (cell connections, and + // optionally whole wires fully inside a seed cone) are added so that + // regions wrapped in extra combinational post-logic are still found. + // Internal candidates are taken round-robin across the seed orders so a + // module with many FFs cannot starve the seeds whose cones hold the + // region. `width_ok` filters candidate widths; `seed_cone_interesting` + // gates internal harvesting per seed cone (e.g. "contains $bmux"). + vector collect_root_candidates( + std::function width_ok, + std::function &)> seed_cone_interesting, + bool wire_roots, int max_cone_cells, int max_leaf_bits, + int max_internal_roots = 128) + { + vector roots; + pool seen; + vector seed_sigs; + + auto consider_root = [&](const SigSpec &sig, const std::string &name, bool seed) -> bool { + if (!width_ok(GetSize(sig))) + return false; + if (!seen.insert(sig).second) + return false; + roots.push_back({sig, name}); + if (seed) + seed_sigs.push_back(sig); + return true; + }; + + for (auto w : module->wires()) { + if (!w->port_output || w->port_input) + continue; + consider_root(sigmap(SigSpec(w)), w->name.str(), true); + } + + for (auto c : module->cells()) { + if (!c->type.in(ID($ff), ID($dff), ID($dffe), ID($adff), ID($adffe), + ID($sdff), ID($sdffe), ID($sdffce), ID($dffsr), ID($dffsre), + ID($dlatch), ID($adlatch), ID($dlatchsr))) + continue; + if (!c->hasPort(ID::D)) + continue; + consider_root(sigmap(c->getPort(ID::D)), stringf("%s.D", log_id(c->name)), true); + } + + pool cone_out_bits; + vector> seed_orders; + for (auto &seed : seed_sigs) { + auto sc = seed_cone(seed, max_cone_cells, max_leaf_bits); + vector order; + if (sc->valid && seed_cone_interesting(sc->cells)) + order = sc->order; + seed_orders.push_back(order); + if (wire_roots) + for (auto c : order) + for (auto &conn : c->connections()) + if (c->output(conn.first)) + for (auto bit : sigmap(conn.second)) + if (bit.wire) + cone_out_bits.insert(bit); + } + + // Internal cell connections in round-robin BFS order (closest to a + // seed first), so post-logic wrappers are peeled off quickly. These + // come before wire roots: connection buses are ordered by proximity + // to the seeds, while wire iteration order is arbitrary. + int internal_roots = 0; + size_t longest_order = 0; + for (auto &order : seed_orders) + longest_order = std::max(longest_order, order.size()); + for (size_t pos = 0; pos < longest_order && internal_roots < max_internal_roots; pos++) { + for (auto &order : seed_orders) { + if (internal_roots >= max_internal_roots) + break; + if (pos >= order.size()) + continue; + Cell *c = order[pos]; + for (auto &conn : c->connections()) { + SigSpec sig = sigmap(conn.second); + if (!width_ok(GetSize(sig))) + continue; + if (!sig_fully_driven(sig)) + continue; + if (consider_root(sig, stringf("%s.%s", log_id(c->name), log_id(conn.first)), false)) + internal_roots++; + } + } + } + + // Whole wires fully inside some seed cone (regions written bit by + // bit, e.g. a |= scatter chain, are only visible as named wires). + int wire_root_count = 0; + if (wire_roots) { + for (auto w : module->wires()) { + if (wire_root_count >= max_internal_roots) + break; + if (!width_ok(GetSize(w))) + continue; + SigSpec sig = sigmap(SigSpec(w)); + bool inside = true; + for (auto bit : sig) + if (!bit.wire || !cone_out_bits.count(bit)) { + inside = false; + break; + } + if (!inside) + continue; + if (consider_root(sig, w->name.str(), false)) + wire_root_count++; + } + } + + return roots; + } + + // --- Small numeric helpers shared by the fingerprints. --- + + static uint64_t lowmask_u64(int w) + { + if (w <= 0) + return 0; + if (w >= 64) + return ~0ULL; + return (1ULL << w) - 1; + } + + static Const const_u64(uint64_t value, int width) + { + vector bits(width, State::S0); + for (int i = 0; i < width && i < 64; i++) + if ((value >> i) & 1ULL) + bits[i] = State::S1; + return Const(bits); + } + + SigSpec zext_sig(SigSpec sig, int width) + { + sig = sigmap(sig); + if (GetSize(sig) > width) + return sig.extract(0, width); + if (GetSize(sig) < width) + sig.append(SigSpec(State::S0, width - GetSize(sig))); + return sig; + } + + // Evaluate `out_sig` under the given input assignments; returns false if + // the cut does not fully determine the output. Charges the eval budget + // by `cone_cells_estimate`. + bool eval_with(ConstEval &ce, const vector> &sets, + const SigSpec &out_sig, uint64_t &result, int64_t cone_cells_estimate) + { + charge_eval(cone_cells_estimate); + ce.push(); + for (auto &s : sets) + ce.set(s.first, s.second); + SigSpec out = out_sig; + SigSpec undef; + bool ok = ce.eval(out, undef); + if (ok && out.is_fully_const()) { + Const cv = out.as_const(); + uint64_t r = 0; + for (int i = 0; i < GetSize(cv) && i < 64; i++) + if (cv[i] == State::S1) + r |= 1ULL << i; + result = r; + } else { + ok = false; + } + ce.pop(); + return ok; + } +}; diff --git a/passes/opt/opt_argmax.cc b/passes/opt/opt_argmax.cc index 59f7bde76..a842b72ee 100644 --- a/passes/opt/opt_argmax.cc +++ b/passes/opt/opt_argmax.cc @@ -20,6 +20,7 @@ #include "kernel/yosys.h" #include "kernel/sigtools.h" #include "kernel/consteval.h" +#include #include #include #include @@ -59,8 +60,12 @@ static Const packed_valid_const(const vector &valid) return Const(bits); } -struct OptArgmaxWorker +#include "passes/opt/cut_region.h" + +struct OptArgmaxWorker : CutRegionWorker { + typedef BusCand InputBus; + struct TestVector { vector valid; vector index; @@ -68,19 +73,24 @@ struct OptArgmaxWorker }; struct Candidate { - Wire *out_wire = nullptr; - Wire *valid_wire = nullptr; + SigSpec out_sig; + std::string out_name; SigSpec valid_sig; SigSpec index_sig; SigSpec values_sig; + std::string valid_name; std::string index_name; std::string values_name; bool identity_index = false; + bool values_const = false; + bool values_learned = false; + vector ok_index; int width = 0; int index_width = 0; int value_width = 0; Cell *anchor = nullptr; IdString anchor_port; + pool cut_cells; }; struct OutputCone { @@ -90,132 +100,31 @@ struct OptArgmaxWorker bool saw_lt = false; }; - struct InputBus { - SigSpec sig; - std::string name; - int entries = 0; - int elem_width = 0; - }; - struct Record { SigBit valid; SigSpec value; SigSpec index; }; - Module *module; - SigMap sigmap; - dict bit_to_driver; - pool input_port_bits; Cell *cell = nullptr; int min_width = 4; int max_width = 64; + int max_select_lanes = 16; + // Strict mode disables rewrites that rely on don't-care freedom (the + // learned const-table mode diverges from the original netlist on + // x-padded out-of-range table indices), so every emitted region is + // provable with equiv_opt -assert. Enabled by the flow's formal mode. + bool strict_mode = false; int regions_rewritten = 0; int cells_added = 0; - OptArgmaxWorker(Module *module) : module(module), sigmap(module) + // Per-root estimate of the cut cone size, used to charge the shared + // eval budget for each fingerprint vector. + int64_t cur_cone_est = 64; + + OptArgmaxWorker(Module *module) : CutRegionWorker(module) { - build_indexes(); - } - - bool is_sequential(Cell *c) - { - return c->type.in( - ID($ff), ID($dff), ID($dffe), ID($adff), ID($adffe), - ID($sdff), ID($sdffe), ID($sdffce), ID($dffsr), ID($dffsre), - ID($_DFF_P_), ID($_DFF_N_), - ID($_DFFE_PP_), ID($_DFFE_PN_), ID($_DFFE_NP_), ID($_DFFE_NN_), - ID($_DFF_PP0_), ID($_DFF_PP1_), ID($_DFF_PN0_), ID($_DFF_PN1_), - ID($_DFF_NP0_), ID($_DFF_NP1_), ID($_DFF_NN0_), ID($_DFF_NN1_), - ID($dlatch), ID($adlatch), ID($dlatchsr), - ID($mem), ID($mem_v2), ID($meminit), ID($meminit_v2), - ID($memrd), ID($memrd_v2), ID($memwr), ID($memwr_v2), - ID($fsm), - ID($assert), ID($assume), ID($cover), ID($live), ID($fair), - ID($print), ID($check), - ID($anyconst), ID($anyseq), ID($allconst), ID($allseq), - ID($initstate)); - } - - void build_indexes() - { - for (auto c : module->cells()) { - if (is_sequential(c)) - continue; - for (auto &conn : c->connections()) { - if (!c->output(conn.first)) - continue; - for (auto bit : sigmap(conn.second)) { - if (!bit.wire) - continue; - auto it = bit_to_driver.find(bit); - if (it == bit_to_driver.end()) - bit_to_driver[bit] = c; - else if (it->second != c) - it->second = nullptr; - } - } - } - - for (auto w : module->wires()) { - if (!w->port_input) - continue; - for (auto bit : sigmap(SigSpec(w))) - if (bit.wire) - input_port_bits.insert(bit); - } - } - - bool get_cone(SigSpec from, pool &cone_cells, pool &leaf_bits, - int max_cone_cells, int max_leaf_bits) - { - pool visited; - std::queue worklist; - for (auto bit : sigmap(from)) { - if (!bit.wire) - continue; - if (visited.insert(bit).second) - worklist.push(bit); - } - - while (!worklist.empty()) { - SigBit bit = worklist.front(); - worklist.pop(); - - if (input_port_bits.count(bit)) { - leaf_bits.insert(bit); - if (GetSize(leaf_bits) > max_leaf_bits) - return false; - continue; - } - - Cell *drv = bit_to_driver.at(bit, nullptr); - if (drv == nullptr) { - leaf_bits.insert(bit); - if (GetSize(leaf_bits) > max_leaf_bits) - return false; - continue; - } - - if (!cone_cells.insert(drv).second) - continue; - if (GetSize(cone_cells) > max_cone_cells) - return false; - - for (auto &conn : drv->connections()) { - if (!drv->input(conn.first)) - continue; - for (auto in_bit : sigmap(conn.second)) { - if (!in_bit.wire) - continue; - if (visited.insert(in_bit).second) - worklist.push(in_bit); - } - } - } - - return true; } OutputCone summarize_output_cone(const pool &cone_cells, pool leaf_bits) @@ -235,7 +144,7 @@ struct OptArgmaxWorker return cone.saw_bmux && (cone.saw_lt || value_width == 1); } - bool leaves_are_candidate_inputs(const pool &leaf_bits, const Candidate &cand) + pool candidate_input_bits(const Candidate &cand) { pool allowed; for (auto bit : sigmap(cand.valid_sig)) @@ -245,19 +154,155 @@ struct OptArgmaxWorker for (auto bit : sigmap(cand.index_sig)) if (bit.wire) allowed.insert(bit); - for (auto bit : sigmap(cand.values_sig)) - if (bit.wire) - allowed.insert(bit); + if (!cand.values_const) + for (auto bit : sigmap(cand.values_sig)) + if (bit.wire) + allowed.insert(bit); + return allowed; + } - for (auto bit : leaf_bits) - if (!allowed.count(bit)) + // The fingerprint drives each candidate bus bit independently, so the + // buses must consist of distinct non-constant bits. Constant value + // tables are not driven and are exempt. + bool candidate_inputs_disjoint(const Candidate &cand) + { + pool seen_bits; + for (auto bit : sigmap(cand.valid_sig)) + if (!bit.wire || !seen_bits.insert(bit).second) return false; + if (!cand.identity_index) + for (auto bit : sigmap(cand.index_sig)) + if (!bit.wire || !seen_bits.insert(bit).second) + return false; + if (!cand.values_const) + for (auto bit : sigmap(cand.values_sig)) + if (!bit.wire || !seen_bits.insert(bit).second) + return false; return true; } - bool find_anchor_driver(Wire *out_wire, Cell *&anchor, IdString &anchor_port) + // Decode a constant value table (e.g. a lookup of fixed split sizes read + // through $bmux) into per-entry integers; x bits decode as zero. + vector decode_const_table(const SigSpec &sig, int entries, int elem_width) { - for (auto bit : sigmap(SigSpec(out_wire))) { + vector table(entries, 0); + for (int k = 0; k < entries; k++) + for (int b = 0; b < elem_width && b < 64; b++) + if (sig[k * elem_width + b] == State::S1) + table[k] |= 1ULL << b; + return table; + } + + // The per-lane values may be produced by constant lookup logic baked + // into the cone (e.g. mux trees over a fixed table, where no explicit + // values bus exists). The induced order relation is observable from the + // winner index alone: probing two valid lanes with indices (a, b) tells + // whether T[a] < T[b]. The table is learned up to order isomorphism + // (ranks) and then verified by the regular fingerprint. + bool learn_const_table(const Candidate &cand, vector &table) + { + int n = cand.width; + ConstEval ce(module); + SigSpec out_sig = sigmap(cand.out_sig); + SigSpec valid_sig = sigmap(cand.valid_sig); + SigSpec index_sig = sigmap(cand.index_sig); + + charge_eval((int64_t)n * n * cur_cone_est); + vector> lt_rel(n, vector(n, false)); + vector> lt_known(n, vector(n, false)); + for (int a = 0; a < n; a++) + for (int b = 0; b < n; b++) { + if (a == b) + continue; + vector valid(n, 0); + valid[0] = 1; + valid[1] = 1; + vector index(n, 0); + index[0] = a; + index[1] = b; + ce.push(); + ce.set(valid_sig, packed_valid_const(valid)); + ce.set(index_sig, packed_table_const(index, cand.index_width)); + SigSpec out = out_sig; + SigSpec undef; + bool ok = ce.eval(out, undef); + ce.pop(); + if (!ok || !out.is_fully_const()) + continue; + int winner = out.as_const().as_int(); + if (winner != 0 && winner != 1) + continue; + lt_rel[a][b] = (winner == 1); + lt_known[a][b] = true; + } + + // The relation must form a strict weak order on the entries that the + // table actually defines. Entries padded with x (out-of-range reads + // of a short lookup table) are lowered under don't-care freedom and + // may behave inconsistently; they are excluded from the table and + // from the verification vectors (ok_index). The frontend already + // lowers x freely (db_preserve_x 0), so diverging on those index + // values keeps to the flow's don't-care semantics. + vector consistent(n, true); + auto compute_ranks = [&]() { + vector rk(n, 0); + for (int a = 0; a < n; a++) + for (int b = 0; b < n; b++) + if (consistent[a] && consistent[b] && lt_rel[b][a]) + rk[a]++; + return rk; + }; + auto count_violations = [&](int x, const vector &rk) { + int v = 0; + for (int b = 0; b < n; b++) { + if (b == x || !consistent[b]) + continue; + if (!lt_known[x][b] || lt_rel[x][b] != (rk[x] < rk[b])) + v++; + if (!lt_known[b][x] || lt_rel[b][x] != (rk[b] < rk[x])) + v++; + } + return v; + }; + + vector rank = compute_ranks(); + for (int round = 0; round < n; round++) { + int worst = -1, worst_v = 0; + for (int a = 0; a < n; a++) { + if (!consistent[a]) + continue; + int v = count_violations(a, rank); + if (v > worst_v) { + worst_v = v; + worst = a; + } + } + if (worst < 0) + break; + consistent[worst] = false; + rank = compute_ranks(); + } + + ok_index.clear(); + for (int a = 0; a < n; a++) + if (consistent[a]) + ok_index.push_back(a); + if (GetSize(ok_index) < 2) { + log_debug(" duel relation inconsistent beyond repair\n"); + return false; + } + table = rank; + return true; + } + + // Index values whose learned table entries behave consistently; the + // fingerprint restricts its index test vectors to these (set by + // learn_const_table, consumed via Candidate::ok_index). + vector ok_index; + + bool find_anchor_driver(const SigSpec &out_sig, Cell *&anchor, IdString &anchor_port) + { + for (auto bit : sigmap(out_sig)) { Cell *drv = bit_to_driver.at(bit, nullptr); if (drv == nullptr) continue; @@ -372,18 +417,35 @@ struct OptArgmaxWorker bool fingerprint(const Candidate &cand) { ConstEval ce(module); - SigSpec out_sig = sigmap(SigSpec(cand.out_wire)); + SigSpec out_sig = sigmap(cand.out_sig); SigSpec valid_sig = sigmap(cand.valid_sig); SigSpec index_sig = cand.identity_index ? SigSpec() : sigmap(cand.index_sig); SigSpec values_sig = sigmap(cand.values_sig); + // Constant value tables are fixed circuit content rather than a + // drivable cut: the expected function is checked against the table + // entries themselves. + vector const_table; + if (cand.values_const) + const_table = decode_const_table(values_sig, cand.width, cand.value_width); + vector vectors = make_test_vectors(cand.width, cand.value_width); + charge_eval((int64_t)vectors.size() * cur_cone_est); for (auto &tv : vectors) { + if (cand.values_const) + tv.values = const_table; + // Learned tables only define an order on the consistent index + // values; restrict the index vectors to those. + if (cand.values_learned && !cand.ok_index.empty()) + for (auto &iv : tv.index) + iv = cand.ok_index[iv % cand.ok_index.size()]; + ce.push(); ce.set(valid_sig, packed_valid_const(tv.valid)); if (!cand.identity_index) ce.set(index_sig, packed_table_const(tv.index, cand.index_width)); - ce.set(values_sig, packed_table_const(tv.values, cand.value_width)); + if (!cand.values_const) + ce.set(values_sig, packed_table_const(tv.values, cand.value_width)); SigSpec out = out_sig; SigSpec undef; @@ -504,37 +566,7 @@ struct OptArgmaxWorker void disconnect_old_output(const Candidate &cand) { - pool target_bits; - for (auto bit : sigmap(SigSpec(cand.out_wire))) - if (bit.wire) - target_bits.insert(bit); - - pool seen_cells; - for (auto target : target_bits) { - Cell *drv = bit_to_driver.at(target, nullptr); - if (drv == nullptr || seen_cells.count(drv)) - continue; - seen_cells.insert(drv); - - for (auto &conn : drv->connections()) { - if (!drv->output(conn.first)) - continue; - - SigSpec orig = conn.second; - SigSpec replacement = orig; - bool changed = false; - Cell *cell = drv; - Wire *dangling = module->addWire(NEW_ID2_SUFFIX("argmax_dangling"), GetSize(orig)); - for (int i = 0; i < GetSize(orig); i++) { - if (target_bits.count(sigmap(orig[i]))) { - replacement[i] = SigBit(dangling, i); - changed = true; - } - } - if (changed) - drv->setPort(conn.first, replacement); - } - } + disconnect_root(cand.out_sig, cand.anchor, "argmax_dangling"); } bool check_candidate(Candidate &cand, const OutputCone &cone) @@ -548,70 +580,505 @@ struct OptArgmaxWorker if (cand.value_width <= 0 || cand.value_width > 62) return false; - if (!cone_has_required_shape(cone, cand.value_width)) + // Learned constant tables imply the compare logic was folded into + // the cone, so the explicit $lt requirement does not apply. + if (!cone_has_required_shape(cone, cand.values_learned ? 1 : cand.value_width)) return false; - if (!leaves_are_candidate_inputs(cone.leaves, cand)) + if (!candidate_inputs_disjoint(cand)) { + log_debug(" valid=%s index=%s values=%s: buses overlap\n", + cand.valid_name.c_str(), cand.index_name.c_str(), cand.values_name.c_str()); return false; - if (!find_anchor_driver(cand.out_wire, cand.anchor, cand.anchor_port)) + } + pool allowed = candidate_input_bits(cand); + if (!cut_cone_walk(cand.out_sig, allowed, GetSize(cone.cells) + 16, nullptr, &cand.cut_cells, + nullptr, &cone.leaves, &cone.cells)) { + log_debug(" valid=%s index=%s values=%s: cut not closed (%s)\n", + cand.valid_name.c_str(), cand.index_name.c_str(), cand.values_name.c_str(), + last_cut_fail.c_str()); + return false; + } + if (!find_anchor_driver(cand.out_sig, cand.anchor, cand.anchor_port)) return false; - return fingerprint(cand); - } - - bool parse_indexed_port_name(Wire *wire, std::string &base, int &index) - { - std::string name = wire->name.str(); - size_t rbrack = name.size(); - if (rbrack == 0 || name[rbrack - 1] != ']') + fp_attempts++; + if (!fingerprint(cand)) { + log_debug(" valid=%s index=%s values=%s: fingerprint mismatch\n", + cand.valid_name.c_str(), cand.index_name.c_str(), cand.values_name.c_str()); return false; - size_t lbrack = name.rfind('['); - if (lbrack == std::string::npos || lbrack + 1 >= rbrack - 1) - return false; - for (size_t i = lbrack + 1; i < rbrack - 1; i++) - if (!isdigit(name[i])) - return false; - base = name.substr(0, lbrack); - index = atoi(name.substr(lbrack + 1, rbrack - lbrack - 2).c_str()); + } return true; } - vector collect_split_input_buses(const vector &inputs) + // ConstEval fingerprints are the expensive step of the candidate search + // and get their own (smaller) per-root budget; cheap closure walks use + // the larger trial budget. + int fp_attempts = 0; + + bool is_compare_cell(Cell *c) { - std::map>> groups; - for (auto w : inputs) { - std::string base; - int index = -1; - if (parse_indexed_port_name(w, base, index)) - groups[base].push_back({index, w}); + return c->type.in(ID($lt), ID($gt), ID($le), ID($ge), ID($sub)); + } + + // Argmax roots are index-width signals whose seed cones contain $bmux; + // max-select roots are value-width signals whose seed cones contain + // compare cells. + vector collect_argmax_roots() + { + int max_w = clog2_int(max_width); + return collect_root_candidates( + [&](int w) { return w >= 2 && w <= max_w; }, + [&](const pool &cone) { + for (auto c : cone) + if (c->type == ID($bmux)) + return true; + return false; + }, + true, std::max(256, max_width * 96), + max_width * (max_w + max_width) + max_width); + } + + vector collect_max_select_roots() + { + return collect_root_candidates( + [&](int w) { return w >= 2 && w <= 62; }, + [&](const pool &cone) { + int cmps = 0; + for (auto c : cone) + if (is_compare_cell(c) && ++cmps >= min_width - 1) + return true; + return false; + }, + true, std::max(256, max_width * 96), + max_width * (62 + max_width) + max_width); + } + + // --- Parallel max-select: out = max over N lanes of (zeroed ? 0 : v[n]). + // Matches hand-written compare-select max trees (e.g. FP exponent max + // stages) and replaces the serial compare levels with all pairwise + // comparisons in parallel, a one-hot winner vector, and an AND-OR select. + + struct MaxCand { + SigSpec out_sig; + std::string out_name; + SigSpec values_sig; + std::string values_name; + SigSpec mask_sig; // empty for the unmasked form + std::string mask_name; + bool zero_when_high = false; // lane forced to 0 when its mask bit is 1 + int lanes = 0; + int value_width = 0; + Cell *anchor = nullptr; + IdString anchor_port; + pool cut_cells; + }; + + int max_regions_rewritten = 0; + + bool fingerprint_max(const MaxCand &cand) + { + int n = cand.lanes; + int vw = cand.value_width; + uint64_t vmask = value_mask(vw); + bool has_mask = !cand.mask_sig.empty(); + + ConstEval ce(module); + SigSpec out_sig = sigmap(cand.out_sig); + SigSpec values_sig = sigmap(cand.values_sig); + SigSpec mask_sig = sigmap(cand.mask_sig); + + // Value patterns paired with mask patterns (zeroing the running max + // must reveal the runner-up, which kills near-miss structures). + vector, uint64_t>> vectors; + auto add = [&](vector vals, uint64_t mask) { + for (auto &v : vals) + v &= vmask; + vectors.push_back({vals, mask}); + }; + + vector asc(n), desc(n), equal(n, 5); + for (int i = 0; i < n; i++) { + asc[i] = uint64_t(i + 1); + desc[i] = uint64_t(n - i); + } + add(vector(n, 0), 0); + add(asc, 0); + add(desc, 0); + add(equal, 0); + for (int i = 0; i < n; i++) { + vector single(n, 0); + single[i] = vmask; + add(single, 0); + } + if (has_mask) { + uint64_t all = lowmask_u64(n); + add(asc, all); + add(desc, all); + for (int i = 0; i < n; i++) { + // Zero the maximal lane: result must become the runner-up. + vector vals = asc; + vals[i] = vmask; + add(vals, 1ULL << i); + add(desc, 1ULL << i); + } + } + uint64_t lfsr = 0x243f6a8885a308d3ULL; + for (int r = 0; r < 24; r++) { + vector vals(n); + for (int i = 0; i < n; i++) { + lfsr ^= lfsr << 13; + lfsr ^= lfsr >> 7; + lfsr ^= lfsr << 17; + vals[i] = lfsr; + } + lfsr ^= lfsr << 13; + lfsr ^= lfsr >> 7; + lfsr ^= lfsr << 17; + add(vals, has_mask ? (lfsr & lowmask_u64(n)) : 0); } - vector buses; - for (auto &it : groups) { - auto entries = it.second; - std::sort(entries.begin(), entries.end(), - [](const std::pair &a, const std::pair &b) { - return a.first < b.first; - }); - if (entries.empty() || entries.front().first != 0) + charge_eval((int64_t)vectors.size() * cur_cone_est); + for (auto &tv : vectors) { + uint64_t expected = 0; + for (int i = 0; i < n; i++) { + bool zeroed = has_mask && + ((((tv.second >> i) & 1ULL) != 0) == cand.zero_when_high); + uint64_t v = zeroed ? 0 : tv.first[i]; + expected = std::max(expected, v); + } + + ce.push(); + ce.set(values_sig, packed_table_const(tv.first, vw)); + if (has_mask) { + vector mask_bits(n); + for (int i = 0; i < n; i++) + mask_bits[i] = (tv.second >> i) & 1ULL; + ce.set(mask_sig, packed_valid_const(mask_bits)); + } + SigSpec out = out_sig; + SigSpec undef; + bool ok = ce.eval(out, undef); + ce.pop(); + if (!ok || !out.is_fully_const()) + return false; + + Const oc = out.as_const(); + uint64_t actual = 0; + for (int i = 0; i < GetSize(oc) && i < 64; i++) + if (oc[i] == State::S1) + actual |= 1ULL << i; + if (actual != expected) + return false; + } + return true; + } + + SigBit emit_ge(Cell *anchor, SigSpec a, SigSpec b) + { + Cell *cell = anchor; + cells_added++; + return module->Ge(NEW_ID2_SUFFIX("maxsel_ge"), a, b)[0]; + } + + SigBit emit_reduce_and(Cell *anchor, SigSpec bits) + { + if (GetSize(bits) == 1) + return bits[0]; + Cell *cell = anchor; + cells_added++; + return module->ReduceAnd(NEW_ID2_SUFFIX("maxsel_win"), bits)[0]; + } + + SigSpec emit_or_tree(Cell *anchor, const vector &terms, int begin, int end) + { + log_assert(begin < end); + if (begin + 1 == end) + return terms[begin]; + int mid = begin + (end - begin) / 2; + SigSpec lhs = emit_or_tree(anchor, terms, begin, mid); + SigSpec rhs = emit_or_tree(anchor, terms, mid, end); + return emit_or(anchor, lhs, rhs); + } + + SigSpec emit_max_select(const MaxCand &cand) + { + int n = cand.lanes; + int vw = cand.value_width; + SigSpec values = sigmap(cand.values_sig); + SigSpec mask = sigmap(cand.mask_sig); + + // Per-lane masked values. + vector m(n); + for (int i = 0; i < n; i++) { + m[i] = values.extract(i * vw, vw); + if (!cand.mask_sig.empty()) { + SigBit en = mask[i]; + if (cand.zero_when_high) + en = emit_not(cand.anchor, SigSpec(en))[0]; + m[i] = emit_and(cand.anchor, m[i], SigSpec(en).repeat(vw)); + } + } + + // All pairwise comparisons in parallel; the winner (lowest index on + // ties) is the lane that is >= all later lanes and strictly above + // all earlier ones. + vector> ge(n, vector(n)); + for (int i = 0; i < n; i++) + for (int j = i + 1; j < n; j++) + ge[i][j] = emit_ge(cand.anchor, m[i], m[j]); + + vector gated(n); + for (int i = 0; i < n; i++) { + SigSpec win_terms; + for (int j = i + 1; j < n; j++) + win_terms.append(ge[i][j]); + for (int j = 0; j < i; j++) + win_terms.append(emit_not(cand.anchor, SigSpec(ge[j][i]))[0]); + SigBit win = emit_reduce_and(cand.anchor, win_terms); + gated[i] = emit_and(cand.anchor, m[i], SigSpec(win).repeat(vw)); + } + + return emit_or_tree(cand.anchor, gated, 0, n); + } + + void run_max_select() + { + if (module->has_processes_warn()) + return; + + // Cheap module prefilter: a compare-select tree needs compares and + // muxes. + bool has_cmp = false, has_mux = false; + for (auto c : module->cells()) { + if (is_compare_cell(c)) + has_cmp = true; + if (c->type.in(ID($mux), ID($pmux))) + has_mux = true; + } + if (!has_cmp || !has_mux) + return; + + const int max_lanes = std::min(max_width, max_select_lanes); + + vector inputs; + for (auto w : module->wires()) + if (w->port_input) + inputs.push_back(w); + + vector rewrites; + vector root_list = collect_max_select_roots(); + for (int ri = 0; ri < GetSize(root_list); ri++) { + auto &root = root_list[ri]; + if (walk_exhausted() || eval_exhausted()) { + note_budget("opt_argmax (max-select)", GetSize(root_list) - ri); + break; + } + if (root_claimed(root.sig)) continue; - bool contiguous = true; - int elem_width = GetSize(entries.front().second); - for (int i = 0; i < GetSize(entries); i++) { - if (entries[i].first != i || GetSize(entries[i].second) != elem_width) { - contiguous = false; + + int vw = GetSize(root.sig); + if (vw < 2 || vw > 62) + continue; + + pool cone_cells; + pool leaf_bits; + vector order; + int max_cone_cells = std::max(256, max_width * 96); + int max_leaf_bits = max_width * (vw + max_width) + max_width; + if (!get_cone(root.sig, cone_cells, leaf_bits, + max_cone_cells, max_leaf_bits, &order)) + continue; + // A max-reduction tree over N >= min_width lanes has at least + // N-1 compares, each steering a mux select; generic datapath + // cones (compares and muxes that are not wired as + // compare-select) are skipped before the expensive bus search. + int cone_cmp_count = 0; + pool mux_sel_bits; + for (auto c : cone_cells) { + if (is_compare_cell(c)) + cone_cmp_count++; + if (c->type == ID($mux)) + for (auto bit : sigmap(c->getPort(ID::S))) + if (bit.wire) + mux_sel_bits.insert(bit); + } + bool cmp_feeds_mux_sel = false; + for (auto c : cone_cells) { + if (!is_compare_cell(c) || !c->hasPort(ID::Y)) + continue; + for (auto bit : sigmap(c->getPort(ID::Y))) + if (bit.wire && mux_sel_bits.count(bit)) { + cmp_feeds_mux_sel = true; + break; + } + if (cmp_feeds_mux_sel) break; + } + if (cone_cmp_count < min_width - 1 || !cmp_feeds_mux_sel) + continue; + cur_cone_est = GetSize(cone_cells); + log_debug("max root %s (vw=%d): cone %d cells, %d leaf bits\n", + root.name.c_str(), vw, GetSize(cone_cells), GetSize(leaf_bits)); + + // Bus candidates: ports first, then in-cone wire runs (longest + // first), then internal cell connections shallowest-first. + vector buses; + pool seen_buses; + for (auto input : inputs) { + SigSpec sig = sigmap(SigSpec(input)); + if (seen_buses.insert(sig).second) + buses.push_back({sig, input->name.str(), 0, 0}); + } + + // Whole wires touching the cone, longest runs first. + pool cone_sig_bits = cone_sig_bit_pool(cone_cells, leaf_bits); + for (auto &run : collect_wire_run_buses(cone_sig_bits, 64)) { + if (!seen_buses.insert(run.sig).second) + continue; + buses.push_back(run); + } + + // Per-lane split wires ("exp_cmp[0]".."exp_cmp[N-1]") grouped + // into lane-ordered buses, same as split ports / array FFs. + { + vector cone_wires; + for (auto wb : module->wires()) { + bool touches = false; + for (auto bit : sigmap(SigSpec(wb))) + if (bit.wire && cone_sig_bits.count(bit)) { + touches = true; + break; + } + if (touches) + cone_wires.push_back(wb); + } + for (auto &bus : collect_split_buses(cone_wires)) { + SigSpec sig = sigmap(bus.sig); + if (seen_buses.insert(sig).second) + buses.push_back({sig, bus.name, 0, 0}); } } - if (!contiguous) - continue; - SigSpec sig; - for (auto &entry : entries) - sig.append(SigSpec(entry.second)); - buses.push_back({sig, it.first, GetSize(entries), elem_width}); + dict depths = compute_cone_depths(cone_cells); + vector by_depth = order; + std::stable_sort(by_depth.begin(), by_depth.end(), + [&](Cell *a, Cell *b) { + return depths.at(a, 1 << 30) < depths.at(b, 1 << 30); + }); + int internal_buses = 0; + for (auto c : by_depth) { + if (internal_buses >= 32) + break; + for (auto &conn : c->connections()) { + SigSpec sig = sigmap(conn.second); + if (GetSize(sig) < 2 || !seen_buses.insert(sig).second) + continue; + if (!sig_fully_driven(sig)) + continue; + buses.push_back({sig, stringf("%s.%s", log_id(c->name), log_id(conn.first)), 0, 0}); + internal_buses++; + } + } + + const int max_closure_attempts = 512; + const int max_fp = 16; + int closure_attempts = 0; + int fps = 0; + bool matched = false; + MaxCand cand; + + for (auto &values : buses) { + if (matched || closure_attempts >= max_closure_attempts || fps >= max_fp || walk_exhausted() || eval_exhausted()) + break; + int total = GetSize(values.sig); + if (total % vw != 0) + continue; + int n = total / vw; + if (n < min_width || n > max_lanes) + continue; + + pool value_bits; + if (!sig_bits_unique(values.sig, value_bits)) + continue; + + cand = MaxCand(); + cand.out_sig = root.sig; + cand.out_name = root.name; + cand.values_sig = values.sig; + cand.values_name = values.name; + cand.lanes = n; + cand.value_width = vw; + if (!find_anchor_driver(root.sig, cand.anchor, cand.anchor_port)) + break; + + // Unmasked form first. + closure_attempts++; + if (cut_cone_walk(root.sig, value_bits, GetSize(cone_cells) + 16, nullptr, &cand.cut_cells, + nullptr, &leaf_bits, &cone_cells) && + GetSize(cand.cut_cells) >= 2 * (n - 1)) { + fps++; + if (fingerprint_max(cand)) { + matched = true; + break; + } + log_debug(" max values=%s: fingerprint mismatch\n", values.name.c_str()); + } + + // Masked form: a lane is forced to zero by its mask bit. + for (auto &mask : buses) { + if (matched || closure_attempts >= max_closure_attempts || fps >= max_fp || walk_exhausted() || eval_exhausted()) + break; + if (GetSize(mask.sig) != n || mask.sig == values.sig) + continue; + pool allowed = value_bits; + if (!sig_bits_unique(mask.sig, allowed)) + continue; + closure_attempts++; + if (!cut_cone_walk(root.sig, allowed, GetSize(cone_cells) + 16, nullptr, &cand.cut_cells, + nullptr, &leaf_bits, &cone_cells)) { + log_debug(" max values=%s mask=%s: cut not closed (%s)\n", + values.name.c_str(), mask.name.c_str(), last_cut_fail.c_str()); + continue; + } + if (GetSize(cand.cut_cells) < 2 * (n - 1)) + continue; + cand.mask_sig = mask.sig; + cand.mask_name = mask.name; + fps++; + cand.zero_when_high = true; + if (fingerprint_max(cand)) { + matched = true; + break; + } + cand.zero_when_high = false; + if (fingerprint_max(cand)) { + matched = true; + break; + } + log_debug(" max values=%s mask=%s: fingerprint mismatch\n", + values.name.c_str(), mask.name.c_str()); + cand.mask_sig = SigSpec(); + cand.mask_name = ""; + } + } + + if (matched) { + rewrites.push_back(cand); + claim_region(root.sig, cand.cut_cells); + log(" %s: %s <- max(values=%s%s%s) [N=%d, VW=%d]\n", + log_id(module), cand.out_name.c_str(), cand.values_name.c_str(), + cand.mask_sig.empty() ? "" : (cand.zero_when_high ? ", zero=" : ", valid="), + cand.mask_sig.empty() ? "" : cand.mask_name.c_str(), + cand.lanes, cand.value_width); + } } - return buses; + for (auto &cand : rewrites) { + cell = cand.anchor; + SigSpec new_out = emit_max_select(cand); + disconnect_root(cand.out_sig, cand.anchor, "argmax_dangling"); + module->connect(cand.out_sig, new_out); + max_regions_rewritten++; + } } void run() @@ -619,113 +1086,319 @@ struct OptArgmaxWorker if (module->has_processes_warn()) return; + bool module_has_bmux = false; + for (auto c : module->cells()) + if (c->type == ID($bmux)) + module_has_bmux = true; + if (!module_has_bmux) + return; + vector inputs; - vector outputs; - for (auto w : module->wires()) { + for (auto w : module->wires()) if (w->port_input) inputs.push_back(w); - if (w->port_output && !w->port_input) - outputs.push_back(w); - } + vector split_buses = collect_split_buses(inputs); vector rewrites; - pool claimed_outputs; - for (auto out : outputs) { - if (claimed_outputs.count(out)) + vector root_list = collect_argmax_roots(); + for (int ri = 0; ri < GetSize(root_list); ri++) { + auto &root = root_list[ri]; + if (walk_exhausted() || eval_exhausted()) { + note_budget("opt_argmax", GetSize(root_list) - ri); + break; + } + if (root_claimed(root.sig)) continue; - int out_width = GetSize(out); - if (out_width < 2) + + int out_width = GetSize(root.sig); + int width = 1 << out_width; + if (width < min_width || width > max_width) continue; pool cone_cells; pool leaf_bits; + vector order; int max_cone_cells = std::max(256, max_width * 96); int max_leaf_bits = max_width * (out_width + max_width) + max_width; - if (!get_cone(SigSpec(out), cone_cells, leaf_bits, - max_cone_cells, max_leaf_bits)) + if (!get_cone(root.sig, cone_cells, leaf_bits, + max_cone_cells, max_leaf_bits, &order)) continue; OutputCone cone = summarize_output_cone(cone_cells, std::move(leaf_bits)); if (!cone.saw_bmux) continue; + cur_cone_est = GetSize(cone.cells); + log_debug("root %s (w=%d, N=%d): cone %d cells, %d leaf bits\n", + root.name.c_str(), out_width, width, GetSize(cone.cells), GetSize(cone.leaves)); - for (auto valid : inputs) { - int width = GetSize(valid); - if (width < min_width || width > max_width) - continue; - if (clog2_int(width) != out_width) - continue; + // Bus candidates: module input ports first (cheap, original + // behavior), then internal cut signals from the root cone in + // reverse BFS order (deepest first) to support pre-logic between + // the ports and the rewritable region. + vector valid_buses; + vector index_buses; + vector values_buses; + auto classify_bus = [&](const SigSpec &sig, const std::string &name) { + int sz = GetSize(sig); + if (sz == width) + valid_buses.push_back({sig, name, width, 1}); + if (sz == width * out_width) + index_buses.push_back({sig, name, width, out_width}); + if (sz >= width && sz % width == 0 && sz / width <= 62) + values_buses.push_back({sig, name, width, sz / width}); + }; - vector index_buses; - vector values_buses; - for (auto input : inputs) { - if (input == valid) - continue; - if (GetSize(input) == width * out_width) - index_buses.push_back({SigSpec(input), input->name.str(), width, out_width}); - if (GetSize(input) % width == 0) - values_buses.push_back({SigSpec(input), input->name.str(), width, GetSize(input) / width}); + pool seen_buses; + for (auto input : inputs) { + SigSpec sig = sigmap(SigSpec(input)); + if (seen_buses.insert(sig).second) + classify_bus(sig, input->name.str()); + } + for (auto &bus : split_buses) { + if (!seen_buses.insert(sigmap(bus.sig)).second) + continue; + if (bus.entries == width && bus.elem_width == out_width) + index_buses.push_back(bus); + if (bus.entries == width) + values_buses.push_back(bus); + if (bus.entries == width && bus.elem_width == 1) + valid_buses.push_back(bus); + } + + // Whole wires touching the cone, longest runs first. + pool cone_sig_bits = cone_sig_bit_pool(cone_cells, cone.leaves); + for (auto &run : collect_wire_run_buses(cone_sig_bits, 64)) { + if (!seen_buses.insert(run.sig).second) + continue; + classify_bus(run.sig, run.name); + } + + // Per-lane split wires ("ids_q[0]".."ids_q[N]") grouped into + // buses: array FFs are lowered into per-entry registers the same + // way multi-dimensional ports are split. + { + vector cone_wires; + for (auto wb : module->wires()) { + bool touches = false; + for (auto bit : sigmap(SigSpec(wb))) + if (bit.wire && cone_sig_bits.count(bit)) { + touches = true; + break; + } + if (touches) + cone_wires.push_back(wb); } - - vector split_buses = collect_split_input_buses(inputs); - for (auto bus : split_buses) { + for (auto &bus : collect_split_buses(cone_wires)) { + if (!seen_buses.insert(sigmap(bus.sig)).second) + continue; if (bus.entries == width && bus.elem_width == out_width) index_buses.push_back(bus); if (bus.entries == width) values_buses.push_back(bus); + if (bus.entries == width && bus.elem_width == 1) + valid_buses.push_back(bus); + classify_bus(sigmap(bus.sig), bus.name); } + } + + // Constant value tables read through $bmux/$shiftx (fixed lookup + // tables, e.g. size classes selected by a per-lane id). + for (auto c : cone_cells) { + if (!c->type.in(ID($bmux), ID($shiftx))) + continue; + SigSpec sig = sigmap(c->getPort(ID::A)); + if (!sig.is_fully_const()) + continue; + if (GetSize(sig) < width || GetSize(sig) % width != 0 || GetSize(sig) / width > 62) + continue; + if (!seen_buses.insert(sig).second) + continue; + values_buses.push_back({sig, stringf("%s.A", log_id(c->name)), + width, GetSize(sig) / width, true}); + } + + // Internal cut buses are taken from the shallowest cone cells + // first (depth measured from the leaves), since the cut points + // introduced by pre-logic sit right above the module inputs. + const int max_internal_buses = 32; + int internal_buses = 0; + dict depths = compute_cone_depths(cone_cells); + vector by_depth = order; + std::stable_sort(by_depth.begin(), by_depth.end(), + [&](Cell *a, Cell *b) { + return depths.at(a, 1 << 30) < depths.at(b, 1 << 30); + }); + for (auto c : by_depth) { + if (internal_buses >= max_internal_buses) + break; + for (auto &conn : c->connections()) { + SigSpec sig = sigmap(conn.second); + if (GetSize(sig) < width || !seen_buses.insert(sig).second) + continue; + if (!sig_fully_driven(sig)) + continue; + classify_bus(sig, stringf("%s.%s", log_id(c->name), log_id(conn.first))); + internal_buses++; + } + } + + // Each matching mode gets its own budget pair so expensive + // fingerprints in one mode cannot starve the others. + const int max_attempts = 768; + const int max_fp_attempts = 24; + int attempts = 0; + fp_attempts = 0; + + // Mode 1: identity-indexed values. + for (auto &valid : valid_buses) { + if (attempts >= max_attempts || fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; for (auto &values : values_buses) { + if (attempts >= max_attempts || fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (values.sig == valid.sig) + continue; + // Identity-indexed constant values would make the winner + // a constant function; only table mode uses const values. + if (values.is_const) + continue; Candidate cand; - cand.out_wire = out; - cand.valid_wire = valid; - cand.valid_sig = SigSpec(valid); + cand.out_sig = root.sig; + cand.out_name = root.name; + cand.valid_sig = valid.sig; cand.values_sig = values.sig; + cand.valid_name = valid.name; cand.index_name = ""; cand.values_name = values.name; cand.identity_index = true; cand.width = width; cand.index_width = out_width; cand.value_width = values.elem_width; + attempts++; if (!check_candidate(cand, cone)) continue; rewrites.push_back(cand); - claimed_outputs.insert(out); + claim_region(root.sig, cand.cut_cells); log(" %s: %s <- argmax(valid=%s, index=, values=%s) [N=%d, IW=%d, VW=%d]\n", - log_id(module), log_id(out), log_id(valid), values.name.c_str(), + log_id(module), cand.out_name.c_str(), valid.name.c_str(), values.name.c_str(), cand.width, cand.index_width, cand.value_width); - goto next_output; + goto next_root; } + } + + // Mode 2: table-indexed values (explicit or constant tables). + attempts = 0; + fp_attempts = 0; + for (auto &valid : valid_buses) { + if (attempts >= max_attempts || fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; for (auto &index : index_buses) { + if (attempts >= max_attempts || fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (index.sig == valid.sig) + continue; for (auto &values : values_buses) { - if (index.sig == values.sig) + if (attempts >= max_attempts || fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (index.sig == values.sig || values.sig == valid.sig) continue; Candidate cand; - cand.out_wire = out; - cand.valid_wire = valid; - cand.valid_sig = SigSpec(valid); + cand.out_sig = root.sig; + cand.out_name = root.name; + cand.valid_sig = valid.sig; cand.index_sig = index.sig; cand.values_sig = values.sig; + cand.valid_name = valid.name; cand.index_name = index.name; cand.values_name = values.name; cand.identity_index = false; + cand.values_const = values.is_const; cand.width = width; cand.index_width = out_width; cand.value_width = values.elem_width; + attempts++; if (!check_candidate(cand, cone)) continue; rewrites.push_back(cand); - claimed_outputs.insert(out); + claim_region(root.sig, cand.cut_cells); log(" %s: %s <- argmax(valid=%s, index=%s, values=%s) [N=%d, IW=%d, VW=%d]\n", - log_id(module), log_id(out), log_id(valid), index.name.c_str(), + log_id(module), cand.out_name.c_str(), valid.name.c_str(), index.name.c_str(), values.name.c_str(), cand.width, cand.index_width, cand.value_width); - goto next_output; + goto next_root; } } } -next_output: + + // Mode 3: table mode with the value table folded into the cone + // as constant logic (no explicit values bus): learn the induced + // order by probing two-lane duels, then verify the learned rank + // table with the regular fingerprint. The learned table only + // defines the order on consistently-behaving index values, so + // the rewrite may diverge on x-padded table reads; strict mode + // skips it to keep every rewrite formally provable. + if (strict_mode) + goto next_root; + attempts = 0; + fp_attempts = 0; + for (auto &valid : valid_buses) { + if (attempts >= max_attempts || fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + + for (auto &index : index_buses) { + if (attempts >= max_attempts || fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (index.sig == valid.sig || index.is_const) + continue; + Candidate cand; + cand.out_sig = root.sig; + cand.out_name = root.name; + cand.valid_sig = valid.sig; + cand.index_sig = index.sig; + cand.valid_name = valid.name; + cand.index_name = index.name; + cand.values_name = ""; + cand.identity_index = false; + cand.values_const = true; + cand.values_learned = true; + cand.width = width; + cand.index_width = out_width; + attempts++; + + if (!candidate_inputs_disjoint(cand)) + continue; + pool allowed = candidate_input_bits(cand); + if (!cut_cone_walk(cand.out_sig, allowed, GetSize(cone.cells) + 16, nullptr, &cand.cut_cells, + nullptr, &cone.leaves, &cone.cells)) { + log_debug(" valid=%s index=%s values=: cut not closed (%s)\n", + valid.name.c_str(), index.name.c_str(), last_cut_fail.c_str()); + continue; + } + + vector table; + fp_attempts++; + if (!learn_const_table(cand, table)) { + log_debug(" valid=%s index=%s values=: order probe failed\n", + valid.name.c_str(), index.name.c_str()); + continue; + } + cand.ok_index = ok_index; + cand.value_width = std::max(1, clog2_int(width)); + cand.values_sig = SigSpec(packed_table_const(table, cand.value_width)); + if (!check_candidate(cand, cone)) + continue; + + rewrites.push_back(cand); + claim_region(root.sig, cand.cut_cells); + log(" %s: %s <- argmax(valid=%s, index=%s, values=) [N=%d, IW=%d, VW=%d]\n", + log_id(module), cand.out_name.c_str(), valid.name.c_str(), index.name.c_str(), + cand.width, cand.index_width, cand.value_width); + goto next_root; + } + } +next_root: ; } @@ -733,7 +1406,7 @@ next_output: cell = cand.anchor; SigSpec new_out = emit_argmax(cand); disconnect_old_output(cand); - module->connect(SigSpec(cand.out_wire), new_out); + module->connect(cand.out_sig, new_out); regions_rewritten++; } } @@ -755,12 +1428,37 @@ struct OptArgmaxPass : public Pass log("comparators. Ties preserve the lower candidate index, matching a\n"); log("strict '<' update condition; all-invalid inputs return index zero.\n"); log("\n"); + log("Regions are matched between cut signals (module ports, FF data pins,\n"); + log("or internal buses) and verified by ConstEval fingerprinting, so the\n"); + log("rewrites apply even when extra combinational logic surrounds the\n"); + log("region. Three argmax modes are tried per candidate root: identity\n"); + log("index, explicit index table, and a learned constant table (the value\n"); + log("lookup was folded into logic; its order is recovered by probing\n"); + log("two-lane duels). A fourth mode rewrites masked max-value reductions\n"); + log("(compare-select trees) into all-pairs parallel comparisons with a\n"); + log("one-hot select.\n"); + log("\n"); log(" -max-width N, -max_width N\n"); log(" maximum candidate count to consider (default 64).\n"); log("\n"); log(" -min-width N, -min_width N\n"); log(" minimum candidate count to consider (default 4).\n"); log("\n"); + log(" -max-lanes N, -max_lanes N\n"); + log(" maximum lane count for the parallel max-select rewrite\n"); + log(" (default 16; the all-pairs comparison area grows with N^2).\n"); + log("\n"); + log(" -strict\n"); + log(" disable rewrites that rely on don't-care freedom (the\n"); + log(" learned const-table mode may diverge on x-padded\n"); + log(" out-of-range table reads). With -strict every rewrite is\n"); + log(" provable with equiv_opt -assert; used by formal flows.\n"); + log("\n"); + log(" -walk-budget N, -eval-budget N, -attempt-budget N\n"); + log(" per-module work limits for the candidate search (defaults\n"); + log(" 20000000 / 20000000 / 65536). When a budget is exhausted\n"); + log(" the remaining candidates are skipped and a note is logged.\n"); + log("\n"); } void execute(std::vector args, RTLIL::Design *design) override @@ -769,6 +1467,9 @@ struct OptArgmaxPass : public Pass int max_width = 64; int min_width = 4; + int max_lanes = 16; + bool strict = false; + int64_t walk_budget = -1, eval_budget = -1, attempt_budget = -1; size_t argidx; for (argidx = 1; argidx < args.size(); argidx++) { if ((args[argidx] == "-max-width" || args[argidx] == "-max_width") && @@ -781,25 +1482,60 @@ struct OptArgmaxPass : public Pass min_width = std::stoi(args[++argidx]); continue; } + if ((args[argidx] == "-max-lanes" || args[argidx] == "-max_lanes") && + argidx + 1 < args.size()) { + max_lanes = std::stoi(args[++argidx]); + continue; + } + if (args[argidx] == "-strict") { + strict = true; + continue; + } + if ((args[argidx] == "-walk-budget" || args[argidx] == "-walk_budget") && + argidx + 1 < args.size()) { + walk_budget = std::stoll(args[++argidx]); + continue; + } + if ((args[argidx] == "-eval-budget" || args[argidx] == "-eval_budget") && + argidx + 1 < args.size()) { + eval_budget = std::stoll(args[++argidx]); + continue; + } + if ((args[argidx] == "-attempt-budget" || args[argidx] == "-attempt_budget") && + argidx + 1 < args.size()) { + attempt_budget = std::stoll(args[++argidx]); + continue; + } break; } extra_args(args, argidx, design); int total_regions = 0; + int total_max_regions = 0; int total_cells_added = 0; for (auto module : design->selected_modules()) { OptArgmaxWorker worker(module); worker.max_width = max_width; worker.min_width = min_width; + worker.max_select_lanes = max_lanes; + worker.strict_mode = strict; + if (walk_budget > 0) + worker.walk_budget = walk_budget; + if (eval_budget > 0) + worker.eval_budget = eval_budget; + if (attempt_budget > 0) + worker.attempt_budget = attempt_budget; worker.run(); + worker.run_max_select(); total_regions += worker.regions_rewritten; + total_max_regions += worker.max_regions_rewritten; total_cells_added += worker.cells_added; } - log("Rewrote %d argmax region(s); emitted %d new cell(s).\n", - total_regions, total_cells_added); + log("Rewrote %d argmax region(s) and %d max-select region(s); emitted %d new cell(s).\n", + total_regions, total_max_regions, total_cells_added); - if (total_regions) + if (total_regions || total_max_regions) Yosys::run_pass("clean -purge"); } } OptArgmaxPass; diff --git a/passes/opt/opt_priority_onehot.cc b/passes/opt/opt_priority_onehot.cc index 367276c70..180a6cb33 100644 --- a/passes/opt/opt_priority_onehot.cc +++ b/passes/opt/opt_priority_onehot.cc @@ -53,13 +53,18 @@ static Const pack_lanes(const vector &vals, int elem_w) return Const(bits); } -struct OptPriorityOnehotWorker { +#include "passes/opt/cut_region.h" + +struct OptPriorityOnehotWorker : CutRegionWorker { + typedef BusCand InputBus; + // One detected priority-onehot region ready to be rewritten. struct Candidate { - Wire *out_wire = nullptr; // one-hot output O (width W = 2^idx_w) - Wire *valid_wire = nullptr; // request / valid bus V (width N) + SigSpec out_sig; // one-hot output O (width W = 2^idx_w) + std::string out_name; SigSpec valid_sig; // N bits SigSpec field_sig; // N * idx_w bits: concatenated per-lane index fields + std::string valid_name; std::string index_name; int n = 0; // number of lanes int w = 0; // output width (power of two) @@ -73,10 +78,6 @@ struct OptPriorityOnehotWorker { vector field; }; - Module *module; - SigMap sigmap; - dict bit_to_driver; - pool input_port_bits; Cell *cell = nullptr; int min_width = 4; @@ -84,111 +85,8 @@ struct OptPriorityOnehotWorker { int regions_rewritten = 0; int cells_added = 0; - OptPriorityOnehotWorker(Module *module) : module(module), sigmap(module) + OptPriorityOnehotWorker(Module *module) : CutRegionWorker(module) { - build_indexes(); - } - - bool is_sequential(Cell *c) - { - return c->type.in( - ID($ff), ID($dff), ID($dffe), ID($adff), ID($adffe), - ID($sdff), ID($sdffe), ID($sdffce), ID($dffsr), ID($dffsre), - ID($_DFF_P_), ID($_DFF_N_), - ID($_DFFE_PP_), ID($_DFFE_PN_), ID($_DFFE_NP_), ID($_DFFE_NN_), - ID($_DFF_PP0_), ID($_DFF_PP1_), ID($_DFF_PN0_), ID($_DFF_PN1_), - ID($_DFF_NP0_), ID($_DFF_NP1_), ID($_DFF_NN0_), ID($_DFF_NN1_), - ID($dlatch), ID($adlatch), ID($dlatchsr), - ID($mem), ID($mem_v2), ID($meminit), ID($meminit_v2), - ID($memrd), ID($memrd_v2), ID($memwr), ID($memwr_v2), - ID($fsm), - ID($assert), ID($assume), ID($cover), ID($live), ID($fair), - ID($print), ID($check), - ID($anyconst), ID($anyseq), ID($allconst), ID($allseq), - ID($initstate)); - } - - void build_indexes() - { - for (auto c : module->cells()) { - if (is_sequential(c)) - continue; - for (auto &conn : c->connections()) { - if (!c->output(conn.first)) - continue; - for (auto bit : sigmap(conn.second)) { - if (!bit.wire) - continue; - auto it = bit_to_driver.find(bit); - if (it == bit_to_driver.end()) - bit_to_driver[bit] = c; - else if (it->second != c) - it->second = nullptr; - } - } - } - - for (auto w : module->wires()) { - if (!w->port_input) - continue; - for (auto bit : sigmap(SigSpec(w))) - if (bit.wire) - input_port_bits.insert(bit); - } - } - - // Combinational fanin cone of `from`. Leaves are port-input bits or bits - // driven by sequential cells / undriven. Returns false if size limits are - // exceeded. - bool get_cone(SigSpec from, pool &cone_cells, pool &leaf_bits, - int max_cone_cells, int max_leaf_bits) - { - pool visited; - std::queue worklist; - for (auto bit : sigmap(from)) { - if (!bit.wire) - continue; - if (visited.insert(bit).second) - worklist.push(bit); - } - - while (!worklist.empty()) { - SigBit bit = worklist.front(); - worklist.pop(); - - if (input_port_bits.count(bit)) { - leaf_bits.insert(bit); - if (GetSize(leaf_bits) > max_leaf_bits) - return false; - continue; - } - - Cell *drv = bit_to_driver.at(bit, nullptr); - if (drv == nullptr) { - leaf_bits.insert(bit); - if (GetSize(leaf_bits) > max_leaf_bits) - return false; - continue; - } - - if (!cone_cells.insert(drv).second) - continue; - if (GetSize(cone_cells) > max_cone_cells) - return false; - - for (auto &conn : drv->connections()) { - if (!drv->input(conn.first)) - continue; - for (auto in_bit : sigmap(conn.second)) { - if (!in_bit.wire) - continue; - if (visited.insert(in_bit).second) - worklist.push(in_bit); - } - } - } - - return true; } // Discover the per-lane index field within candidate bus `d_sig_in`. The bus @@ -243,106 +141,6 @@ struct OptPriorityOnehotWorker { return true; } - bool leaves_are_candidate_inputs(const pool &leaf_bits, const SigSpec &valid_sig, - const SigSpec &field_sig) - { - pool allowed; - for (auto bit : sigmap(valid_sig)) - if (bit.wire) - allowed.insert(bit); - for (auto bit : sigmap(field_sig)) - if (bit.wire) - allowed.insert(bit); - for (auto bit : leaf_bits) - if (!allowed.count(bit)) - return false; - return true; - } - - struct InputBus { - SigSpec sig; - std::string name; - int entries = 0; - int elem_width = 0; - }; - - // Parse a split-port name of the form "base[index]" (Verific lowers packed - // multi-dimensional ports into per-lane wires named this way). - bool parse_indexed_port_name(Wire *wire, std::string &base, int &index) - { - std::string name = wire->name.str(); - size_t rbrack = name.size(); - if (rbrack == 0 || name[rbrack - 1] != ']') - return false; - size_t lbrack = name.rfind('['); - if (lbrack == std::string::npos || lbrack + 1 >= rbrack - 1) - return false; - for (size_t i = lbrack + 1; i < rbrack - 1; i++) - if (!isdigit(name[i])) - return false; - base = name.substr(0, lbrack); - index = atoi(name.substr(lbrack + 1, rbrack - lbrack - 2).c_str()); - return true; - } - - // Group per-lane split-port wires into contiguous, equal-width buses. The - // run may start at any base index (Verific lowers both [N-1:0] -> id[0..N-1] - // and [N:1] -> id[1..N]); we only require consecutive indices and matching - // widths. The resulting sig is the ascending-index concatenation, so lane k - // is sig[k*elem_width ...] = the (base+k)-th port. - vector collect_split_input_buses(const vector &inputs) - { - std::map>> groups; - for (auto w : inputs) { - std::string base; - int index = -1; - if (parse_indexed_port_name(w, base, index)) - groups[base].push_back({index, w}); - } - - vector buses; - for (auto &it : groups) { - auto entries = it.second; - std::sort(entries.begin(), entries.end(), - [](const std::pair &a, const std::pair &b) { - return a.first < b.first; - }); - if (entries.empty()) - continue; - bool contiguous = true; - int base_index = entries.front().first; - int elem_width = GetSize(entries.front().second); - for (int i = 0; i < GetSize(entries); i++) { - if (entries[i].first != base_index + i || - GetSize(entries[i].second) != elem_width) { - contiguous = false; - break; - } - } - if (!contiguous) - continue; - - SigSpec sig; - for (auto &entry : entries) - sig.append(SigSpec(entry.second)); - buses.push_back({sig, it.first, GetSize(entries), elem_width}); - } - - return buses; - } - - bool find_anchor_driver(Wire *out_wire, Cell *&anchor) - { - for (auto bit : sigmap(SigSpec(out_wire))) { - Cell *drv = bit_to_driver.at(bit, nullptr); - if (drv != nullptr) { - anchor = drv; - return true; - } - } - return false; - } - vector make_test_vectors(int n, int w) { vector vs; @@ -459,7 +257,7 @@ struct OptPriorityOnehotWorker { bool fingerprint(const Candidate &cand, bool msb_first) { ConstEval ce(module); - SigSpec out_sig = sigmap(SigSpec(cand.out_wire)); + SigSpec out_sig = sigmap(cand.out_sig); SigSpec valid_sig = sigmap(cand.valid_sig); SigSpec field_sig = sigmap(cand.field_sig); @@ -488,18 +286,29 @@ struct OptPriorityOnehotWorker { return true; } - bool check_candidate(Candidate &cand, const pool &leaf_bits) + // The fingerprint drives each candidate bus bit independently, so the + // buses must consist of distinct non-constant bits. + bool candidate_inputs_disjoint(const Candidate &cand) + { + pool seen_bits; + for (auto bit : sigmap(cand.valid_sig)) + if (!bit.wire || !seen_bits.insert(bit).second) + return false; + for (auto bit : sigmap(cand.field_sig)) + if (!bit.wire || !seen_bits.insert(bit).second) + return false; + return true; + } + + bool check_candidate(Candidate &cand) { if (cand.n < min_width || cand.n > max_width) return false; if (!is_power_of_two(cand.w) || (1 << cand.idx_w) != cand.w) return false; - if (!leaves_are_candidate_inputs(leaf_bits, cand.valid_sig, cand.field_sig)) { - log_debug(" reject %s (valid=%s index=%s): cone leaves outside valid+field\n", - log_id(cand.out_wire), log_id(cand.valid_wire), cand.index_name.c_str()); + if (!candidate_inputs_disjoint(cand)) return false; - } - if (!find_anchor_driver(cand.out_wire, cand.anchor)) + if (!find_anchor_driver(cand.out_sig, cand.anchor)) return false; if (fingerprint(cand, false)) { cand.msb_first = false; @@ -510,7 +319,7 @@ struct OptPriorityOnehotWorker { return true; } log_debug(" reject %s (valid=%s index=%s): fingerprint mismatch (both directions)\n", - log_id(cand.out_wire), log_id(cand.valid_wire), cand.index_name.c_str()); + cand.out_name.c_str(), cand.valid_name.c_str(), cand.index_name.c_str()); return false; } @@ -596,39 +405,14 @@ struct OptPriorityOnehotWorker { return emit_decode(cand.anchor, root.index, root.valid, cand.w, cand.idx_w); } - void disconnect_old_output(const Candidate &cand) + vector collect_roots() { - pool target_bits; - for (auto bit : sigmap(SigSpec(cand.out_wire))) - if (bit.wire) - target_bits.insert(bit); - - pool seen_cells; - for (auto target : target_bits) { - Cell *drv = bit_to_driver.at(target, nullptr); - if (drv == nullptr || seen_cells.count(drv)) - continue; - seen_cells.insert(drv); - - for (auto &conn : drv->connections()) { - if (!drv->output(conn.first)) - continue; - SigSpec orig = conn.second; - SigSpec replacement = orig; - bool changed = false; - Cell *cell = drv; - Wire *dangling = module->addWire(NEW_ID2_SUFFIX("prionehot_dangling"), - GetSize(orig)); - for (int i = 0; i < GetSize(orig); i++) { - if (target_bits.count(sigmap(orig[i]))) { - replacement[i] = SigBit(dangling, i); - changed = true; - } - } - if (changed) - drv->setPort(conn.first, replacement); - } - } + int max_cone_cells = std::max(256, max_width * 96); + int max_leaf_bits = max_width * max_width + max_width * 64 + max_width; + return collect_root_candidates( + [&](int w) { return w >= 2 && is_power_of_two(w); }, + [&](const pool &) { return true; }, + true, max_cone_cells, max_leaf_bits, 64); } void run() @@ -637,97 +421,242 @@ struct OptPriorityOnehotWorker { return; vector inputs; - vector outputs; - for (auto w : module->wires()) { + for (auto w : module->wires()) if (w->port_input) inputs.push_back(w); - if (w->port_output && !w->port_input) - outputs.push_back(w); - } // Per-lane split-port buses ("id[0]".."id[N-1]") are grouped once. - vector split_buses = collect_split_input_buses(inputs); + vector split_buses = collect_split_buses(inputs); vector rewrites; - pool claimed_outputs; - for (auto out : outputs) { - if (claimed_outputs.count(out)) + vector root_list = collect_roots(); + for (int ri = 0; ri < GetSize(root_list); ri++) { + auto &root = root_list[ri]; + if (walk_exhausted() || eval_exhausted()) { + note_budget("opt_priority_onehot", GetSize(root_list) - ri); + break; + } + if (root_claimed(root.sig)) continue; - int w = GetSize(out); + + int w = GetSize(root.sig); if (w < 2 || !is_power_of_two(w)) continue; int idx_w = clog2_int(w); pool cone_cells; pool leaf_bits; + vector order; int max_cone_cells = std::max(256, max_width * 96); int max_leaf_bits = max_width * max_width + max_width * w + max_width; - if (!get_cone(SigSpec(out), cone_cells, leaf_bits, max_cone_cells, max_leaf_bits)) { - log_debug("output %s (w=%d): cone exceeds size limits\n", log_id(out), w); + if (!get_cone(root.sig, cone_cells, leaf_bits, max_cone_cells, max_leaf_bits, &order)) { + log_debug("root %s (w=%d): cone exceeds size limits\n", root.name.c_str(), w); continue; } if (cone_cells.empty()) continue; - log_debug("output %s (w=%d, idx_w=%d): cone %d cells, %d leaf bits\n", - log_id(out), w, idx_w, GetSize(cone_cells), GetSize(leaf_bits)); + log_debug("root %s (w=%d, idx_w=%d): cone %d cells, %d leaf bits\n", + root.name.c_str(), w, idx_w, GetSize(cone_cells), GetSize(leaf_bits)); - bool done = false; - for (auto valid : inputs) { - if (done) + // Bus candidates: module input ports first (cheap, original + // behavior), then internal cut signals from the root cone in + // reverse BFS order (deepest first) to support pre-logic between + // the ports and the rewritable region. + vector buses; + for (auto input : inputs) + buses.push_back({sigmap(SigSpec(input)), input->name.str(), 0, 0}); + for (auto &bus : split_buses) + buses.push_back(bus); + + // Internal cut buses are taken from the shallowest cone cells + // first (depth measured from the leaves), since the cut points + // introduced by pre-logic sit right above the module inputs. + const int max_internal_buses = 32; + int internal_buses = 0; + pool seen_buses; + for (auto &bus : buses) + seen_buses.insert(bus.sig); + dict depths = compute_cone_depths(cone_cells); + vector by_depth = order; + std::stable_sort(by_depth.begin(), by_depth.end(), + [&](Cell *a, Cell *b) { + return depths.at(a, 1 << 30) < depths.at(b, 1 << 30); + }); + for (auto c : by_depth) { + if (internal_buses >= max_internal_buses) break; - int n = GetSize(valid); - if (n < min_width || n > max_width) - continue; - - // Candidate index sources: a flat input bus split into n lanes, - // or a per-lane split-port bus with n entries. - vector> sources; - for (auto d : inputs) { - if (d == valid) + for (auto &conn : c->connections()) { + SigSpec sig = sigmap(conn.second); + if (GetSize(sig) < min_width || !seen_buses.insert(sig).second) continue; - if (GetSize(d) % n == 0) - sources.push_back({SigSpec(d), d->name.str()}); + if (!sig_bus_ok(sig)) + continue; + buses.push_back({sig, stringf("%s.%s", log_id(c->name), log_id(conn.first)), 0, 0}); + internal_buses++; } - for (auto &bus : split_buses) - if (bus.entries == n) - sources.push_back({bus.sig, bus.name}); + } - for (auto &src : sources) { - int total = GetSize(src.first); - if (total % n != 0) - continue; - int s = total / n; - if (s < idx_w) - continue; + // Whole wires touching the cone (outputs of cone cells or cone + // leaves, e.g. FF-driven valid/index buses); constant edge bits + // (never-written vector heads) are trimmed off as maximal runs. + // Whole wires touching the cone, longest runs first. + pool cone_sig_bits = cone_sig_bit_pool(cone_cells, leaf_bits); + for (auto &run : collect_wire_run_buses(cone_sig_bits, 64)) { + if (!seen_buses.insert(run.sig).second) + continue; + buses.push_back(run); + } - SigSpec field_sig; - if (!infer_field(src.first, n, idx_w, s, leaf_bits, field_sig)) { - log_debug(" valid=%s index=%s (n=%d, s=%d): field layout inference failed\n", - log_id(valid), src.second.c_str(), n, s); + // Per-lane split wires grouped into buses (array FFs and + // multi-dimensional nets are lowered into per-entry wires). + for (auto &bus : collect_cone_split_buses(cone_sig_bits)) { + SigSpec sig = sigmap(bus.sig); + if (seen_buses.insert(sig).second) + buses.push_back({sig, bus.name, bus.entries, bus.elem_width}); + } + + // Valid candidates: each bus as-is first (covers the simple + // cases cheaply), then one-lane-short windows (regions often use + // N of an N+1-entry vector, e.g. fits[N:1]). + vector valid_views; + for (auto &bus : buses) { + int nv = GetSize(bus.sig); + if (nv >= min_width && nv <= max_width) + valid_views.push_back(bus); + } + for (auto &bus : buses) { + int nv = GetSize(bus.sig); + if (nv - 1 >= min_width && nv - 1 <= max_width) { + valid_views.push_back({bus.sig.extract(0, nv - 1), + stringf("%s[%d:0]", bus.name.c_str(), nv - 2), 0, 0}); + valid_views.push_back({bus.sig.extract(1, nv - 1), + stringf("%s[%d:1]", bus.name.c_str(), nv - 1), 0, 0}); + } + } + // Power-of-two lane counts (the common region shape) and wider + // buses first, so the attempt budget is not exhausted on junk + // pairings in modules with many other wires. + std::stable_sort(valid_views.begin(), valid_views.end(), + [](const InputBus &a, const InputBus &b) { + int na = GetSize(a.sig), nb = GetSize(b.sig); + bool pa = is_power_of_two(na), pb = is_power_of_two(nb); + if (pa != pb) + return pa; + return na > nb; + }); + + // Closure walks are cheap graph traversals; the full candidate + // check (ConstEval fingerprint) gets its own smaller budget. + const int max_attempts = 2048; + const int max_fp_attempts = 24; + int attempts = 0; + int fp_attempts = 0; + bool done = false; + for (auto &valid : valid_views) { + if (done || attempts >= max_attempts || fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + int n = GetSize(valid.sig); + + pool valid_bits; + for (auto bit : sigmap(valid.sig)) + if (bit.wire) + valid_bits.insert(bit); + + for (auto &src : buses) { + if (done || attempts >= max_attempts || fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (src.sig == valid.sig) continue; + int total = GetSize(src.sig); + + // Index source views: the bus split into n lanes, or n + // contiguous lanes of an (n+1)-lane bus (e.g. ids[N:1] + // of an [N:0] table). + vector> src_views; + if (total % n == 0 && total / n >= idx_w) + src_views.push_back({src.sig, src.name}); + else if (total % (n + 1) == 0 && total / (n + 1) >= idx_w) { + int s2 = total / (n + 1); + src_views.push_back({src.sig.extract(0, n * s2), + stringf("%s[lane0+]", src.name.c_str())}); + src_views.push_back({src.sig.extract(s2, n * s2), + stringf("%s[lane1+]", src.name.c_str())}); } - Candidate cand; - cand.out_wire = out; - cand.valid_wire = valid; - cand.valid_sig = SigSpec(valid); - cand.field_sig = field_sig; - cand.index_name = src.second; - cand.n = n; - cand.w = w; - cand.idx_w = idx_w; - if (!check_candidate(cand, leaf_bits)) - continue; + for (auto &sv : src_views) { + if (done || attempts >= max_attempts || fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + int s = GetSize(sv.first) / n; - rewrites.push_back(cand); - claimed_outputs.insert(out); - log(" %s: %s <- priority_onehot(valid=%s, index=%s) " - "[N=%d, W=%d, IDX_W=%d, %s]\n", - log_id(module), log_id(out), log_id(valid), src.second.c_str(), - cand.n, cand.w, cand.idx_w, - cand.msb_first ? "MSB-first" : "LSB-first"); - done = true; - break; + // Cut the root cone at valid+index bits: it must close + // without reaching other inputs, and the index bits the + // cone actually uses define the per-lane field layout. + pool allowed = valid_bits; + for (auto bit : sigmap(sv.first)) + if (bit.wire) + allowed.insert(bit); + pool hit_bits; + pool cut_cells; + attempts++; + if (!cut_cone_walk(root.sig, allowed, GetSize(cone_cells) + 16, &hit_bits, &cut_cells, + nullptr, &leaf_bits, &cone_cells)) { + log_debug(" valid=%s index=%s: cut not closed (%s)\n", + valid.name.c_str(), sv.second.c_str(), last_cut_fail.c_str()); + continue; + } + + SigSpec field_sig; + if (!infer_field(sv.first, n, idx_w, s, hit_bits, field_sig)) { + log_debug(" valid=%s index=%s (n=%d, s=%d): field layout inference failed\n", + valid.name.c_str(), sv.second.c_str(), n, s); + continue; + } + + // The fingerprint forces every valid+field bit, and + // ConstEval caches whole cell outputs: no forced bit + // may be driven by a cell inside the cut cone, even + // one the cone never reads. + bool forced_conflict = false; + for (auto bit : sigmap(valid.sig)) { + Cell *drv = bit.wire ? bit_to_driver.at(bit, nullptr) : nullptr; + if (drv != nullptr && cut_cells.count(drv)) + forced_conflict = true; + } + for (auto bit : sigmap(field_sig)) { + Cell *drv = bit.wire ? bit_to_driver.at(bit, nullptr) : nullptr; + if (drv != nullptr && cut_cells.count(drv)) + forced_conflict = true; + } + if (forced_conflict) { + log_debug(" valid=%s index=%s: forced bit driven inside cone\n", + valid.name.c_str(), sv.second.c_str()); + continue; + } + + Candidate cand; + cand.out_sig = root.sig; + cand.out_name = root.name; + cand.valid_sig = valid.sig; + cand.field_sig = field_sig; + cand.valid_name = valid.name; + cand.index_name = sv.second; + cand.n = n; + cand.w = w; + cand.idx_w = idx_w; + fp_attempts++; + if (!check_candidate(cand)) + continue; + + rewrites.push_back(cand); + claim_region(root.sig, cut_cells); + log(" %s: %s <- priority_onehot(valid=%s, index=%s) " + "[N=%d, W=%d, IDX_W=%d, %s]\n", + log_id(module), cand.out_name.c_str(), valid.name.c_str(), sv.second.c_str(), + cand.n, cand.w, cand.idx_w, + cand.msb_first ? "MSB-first" : "LSB-first"); + done = true; + break; + } } } } @@ -735,8 +664,8 @@ struct OptPriorityOnehotWorker { for (auto &cand : rewrites) { cell = cand.anchor; SigSpec new_out = emit_priority_onehot(cand); - disconnect_old_output(cand); - module->connect(SigSpec(cand.out_wire), new_out); + disconnect_root(cand.out_sig, cand.anchor, "prionehot_dangling"); + module->connect(cand.out_sig, new_out); regions_rewritten++; } } @@ -779,6 +708,11 @@ struct OptPriorityOnehotPass : public Pass { log(" -min-width N, -min_width N\n"); log(" minimum lane count to consider (default 4).\n"); log("\n"); + log(" -walk-budget N, -eval-budget N, -attempt-budget N\n"); + log(" per-module work limits for the candidate search (defaults\n"); + log(" 20000000 / 20000000 / 65536). When a budget is exhausted\n"); + log(" the remaining candidates are skipped and a note is logged.\n"); + log("\n"); log("After rewriting, the original cone cells become unused and are removed\n"); log("by the trailing 'clean -purge'.\n"); log("\n"); @@ -790,6 +724,7 @@ struct OptPriorityOnehotPass : public Pass { int max_width = 64; int min_width = 4; + int64_t walk_budget = -1, eval_budget = -1, attempt_budget = -1; size_t argidx; for (argidx = 1; argidx < args.size(); argidx++) { if ((args[argidx] == "-max-width" || args[argidx] == "-max_width") && @@ -802,6 +737,21 @@ struct OptPriorityOnehotPass : public Pass { min_width = std::stoi(args[++argidx]); continue; } + if ((args[argidx] == "-walk-budget" || args[argidx] == "-walk_budget") && + argidx + 1 < args.size()) { + walk_budget = std::stoll(args[++argidx]); + continue; + } + if ((args[argidx] == "-eval-budget" || args[argidx] == "-eval_budget") && + argidx + 1 < args.size()) { + eval_budget = std::stoll(args[++argidx]); + continue; + } + if ((args[argidx] == "-attempt-budget" || args[argidx] == "-attempt_budget") && + argidx + 1 < args.size()) { + attempt_budget = std::stoll(args[++argidx]); + continue; + } break; } extra_args(args, argidx, design); @@ -812,6 +762,12 @@ struct OptPriorityOnehotPass : public Pass { OptPriorityOnehotWorker worker(module); worker.max_width = max_width; worker.min_width = min_width; + if (walk_budget > 0) + worker.walk_budget = walk_budget; + if (eval_budget > 0) + worker.eval_budget = eval_budget; + if (attempt_budget > 0) + worker.attempt_budget = attempt_budget; worker.run(); total_regions += worker.regions_rewritten; total_cells_added += worker.cells_added; diff --git a/passes/silimate/opt_compact_prefix.cc b/passes/silimate/opt_compact_prefix.cc index 59b8a62c4..f3651d6e2 100644 --- a/passes/silimate/opt_compact_prefix.cc +++ b/passes/silimate/opt_compact_prefix.cc @@ -20,6 +20,8 @@ #include "kernel/yosys.h" #include "kernel/sigtools.h" #include "kernel/consteval.h" +#include +#include USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN @@ -35,12 +37,28 @@ static int ceil_log2_int(int v) return r; } -struct OptCompactPrefixWorker +#include "passes/opt/cut_region.h" + +struct OptCompactPrefixWorker : CutRegionWorker { - Module *module; - SigMap sigmap; + // One matched compaction region ready to be rewritten. Kind 0 is the + // forward dense pack (a = packed input), kind 1 the reverse suffix read + // (a = disable, b = data), kind 2 the modulo decimation (a = en, b = n), + // kind 3 the forward gather/compress (a = en, b = data). + struct Rewrite { + int kind = 0; + SigSpec root; + std::string root_name; + SigSpec a, b; + std::string a_name, b_name; + int loop_width = 0; + bool msb_first = false; + int mod_offset = 0; + Cell *anchor = nullptr; + int cone_cells = 0; + }; + int max_width; - dict bit_drivers; Cell *ref_cell = nullptr; int forward_rewrites = 0; @@ -50,206 +68,232 @@ struct OptCompactPrefixWorker int new_cells_emitted = 0; OptCompactPrefixWorker(Module *module, int max_width) - : module(module), sigmap(module), max_width(max_width) + : CutRegionWorker(module), max_width(max_width) { - for (auto cell : module->cells()) { - for (auto &conn : cell->connections()) { - if (!cell->output(conn.first)) - continue; - for (auto bit : sigmap(conn.second)) - bit_drivers[bit] = cell; - } - } } - vector input_ports() + // Cut buses only need to consist of wire bits; FF outputs (cone leaves) + // are valid region boundaries even though they have no comb driver. + bool sig_bus_ok(const SigSpec &sig) { - vector ports; - for (auto wire : module->wires()) - if (wire->port_input) - ports.push_back(wire); - return ports; - } - - vector output_ports() - { - vector ports; - for (auto wire : module->wires()) - if (wire->port_output) - ports.push_back(wire); - return ports; - } - - int count_cells(IdString type) - { - int n = 0; - for (auto cell : module->cells()) - if (cell->type == type) - n++; - return n; - } - - bool sig_is_const_value(SigSpec sig, int64_t value) - { - sig = sigmap(sig); - if (!sig.is_fully_const()) - return false; - uint64_t uvalue = (uint64_t)value; - for (int i = 0; i < GetSize(sig); i++) { - bool want = (i < 64) ? ((uvalue >> i) & 1) : (value < 0); - if (sig[i] != (want ? State::S1 : State::S0)) + for (auto bit : sigmap(sig)) + if (!bit.wire) return false; - } return true; } - int count_binop_const(IdString type, int64_t value) + // Lane-ordered bus extraction: for unrolled scan loops the i-th output + // bit is produced at the i-th loop step, so a boundary signal that first + // influences root bit i (LSB-first scans; last, for MSB-first scans) is + // that step's per-lane input. This recovers enable buses that are + // permutations or replications of other signals and no longer exist as + // wires or cell connections after elaboration. + void lane_ordered_buses(const SigSpec &root, const pool &cone_cells, + vector &buses, pool &seen_buses) { - int n = 0; - for (auto cell : module->cells()) { - if (cell->type != type) - continue; - if (sig_is_const_value(cell->getPort(ID::A), value) || - sig_is_const_value(cell->getPort(ID::B), value)) - n++; - } - return n; - } + int width = GetSize(root); + if (width < 4 || width > 62) + return; - bool has_binop_const_other_than(IdString type, int64_t value) - { - for (auto cell : module->cells()) { - if (cell->type != type) - continue; - bool a_const = sigmap(cell->getPort(ID::A)).is_fully_const(); - bool b_const = sigmap(cell->getPort(ID::B)).is_fully_const(); - if (a_const && !sig_is_const_value(cell->getPort(ID::A), value)) - return true; - if (b_const && !sig_is_const_value(cell->getPort(ID::B), value)) - return true; - } - return false; - } + // Per root bit: boundary bits (not driven inside the cone) feed the + // combinational-lane reconstruction; all visited bits additionally + // feed the per-wire reconstruction (a bus may be an internal hub + // signal, e.g. a decoder output read through wiring permutations). + dict min_idx, max_idx; + dict all_min_idx, all_max_idx; + const int max_boundary = 8 * width + 64; + const int max_tracked = 256 * width + 1024; + bool track_all = true; + if (walk_exhausted()) + return; + for (int i = 0; i < width; i++) { + SigBit rb = sigmap(root[i]); + if (!rb.wire) + return; + pool visited; + std::queue wl; + visited.insert(rb); + wl.push(rb); + while (!wl.empty()) { + SigBit bit = wl.front(); + wl.pop(); + charge_walk(1); + Cell *drv = bit_to_driver.at(bit, nullptr); - int eval_bit_at_zero(SigBit bit, dict &cache, int depth) - { - bit = sigmap(bit); - if (bit == State::S0) return 0; - if (bit == State::S1) return 1; - if (!bit.wire) return 0; + if (track_all) { + if (all_min_idx.count(bit) == 0) { + all_min_idx[bit] = i; + all_max_idx[bit] = i; + } else { + all_min_idx[bit] = std::min(all_min_idx[bit], i); + all_max_idx[bit] = std::max(all_max_idx[bit], i); + } + if (GetSize(all_min_idx) > max_tracked) { + track_all = false; + all_min_idx.clear(); + all_max_idx.clear(); + } + } - auto it = cache.find(bit); - if (it != cache.end()) - return it->second; - if (depth > 64) - return 0; - - cache[bit] = 0; - Cell *drv = bit_drivers.at(bit, nullptr); - if (!drv || !drv->hasPort(ID::Y) || !drv->hasPort(ID::A)) - return 0; - - int bit_pos = -1; - SigSpec y = sigmap(drv->getPort(ID::Y)); - for (int i = 0; i < GetSize(y); i++) { - if (y[i] == bit) { - bit_pos = i; - break; + if (drv == nullptr || !cone_cells.count(drv)) { + if (min_idx.count(bit) == 0) { + min_idx[bit] = i; + max_idx[bit] = i; + } else { + min_idx[bit] = std::min(min_idx[bit], i); + max_idx[bit] = std::max(max_idx[bit], i); + } + continue; + } + for (auto &conn : drv->connections()) { + if (!drv->input(conn.first)) + continue; + for (auto in_bit : sigmap(conn.second)) { + if (!in_bit.wire) + continue; + if (visited.insert(in_bit).second) + wl.push(in_bit); + } + } } + if (GetSize(min_idx) > max_boundary) + return; } - if (bit_pos < 0) - return 0; - auto eval_sig = [&](SigSpec sig) -> int64_t { - int64_t result = 0; - for (int i = 0; i < GetSize(sig) && i < 62; i++) - result |= ((int64_t)eval_bit_at_zero(sig[i], cache, depth + 1) << i); - return result; + auto build = [&](const dict &idx_of, const char *tag) { + vector> lanes(width); + for (auto &it : idx_of) + lanes[it.second].push_back(it.first); + + // Every lane needs at least one bit; ambiguous lanes (shared + // control like the modulus also lands in the first lane) are + // expanded into a few alternative buses. + long combos = 1; + for (int i = 0; i < width; i++) { + if (lanes[i].empty()) + return; + combos *= GetSize(lanes[i]); + if (combos > 8) + return; + } + + vector cands(1, SigSpec()); + for (int i = 0; i < width; i++) { + vector next; + for (auto &prefix : cands) + for (auto bit : lanes[i]) { + SigSpec ext = prefix; + ext.append(bit); + next.push_back(ext); + } + cands.swap(next); + } + for (int c = 0; c < GetSize(cands); c++) + if (seen_buses.insert(cands[c]).second) + buses.push_back({cands[c], stringf("", tag, c)}); }; - int64_t av = eval_sig(drv->getPort(ID::A)); - int64_t bv = drv->hasPort(ID::B) ? eval_sig(drv->getPort(ID::B)) : 0; - int64_t rv = 0; - if (drv->type == ID($add)) rv = av + bv; - else if (drv->type == ID($sub)) rv = av - bv; - else if (drv->type == ID($and) || drv->type == ID($_AND_)) rv = av & bv; - else if (drv->type == ID($or) || drv->type == ID($_OR_)) rv = av | bv; - else if (drv->type == ID($xor) || drv->type == ID($_XOR_)) rv = av ^ bv; - else if (drv->type == ID($not) || drv->type == ID($_NOT_)) rv = ~av; - else if (drv->type == ID($logic_not)) rv = !av; - else if (drv->type == ID($reduce_or)) rv = av != 0; - else if (drv->type == ID($gt)) rv = av > bv; - else if (drv->type == ID($eq)) rv = av == bv; - else if (drv->type == ID($shl) || drv->type == ID($sshl)) rv = av << bv; - else if (drv->type == ID($shr) || drv->type == ID($sshr)) rv = av >> bv; - else if (drv->type == ID($mux)) { - int sv = eval_bit_at_zero(drv->getPort(ID::S)[0], cache, depth + 1); - rv = sv ? bv : av; - } + build(min_idx, "lsb"); + build(max_idx, "msb"); - int val = (rv >> bit_pos) & 1; - cache[bit] = val; - return val; + // Per-wire lane buses: one wire contributing exactly one bit per + // lane (in lane order) recovers buses that only exist as wiring + // permutations of another signal. + log_debug(" lanes: %d boundary bits, %d tracked bits (track_all=%d)\n", + GetSize(min_idx), GetSize(all_min_idx), track_all); + auto build_per_wire = [&](const dict &idx_of, const char *tag) { + dict>> by_wire; + for (auto &it : idx_of) { + auto &lanes = by_wire[it.first.wire]; + if (lanes.empty()) + lanes.resize(width); + lanes[it.second].push_back(it.first); + } + for (auto &it : by_wire) { + SigSpec bus; + bool ok = true; + int covered = 0; + for (int i = 0; i < width; i++) { + if (GetSize(it.second[i]) == 1) { + covered++; + if (ok) + bus.append(it.second[i][0]); + } else { + ok = false; + } + } + if (covered >= width - 2 && !ok) + log_debug(" wlanes-%s %s: %d/%d lanes covered\n", + tag, log_id(it.first->name), covered, width); + if (ok && seen_buses.insert(bus).second) + buses.push_back({bus, stringf("", tag, log_id(it.first->name))}); + } + }; + + build_per_wire(all_min_idx, "lsb"); + build_per_wire(all_max_idx, "msb"); } - bool eval_sig_is_zero(SigSpec sig) + // Learn the enable wiring of a gather given its data bus: with all + // enable candidates forced to 1 the gather is the identity, and zeroing + // one candidate u deletes exactly the scan positions gated by u (probed + // with one-hot data). Returns the enable bus in scan order. + bool learn_gather_enable(const SigSpec &root_sig, const SigSpec &data_sig, + const pool &en_cands, SigSpec &en_bus) { - dict cache; - for (auto bit : sigmap(sig)) - if (eval_bit_at_zero(bit, cache, 0) != 0) + int width = GetSize(root_sig); + vector cand_bits; + for (auto bit : en_cands) + cand_bits.push_back(bit); + + ConstEval ce(module); + SigSpec out_s = sigmap(root_sig); + SigSpec data_s = sigmap(data_sig); + SigSpec cand_s; + for (auto bit : cand_bits) + cand_s.append(bit); + int nc = GetSize(cand_bits); + + // Prefilter: all candidates enabled must give the identity. + uint64_t pattern = lowmask_u64(width) & 0xAAAAAAAAAAAAAAAAULL; + uint64_t actual; + if (!eval_with(ce, {{cand_s, const_u64(lowmask_u64(nc), nc)}, + {data_s, const_u64(pattern, width)}}, + out_s, actual)) + return false; + if (actual != pattern) + return false; + + // Gating matrix: position i is gated by candidate u iff zeroing u + // (alone) makes one-hot data at position i disappear. + vector gate_of(width, -1); + for (int u = 0; u < nc; u++) { + uint64_t en_val = lowmask_u64(nc) & ~(1ULL << u); + for (int i = 0; i < width; i++) { + if (!eval_with(ce, {{cand_s, const_u64(en_val, nc)}, + {data_s, const_u64(1ULL << i, width)}}, + out_s, actual)) + return false; + if (actual != 0) + continue; + if (gate_of[i] >= 0 && gate_of[i] != u) + return false; + gate_of[i] = u; + } + } + for (int i = 0; i < width; i++) + if (gate_of[i] < 0) return false; + + en_bus = SigSpec(); + for (int i = 0; i < width; i++) + en_bus.append(cand_bits[gate_of[i]]); return true; } - int64_t eval_sig_at_zero(SigSpec sig) - { - dict cache; - int64_t result = 0; - for (int i = 0; i < GetSize(sig) && i < 62; i++) - result |= ((int64_t)eval_bit_at_zero(sig[i], cache, 0) << i); - return result; - } - - int output_bits_controlled_by(Wire *control, Wire *output) - { - pool bits; - for (auto cell : module->cells()) { - if (cell->type != ID($mux)) - continue; - SigSpec sel = sigmap(cell->getPort(ID::S)); - if (GetSize(sel) != 1 || sel[0].wire != control) - continue; - for (auto bit : sigmap(cell->getPort(ID::Y))) - if (bit.wire == output) - bits.insert(bit.offset); - } - return GetSize(bits); - } - - bool indexed_reads_stay_in_range(Wire *data, int loop_width) - { - bool saw_data_read = false; - for (auto cell : module->cells()) { - if (!cell->type.in(ID($bmux), ID($shiftx))) - continue; - if (sigmap(cell->getPort(ID::A)) != sigmap(SigSpec(data))) - continue; - saw_data_read = true; - IdString select_port = cell->type == ID($bmux) ? ID::S : ID::B; - if (eval_sig_at_zero(cell->getPort(select_port)) >= loop_width) - return false; - } - return saw_data_read; - } - SigSpec zext(SigSpec sig, int width) { - sig = sigmap(sig); - if (GetSize(sig) > width) - return sig.extract(0, width); - if (GetSize(sig) < width) - sig.append(SigSpec(State::S0, width - GetSize(sig))); - return sig; + return zext_sig(sig, width); } SigSpec balanced_sum_rec(const vector &terms, int begin, int end, int width) @@ -341,159 +385,176 @@ struct OptCompactPrefixWorker return SigBit(out); } - static Const const_u64(uint64_t value, int width) + // Per-root estimate of the cut cone size, used to charge the shared + // eval budget for each fingerprint vector. + int64_t cur_cone_est = 64; + + bool eval_with(ConstEval &ce, const vector> &sets, + const SigSpec &out_sig, uint64_t &result) { - vector bits(width, State::S0); - for (int i = 0; i < width && i < 64; i++) - if ((value >> i) & 1ULL) - bits[i] = State::S1; - return Const(bits); + return CutRegionWorker::eval_with(ce, sets, out_sig, result, cur_cone_est); } - void remove_old_cells(const vector &old_cells) + vector make_value_set(int width) { - for (auto cell : old_cells) { - if (module->cell(cell->name) == nullptr) - continue; - module->remove(cell); - old_cells_removed++; + uint64_t full = lowmask_u64(width); + vector vals; + vals.push_back(0); + vals.push_back(full); + vals.push_back(full & 0x5555555555555555ULL); + vals.push_back(full & 0xAAAAAAAAAAAAAAAAULL); + for (int i = 0; i < width; i++) + vals.push_back(1ULL << i); + uint64_t lfsr = 0x1234567089abcdefULL; + for (int r = 0; r < 32; r++) { + lfsr ^= lfsr << 13; + lfsr ^= lfsr >> 7; + lfsr ^= lfsr << 17; + vals.push_back(lfsr & full); } + return vals; } - bool rewrite_forward_dense_pack() + // --- Forward dense pack: out = thermometer(popcount(in)). --- + + bool fingerprint_forward(const SigSpec &in_sig, const SigSpec &out_sig, int width) { - vector inputs = input_ports(); - vector outputs = output_ports(); - if (GetSize(inputs) != 1 || GetSize(outputs) != 1) - return false; - Wire *sig = inputs[0]; - Wire *sig2 = outputs[0]; - if (GetSize(sig) != GetSize(sig2)) - return false; - if (GetSize(sig) < 4 || GetSize(sig) > max_width) - return false; - if (GetSize(module->ports) != 2) - return false; - if (count_binop_const(ID($add), 1) < GetSize(sig) - 2) - return false; - if (count_cells(ID($mux)) < GetSize(sig)) - return false; - if (!eval_sig_is_zero(SigSpec(sig2))) + if (width < 4 || width > 62) return false; + ConstEval ce(module); + SigSpec in_s = sigmap(in_sig); + SigSpec out_s = sigmap(out_sig); - vector old_cells(module->cells().begin(), module->cells().end()); - ref_cell = old_cells.front(); - - int width = GetSize(sig); - int count_width = ceil_log2_int(width + 1); - vector bits; - for (int i = 0; i < width; i++) - bits.push_back(SigSpec(sigmap(SigBit(sig, i)))); - - SigSpec count = balanced_sum(bits, count_width); - SigSpec packed; - for (int i = 0; i < width; i++) - packed.append(emit_gt(count, i, count_width)); - - module->connect(SigSpec(sig2), packed); - remove_old_cells(old_cells); - - log(" Forward dense pack: %s -> %s, width=%d, count_width=%d.\n", - log_id(sig->name), log_id(sig2->name), width, count_width); - forward_rewrites++; - return true; - } - - bool rewrite_reverse_suffix_read() - { - vector inputs = input_ports(); - vector outputs = output_ports(); - if (GetSize(inputs) != 2 || GetSize(outputs) != 1) - return false; - Wire *mask = outputs[0]; - if (GetSize(inputs[0]) != GetSize(inputs[1]) || GetSize(mask) != GetSize(inputs[0])) - return false; - - int dec_count = std::max(count_binop_const(ID($sub), 1), - count_binop_const(ID($add), -1)); - int loop_width = dec_count + 1; - if (count_cells(ID($mux)) < loop_width) - return false; - if (has_binop_const_other_than(ID($sub), 1) || - has_binop_const_other_than(ID($add), -1)) - return false; - - Wire *disable = nullptr; - int controlled_width = 0; - for (auto input : inputs) { - int n = output_bits_controlled_by(input, mask); - if (n == 0) - continue; - if (disable != nullptr) + for (uint64_t v : make_value_set(width)) { + uint64_t actual; + if (!eval_with(ce, {{in_s, const_u64(v, width)}}, out_s, actual)) + return false; + uint64_t expected = lowmask_u64(__builtin_popcountll(v)); + if (actual != expected) return false; - disable = input; - controlled_width = n; } - if (disable == nullptr) - return false; - if (controlled_width > 0) - loop_width = controlled_width; - if (loop_width < 4 || loop_width > max_width || loop_width > GetSize(mask)) - return false; - if (dec_count < loop_width - 1) - return false; - Wire *data = (inputs[0] == disable) ? inputs[1] : inputs[0]; - - if (!indexed_reads_stay_in_range(data, loop_width)) - return false; - if (!eval_sig_is_zero(SigSpec(mask))) - return false; - - vector old_cells(module->cells().begin(), module->cells().end()); - ref_cell = old_cells.front(); - - int count_width = ceil_log2_int(loop_width + 1); - vector valid(loop_width); - for (int i = 0; i < loop_width; i++) - valid[i] = emit_not(sigmap(SigBit(disable, i))); - - SigSpec out_bits; - for (int j = 0; j < loop_width; j++) { - vector suffix_terms; - for (int k = j + 1; k < loop_width; k++) - suffix_terms.push_back(SigSpec(valid[k])); - SigSpec suffix_count = balanced_sum(suffix_terms, count_width); - - SigSpec candidates; - for (int k = 0; k < loop_width; k++) { - int needed_count = loop_width - 1 - k; - SigBit is_source = emit_eq(suffix_count, needed_count, count_width); - SigBit gated_data = emit_and(sigmap(SigBit(data, k)), is_source); - candidates.append(gated_data); - } - - SigBit selected = emit_reduce_or(candidates); - out_bits.append(emit_and(valid[j], selected)); - } - - if (GetSize(mask) > loop_width) - out_bits.append(SigSpec(State::S0, GetSize(mask) - loop_width)); - - module->connect(SigSpec(mask), out_bits); - remove_old_cells(old_cells); - - log(" Reverse suffix read: %s/%s -> %s, loop_width=%d, count_width=%d.\n", - log_id(disable->name), log_id(data->name), log_id(mask->name), - loop_width, count_width); - reverse_rewrites++; return true; } - // Reference function for the modulo-n decimation loop: scanning the enable - // vector from one end, mark every n-th enabled bit. Equivalent closed form: + // --- Reverse suffix read (MSB-first scan, cf. qor_spi_ra_sub_chain): + // indx = lw; for (I = lw; I > 0; I--) + // if (!disable[I-1]) { mask[I-1] = data[--indx]; } else mask[I-1] = 0; + // Only the low `lw` bits of disable/data participate; high mask bits are 0. + + // `en_high` selects the disable-bus polarity: false means a set bit + // disables the lane (the original form), true means a set bit enables it + // (frontends often push the inversion into the muxes). + uint64_t expected_reverse_mask(uint64_t dis, uint64_t data, int lw, bool en_high) + { + uint64_t mask = 0; + int indx = lw; + for (int I = lw; I > 0; I--) { + int i = I - 1; + bool enabled = (((dis >> i) & 1ULL) != 0) == en_high; + if (enabled) { + if ((data >> (indx - 1)) & 1ULL) + mask |= 1ULL << i; + indx--; + } + } + return mask; + } + + // The loop width is hidden state: probe with all lanes enabled and + // data=all-ones, which yields mask = lowmask(loop_width). The + // disable/data buses may be wider than the loop (e.g. 32-bit ports whose + // loop only touches [15:0]) or exactly loop-sized (when the frontend + // already narrowed pre-logic). + bool derive_reverse_loop_width(const SigSpec &dis_s, const SigSpec &data_s, + const SigSpec &out_s, int width, int &lw, bool &en_high) + { + int wd = GetSize(dis_s); + int wt = GetSize(data_s); + bool self = sigmap(dis_s) == sigmap(data_s); + ConstEval ce(module); + for (int pol = 0; pol < 2; pol++) { + // A self-pair (data == disable bus) can only be probed with all + // lanes enabled AND data all-ones, i.e. enable-high polarity. + if (self && pol == 0) + continue; + uint64_t all_enabled = pol ? lowmask_u64(wd) : 0; + vector> sets; + sets.push_back({dis_s, const_u64(all_enabled, wd)}); + if (!self) + sets.push_back({data_s, const_u64(lowmask_u64(wt), wt)}); + uint64_t actual; + if (!eval_with(ce, sets, out_s, actual)) + return false; + int n = __builtin_popcountll(actual); + if (n < 4 || n > width || n > wd || n > wt || actual != lowmask_u64(n)) + continue; + if (n > max_width || n > 62) + continue; + lw = n; + en_high = pol != 0; + return true; + } + return false; + } + + bool fingerprint_reverse(const SigSpec &dis_sig, const SigSpec &data_sig, + const SigSpec &out_sig, int width, int lw, bool en_high) + { + int wd = GetSize(dis_sig); + int wt = GetSize(data_sig); + if (width < 4 || width > 62 || lw < 4 || lw > width) + return false; + if (lw > wd || lw > wt || wd > 62 || wt > 62) + return false; + ConstEval ce(module); + SigSpec dis_s = sigmap(dis_sig); + SigSpec data_s = sigmap(data_sig); + SigSpec out_s = sigmap(out_sig); + bool self = dis_s == data_s; + + vector disvals = make_value_set(wd); + uint64_t full = lowmask_u64(wt); + vector datavals; + datavals.push_back(0); + datavals.push_back(full); + datavals.push_back(full & 0x5555555555555555ULL); + datavals.push_back(full & 0xAAAAAAAAAAAAAAAAULL); + uint64_t lfsr = 0xfedcba9876543210ULL; + for (int r = 0; r < 6; r++) { + lfsr ^= lfsr << 13; + lfsr ^= lfsr >> 7; + lfsr ^= lfsr << 17; + datavals.push_back(lfsr & full); + } + + for (uint64_t dv : disvals) + for (uint64_t tv : datavals) { + if (self) + tv = dv; + vector> sets; + sets.push_back({dis_s, const_u64(dv, wd)}); + if (!self) + sets.push_back({data_s, const_u64(tv, wt)}); + uint64_t actual; + if (!eval_with(ce, sets, out_s, actual)) + return false; + if (actual != expected_reverse_mask(dv, tv, lw, en_high)) + return false; + if (self) + break; // data values collapse onto the disable values + } + return true; + } + + // --- Modulo decimation: scanning the enable vector from one end, mark + // every n-th enabled bit. Equivalent closed form: // mask[I] = en[I] && n>0 && (inclusive_popcount(I) % n == 0) // where the popcount runs from the scan's leading end down to (incl.) I. - uint64_t expected_modulo_mask(uint64_t en, uint64_t n, int width, bool msb_first) + + // `offset` distinguishes "mark every n-th enabled bit" (offset 0, the + // counter resets when it hits n-1) from "mark the first of each group of + // n" (offset 1, the counter is checked against 0 before incrementing). + uint64_t expected_modulo_mask(uint64_t en, uint64_t n, int width, bool msb_first, int offset) { if (n == 0) return 0; @@ -508,24 +569,26 @@ struct OptCompactPrefixWorker else for (int k = 0; k <= i; k++) v += (en >> k) & 1ULL; - if (v % n == 0) + if ((v - offset) % n == 0) mask |= (1ULL << i); } return mask; } - // Confirm the combinational function from (en, n) to mask matches the modulo - // decimation reference for the given scan direction, using ConstEval over a - // structured + pseudo-random vector set (cf. opt_argmax's fingerprint). - bool fingerprint_modulo(Wire *en, Wire *n, Wire *mask, int width, bool msb_first) + bool fingerprint_modulo(const SigSpec &en_sig, const SigSpec &n_sig, + const SigSpec &mask_sig, int width, bool msb_first, int offset) { if (width <= 0 || width > 62) return false; + // A 1-bit modulus degenerates the function to (n ? en : 0), which + // also matches plain gating muxes; require a real counter range. + if (GetSize(n_sig) < 2) + return false; ConstEval ce(module); - SigSpec en_sig = sigmap(SigSpec(en)); - SigSpec n_sig = sigmap(SigSpec(n)); - SigSpec mask_sig = sigmap(SigSpec(mask)); - int nw = GetSize(n); + SigSpec en_s = sigmap(en_sig); + SigSpec n_s = sigmap(n_sig); + SigSpec mask_s = sigmap(mask_sig); + int nw = GetSize(n_sig); vector nvals; uint64_t nmax = (nw >= 64) ? ~0ULL : ((1ULL << nw) - 1); @@ -534,7 +597,7 @@ struct OptCompactPrefixWorker if (nmax > (uint64_t)width + 1) nvals.push_back(nmax); - uint64_t full = (width >= 64) ? ~0ULL : ((1ULL << width) - 1); + uint64_t full = lowmask_u64(width); vector envals; envals.push_back(0); envals.push_back(full); @@ -552,71 +615,188 @@ struct OptCompactPrefixWorker for (uint64_t nv : nvals) for (uint64_t ev : envals) { - ce.push(); - ce.set(en_sig, const_u64(ev, width)); - ce.set(n_sig, const_u64(nv, nw)); - SigSpec out = mask_sig; - SigSpec undef; - bool ok = ce.eval(out, undef); - ce.pop(); - if (!ok || !out.is_fully_const()) + uint64_t actual; + if (!eval_with(ce, {{en_s, const_u64(ev, width)}, + {n_s, const_u64(nv, nw)}}, + mask_s, actual)) return false; - Const cv = out.as_const(); - uint64_t actual = 0; - for (int i = 0; i < width && i < 64; i++) - if (cv[i] == State::S1) - actual |= (1ULL << i); - if (actual != expected_modulo_mask(ev, nv, width, msb_first)) + if (actual != expected_modulo_mask(ev, nv, width, msb_first, offset)) return false; } return true; } - bool rewrite_modulo_decimation() + // --- Forward gather/compress: out[k] = data[i_k] where i_k is the + // position of the (k+1)-th set bit of en; out bits beyond popcount(en) + // are zero. The forward dense pack is the data==en special case. + + uint64_t expected_gather_mask(uint64_t en, uint64_t data, int width) { - vector inputs = input_ports(); - vector outputs = output_ports(); - if (GetSize(inputs) != 2 || GetSize(outputs) != 1) - return false; - if (GetSize(module->ports) != 3) - return false; + uint64_t out = 0; + int indx = 0; + for (int i = 0; i < width; i++) + if ((en >> i) & 1ULL) { + if ((data >> i) & 1ULL) + out |= 1ULL << indx; + indx++; + } + return out; + } - Wire *mask = outputs[0]; - int width = GetSize(mask); - if (width < 4 || width > max_width) + // The en/data buses may contain repeated bits (e.g. an enable vector + // built by replicating a per-group mask bit across the group's lanes) or + // share bits with each other. The fingerprint drives the distinct + // underlying bits and expands each test value onto the bus patterns, so + // the function is verified exactly on the reachable input subspace. + bool fingerprint_gather(const SigSpec &en_sig, const SigSpec &data_sig, + const SigSpec &out_sig, int width) + { + if (width < 4 || width > 62) return false; + ConstEval ce(module); + SigSpec en_s = sigmap(en_sig); + SigSpec data_s = sigmap(data_sig); + SigSpec out_s = sigmap(out_sig); - // en matches the mask width; n (the modulus) is a distinct, narrower bus. - // Requiring different widths also disambiguates from the reverse suffix - // read form, whose two inputs share the mask width. - Wire *en = nullptr, *n = nullptr; - for (auto in : inputs) { - if (GetSize(in) == width && en == nullptr) - en = in; - else - n = in; + dict unique_idx; + SigSpec unique_sig; + vector en_map(width), data_map(width); + for (int i = 0; i < width; i++) { + auto it = unique_idx.find(en_s[i]); + if (it == unique_idx.end()) { + unique_idx[en_s[i]] = GetSize(unique_sig); + unique_sig.append(en_s[i]); + } + en_map[i] = unique_idx.at(en_s[i]); } - if (en == nullptr || n == nullptr || en == n || GetSize(n) == width) + for (int i = 0; i < width; i++) { + auto it = unique_idx.find(data_s[i]); + if (it == unique_idx.end()) { + unique_idx[data_s[i]] = GetSize(unique_sig); + unique_sig.append(data_s[i]); + } + data_map[i] = unique_idx.at(data_s[i]); + } + int nu = GetSize(unique_sig); + if (nu > 62) return false; - bool msb_first; - if (fingerprint_modulo(en, n, mask, width, true)) - msb_first = true; - else if (fingerprint_modulo(en, n, mask, width, false)) - msb_first = false; - else - return false; + auto expand = [&](uint64_t v, const vector &map) { + uint64_t r = 0; + for (int i = 0; i < width; i++) + if ((v >> map[i]) & 1ULL) + r |= 1ULL << i; + return r; + }; - vector old_cells(module->cells().begin(), module->cells().end()); - if (old_cells.empty()) - return false; - ref_cell = old_cells.front(); + for (uint64_t v : make_value_set(nu)) { + uint64_t actual; + if (!eval_with(ce, {{unique_sig, const_u64(v, nu)}}, out_s, actual)) + return false; + if (actual != expected_gather_mask(expand(v, en_map), expand(v, data_map), width)) + return false; + } + // A second value sweep decorrelates en from data when they share no + // bits; with shared bits it simply adds more reachable patterns. + uint64_t lfsr = 0x0f1e2d3c4b5a6978ULL; + for (int r = 0; r < 48; r++) { + lfsr ^= lfsr << 13; + lfsr ^= lfsr >> 7; + lfsr ^= lfsr << 17; + uint64_t v = lfsr & lowmask_u64(nu); + uint64_t actual; + if (!eval_with(ce, {{unique_sig, const_u64(v, nu)}}, out_s, actual)) + return false; + if (actual != expected_gather_mask(expand(v, en_map), expand(v, data_map), width)) + return false; + } + return true; + } + + // --- Emission --- + + void apply_forward(const Rewrite &rw) + { + ref_cell = rw.anchor; + int width = GetSize(rw.root); + int count_width = ceil_log2_int(width + 1); + SigSpec in_s = sigmap(rw.a); + + vector bits; + for (int i = 0; i < width; i++) + bits.push_back(SigSpec(in_s[i])); + + SigSpec count = balanced_sum(bits, count_width); + SigSpec packed; + for (int i = 0; i < width; i++) + packed.append(emit_gt(count, i, count_width)); + + disconnect_root(rw.root, rw.anchor, "compact_dangling"); + module->connect(rw.root, packed); + old_cells_removed += rw.cone_cells; + + log(" Forward dense pack: %s -> %s, width=%d, count_width=%d.\n", + rw.a_name.c_str(), rw.root_name.c_str(), width, count_width); + forward_rewrites++; + } + + void apply_reverse(const Rewrite &rw) + { + ref_cell = rw.anchor; + int width = GetSize(rw.root); + int loop_width = rw.loop_width; + bool en_high = rw.msb_first; // polarity flag (see matching loop) + int count_width = ceil_log2_int(loop_width + 1); + SigSpec dis_s = sigmap(rw.a); + SigSpec data_s = sigmap(rw.b); + + vector valid(loop_width); + for (int i = 0; i < loop_width; i++) + valid[i] = en_high ? dis_s[i] : emit_not(dis_s[i]); + + SigSpec out_bits; + for (int j = 0; j < loop_width; j++) { + vector suffix_terms; + for (int k = j + 1; k < loop_width; k++) + suffix_terms.push_back(SigSpec(valid[k])); + SigSpec suffix_count = balanced_sum(suffix_terms, count_width); + + SigSpec candidates; + for (int k = 0; k < loop_width; k++) { + int needed_count = loop_width - 1 - k; + SigBit is_source = emit_eq(suffix_count, needed_count, count_width); + SigBit gated_data = emit_and(data_s[k], is_source); + candidates.append(gated_data); + } + + SigBit selected = emit_reduce_or(candidates); + out_bits.append(emit_and(valid[j], selected)); + } + + if (width > loop_width) + out_bits.append(SigSpec(State::S0, width - loop_width)); + + disconnect_root(rw.root, rw.anchor, "compact_dangling"); + module->connect(rw.root, out_bits); + old_cells_removed += rw.cone_cells; + + log(" Reverse suffix read: %s/%s -> %s, loop_width=%d, count_width=%d.\n", + rw.a_name.c_str(), rw.b_name.c_str(), rw.root_name.c_str(), + loop_width, count_width); + reverse_rewrites++; + } + + void apply_modulo(const Rewrite &rw) + { + ref_cell = rw.anchor; + int width = GetSize(rw.root); + SigSpec en_s = sigmap(rw.a); + SigSpec n_s = sigmap(rw.b); + bool msb_first = rw.msb_first; int cnt_width = ceil_log2_int(width + 1); int table_size = 1 << cnt_width; - int cmp_width = std::max(GetSize(n), cnt_width); - - auto en_bit = [&](int i) { return sigmap(SigBit(en, i)); }; + int cmp_width = std::max(GetSize(n_s), cnt_width); // 1) Inclusive prefix popcount as a naive linear $add cascade. Each // running sum is consumed below, so the downstream opt_parallel_prefix @@ -626,11 +806,11 @@ struct OptCompactPrefixWorker int start = msb_first ? width - 1 : 0; int step = msb_first ? -1 : 1; int last = msb_first ? 0 : width - 1; - SigSpec acc = SigSpec(en_bit(start)); + SigSpec acc = SigSpec(en_s[start]); popcount[start] = zext(acc, cnt_width); for (int i = start + step; i != last + step; i += step) { Wire *sum = module->addWire(NEW_ID2_SUFFIX("compact_pop"), cnt_width); - module->addAdd(NEW_ID2_SUFFIX("compact_pop_add"), acc, SigSpec(en_bit(i)), sum); + module->addAdd(NEW_ID2_SUFFIX("compact_pop_add"), acc, SigSpec(en_s[i]), sum); new_cells_emitted++; acc = SigSpec(sum); popcount[i] = SigSpec(sum); @@ -639,16 +819,22 @@ struct OptCompactPrefixWorker // 2) Decode the modulus once: eq_d = (n == d) for d in [1..width]. vector eqd(width + 1, State::S0); for (int d = 1; d <= width; d++) - eqd[d] = emit_eq(SigSpec(n), d, cmp_width); + eqd[d] = emit_eq(n_s, d, cmp_width); - // 3) Divisibility per popcount value: div_k = OR_{d | k} eq_d. n>0 and - // n>width fall out for free (no divisor in range matches). + // 3) Divisibility per popcount value: div_k = OR_{d | k-offset} eq_d. + // n>0 and n>width fall out for free (no divisor in range matches), + // except k-offset == 0 which is divisible by any n > 0. vector divisible(table_size, State::S0); divisible[0] = State::S1; // gated away by en[]; defined to size the table for (int k = 1; k <= width; k++) { + int target = k - rw.mod_offset; + if (target == 0) { + divisible[k] = emit_reduce_or(n_s); + continue; + } SigSpec terms; - for (int d = 1; d <= k; d++) - if (k % d == 0) + for (int d = 1; d <= target; d++) + if (target % d == 0) terms.append(SigSpec(eqd[d])); divisible[k] = emit_reduce_or(terms); } @@ -661,28 +847,534 @@ struct OptCompactPrefixWorker SigSpec out_bits; for (int i = 0; i < width; i++) { SigBit sel_divisible = emit_bmux(table, popcount[i]); - out_bits.append(emit_and(en_bit(i), sel_divisible)); + out_bits.append(emit_and(en_s[i], sel_divisible)); } - module->connect(SigSpec(mask), out_bits); - remove_old_cells(old_cells); + disconnect_root(rw.root, rw.anchor, "compact_dangling"); + module->connect(rw.root, out_bits); + old_cells_removed += rw.cone_cells; - log(" Modulo decimation: en=%s, n=%s -> %s, width=%d, %s scan.\n", - log_id(en->name), log_id(n->name), log_id(mask->name), width, - msb_first ? "MSB-first" : "LSB-first"); + log(" Modulo decimation: en=%s, n=%s -> %s, width=%d, %s scan, offset=%d.\n", + rw.a_name.c_str(), rw.b_name.c_str(), rw.root_name.c_str(), width, + msb_first ? "MSB-first" : "LSB-first", rw.mod_offset); modulo_rewrites++; - return true; + } + + void apply_gather(const Rewrite &rw) + { + ref_cell = rw.anchor; + int width = GetSize(rw.root); + int cnt_width = ceil_log2_int(width + 1); + SigSpec en_s = sigmap(rw.a); + SigSpec data_s = sigmap(rw.b); + + // Exclusive prefix popcount as a naive linear $add cascade (rebuilt + // into a shared log-depth prefix network by opt_parallel_prefix). + vector prefix(width); + Cell *cell = ref_cell; + prefix[0] = SigSpec(Const(0, cnt_width)); + SigSpec acc = prefix[0]; + for (int i = 1; i < width; i++) { + Wire *sum = module->addWire(NEW_ID2_SUFFIX("compact_pre"), cnt_width); + module->addAdd(NEW_ID2_SUFFIX("compact_pre_add"), acc, SigSpec(en_s[i - 1]), sum); + new_cells_emitted++; + acc = SigSpec(sum); + prefix[i] = acc; + } + + SigSpec out_bits; + for (int k = 0; k < width; k++) { + SigSpec candidates; + for (int i = k; i < width; i++) { + SigBit is_source = emit_eq(prefix[i], k, cnt_width); + SigBit gated = emit_and(emit_and(en_s[i], data_s[i]), is_source); + candidates.append(gated); + } + out_bits.append(emit_reduce_or(candidates)); + } + + disconnect_root(rw.root, rw.anchor, "compact_dangling"); + module->connect(rw.root, out_bits); + old_cells_removed += rw.cone_cells; + + log(" Forward gather: en=%s, data=%s -> %s, width=%d, count_width=%d.\n", + rw.a_name.c_str(), rw.b_name.c_str(), rw.root_name.c_str(), width, cnt_width); + forward_rewrites++; + } + + // --- Candidate enumeration --- + + // Candidate root signals: module output ports and FF data inputs are + // seeds; internal signals inside seed cones are added so that regions + // wrapped in extra combinational post-logic are still found. + vector collect_roots() + { + int min_w = 4; + int max_w = std::min(max_width, 62); + return collect_root_candidates( + [&](int w) { return w >= min_w && w <= max_w; }, + [&](const pool &) { return true; }, + true, std::max(256, max_width * 96), max_width * (max_width + 8)); } void run() { if (module->has_processes_warn()) return; - if (rewrite_forward_dense_pack()) + + // All three forms come from unrolled loops with conditional updates; + // modules without muxes can be skipped cheaply. + bool has_mux = false; + for (auto c : module->cells()) + if (c->type.in(ID($mux), ID($pmux))) + has_mux = true; + if (!has_mux) return; - if (rewrite_reverse_suffix_read()) - return; - rewrite_modulo_decimation(); + + vector port_buses; + for (auto w : module->wires()) + if (w->port_input) + port_buses.push_back({sigmap(SigSpec(w)), w->name.str()}); + + vector rewrites; + pool claimed_bits; + vector root_list = collect_roots(); + for (int ri = 0; ri < GetSize(root_list); ri++) { + auto &root = root_list[ri]; + if (walk_exhausted() || eval_exhausted()) { + note_budget("opt_compact_prefix", GetSize(root_list) - ri); + break; + } + if (root_claimed(root.sig)) + continue; + + int width = GetSize(root.sig); + if (width < 4 || width > max_width || width > 62) + continue; + + pool cone_cells; + pool leaf_bits; + vector order; + int max_cone_cells = std::max(256, max_width * 96); + int max_leaf_bits = max_width * (max_width + 8); + if (!get_cone(root.sig, cone_cells, leaf_bits, max_cone_cells, max_leaf_bits, &order)) + continue; + if (cone_cells.empty()) + continue; + cur_cone_est = GetSize(cone_cells); + + Rewrite rw; + rw.root = root.sig; + rw.root_name = root.name; + rw.cone_cells = GetSize(cone_cells); + if (!find_anchor_driver(root.sig, rw.anchor)) + continue; + + // Bus candidates: module input ports first, then internal cut + // signals from the root cone ordered shallowest-first (depth + // measured from the leaves), since the cut points introduced by + // pre-logic sit right above the module inputs. + vector buses = port_buses; + const int max_internal_buses = 32; + int internal_buses = 0; + pool seen_buses; + for (auto &bus : port_buses) + seen_buses.insert(bus.sig); + dict depths = compute_cone_depths(cone_cells); + vector by_depth = order; + std::stable_sort(by_depth.begin(), by_depth.end(), + [&](Cell *a, Cell *b) { + return depths.at(a, 1 << 30) < depths.at(b, 1 << 30); + }); + for (auto c : by_depth) { + if (internal_buses >= max_internal_buses) + break; + for (auto &conn : c->connections()) { + SigSpec sig = sigmap(conn.second); + if (GetSize(sig) < 2 || !seen_buses.insert(sig).second) + continue; + if (!sig_bus_ok(sig)) + continue; + buses.push_back({sig, stringf("%s.%s", log_id(c->name), log_id(conn.first))}); + internal_buses++; + } + } + + // Whole wires touching the cone (FF-driven masks etc.), longest + // runs first, plus per-lane split wires grouped into buses + // (regions written bit-by-bit only expose their buses as named + // wires). + pool cone_sig_bits = cone_sig_bit_pool(cone_cells, leaf_bits); + for (auto &run : collect_wire_run_buses(cone_sig_bits, 64)) { + if (!seen_buses.insert(run.sig).second) + continue; + buses.push_back({run.sig, run.name}); + } + for (auto &bus : collect_cone_split_buses(cone_sig_bits)) { + SigSpec sig = sigmap(bus.sig); + if (seen_buses.insert(sig).second) + buses.push_back({sig, bus.name}); + } + + // Width-targeted windows of slightly wider buses (the loop may + // cover only part of a vector, e.g. actv[15:0] of a [W:0] bus). + { + vector windows; + for (auto &bus : buses) { + int sz = GetSize(bus.sig); + if (sz <= width || sz - width > 4) + continue; + for (int off = 0; off + width <= sz; off++) { + SigSpec slice = bus.sig.extract(off, width); + if (!seen_buses.insert(slice).second) + continue; + windows.push_back({slice, stringf("%s[%d+:%d]", bus.name.c_str(), off, width)}); + } + } + for (auto &wnd : windows) + buses.push_back(wnd); + } + + // Lane-ordered buses recovered from the per-root-bit cones. + lane_ordered_buses(root.sig, cone_cells, buses, seen_buses); + + // Buses overlapping the root would let the fingerprint force the + // output it is checking; drop them up front. + pool root_bits; + for (auto bit : root.sig) + if (bit.wire) + root_bits.insert(bit); + { + vector filtered; + for (auto &bus : buses) { + bool overlap = false; + for (auto bit : bus.sig) + if (root_bits.count(bit)) { + overlap = true; + break; + } + if (!overlap) + filtered.push_back(bus); + } + buses.swap(filtered); + } + + // Closure walks are cheap graph traversals; ConstEval fingerprints + // are the expensive step and get their own (smaller) budget. + const int max_closure_attempts = 768; + const int max_fp_attempts = 24; + int closure_attempts = 0; + int fp_attempts = 0; + bool matched = false; + pool matched_cone; + + log_debug("root %s (w=%d): cone %d cells, %d buses\n", + root.name.c_str(), width, GetSize(cone_cells), GetSize(buses)); + + // 1) Forward dense pack: single input bus of the same width. + for (auto &bus : buses) { + if (closure_attempts >= max_closure_attempts || fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (GetSize(bus.sig) != width) + continue; + pool allowed; + if (!sig_bits_unique(bus.sig, allowed)) + continue; + closure_attempts++; + if (!cut_cone_walk(root.sig, allowed, GetSize(cone_cells) + 16, nullptr, &matched_cone, + nullptr, &leaf_bits, &cone_cells)) { + log_debug(" fwd %s: cut not closed\n", bus.name.c_str()); + continue; + } + fp_attempts++; + if (!fingerprint_forward(bus.sig, root.sig, width)) { + log_debug(" fwd %s: fingerprint mismatch\n", bus.name.c_str()); + continue; + } + rw.kind = 0; + rw.a = bus.sig; + rw.a_name = bus.name; + matched = true; + break; + } + + // 1b) Forward gather/compress: en/data buses of the same width + // (the data==en special case is the forward pack above). + // Repeated or shared bits between the buses are allowed: the + // fingerprint drives the distinct underlying bits and checks + // the function on the reachable input subspace. + if (!matched) { + closure_attempts = 0; + fp_attempts = 0; + for (auto &en : buses) { + if (matched || closure_attempts >= max_closure_attempts || + fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (GetSize(en.sig) != width) + continue; + for (auto &data : buses) { + if (closure_attempts >= max_closure_attempts || + fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (GetSize(data.sig) != width || data.sig == en.sig) + continue; + bool en_const = false; + bool data_const = false; + pool allowed; + for (auto bit : sigmap(en.sig)) { + if (!bit.wire) + en_const = true; + else + allowed.insert(bit); + } + if (en_const) + break; + for (auto bit : sigmap(data.sig)) { + if (!bit.wire) + data_const = true; + else + allowed.insert(bit); + } + if (data_const) + continue; + closure_attempts++; + if (!cut_cone_walk(root.sig, allowed, GetSize(cone_cells) + 16, nullptr, &matched_cone, + nullptr, &leaf_bits, &cone_cells)) { + log_debug(" gat %s/%s: cut not closed\n", en.name.c_str(), data.name.c_str()); + continue; + } + // A real serial gather burns at least a mux and a + // counter increment per lane; small cones are + // degenerate matches (e.g. write-enable muxes) where + // the rewrite would only add area. + if (GetSize(matched_cone) < 2 * width) { + log_debug(" gat %s/%s: cone too small (%d cells)\n", + en.name.c_str(), data.name.c_str(), GetSize(matched_cone)); + continue; + } + fp_attempts++; + if (!fingerprint_gather(en.sig, data.sig, root.sig, width)) { + log_debug(" gat %s/%s: fingerprint mismatch\n", en.name.c_str(), data.name.c_str()); + continue; + } + rw.kind = 3; + rw.a = en.sig; + rw.a_name = en.name; + rw.b = data.sig; + rw.b_name = data.name; + matched = true; + break; + } + } + } + + // 1c) Forward gather with an implicit enable: the enable side of + // the gather may be arbitrary wiring (replication of a group + // mask, etc.) that exists neither as a wire nor as a cell + // connection. Once a data bus is known, the enable wiring is + // learned by probing and verified by the same fingerprint. + if (!matched) { + closure_attempts = 0; + fp_attempts = 0; + for (auto &data : buses) { + if (matched || closure_attempts >= max_closure_attempts || + fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (GetSize(data.sig) != width) + continue; + // The probe drives one-hot patterns onto the data bus, so + // its bits must be unique (the learned enable side is + // what handles replicated wiring). + bool data_const = false; + pool data_bits; + for (auto bit : sigmap(data.sig)) { + if (!bit.wire) + data_const = true; + else + data_bits.insert(bit); + } + if (data_const || GetSize(data_bits) != width) + continue; + + closure_attempts++; + pool en_cands; + if (!cut_cone_extra_leaves(root.sig, data_bits, GetSize(cone_cells) + 16, en_cands, 24)) + continue; + if (en_cands.empty()) + continue; + + // The closure check also guards against forced bits whose + // drivers sit inside the cone (ConstEval conflicts), so it + // must pass before any probing evaluates the cut. + pool allowed = data_bits; + for (auto bit : en_cands) + allowed.insert(bit); + if (!cut_cone_walk(root.sig, allowed, GetSize(cone_cells) + 16, nullptr, &matched_cone, + nullptr, &leaf_bits, &cone_cells)) { + log_debug(" gat /%s: cut not closed (%s)\n", data.name.c_str(), + last_cut_fail.c_str()); + continue; + } + // See the substantiality guard in the explicit gather + // loop above: tiny cones are degenerate matches. + if (GetSize(matched_cone) < 2 * width) { + log_debug(" gat /%s: cone too small (%d cells)\n", + data.name.c_str(), GetSize(matched_cone)); + continue; + } + + fp_attempts++; + SigSpec en_bus; + if (!learn_gather_enable(root.sig, data.sig, en_cands, en_bus)) { + log_debug(" gat /%s: enable probe failed\n", data.name.c_str()); + continue; + } + if (!fingerprint_gather(en_bus, data.sig, root.sig, width)) { + log_debug(" gat /%s: fingerprint mismatch\n", data.name.c_str()); + continue; + } + rw.kind = 3; + rw.a = en_bus; + rw.a_name = ""; + rw.b = data.sig; + rw.b_name = data.name; + matched = true; + break; + } + } + + // 2) Reverse suffix read: disable/data buses (the loop may cover + // only the low bits of wider buses, so widths are independent). + if (!matched) { + closure_attempts = 0; + fp_attempts = 0; + for (auto &dis : buses) { + if (matched || closure_attempts >= max_closure_attempts || + fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (GetSize(dis.sig) < 4 || GetSize(dis.sig) > 62) + continue; + for (auto &data : buses) { + if (closure_attempts >= max_closure_attempts || + fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (GetSize(data.sig) < 4 || GetSize(data.sig) > 62) + continue; + bool self = data.sig == dis.sig; + pool allowed; + if (!sig_bits_unique(dis.sig, allowed)) + break; + if (!self && !sig_bits_unique(data.sig, allowed)) + continue; + closure_attempts++; + if (!cut_cone_walk(root.sig, allowed, GetSize(cone_cells) + 16, nullptr, &matched_cone, + nullptr, &leaf_bits, &cone_cells)) { + log_debug(" rev %s/%s: cut not closed (%s)\n", dis.name.c_str(), data.name.c_str(), + last_cut_fail.c_str()); + continue; + } + fp_attempts++; + int lw = 0; + bool en_high = false; + if (!derive_reverse_loop_width(dis.sig, data.sig, root.sig, width, lw, en_high)) { + log_debug(" rev %s/%s: loop width probe failed\n", dis.name.c_str(), data.name.c_str()); + continue; + } + if (!fingerprint_reverse(dis.sig, data.sig, root.sig, width, lw, en_high)) { + log_debug(" rev %s/%s: fingerprint mismatch (lw=%d)\n", dis.name.c_str(), data.name.c_str(), lw); + continue; + } + rw.kind = 1; + rw.a = dis.sig; + rw.a_name = dis.name; + rw.b = data.sig; + rw.b_name = data.name; + rw.loop_width = lw; + rw.msb_first = en_high; // reused as the polarity flag for reverse + matched = true; + break; + } + } + } + + // 3) Modulo decimation: en bus of the same width plus a narrower + // modulus bus (the width mismatch also disambiguates from the + // reverse suffix read form). + if (!matched) { + closure_attempts = 0; + fp_attempts = 0; + for (auto &en : buses) { + if (matched || closure_attempts >= max_closure_attempts || + fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (GetSize(en.sig) != width) + continue; + for (auto &n : buses) { + if (closure_attempts >= max_closure_attempts || + fp_attempts >= max_fp_attempts || walk_exhausted() || eval_exhausted()) + break; + if (GetSize(n.sig) == width || GetSize(n.sig) > 62) + continue; + pool allowed; + if (!sig_bits_unique(en.sig, allowed)) + break; + if (!sig_bits_unique(n.sig, allowed)) + continue; + closure_attempts++; + if (!cut_cone_walk(root.sig, allowed, GetSize(cone_cells) + 16, nullptr, &matched_cone, + nullptr, &leaf_bits, &cone_cells)) { + log_debug(" mod %s/%s: cut not closed\n", en.name.c_str(), n.name.c_str()); + continue; + } + fp_attempts++; + bool msb_first = false; + int offset = -1; + if (fingerprint_modulo(en.sig, n.sig, root.sig, width, true, 0)) { + msb_first = true; + offset = 0; + } else if (fingerprint_modulo(en.sig, n.sig, root.sig, width, false, 0)) { + msb_first = false; + offset = 0; + } else if (fingerprint_modulo(en.sig, n.sig, root.sig, width, false, 1)) { + msb_first = false; + offset = 1; + } else if (fingerprint_modulo(en.sig, n.sig, root.sig, width, true, 1)) { + msb_first = true; + offset = 1; + } else { + log_debug(" mod %s/%s: fingerprint mismatch\n", en.name.c_str(), n.name.c_str()); + continue; + } + rw.kind = 2; + rw.a = en.sig; + rw.a_name = en.name; + rw.b = n.sig; + rw.b_name = n.name; + rw.msb_first = msb_first; + rw.mod_offset = offset; + matched = true; + break; + } + } + } + + if (matched) { + rw.cone_cells = GetSize(matched_cone); + rewrites.push_back(rw); + claim_region(root.sig, matched_cone); + } + } + + for (auto &rw : rewrites) { + if (rw.kind == 0) + apply_forward(rw); + else if (rw.kind == 1) + apply_reverse(rw); + else if (rw.kind == 2) + apply_modulo(rw); + else + apply_gather(rw); + } } }; @@ -705,6 +1397,11 @@ struct OptCompactPrefixPass : public Pass log("qor_spi_ra_sub_chain, and qor_spi_ra_add_chain2 regressions.\n"); log("Non-matching modules are left unchanged.\n"); log("\n"); + log("Matching is performed by functionally fingerprinting (ConstEval)\n"); + log("candidate regions between cut signals. Cut points include module\n"); + log("ports, FF data inputs, and internal signals, so the rewrites still\n"); + log("apply when extra combinational logic surrounds the compaction loop.\n"); + log("\n"); log("The modulo decimation form (mark every n-th enabled bit while scanning\n"); log("the enable vector) is lowered to a prefix-popcount plus divisor-decode\n"); log("divisibility check. The popcount is emitted as a plain linear $add\n"); @@ -714,6 +1411,11 @@ struct OptCompactPrefixPass : public Pass log(" -max_width \n"); log(" Maximum compaction width to rewrite. Default: 64.\n"); log("\n"); + log(" -walk-budget N, -eval-budget N, -attempt-budget N\n"); + log(" per-module work limits for the candidate search (defaults\n"); + log(" 20000000 / 20000000 / 65536). When a budget is exhausted\n"); + log(" the remaining candidates are skipped and a note is logged.\n"); + log("\n"); } void execute(std::vector args, RTLIL::Design *design) override @@ -721,12 +1423,28 @@ struct OptCompactPrefixPass : public Pass log_header(design, "Executing OPT_COMPACT_PREFIX pass (monotonic compaction rewrites).\n"); int max_width = 64; + int64_t walk_budget = -1, eval_budget = -1, attempt_budget = -1; size_t argidx; for (argidx = 1; argidx < args.size(); argidx++) { if (args[argidx] == "-max_width" && argidx + 1 < args.size()) { max_width = atoi(args[++argidx].c_str()); continue; } + if ((args[argidx] == "-walk-budget" || args[argidx] == "-walk_budget") && + argidx + 1 < args.size()) { + walk_budget = std::stoll(args[++argidx]); + continue; + } + if ((args[argidx] == "-eval-budget" || args[argidx] == "-eval_budget") && + argidx + 1 < args.size()) { + eval_budget = std::stoll(args[++argidx]); + continue; + } + if ((args[argidx] == "-attempt-budget" || args[argidx] == "-attempt_budget") && + argidx + 1 < args.size()) { + attempt_budget = std::stoll(args[++argidx]); + continue; + } break; } extra_args(args, argidx, design); @@ -739,6 +1457,12 @@ struct OptCompactPrefixPass : public Pass for (auto module : design->selected_modules()) { OptCompactPrefixWorker worker(module, max_width); + if (walk_budget > 0) + worker.walk_budget = walk_budget; + if (eval_budget > 0) + worker.eval_budget = eval_budget; + if (attempt_budget > 0) + worker.attempt_budget = attempt_budget; worker.run(); total_forward += worker.forward_rewrites; total_reverse += worker.reverse_rewrites;