Improve arith_tree: FMA add, elarith WIP.

kernel/compressor_tree.h

@@ -0,0 +1,332 @@
+/**
+ * Generalized compressor-tree utilities for multi-operand addition
+ *
+ * Terminology:
+ * - compressor: $fa viewed as reducing N inputs to M outputs (sum + shifted carry) (N:M compressor)
+ * - level:      A stage of parallel compression operations
+ * - depth:      Maximum number of N:M compressor levels from any input to a signal
+ *
+ * Supported compressors:
+ * - 3:2 compressor
+ * - 4:2 compressor
+ *
+ * References:
+ * - "Some schemes for parallel multipliers" (https://www.acsel-lab.com/arithmetic/arith6/papers/ARITH6_Dadda.pdf)
+ * - "Binary Adder Architectures for Cell-Based VLSI" (https://iis-people.ee.ethz.ch/~zimmi/publications/adder_arch.pdf)
+ * - "Basilisk: Achieving Competitive Performance with Open EDA Tools" (https://arxiv.org/pdf/2405.03523)
+ * - "Binary Adder Architectures for Cell-Based VLSI and their Synthesis" (https://iis-people.ee.ethz.ch/~zimmi/publications/adder_arch.pdf)
+ * - "A Suggestion for a Fast Multiplier" (https://www.ece.ucdavis.edu/~vojin/CLASSES/EEC280/Web-page/papers/Arithmetic/Wallace_mult.pdf)
+ */
+
+#ifndef COMPRESSOR_TREE_H
+#define COMPRESSOR_TREE_H
+
+#include "kernel/sigtools.h"
+#include "kernel/yosys.h"
+
+YOSYS_NAMESPACE_BEGIN
+
+namespace CompressorTree
+{
+
+// Width threshold below which a ripple is preferred over parallel-prefix
+constexpr int RIPPLE_PREFIX_THRESHOLD = 16;
+
+enum class Strategy {
+	FA_ONLY,   // 3:2 compressors
+	PREFER_42, // Prefer 4:2 grouping when >=4 operands ready
+	DADDA,     // Defer compression until column counts exceed
+};
+
+struct DepthSig {
+	SigSpec sig;
+	int depth;
+};
+
+enum class FinalAdder {
+	DEFAULT,         // emit $add and let downstream techmap pick
+	RIPPLE,          // emit $add with explicit narrow hint
+	PARALLEL_PREFIX, // emit $add with PARALLEL_PREFIX
+	ELARITH_FAST,    // black-box instance of \AddCfast
+	ELARITH_MOP_CSV, // black-box instance of \AddMopCsv
+};
+
+enum class FinalMode {
+	AUTO,
+	RIPPLE,
+	PREFIX,
+	ELARITH
+};
+
+inline std::pair<SigSpec, SigSpec> emit_compressor_32(Module *module, SigSpec a, SigSpec b, SigSpec c, int width)
+{
+	SigSpec sum = module->addWire(NEW_ID, width);
+	SigSpec cout = module->addWire(NEW_ID, width);
+	module->addFa(NEW_ID, a, b, c, cout, sum);
+
+	SigSpec carry;
+	carry.append(State::S0);
+	carry.append(cout.extract(0, width - 1));
+	return {sum, carry};
+}
+
+inline std::pair<SigSpec, SigSpec> emit_compressor_42(Module *module, SigSpec a, SigSpec b, SigSpec c, SigSpec d, int width)
+{
+	// First FA: a + b + c -> s0
+	SigSpec s0 = module->addWire(NEW_ID, width);
+	SigSpec cout_h_full = module->addWire(NEW_ID, width);
+	module->addFa(NEW_ID, a, b, c, cout_h_full, s0);
+
+	// cin[0] = 0, cin[i] = cout_h_full[i-1]
+	SigSpec cin;
+	cin.append(State::S0);
+	if (width > 1)
+		cin.append(cout_h_full.extract(0, width - 1));
+
+	// Second FA: s0 + d + cin -> sum
+	SigSpec sum = module->addWire(NEW_ID, width);
+	SigSpec carry_full = module->addWire(NEW_ID, width);
+	module->addFa(NEW_ID, s0, d, cin, carry_full, sum);
+
+	SigSpec carry;
+	carry.append(State::S0);
+	if (width > 1)
+		carry.append(carry_full.extract(0, width - 1));
+
+	return {sum, carry};
+}
+
+inline SigSpec normalize_to_width(SigSpec sig, bool is_signed, int width)
+{
+	// Zero/sign-extend to width
+	if (GetSize(sig) < width) {
+		SigBit pad;
+		if (is_signed && GetSize(sig) > 0)
+			pad = sig[GetSize(sig) - 1];
+		else
+			pad = State::S0;
+		sig.append(SigSpec(pad, width - GetSize(sig)));
+	}
+	// Truncate to width
+	if (GetSize(sig) > width)
+		sig = sig.extract(0, width);
+	return sig;
+}
+
+inline bool supports_signedness(bool a_signed, bool b_signed) {
+	return !(a_signed || b_signed);
+}
+
+/**
+ * generate_partial_products() - Generate partial products for FMA concat
+ * @module:The Yosys module to which the compressors will be added
+ * @a: Signal A
+ * @b: Signal B
+ * @a_signed: Whether signal A is signed
+ * @b_signed: Whether signal B is signed
+ * @width: Target width
+ *
+ * Return: Radix-2 partial product matrix as a set of depth-0 vectors
+ */
+inline std::vector<DepthSig> generate_partial_products(Module *module, SigSpec a, SigSpec b, bool a_signed, bool b_signed, int width) {
+	// TODO: Baugh-Wooley sign extension for mixed sign and sign*sign cases, don't bail out to non-FMA
+	log_assert(supports_signedness(a_signed, b_signed) && "CompressorTree::generate_partial_products: signed inputs unsupported");
+
+	int width_a = GetSize(a);
+	std::vector<DepthSig> products;
+	products.reserve(width_a);
+
+	for (int i = 0; i < width_a; i++) {
+		SigBit ai = a[i];
+
+		// b_shifted = (0_i ## b)
+		SigSpec b_shifted = SigSpec(State::S0, i);
+		b_shifted.append(b);
+		b_shifted = normalize_to_width(b_shifted, false, width);
+
+		// product = b_shifted & replicate(a[i], width)
+		SigSpec ai_rep = SigSpec(ai, width);
+		SigSpec product = module->addWire(NEW_ID, width);
+		module->addAnd(NEW_ID, b_shifted, ai_rep, product);
+
+		products.push_back({product, 0});
+	}
+
+	return products;
+}
+
+/**
+ * reduce_scheduled() - Reduce multiple operands to two using a compressor tree
+ * @module: The Yosys module to which the compressors will be added
+ * @operands: Vector of operands to be reduced
+ * @sigs: Vector of input signals (operands) to be reduced
+ * @width: Target bit-width to which all operands will be zero-extended
+ * @strategy: Compression strategy to use
+ * @compressor_count: Optional pointer to return the number of $fa cells emitted
+ *
+ * Return: The final two reduced operands, that are to be fed into an adder
+ */
+inline std::pair<SigSpec, SigSpec> reduce_scheduled(Module *module, std::vector<DepthSig> operands, int width, Strategy strategy, int *compressor_count = nullptr) {
+	int levels = 0;
+	int fa_count = 0;
+	int c42_count = 0;
+	int final_depth = 0;
+
+	for (auto &op : operands)
+		op.sig.extend_u0(width);
+
+	// Only compress operands ready at current level
+	for (int level = 0; operands.size() > 2; level++) {
+		// Partition operands into ready and waiting
+		std::vector<DepthSig> ready;
+		std::vector<DepthSig> waiting;
+		ready.reserve(operands.size());
+		for (auto &op : operands) {
+			if (op.depth <= level)
+				ready.push_back(op);
+			else
+				waiting.push_back(op);
+		}
+
+		if (ready.size() < 3) {
+			levels++;
+			continue;
+		}
+
+		// Apply compressors to ready operands
+		std::vector<DepthSig> compressed;
+		compressed.reserve(ready.size());
+		size_t i = 0;
+
+		// PREFER_42 attempts 4:2 grouping greedily (falls back to 3:2 for the residual)
+		// FA_ONLY skips
+		// DADDA = PREFER_42 (TODO: inspect column heights?)
+		bool try_42 = (strategy == Strategy::PREFER_42 || strategy == Strategy::DADDA);
+
+		while (i < ready.size()) {
+			size_t remaining = ready.size() - i;
+
+			if (try_42 && remaining >= 4) {
+				DepthSig a = ready[i + 0];
+				DepthSig b = ready[i + 1];
+				DepthSig c = ready[i + 2];
+				DepthSig d = ready[i + 3];
+
+				auto [sum, carry] = emit_compressor_42(module, a.sig, b.sig, c.sig, d.sig, width);
+				int dmax = std::max({a.depth, a.depth, a.depth, a.depth});
+
+				compressed.push_back({sum, dmax + 2});
+				compressed.push_back({carry, dmax + 2});
+
+				fa_count += 2;
+				c42_count += 1;
+				i += 4;
+			} else if (remaining >= 3) {
+				DepthSig a = ready[i + 0];
+				DepthSig b = ready[i + 1];
+				DepthSig c = ready[i + 2];
+
+				auto [sum, carry] = emit_compressor_32(module, a.sig, b.sig, c.sig, width);
+				int dmax = std::max({a.depth, b.depth, c.depth});
+
+				compressed.push_back({sum, dmax + 1});
+				compressed.push_back({carry, dmax + 1});
+
+				fa_count += 1;
+				i += 3;
+			} else {
+				// Uncompressed operands pass through to next level
+				for (; i < ready.size(); i++)
+					compressed.push_back(ready[i]);
+				break;
+			}
+		}
+
+		// Merge compressed with waiting operands
+		for (auto &op : waiting)
+			compressed.push_back(op);
+
+		operands = std::move(compressed);
+		levels++;
+	}
+
+	if(compressor_count)
+		*compressor_count = fa_count;
+
+	if (operands.size() == 0)
+		return {SigSpec(State::S0, width), SigSpec(State::S0, width)};
+	if (operands.size() == 1)
+		return {operands[0].sig, SigSpec(State::S0, width)};
+
+	final_depth = std::max(operands[0].depth, operands[1].depth);
+	log_assert(operands.size() == 2);
+	log("    CompressorTree::reduce_scheduled: %d levels, %d $fa (%d as 4:2), final depth %d\n", levels, fa_count, c42_count, final_depth);
+	return {operands[0].sig, operands[1].sig};
+}
+
+/**
+ * emit_final_adder() - Emit the final carry-propagate addition between the two reduced vectors
+ * @module:The Yosys module to which the compressors will be added
+ * @a: Signal A
+ * @b: Signal B
+ * @y: Signal Y
+ * @choice: Adder type to instantiate
+ * @any_signed: Signed info for library macros
+ *
+ * Return: Cell* of the emitted instance
+ */
+inline Cell *emit_final_adder(Module *module, SigSpec a, SigSpec b, SigSpec y, FinalAdder choice, bool any_signed) {
+	switch (choice) {
+		case FinalAdder::DEFAULT:
+		case FinalAdder::RIPPLE: {
+			return module->addAdd(NEW_ID, a, b, y, false);
+		}
+		case FinalAdder::PARALLEL_PREFIX: {
+			Cell *c = module->addAdd(NEW_ID, a, b, y,false);
+			c->set_string_attribute(ID(adder_arch), "parallel_prefix");
+			return c;
+		}
+		case FinalAdder::ELARITH_FAST: {
+			Cell *c = module->addCell(NEW_ID, IdString("\\AddCfast"));
+			int w = GetSize(y);
+			c->setParam(IdString("\\WIDTH"), w);
+			c->setParam(IdString("\\SPEED"), Const("fast"));
+			c->setParam(IdString("\\SIGNED"), any_signed ? 1 : 0);
+			c->setPort(IdString("\\A"), a);
+			c->setPort(IdString("\\B"), b);
+			c->setPort(IdString("\\Cin"), State::S0);
+			c->setPort(IdString("\\Sum"), y);
+			c->setPort(IdString("\\Cout"), module->addWire(NEW_ID));
+			return c;
+		}
+		case FinalAdder::ELARITH_MOP_CSV: {
+			Cell *c = module->addCell(NEW_ID, IdString("\\AddMopCsv"));
+			int w = GetSize(y);
+			c->setParam(IdString("\\WIDTH"), w);
+			c->setParam(IdString("\\NUM_OPERANDS"), 2);
+			c->setParam(IdString("\\SIGNED"), any_signed ? 1 : 0);
+			c->setParam(IdString("\\SPEED"), Const("fast"));
+			c->setPort(IdString("\\Operands"), {a, b});
+			c->setPort(IdString("\\Sum"), y);
+			return c;
+		}
+	}
+	log_assert(false && "CompressorTree::emit_final_adder: invalid choice");
+	return nullptr;
+}
+
+inline FinalAdder pick_final_adder(int width, FinalMode mode) {
+	switch (mode) {
+		case FinalMode::RIPPLE:  return FinalAdder::RIPPLE;
+		case FinalMode::PREFIX:  return FinalAdder::PARALLEL_PREFIX;
+		case FinalMode::ELARITH: return FinalAdder::ELARITH_FAST;
+		case FinalMode::AUTO:
+		default:                 return (width < RIPPLE_PREFIX_THRESHOLD) ? FinalAdder::DEFAULT : FinalAdder::PARALLEL_PREFIX;
+	}
+}
+
+} // namespace CompressorTree
+
+YOSYS_NAMESPACE_END
+
+#endif // COMPRESSOR_TREE_H

kernel/wallace_tree.h

@@ -1,112 +0,0 @@
-/**
- * Wallace tree utilities for multi-operand addition using carry-save adders
- *
- * Terminology:
- * - compressor: $fa viewed as reducing 3 inputs to 2 outputs (sum + shifted carry) (3:2 compressor)
- * - level:      A stage of parallel compression operations
- * - depth:      Maximum number of 3:2 compressor levels from any input to a signal
- *
- * References:
- * - "Binary Adder Architectures for Cell-Based VLSI and their Synthesis" (https://iis-people.ee.ethz.ch/~zimmi/publications/adder_arch.pdf)
- * - "A Suggestion for a Fast Multiplier" (https://www.ece.ucdavis.edu/~vojin/CLASSES/EEC280/Web-page/papers/Arithmetic/Wallace_mult.pdf)
- */
-
-#ifndef WALLACE_TREE_H
-#define WALLACE_TREE_H
-
-#include "kernel/sigtools.h"
-#include "kernel/yosys.h"
-
-YOSYS_NAMESPACE_BEGIN
-
-inline std::pair<SigSpec, SigSpec> emit_fa(Module *module, SigSpec a, SigSpec b, SigSpec c, int width)
-{
-	SigSpec sum = module->addWire(NEW_ID, width);
-	SigSpec cout = module->addWire(NEW_ID, width);
-
-	module->addFa(NEW_ID, a, b, c, cout, sum);
-
-	SigSpec carry;
-	carry.append(State::S0);
-	carry.append(cout.extract(0, width - 1));
-	return {sum, carry};
-}
-
-/**
- * wallace_reduce_scheduled() - Reduce multiple operands to two using a Wallace tree
- * @module: The Yosys module to which the compressors will be added
- * @sigs: Vector of input signals (operands) to be reduced
- * @width: Target bit-width to which all operands will be zero-extended
- * @compressor_count: Optional pointer to return the number of $fa cells emitted
- *
- * Return: The final two reduced operands, that are to be fed into an adder
- */
-inline std::pair<SigSpec, SigSpec> wallace_reduce_scheduled(Module *module, std::vector<SigSpec> &sigs, int width, int *compressor_count = nullptr)
-{
-	struct DepthSig {
-		SigSpec sig;
-		int depth;
-	};
-
-	for (auto &s : sigs)
-		s.extend_u0(width);
-
-	std::vector<DepthSig> operands;
-	operands.reserve(sigs.size());
-	for (auto &s : sigs)
-		operands.push_back({s, 0});
-
-	// Number of $fa's emitted
-	if (compressor_count)
-		*compressor_count = 0;
-
-	// Only compress operands ready at current level
-	for (int level = 0; operands.size() > 2; level++) {
-		// Partition operands into ready and waiting
-		std::vector<DepthSig> ready, waiting;
-		for (auto &op : operands) {
-			if (op.depth <= level)
-				ready.push_back(op);
-			else
-				waiting.push_back(op);
-		}
-
-		if (ready.size() < 3)
-			continue;
-
-		// Apply compressors to ready operands
-		std::vector<DepthSig> compressed;
-		size_t i = 0;
-		while (i + 2 < ready.size()) {
-			auto [sum, carry] = emit_fa(module, ready[i].sig, ready[i + 1].sig, ready[i + 2].sig, width);
-			int new_depth = std::max({ready[i].depth, ready[i + 1].depth, ready[i + 2].depth}) + 1;
-			compressed.push_back({sum, new_depth});
-			compressed.push_back({carry, new_depth});
-			if (compressor_count)
-				(*compressor_count)++;
-			i += 3;
-		}
-		// Uncompressed operands pass through to next level
-		for (; i < ready.size(); i++)
-			compressed.push_back(ready[i]);
-		// Merge compressed with waiting operands
-		for (auto &op : waiting)
-			compressed.push_back(op);
-
-		operands = std::move(compressed);
-	}
-
-	if (operands.size() == 0)
-		return {SigSpec(State::S0, width), SigSpec(State::S0, width)};
-	else if (operands.size() == 1)
-		return {operands[0].sig, SigSpec(State::S0, width)};
-	else {
-		log_assert(operands.size() == 2);
-		log("    Wallace tree depth: %d levels of $fa + 1 final $add\n", std::max(operands[0].depth, operands[1].depth));
-		return {operands[0].sig, operands[1].sig};
-	}
-}
-
-YOSYS_NAMESPACE_END
-
-#endif

passes/techmap/arith_tree.cc

@@ -1,5 +1,5 @@
 /**
- * Replaces chains of $add/$sub and $macc cells with carry-save adder trees
+ * Replaces chains of $add/$sub/$alu and $macc cells with carry-save compression trees
  *
  * Terminology:
  * - parent:    Cells that consume another cell's output
@@ -7,9 +7,9 @@
  * - chain:     Connected path of chainable cells
  */
 
+#include "kernel/compressor_tree.h"
 #include "kernel/macc.h"
 #include "kernel/sigtools.h"
-#include "kernel/wallace_tree.h"
 #include "kernel/yosys.h"
 
 #include <queue>
@@ -17,49 +17,58 @@
 USING_YOSYS_NAMESPACE
 PRIVATE_NAMESPACE_BEGIN
 
-struct Operand {
+struct ArithTreeOptions {
-	SigSpec sig;
+	CompressorTree::Strategy strategy = CompressorTree::Strategy::PREFER_42;
-	bool is_signed;
+	CompressorTree::FinalMode final_mode = CompressorTree::FinalMode::AUTO;
-	bool negate;
+	bool fma_fusion = true;
+	bool elarith_macro = false;
 };
 
-struct Traversal {
+struct ArithTreeWorker {
+	const ArithTreeOptions &opt;
+	Module *module;
 	SigMap sigmap;
+
 	dict<SigBit, pool<Cell *>> bit_consumers;
 	dict<SigBit, int> fanout;
-	Traversal(Module *module) : sigmap(module)
-	{
-		for (auto cell : module->cells())
-			for (auto &conn : cell->connections())
-				if (cell->input(conn.first))
-					for (auto bit : sigmap(conn.second))
-						bit_consumers[bit].insert(cell);
 
-		for (auto &pair : bit_consumers)
+	pool<Cell *> addsub;
-			fanout[pair.first] = pair.second.size();
+	pool<Cell *> alu;
+	pool<Cell *> macc;
+
+	struct Operand {
+		SigSpec sig;
+		bool is_signed;
+		bool negate;
+		// With FMA, when both factors are set, the operand represents a product to
+		// be expanded into partial products at extraction time, is_signed then
+		// applies to factor_a, and factor_b carries its own signedness
+		SigSpec factor_b; // empty for regular operands
+		bool factor_b_signed = false;
+	};
+
+	ArithTreeWorker(const ArithTreeOptions &opt, Module *module) : opt(opt), module(module), sigmap(module)
+	{
+		// Build traversal data
+		for (auto cell : module->cells()) {
+			for (auto &[name, sig] : cell->connections()) {
+				if (cell->input(name)) {
+					for (auto bit : sigmap(sig)) {
+						bit_consumers[bit].insert(cell);
+					}
+				}
+			}
+		}
+
+		for (auto &[sig, consumers] : bit_consumers)
+			fanout[sig] = consumers.size();
 
 		for (auto wire : module->wires())
 			if (wire->port_output)
 				for (auto bit : sigmap(SigSpec(wire)))
 					fanout[bit]++;
-	}
-};
 
-struct Cells {
+		// Collect cell data
-	pool<Cell *> addsub;
-	pool<Cell *> alu;
-	pool<Cell *> macc;
-
-	static bool is_addsub(Cell *cell) { return cell->type == ID($add) || cell->type == ID($sub); }
-
-	static bool is_alu(Cell *cell) { return cell->type == ID($alu); }
-
-	static bool is_macc(Cell *cell) { return cell->type == ID($macc) || cell->type == ID($macc_v2); }
-
-	bool empty() { return addsub.empty() && alu.empty() && macc.empty(); }
-
-	Cells(Module *module)
-	{
 		for (auto cell : module->cells()) {
 			if (is_addsub(cell))
 				addsub.insert(cell);
@@ -69,59 +78,55 @@ struct Cells {
 				macc.insert(cell);
 		}
 	}
-};
 
-struct AluInfo {
+	bool is_addsub(Cell *cell) {
-	Cells &cells;
+		return cell->type == ID($add) || cell->type == ID($sub);
-	Traversal &traversal;
+	}
-	bool is_subtract(Cell *cell)
+
-	{
+	bool is_alu(Cell *cell) {
-		SigSpec bi = traversal.sigmap(cell->getPort(ID::BI));
+		return cell->type == ID($alu);
-		SigSpec ci = traversal.sigmap(cell->getPort(ID::CI));
+	}
+
+	bool is_macc(Cell *cell) {
+		return cell->type == ID($macc) || cell->type == ID($macc_v2);
+	}
+
+	bool is_sub(Cell *cell) {
+		SigSpec bi = sigmap(cell->getPort(ID::BI));
+		SigSpec ci = sigmap(cell->getPort(ID::CI));
 		return GetSize(bi) == 1 && bi[0] == State::S1 && GetSize(ci) == 1 && ci[0] == State::S1;
 	}
 
 	bool is_add(Cell *cell)
 	{
-		SigSpec bi = traversal.sigmap(cell->getPort(ID::BI));
+		SigSpec bi = sigmap(cell->getPort(ID::BI));
-		SigSpec ci = traversal.sigmap(cell->getPort(ID::CI));
+		SigSpec ci = sigmap(cell->getPort(ID::CI));
 		return GetSize(bi) == 1 && bi[0] == State::S0 && GetSize(ci) == 1 && ci[0] == State::S0;
 	}
 
 	bool is_chainable(Cell *cell)
 	{
-		if (!(is_add(cell) || is_subtract(cell)))
+		if (!(is_add(cell) || is_sub(cell)))
 			return false;
-
+		for (auto bit : sigmap(cell->getPort(ID::X)))
-		for (auto bit : traversal.sigmap(cell->getPort(ID::X)))
+			if (fanout.count(bit) && fanout[bit] > 0)
-			if (traversal.fanout.count(bit) && traversal.fanout[bit] > 0)
 				return false;
-		for (auto bit : traversal.sigmap(cell->getPort(ID::CO)))
+		for (auto bit : sigmap(cell->getPort(ID::CO)))
-			if (traversal.fanout.count(bit) && traversal.fanout[bit] > 0)
+			if (fanout.count(bit) && fanout[bit] > 0)
 				return false;
-
 		return true;
 	}
-};
-
-struct Rewriter {
-	Module *module;
-	Cells &cells;
-	Traversal traversal;
-	AluInfo alu_info;
-
-	Rewriter(Module *module, Cells &cells) : module(module), cells(cells), traversal(module), alu_info{cells, traversal} {}
 
 	Cell *sole_chainable_consumer(SigSpec sig, const pool<Cell *> &candidates)
 	{
 		Cell *consumer = nullptr;
 		for (auto bit : sig) {
-			if (!traversal.fanout.count(bit) || traversal.fanout[bit] != 1)
+			if (!fanout.count(bit) || fanout[bit] != 1)
 				return nullptr;
-			if (!traversal.bit_consumers.count(bit) || traversal.bit_consumers[bit].size() != 1)
+			if (!bit_consumers.count(bit) || bit_consumers[bit].size() != 1)
 				return nullptr;
 
-			Cell *c = *traversal.bit_consumers[bit].begin();
+			Cell *c = *bit_consumers[bit].begin();
 			if (!candidates.count(c))
 				return nullptr;
 
@@ -137,7 +142,7 @@ struct Rewriter {
 	{
 		dict<Cell *, Cell *> parent_of;
 		for (auto cell : candidates) {
-			Cell *consumer = sole_chainable_consumer(traversal.sigmap(cell->getPort(ID::Y)), candidates);
+			Cell *consumer = sole_chainable_consumer(sigmap(cell->getPort(ID::Y)), candidates);
 			if (consumer && consumer != cell)
 				parent_of[cell] = consumer;
 		}
@@ -177,12 +182,12 @@ struct Rewriter {
 	{
 		pool<SigBit> bits;
 		for (auto cell : chain)
-			for (auto bit : traversal.sigmap(cell->getPort(ID::Y)))
+			for (auto bit : sigmap(cell->getPort(ID::Y)))
 				bits.insert(bit);
 		return bits;
 	}
 
-	static bool overlaps(SigSpec sig, const pool<SigBit> &bits)
+	bool overlaps(SigSpec sig, const pool<SigBit> &bits)
 	{
 		for (auto bit : sig)
 			if (bits.count(bit))
@@ -195,17 +200,16 @@ struct Rewriter {
 		bool parent_subtracts;
 		if (parent->type == ID($sub))
 			parent_subtracts = true;
-		else if (cells.is_alu(parent))
+		else if (is_alu(parent))
-			parent_subtracts = alu_info.is_subtract(parent);
+			parent_subtracts = is_sub(parent);
 		else
 			return false;
 
 		if (!parent_subtracts)
 			return false;
 
-		// Check if any bit of child's Y connects to parent's B
+		SigSpec child_y = sigmap(child->getPort(ID::Y));
-		SigSpec child_y = traversal.sigmap(child->getPort(ID::Y));
+		SigSpec parent_b = sigmap(parent->getPort(ID::B));
-		SigSpec parent_b = traversal.sigmap(parent->getPort(ID::B));
 		for (auto bit : child_y)
 			for (auto pbit : parent_b)
 				if (bit == pbit)
@@ -244,21 +248,20 @@ struct Rewriter {
 		for (auto cell : chain) {
 			bool cell_neg = negated.count(cell) ? negated[cell] : false;
 
-			SigSpec a = traversal.sigmap(cell->getPort(ID::A));
+			SigSpec a = sigmap(cell->getPort(ID::A));
-			SigSpec b = traversal.sigmap(cell->getPort(ID::B));
+			SigSpec b = sigmap(cell->getPort(ID::B));
 			bool a_signed = cell->getParam(ID::A_SIGNED).as_bool();
 			bool b_signed = cell->getParam(ID::B_SIGNED).as_bool();
-			bool b_sub = (cell->type == ID($sub)) || (cells.is_alu(cell) && alu_info.is_subtract(cell));
+			bool b_sub = (cell->type == ID($sub)) || (is_alu(cell) && is_sub(cell));
 
-			// Only add operands not produced by other chain cells
 			if (!overlaps(a, chain_bits)) {
-				operands.push_back({a, a_signed, cell_neg});
+				operands.push_back({a, a_signed, cell_neg, SigSpec(), false});
 				if (cell_neg)
 					neg_compensation++;
 			}
 			if (!overlaps(b, chain_bits)) {
 				bool neg = cell_neg ^ b_sub;
-				operands.push_back({b, b_signed, neg});
+				operands.push_back({b, b_signed, neg, SigSpec(), false});
 				if (neg)
 					neg_compensation++;
 			}
@@ -272,63 +275,123 @@ struct Rewriter {
 		neg_compensation = 0;
 
 		for (auto &term : macc.terms) {
-			// Bail on multiplication
+			if (GetSize(term.in_b) != 0) {
-			if (GetSize(term.in_b) != 0)
+				// TODO: Baugh-Wooley sign extension for mixed sign and sign*sign cases, don't bail out to non-FMA
-				return false;
+				if (!opt.fma_fusion)
-			operands.push_back({term.in_a, term.is_signed, term.do_subtract});
+					return false;
+				if (term.is_signed || !CompressorTree::supports_signedness(term.is_signed, term.is_signed))
+					return false;
+
+				// Preserve term as a multiplicative operand which is expanded into partial products
+				Operand op;
+				op.sig = term.in_a;
+				op.is_signed = false;
+				op.negate = term.do_subtract;
+				op.factor_b = term.in_b;
+				op.factor_b_signed = false;
+				operands.push_back(op);
+				continue;
+			}
+			operands.push_back({term.in_a, term.is_signed, term.do_subtract, SigSpec(), false});
 			if (term.do_subtract)
 				neg_compensation++;
 		}
 		return true;
 	}
 
-	SigSpec extend_operand(SigSpec sig, bool is_signed, int width)
+	std::vector<CompressorTree::DepthSig> build_operand_pool(std::vector<Operand> &operands, int width, int &neg_compensation)
 	{
-		if (GetSize(sig) < width) {
+		// Expand operands into a flat list of signals for reduction
-			SigBit pad;
+		std::vector<CompressorTree::DepthSig> pool;
-			if (is_signed && GetSize(sig) > 0)
+		pool.reserve(operands.size() * 2);
-				pad = sig[GetSize(sig) - 1];
-			else
-				pad = State::S0;
-			sig.append(SigSpec(pad, width - GetSize(sig)));
-		}
-		if (GetSize(sig) > width)
-			sig = sig.extract(0, width);
-		return sig;
-	}
-
-	void replace_with_carry_save_tree(std::vector<Operand> &operands, SigSpec result_y, int neg_compensation, const char *desc)
-	{
-		int width = GetSize(result_y);
-		std::vector<SigSpec> extended;
-		extended.reserve(operands.size() + 1);
 
 		for (auto &op : operands) {
-			SigSpec s = extend_operand(op.sig, op.is_signed, width);
+			if (GetSize(op.factor_b) == 0) {
-			if (op.negate)
+				// Additive operand
-				s = module->Not(NEW_ID, s);
+				SigSpec s = CompressorTree::normalize_to_width(op.sig, op.is_signed, width);
-			extended.push_back(s);
+				if (op.negate)
+					s = module->Not(NEW_ID, s);
+				pool.push_back({s, 0});
+			} else {
+				// Multiplicative operand
+				// TODO: Negate product instead of factor
+				auto pps =
+				  CompressorTree::generate_partial_products(module, op.sig, op.factor_b, op.is_signed, op.factor_b_signed, width);
+
+				if (op.negate) {
+					for (auto &pp : pps) {
+						SigSpec inv = module->addWire(NEW_ID, width);
+						module->addNot(NEW_ID, pp.sig, inv);
+						pp.sig = inv;
+						neg_compensation++;
+					}
+				}
+
+				for (auto &pp : pps)
+					pool.push_back(pp);
+			}
 		}
 
-		// Add correction for negated operands (-x = ~x + 1 so 1 per negation)
 		if (neg_compensation > 0)
-			extended.push_back(SigSpec(neg_compensation, width));
+			pool.push_back({SigSpec(neg_compensation, width), 0});
 
-		int compressor_count;
+		return pool;
-		auto [a, b] = wallace_reduce_scheduled(module, extended, width, &compressor_count);
+	}
-		log("  %s -> %d $fa + 1 $add (%d operands, module %s)\n", desc, compressor_count, (int)operands.size(), module);
 
-		// Emit final add
+	void emit_tree(std::vector<Operand> &operands, SigSpec result_y, int neg_compensation, bool any_signed, const char *desc)
-		module->addAdd(NEW_ID, a, b, result_y, false);
+	{
+		int width = GetSize(result_y);
+
+		if (opt.elarith_macro) {
+			// Bypass the compressor
+			emit_elarith_macro(operands, result_y, neg_compensation, any_signed, desc);
+			return;
+		}
+
+		auto pool = build_operand_pool(operands, width, neg_compensation);
+		auto [a, b] = CompressorTree::reduce_scheduled(module, std::move(pool), width, opt.strategy);
+		auto final_choice = CompressorTree::pick_final_adder(width, opt.final_mode);
+		CompressorTree::emit_final_adder(module, a, b, result_y, final_choice, any_signed);
+	}
+
+	void emit_elarith_macro(std::vector<Operand> &operands, SigSpec result_y, int neg_compensation, bool any_signed, const char *desc)
+	{
+		int width = GetSize(result_y);
+		auto pool = build_operand_pool(operands, width, neg_compensation);
+
+		log("  arith_tree::elarith: %s -> \\AddMopCsv macro, %d operands, width %d (module %s)\n", desc, (int)pool.size(), width, log_id(module));
+
+		// Pack all operands
+		SigSpec flat;
+		for (auto &dp : pool) {
+			SigSpec ext = CompressorTree::normalize_to_width(dp.sig, false, width);
+			flat.append(ext);
+		}
+
+		Cell *c = module->addCell(NEW_ID, IdString("\\AddMopCsv"));
+		c->setParam(IdString("\\WIDTH"), width);
+		c->setParam(IdString("\\NUM_OPERANDS"), (int)pool.size());
+		c->setParam(IdString("\\SIGNED"), any_signed ? 1 : 0);
+		c->setParam(IdString("\\SPEED"), Const("fast"));
+		c->setPort(IdString("\\Operands"), flat);
+		c->setPort(IdString("\\Sum"), result_y);
+	}
+
+	bool any_operand_signed(const std::vector<Operand> &operands)
+	{
+		for (auto &op : operands)
+			if (op.is_signed)
+				return true;
+		return false;
 	}
 
 	void process_chains()
 	{
 		pool<Cell *> candidates;
-		for (auto cell : cells.addsub)
+		for (auto cell : addsub)
 			candidates.insert(cell);
-		for (auto cell : cells.alu)
+		for (auto cell : alu)
-			if (alu_info.is_chainable(cell))
+			if (is_chainable(cell))
 				candidates.insert(cell);
 
 		if (candidates.empty())
@@ -354,7 +417,7 @@ struct Rewriter {
 			for (auto c : chain)
 				to_remove.insert(c);
 
-			replace_with_carry_save_tree(operands, root->getPort(ID::Y), neg_compensation, "Replaced add/sub chain");
+			emit_tree(operands, root->getPort(ID::Y), neg_compensation, any_operand_signed(operands), "Replaced $add/$sub chain");
 		}
 
 		for (auto cell : to_remove)
@@ -363,48 +426,76 @@ struct Rewriter {
 
 	void process_maccs()
 	{
-		for (auto cell : cells.macc) {
+		pool<Cell *> to_remove;
+		for (auto cell : macc) {
 			std::vector<Operand> operands;
 			int neg_compensation;
 			if (!extract_macc_operands(cell, operands, neg_compensation))
 				continue;
-			if (operands.size() < 3)
+			if (operands.size() < 1)
+				continue;
+			bool has_mul = false;
+			for (auto &op : operands)
+				if (GetSize(op.factor_b) > 0) {
+					has_mul = true;
+					break;
+				}
+
+			if (!has_mul && operands.size() < 3)
 				continue;
 
-			replace_with_carry_save_tree(operands, cell->getPort(ID::Y), neg_compensation, "Replaced $macc");
+			emit_tree(operands, cell->getPort(ID::Y), neg_compensation, any_operand_signed(operands), has_mul ? "Replaced $macc (FMA)" : "Replaced $macc");
-			module->remove(cell);
+			to_remove.insert(cell);
 		}
+		for (auto cell : to_remove)
+			module->remove(cell);
+	}
+
+	void run()
+	{
+		if (addsub.empty() && alu.empty() && macc.empty())
+			return;
+
+		process_chains();
+		process_maccs();
 	}
 };
 
-void run(Module *module)
-{
-	Cells cells(module);
-
-	if (cells.empty())
-		return;
-
-	Rewriter rewriter{module, cells};
-	rewriter.process_chains();
-	rewriter.process_maccs();
-}
-
 struct ArithTreePass : public Pass {
-	ArithTreePass() : Pass("arith_tree", "convert add/sub/macc chains to carry-save adder trees") {}
+	ArithTreePass() : Pass("arith_tree", "convert add/sub/macc/alu chains to carry-save adder trees") {}
 
 	void help() override
 	{
 		//   |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|
 		log("\n");
-		log("    arith_tree [selection]\n");
+		log("    arith_tree [options] [selection]\n");
 		log("\n");
 		log("This pass replaces chains of $add/$sub cells, $alu cells (with constant\n");
-		log("BI/CI), and $macc/$macc_v2 cells (without multiplications) with carry-save\n");
+		log("BI/CI), and $macc/$macc_v2 cells with carry-save adder trees \n");
-		log("adder trees using $fa cells and a single final $add.\n");
+		log("using $fa cells and a single final adder.\n");
 		log("\n");
-		log("The tree uses Wallace-tree scheduling: at each level, ready operands are\n");
+		log("    -strategy <fa|42>\n");
-		log("grouped into triplets and compressed via full adders, giving\n");
+		log("        Compressor strategy. 'fa' uses only 3:2 full-adder groupings\n");
-		log("O(log_{1.5} N) depth for N input operands.\n");
+		log("        '42' (the default) prefers 4:2 compressor groupings, with\n");
+		log("        fallback to 3:2 compressors for residuals\n");
+		log("\n");
+		log("    -final <auto|ripple|prefix|elarith>\n");
+		log("        Selects the architecture used for the final two-vector add.\n");
+		log("        'auto' (default) emits a ripple-style $add for narrow widths\n");
+		log("        (< 16 bits) and a parallel prefix hinted $add for wider ones.\n");
+		log("        'elarith' emits an \\AddCfast black-box from the ELArith\n");
+		log("        library; the surrounding flow must provide that module.\n");
+		log("\n");
+		log("    -no-fma\n");
+		log("        Disable fused multiply-add expansion in $macc cells\n");
+		log("\n");
+		log("    -elarith-macro\n");
+		log("        Replace each detected chain with a single \\AddMopCsv black-box\n");
+		log("        instance instead of expanding it into $fa cells. The downstream\n");
+		log("        flow must provide an \\AddMopCsv implementation\n");
+		log("\n");
+		log("The default behaviour delivers 4:2 compression, FMA fusion, and a\n");
+		log("width-adaptive final adder\n");
 		log("\n");
 	}
 
@@ -412,15 +503,44 @@ struct ArithTreePass : public Pass {
 	{
 		log_header(design, "Executing ARITH_TREE pass.\n");
 
+		ArithTreeOptions opt;
+
 		size_t argidx;
-		for (argidx = 1; argidx < args.size(); argidx++)
+		for (argidx = 1; argidx < args.size(); argidx++) {
+			const std::string &arg = args[argidx];
+			if (arg == "-strategy" && argidx + 1 < args.size()) {
+				const std::string &v = args[++argidx];
+				if (v == "fa") { opt.strategy = CompressorTree::Strategy::FA_ONLY; }
+				else if (v == "42") { opt.strategy = CompressorTree::Strategy::PREFER_42; }
+				else { log_cmd_error("arith_tree: unknown -strategy '%s'\n", v.c_str()); }
+				continue;
+			}
+			if (arg == "-final" && argidx + 1 < args.size()) {
+				const std::string &v = args[++argidx];
+				if (v == "auto") { opt.final_mode = CompressorTree::FinalMode::AUTO; }
+				else if (v == "ripple") { opt.final_mode = CompressorTree::FinalMode::RIPPLE; }
+				else if (v == "prefix") { opt.final_mode = CompressorTree::FinalMode::PREFIX; }
+				else if (v == "elarith") { opt.final_mode = CompressorTree::FinalMode::ELARITH; }
+				else { log_cmd_error("arith_tree: unknown -final '%s'\n", v.c_str()); }
+				continue;
+			}
+			if (arg == "-no-fma") {
+				opt.fma_fusion = false;
+				continue;
+			}
+			if (arg == "-elarith-macro") {
+				opt.elarith_macro = true;
+				continue;
+			}
 			break;
+		}
 		extra_args(args, argidx, design);
 
-		for (auto module : design->selected_modules()) {
+		for (auto mod : design->selected_modules()) {
-			run(module);
+			ArithTreeWorker worker(opt, mod);
+			worker.run();
 		}
 	}
 } ArithTreePass;
 
-PRIVATE_NAMESPACE_END
+PRIVATE_NAMESPACE_END

passes/techmap/booth.cc

@@ -58,7 +58,7 @@ synth -top my_design -booth
 #include "kernel/sigtools.h"
 #include "kernel/yosys.h"
 #include "kernel/macc.h"
-#include "kernel/wallace_tree.h"
+#include "kernel/compressor_tree.h"
 
 USING_YOSYS_NAMESPACE
 PRIVATE_NAMESPACE_BEGIN
@@ -386,7 +386,11 @@ struct BoothPassWorker {
 		// Later on yosys will clean up unused constants
 		//  DebugDumpAlignPP(aligned_pp);
 
-		auto [wtree_a, wtree_b] = wallace_reduce_scheduled(module, aligned_pp, z_sz);
+		std::vector<CompressorTree::DepthSig> operands;
+		operands.reserve(aligned_pp.size());
+		for (auto &s : aligned_pp)
+			operands.push_back({s, 0});
+		auto [wtree_a, wtree_b] = CompressorTree::reduce_scheduled(module, std::move(operands), z_sz, CompressorTree::Strategy::FA_ONLY);
 
 		// Debug code: Dump out the csa trees
 		// DumpCSATrees(debug_csa_trees);