Make NEW_ID create IDs whose string allocation is delayed

kernel/io.cc

@@ -587,7 +587,7 @@ void format_emit_idstring(std::string &result, std::string_view spec, int *dynam
 {
 	if (spec == "%s") {
 		// Format checking will have guaranteed num_dynamic_ints == 0.
-		result += arg.str_view();
+		arg.append_to(&result);
 		return;
 	}
 	format_emit_stringf(result, spec, dynamic_ints, num_dynamic_ints, arg.c_str());

kernel/rtlil.cc

@@ -28,6 +28,7 @@
 
 #include <string.h>
 #include <algorithm>
+#include <charconv>
 #include <optional>
 #include <string_view>
 
@@ -37,6 +38,8 @@ bool RTLIL::IdString::destruct_guard_ok = false;
 RTLIL::IdString::destruct_guard_t RTLIL::IdString::destruct_guard;
 std::vector<RTLIL::IdString::Storage> RTLIL::IdString::global_id_storage_;
 std::unordered_map<std::string_view, int> RTLIL::IdString::global_id_index_;
+std::unordered_map<int, const std::string*> RTLIL::IdString::global_autoidx_id_prefix_storage_;
+std::unordered_map<int, char*> RTLIL::IdString::global_autoidx_id_storage_;
 #ifndef YOSYS_NO_IDS_REFCNT
 std::unordered_map<int, int> RTLIL::IdString::global_refcount_storage_;
 std::vector<int> RTLIL::IdString::global_free_idx_list_;
@@ -64,6 +67,74 @@ void RTLIL::IdString::prepopulate()
 #undef X
 }
 
+static std::optional<int> parse_autoidx(std::string_view v)
+{
+	// autoidx values can never be <= 0, so there can never be a leading 0 digit.
+	if (v.empty() || v[0] == '0')
+		return std::nullopt;
+	for (char ch : v) {
+		if (ch < '0' || ch > '9')
+			return std::nullopt;
+	}
+	int p_autoidx;
+	if (std::from_chars(v.data(), v.data() + v.size(), p_autoidx).ec != std::errc())
+		return std::nullopt;
+	return p_autoidx;
+}
+
+int RTLIL::IdString::really_insert(std::string_view p, std::unordered_map<std::string_view, int>::iterator &it)
+{
+	ensure_prepopulated();
+
+	log_assert(p[0] == '$' || p[0] == '\\');
+	for (char ch : p)
+		if ((unsigned)ch <= (unsigned)' ')
+			log_error("Found control character or space (0x%02x) in string '%s' which is not allowed in RTLIL identifiers\n", ch, std::string(p).c_str());
+
+	if (p.substr(0, 6) == "$auto$") {
+		size_t autoidx_pos = p.find_last_of('$') + 1;
+		std::optional<int> p_autoidx = parse_autoidx(p.substr(autoidx_pos));
+		if (p_autoidx.has_value()) {
+			auto prefix_it = global_autoidx_id_prefix_storage_.find(-*p_autoidx);
+			if (prefix_it != global_autoidx_id_prefix_storage_.end() && p.substr(0, autoidx_pos) == *prefix_it->second)
+				return -*p_autoidx;
+			// Ensure NEW_ID/NEW_ID_SUFFIX will not create collisions with the ID
+			// we're about to create.
+			autoidx = std::max(autoidx, *p_autoidx + 1);
+		}
+	}
+
+#ifndef YOSYS_NO_IDS_REFCNT
+	if (global_free_idx_list_.empty()) {
+		log_assert(global_id_storage_.size() < 0x40000000);
+		global_free_idx_list_.push_back(global_id_storage_.size());
+		global_id_storage_.push_back({nullptr, 0});
+	}
+
+	int idx = global_free_idx_list_.back();
+	global_free_idx_list_.pop_back();
+#else
+	int idx = global_id_storage_.size();
+	global_id_index_[global_id_storage_.back()] = idx;
+#endif
+	char* buf = static_cast<char*>(malloc(p.size() + 1));
+	memcpy(buf, p.data(), p.size());
+	buf[p.size()] = 0;
+	global_id_storage_.at(idx) = {buf, GetSize(p)};
+	global_id_index_.insert(it, {std::string_view(buf, p.size()), idx});
+
+	if (yosys_xtrace) {
+		log("#X# New IdString '%s' with index %d.\n", global_id_storage_.at(idx).buf, idx);
+		log_backtrace("-X- ", yosys_xtrace-1);
+	}
+
+#ifdef YOSYS_XTRACE_GET_PUT
+	if (yosys_xtrace)
+		log("#X# GET-BY-NAME '%s' (index %d, refcount %u)\n", global_id_storage_.at(idx), idx, refcount(idx));
+#endif
+	return idx;
+}
+
 static constexpr bool check_well_known_id_order()
 {
 	int size = sizeof(IdTable) / sizeof(IdTable[0]);
@@ -78,9 +149,9 @@ static constexpr bool check_well_known_id_order()
 static_assert(check_well_known_id_order());
 
 struct IdStringCollector {
-	IdStringCollector(int size) : live(size, false) {}
+	void trace(IdString id) {
-
+		live.insert(id.index_);
-	void trace(IdString id) { live[id.index_] = true; }
+	}
 	template <typename T> void trace(const T* v) {
 		trace(*v);
 	}
@@ -169,23 +240,23 @@ struct IdStringCollector {
 		trace(action.memid);
 	}
 
-	std::vector<bool> live;
+	std::unordered_set<int> live;
 };
 
 void RTLIL::OwningIdString::collect_garbage()
 {
 #ifndef YOSYS_NO_IDS_REFCNT
-	int size = GetSize(global_id_storage_);
+	IdStringCollector collector;
-	IdStringCollector collector(size);
 	for (auto &[idx, design] : *RTLIL::Design::get_all_designs()) {
 		collector.trace(*design);
 	}
+	int size = GetSize(global_id_storage_);
 	for (int i = static_cast<int>(StaticId::STATIC_ID_END); i < size; ++i) {
-		if (collector.live[i])
-			continue;
 		RTLIL::IdString::Storage &storage = global_id_storage_.at(i);
 		if (storage.buf == nullptr)
 			continue;
+		if (collector.live.find(i) != collector.live.end())
+			continue;
 		if (global_refcount_storage_.find(i) != global_refcount_storage_.end())
 			continue;
 
@@ -199,6 +270,23 @@ void RTLIL::OwningIdString::collect_garbage()
 		storage = {nullptr, 0};
 		global_free_idx_list_.push_back(i);
 	}
+
+	for (auto it = global_autoidx_id_prefix_storage_.begin(); it != global_autoidx_id_prefix_storage_.end();) {
+		if (collector.live.find(it->first) != collector.live.end()) {
+			++it;
+			continue;
+		}
+		if (global_refcount_storage_.find(it->first) != global_refcount_storage_.end()) {
+			++it;
+			continue;
+		}
+		auto str_it = global_autoidx_id_storage_.find(it->first);
+		if (str_it != global_autoidx_id_storage_.end()) {
+			delete[] str_it->second;
+			global_autoidx_id_storage_.erase(str_it);
+		}
+		it = global_autoidx_id_prefix_storage_.erase(it);
+	}
 #endif
 }
 

kernel/rtlil.h

@@ -23,7 +23,6 @@
 #include "kernel/yosys_common.h"
 #include "kernel/yosys.h"
 
-#include <charconv>
 #include <string_view>
 #include <unordered_map>
 
@@ -145,8 +144,16 @@ struct RTLIL::IdString
 		~destruct_guard_t() { destruct_guard_ok = false; }
 	} destruct_guard;
 
+	// String storage for non-autoidx IDs
 	static std::vector<Storage> global_id_storage_;
+	// Lookup table for non-autoidx IDs
 	static std::unordered_map<std::string_view, int> global_id_index_;
+	// Shared prefix string storage for autoidx IDs, which have negative
+	// indices. Append the negated (i.e. positive) ID to this string to get
+	// the real string. The prefix strings must live forever.
+	static std::unordered_map<int, const std::string*> global_autoidx_id_prefix_storage_;
+	// Explicit string storage for autoidx IDs
+	static std::unordered_map<int, char*> global_autoidx_id_storage_;
 #ifndef YOSYS_NO_IDS_REFCNT
 	// All (index, refcount) pairs in this map have refcount > 0.
 	static std::unordered_map<int, int> global_refcount_storage_;
@@ -192,54 +199,15 @@ struct RTLIL::IdString
 	#endif
 			return it->second;
 		}
-
+		return really_insert(p, it);
-		ensure_prepopulated();
-
-		log_assert(p[0] == '$' || p[0] == '\\');
-		for (char ch : p)
-			if ((unsigned)ch <= (unsigned)' ')
-				log_error("Found control character or space (0x%02x) in string '%s' which is not allowed in RTLIL identifiers\n", ch, std::string(p).c_str());
-
-		if (p.substr(0, 6) == "$auto$") {
-			// Ensure new_id(_suffix) will not create collisions.
-			size_t autoidx_pos = p.find_last_of('$');
-			int p_autoidx;
-			std::string_view v = p.substr(autoidx_pos + 1);
-			if (std::from_chars(v.begin(), v.end(), p_autoidx).ec == std::errc()) {
-				autoidx = std::max(autoidx, p_autoidx + 1);
-			}
 	}
 
-	#ifndef YOSYS_NO_IDS_REFCNT
+	// Inserts an ID with string `prefix + autoidx', incrementing autoidx.
-		if (global_free_idx_list_.empty()) {
+	// `prefix` must start with '$auto$', end with '$', and live forever.
-			log_assert(global_id_storage_.size() < 0x40000000);
+	static IdString new_autoidx_with_prefix(const std::string *prefix) {
-			global_free_idx_list_.push_back(global_id_storage_.size());
+		int index = -(autoidx++);
-			global_id_storage_.push_back({nullptr, 0});
+		global_autoidx_id_prefix_storage_.insert({index, prefix});
-		}
+		return from_index(index);
-
-		int idx = global_free_idx_list_.back();
-		global_free_idx_list_.pop_back();
-	#else
-		int idx = global_id_storage_.size();
-		global_id_storage_.push_back({nullptr, 0});
-	#endif
-		char* buf = static_cast<char*>(malloc(p.size() + 1));
-		memcpy(buf, p.data(), p.size());
-		buf[p.size()] = 0;
-		global_id_storage_.at(idx) = {buf, GetSize(p)};
-		global_id_index_.insert(it, {std::string_view(buf, p.size()), idx});
-
-		if (yosys_xtrace) {
-			log("#X# New IdString '%s' with index %d.\n", global_id_storage_.at(idx).buf, idx);
-			log_backtrace("-X- ", yosys_xtrace-1);
-		}
-
-	#ifdef YOSYS_XTRACE_GET_PUT
-		if (yosys_xtrace)
-			log("#X# GET-BY-NAME '%s' (index %d, refcount %u)\n", global_id_storage_.at(idx).buf, idx, refcount(idx));
-	#endif
-
-		return idx;
 	}
 
 	// the actual IdString object is just is a single int
@@ -269,17 +237,35 @@ struct RTLIL::IdString
 	constexpr inline const IdString &id_string() const { return *this; }
 
 	inline const char *c_str() const {
+		if (index_ >= 0)
 			return global_id_storage_.at(index_).buf;
+		auto it = global_autoidx_id_storage_.find(index_);
+		if (it != global_autoidx_id_storage_.end())
+			return it->second;
+
+		const std::string &prefix = *global_autoidx_id_prefix_storage_.at(index_);
+		std::string suffix = std::to_string(-index_);
+		char *c = new char[prefix.size() + suffix.size() + 1];
+		memcpy(c, prefix.data(), prefix.size());
+		memcpy(c + prefix.size(), suffix.c_str(), suffix.size() + 1);
+		global_autoidx_id_storage_.insert(it, {index_, c});
+		return c;
 	}
 
 	inline std::string str() const {
-		const Storage &storage = global_id_storage_.at(index_);
+		std::string result;
-		return std::string(storage.buf, storage.size);
+		append_to(&result);
+		return result;
 	}
 
-	inline std::string_view str_view() const {
+	inline void append_to(std::string *out) const {
+		if (index_ >= 0) {
 			const Storage &storage = global_id_storage_.at(index_);
-		return std::string_view(storage.buf, storage.size);
+			*out += std::string_view(storage.buf, storage.size);
+			return;
+		}
+		*out += *global_autoidx_id_prefix_storage_.at(index_);
+		*out += std::to_string(-index_);
 	}
 
 	inline bool operator<(const IdString &rhs) const {
@@ -335,7 +321,9 @@ struct RTLIL::IdString
 	}
 
 	size_t size() const {
+		if (index_ >= 0)
 			return global_id_storage_.at(index_).size;
+		return strlen(c_str());
 	}
 
 	bool empty() const {
@@ -381,6 +369,14 @@ struct RTLIL::IdString
 
 private:
 	static void prepopulate();
+	static int really_insert(std::string_view p, std::unordered_map<std::string_view, int>::iterator &it);
+
+protected:
+	static IdString from_index(int index) {
+		IdString result;
+		result.index_ = index;
+		return result;
+	}
 
 public:
 	static void ensure_prepopulated() {
@@ -449,7 +445,7 @@ private:
 	#endif
 	#ifdef YOSYS_XTRACE_GET_PUT
 		if (yosys_xtrace && idx >= static_cast<short>(StaticId::STATIC_ID_END))
-			log("#X# GET-BY-INDEX '%s' (index %d, refcount %u)\n", global_id_storage_.at(idx), idx, refcount(idx));
+			log("#X# GET-BY-INDEX '%s' (index %d, refcount %u)\n", from_index(idx), idx, refcount(idx));
 	#endif
 	}
 
@@ -462,7 +458,7 @@ private:
 			return;
 	#ifdef YOSYS_XTRACE_GET_PUT
 		if (yosys_xtrace)
-			log("#X# PUT '%s' (index %d, refcount %u)\n", global_id_storage_.at(index_), index_, refcount(index_));
+			log("#X# PUT '%s' (index %d, refcount %u)\n", from_index(index_), index_, refcount(index_));
 	#endif
 		auto it = global_refcount_storage_.find(index_);
 		log_assert(it != global_refcount_storage_.end() && it->second >= 1);

kernel/yosys.cc

@@ -295,7 +295,7 @@ void yosys_shutdown()
 #endif
 }
 
-RTLIL::IdString new_id(std::string_view file, int line, std::string_view func)
+const std::string *create_id_prefix(std::string_view file, int line, std::string_view func)
 {
 #ifdef _WIN32
 	size_t pos = file.find_last_of("/\\");
@@ -309,7 +309,7 @@ RTLIL::IdString new_id(std::string_view file, int line, std::string_view func)
 	if (pos != std::string_view::npos)
 		func = func.substr(pos+1);
 
-	return stringf("$auto$%s:%d:%s$%d", file, line, func, autoidx++);
+	return new std::string(stringf("$auto$%s:%d:%s$", file, line, func));
 }
 
 RTLIL::IdString new_id_suffix(std::string_view file, int line, std::string_view func, std::string_view suffix)

kernel/yosys_common.h

@@ -271,11 +271,14 @@ extern int autoidx;
 extern int yosys_xtrace;
 extern bool yosys_write_versions;
 
-RTLIL::IdString new_id(std::string_view file, int line, std::string_view func);
+const std::string *create_id_prefix(std::string_view file, int line, std::string_view func);
 RTLIL::IdString new_id_suffix(std::string_view file, int line, std::string_view func, std::string_view suffix);
 
 #define NEW_ID \
-	YOSYS_NAMESPACE_PREFIX new_id(__FILE__, __LINE__, __FUNCTION__)
+	YOSYS_NAMESPACE_PREFIX RTLIL::IdString::new_autoidx_with_prefix([](std::string_view func) -> const std::string * { \
+		static const std::string *prefix = create_id_prefix(__FILE__, __LINE__, func); \
+		return prefix; \
+	}(__FUNCTION__))
 #define NEW_ID_SUFFIX(suffix) \
 	YOSYS_NAMESPACE_PREFIX new_id_suffix(__FILE__, __LINE__, __FUNCTION__, suffix)
 

pyosys/generator.py

@@ -163,6 +163,8 @@ pyosys_headers = [
                     {
                         "global_id_storage_",
                         "global_id_index_",
+                        "global_negative_id_storage_",
+                        "global_negative_id_prefix_storage_",
                         "global_refcount_storage_",
                         "global_free_idx_list_",
                         "last_created_idx_ptr_",

tests/unit/kernel/rtlilTest.cc

@@ -367,6 +367,12 @@ namespace RTLIL {
 		EXPECT_EQ(own.str(), "\\figblortle");
 	}
 
+	TEST_F(KernelRtlilTest, LookupAutoidxId) {
+		IdString id = NEW_ID;
+		IdString id2 = IdString(id.str());
+		EXPECT_EQ(id, id2);
+	}
+
 	class WireRtlVsHdlIndexConversionTest :
 		public KernelRtlilTest,
 		public testing::WithParamInterface<std::tuple<bool, int, int>>