pyosys: globals, set operators for opaque types

There is so much templating going on that compiling wrappers.cc now takes 1m1.668s on an Apple M4…

Makefile

@@ -102,7 +102,7 @@ all: top-all
 YOSYS_SRC := $(dir $(firstword $(MAKEFILE_LIST)))
 VPATH := $(YOSYS_SRC)
 
-CXXSTD ?= c++17
+export CXXSTD ?= c++17
 CXXFLAGS := $(CXXFLAGS) -Wall -Wextra -ggdb -I. -I"$(YOSYS_SRC)" -MD -MP -D_YOSYS_ -fPIC -I$(PREFIX)/include
 LIBS := $(LIBS) -lstdc++ -lm
 PLUGIN_LINKFLAGS :=

kernel/yosys.cc

@@ -185,13 +185,12 @@ bool already_setup = false;
 bool already_shutdown = false;
 
 #ifdef WITH_PYTHON
-// Include pyosys as a module so 'from pyosys import libyosys' also works
+// Include pyosys as a package for some compatibility with wheels.
-// in interpreter mode.
 //
 // This should not affect using wheels as the dylib has to actually be called
-// pyosys.so for this module to be interacted with at all.
+// pyosys.so for this function to be interacted with at all.
 PYBIND11_MODULE(pyosys, m) {
-	m.add_object("libyosys", m.import("libyosys"));
+	m.add_object("__path__", py::list());
 }
 #endif
 
@@ -209,7 +208,8 @@ void yosys_setup()
 	// initialized platform fails (such as when libyosys is imported
 	// from a Python interpreter)
 	if (!Py_IsInitialized()) {
-		PyImport_AppendInittab((char*)"libyosys", PyInit_libyosys);
+		PyImport_AppendInittab((char*)"pyosys.libyosys", PyInit_libyosys);
+		// compatibility with wheels
 		PyImport_AppendInittab((char*)"pyosys", PyInit_pyosys);
 		Py_Initialize();
 		PyRun_SimpleString("import sys");

pyosys/generator.py

@@ -18,7 +18,19 @@
 #  Written by Mohamed Gaber <me@donn.website>
 #
 #  Inspired by py_wrap_generator.py by Benedikt Tutzer
+"""
+This generates:
+- Wrapper calls for opaque container types
+- Full wrappers for select classes and all enums, global functions and global
+  variables
 
+Jump to "MARK: Inclusion and Exclusion" to control the above.
+
+Please run ruff on this file in particular to make sure it parses with Python
+<= 3.12. There is so much f-string use here that the outer quote thing
+is a common problem. ``ruff check pyosys/generator.py`` suffices.
+"""
+import os
 import io
 import shutil
 from pathlib import Path
@@ -58,15 +70,24 @@ class PyosysClass:
     :param name: The base name of the class (without extra qualifiers)
     :param ref_only: Whether this class can be copied to Python, or only
         referenced.
-    :param string_expr: A C++ expression that will be used for the __str__ method in Python
+    :param string_expr:
-    :param hash_expr: A C++ expression that will be fed to ``run_hash`` to declare a __hash__ method for Python
+        A C++ expression that will be used for the ``__str__`` method in Python.
+
+        The object will be available as a const reference with the identifier
+        `s`.
+    :param hash_expr:
+        A C++ expression that will be fed to ``run_hash`` to declare a
+        ``__hash__`` method for Python.
+
+        The object will be vailable as a const reference with the identifier
+        `s`.
     :param denylist: If specified, one or more methods can be excluded from
         wrapping.
     """
     name: str
     ref_only: bool = False
 
-    # in the format s.(method or property)
+    # in the format s.(method or property) (or just s)
     string_expr: Optional[str] = None
     hash_expr: Optional[str] = None
 
@@ -89,9 +110,7 @@ class PyosysHeader:
             for cls in classes:
                 self.classes_by_name[cls.name] = cls
 
-"""
+# MARK: Inclusion and Exclusion
-Add headers and classes here!
-"""
 global_denylist = frozenset(
     {
         # deprecated
@@ -327,10 +346,11 @@ class PyosysWrapperGenerator(object):
     def make_preprocessor_options(self):
         py_include = get_paths()["include"]
         preprocessor_bin = shutil.which("clang++") or "g++"
+        cxx_std = os.getenv("CXX_STD", "c++17")
         return ParserOptions(
             preprocessor=make_gcc_preprocessor(
                 defines=["_YOSYS_", "WITH_PYTHON"],
-                gcc_args=[preprocessor_bin, "-fsyntax-only"],
+                gcc_args=[preprocessor_bin, "-fsyntax-only", f"-std={cxx_std}"],
                 include_paths=[str(__yosys_root__), py_include, pybind11.get_include()],
             ),
         )
@@ -476,7 +496,7 @@ class PyosysWrapperGenerator(object):
         if function.static:
             definition_fn = "def_static"
 
-        print(f"\t\t\t.{definition_fn}({", ".join(self.get_definition_args(function, class_basename, python_name_override))})", file=self.f)
+        print(f"\t\t\t.{definition_fn}({', '.join(self.get_definition_args(function, class_basename, python_name_override))})", file=self.f)
 
     def process_function(self, function: Function):
         if (
@@ -496,7 +516,7 @@ class PyosysWrapperGenerator(object):
 
         self.register_containers(function)
 
-        print(f"\t\t\tm.def({", ".join(self.get_definition_args(function, None))});", file=self.f)
+        print(f"\t\t\tm.def({', '.join(self.get_definition_args(function, None))});", file=self.f)
 
     def process_field(self, field: Field, class_basename: str):
         if field.access != "public":
@@ -511,7 +531,8 @@ class PyosysWrapperGenerator(object):
 
         unique_ptrs = self.find_containers(("unique_ptr",), field.type)
         if len(unique_ptrs):
-            # TODO: figure out how to bridge unique pointers
+            # TODO: figure out how to bridge unique pointers maybe does anyone
+            # care
             return
 
         self.register_containers(field)
@@ -559,10 +580,10 @@ class PyosysWrapperGenerator(object):
             self.process_method(method, basename)
 
         visited_anonymous_unions = set()
-        for field in cls.fields:
+        for field_ in cls.fields:
-            if field.name in metadata.denylist:
+            if field_.name in metadata.denylist:
                 continue
-            self.process_field(field, basename)
+            self.process_field(field_, basename)
 
             # Handle anonymous unions
             for subclass in cls.classes:
@@ -663,7 +684,6 @@ class PyosysWrapperGenerator(object):
 
 
 keyword_aliases = {
-    "in": "in_",
     "False": "False_",
     "None": "None_",
     "True": "True_",

pyosys/hashlib.h

@@ -37,7 +37,7 @@
 // things like mutating containers that are class properties.
 //
 // All methods should be vaguely in the same order as the python reference
-// https://docs.python.org/3/library/stdtypes.html
+// https://docs.python.org/3.13/library/stdtypes.html
 //
 #include <optional> // optional maps cleanest to methods that accept None in Python
 
@@ -60,26 +60,188 @@ bool is_mapping(object obj) {
 	return isinstance(obj, mapping);
 }
 
+// Set Operations
+bool is_subset(const iterable &lhs, const iterable &rhs, bool strict = false) {
+	for (auto &element: lhs) {
+		if (!rhs.contains(element)) {
+			return false;
+		}
+	}
+	if (strict) {
+		return len(rhs) > len(lhs);
+	}
+	return true;
+}
+
+template <typename C, typename T>
+void unionize(C &lhs, const iterable &rhs) {
+	for (auto &element: rhs) {
+		lhs.insert(cast<T>(element));
+	}
+}
+
+template <typename C, typename T>
+void difference(C &lhs, const iterable &rhs) {
+	for (auto &element: rhs) {
+		auto element_cxx = cast<T>(element);
+		if (lhs.count(element_cxx)) {
+			lhs.erase(element_cxx);
+		}
+	}
+}
+
+template <typename C, typename T>
+void intersect(C &lhs, const iterable &rhs) {
+	// Doing it in-place is a lot slower
+	// TODO?: Leave modifying lhs to caller (saves a copy) but complicates
+	//        chaining intersections.
+	C storage(lhs);
+
+	for (auto &element_cxx: lhs) {
+		if (!rhs.contains(cast(element_cxx))) {
+			storage.erase(element_cxx);
+		}
+	}
+	lhs = std::move(storage);
+}
+
+template <typename C, typename T>
+void symmetric_difference(C &lhs, const iterable &rhs) {
+	C storage(lhs);
+
+	for (auto &element: rhs) {
+		auto element_cxx = cast<T>(element);
+		if (lhs.count(element_cxx)) {
+			storage.erase(element_cxx);
+		} else {
+			storage.insert(element_cxx);
+		}
+	}
+	for (auto &element_cxx: lhs) {
+		if (rhs.contains(cast(element_cxx))) {
+			storage.erase(element_cxx);
+		}
+	}
+	lhs = std::move(storage);
+}
+
 // shim
 template <typename C, typename V>
 void bind_vector(module &m, const char *name_cstr) {
 	pybind11::bind_vector<C>(m, name_cstr);
 }
 
-// also used for std::set because the semantics are close enough
+// also used for hashlib pool because the semantics are close enough
 template <typename C, typename T>
-void bind_pool(module &m, const char *name_cstr) {
+void bind_set(module &m, const char *name_cstr) {
-	std::string {name_cstr};
-
 	class_<C>(m, name_cstr)
 		.def(init<>())
+		.def(init<const C &>()) // copy constructor
+		.def(init([](const iterable &other){ // copy instructor from arbitrary iterables
+			auto s = new C();
+			unionize<C, T>(*s, other);
+			return s;
+		}))
 		.def("__len__", [](const C &s){ return (size_t)s.size(); })
 		.def("__contains__", [](const C &s, const T &v){ return s.count(v); })
 		.def("__delitem__", [](C &s, const T &v) {
 			auto n = s.erase(v);
 			if (n == 0) throw key_error(str(cast(v)));
 		})
-		// TODO: disjoint, subset, union, intersection, difference, symdif
+		.def("disjoint", [](const C &s, const iterable &other) {
+			for (const auto &element: other) {
+				if (s.count(cast<T>(element))) {
+					return false;
+				}
+			}
+			return true;
+		})
+		.def("issubset", [](const iterable &s, const iterable &other) {
+			return is_subset(s, other);
+		})
+		.def("__eq__", [](const iterable &s, const iterable &other) {
+			return is_subset(s, other) && len(s) == len(other);
+		})
+		.def("__le__", [](const iterable &s, const iterable &other) {
+			return is_subset(s, other);
+		})
+		.def("__lt__", [](const iterable &s, const iterable &other) {
+			return is_subset(s, other, true);
+		})
+		.def("issuperset", [](const iterable &s, const iterable &other) {
+			return is_subset(other, s);
+		})
+		.def("__ge__",  [](const iterable &s, const iterable &other) {
+			return is_subset(other, s);
+		})
+		.def("__gt__",  [](const iterable &s, const iterable &other) {
+			return is_subset(other, s, true);
+		})
+		.def("union",  [](const C &s, const args &others) {
+			auto result = new C(s);
+			for (const auto &arg: others) {
+				auto arg_iterable = reinterpret_borrow<iterable>(arg);
+				unionize<C, T>(*result, arg_iterable);
+			}
+			return result;
+		})
+		.def("__or__",  [](const C &s, const iterable &other) {
+			auto result = new C(s);
+			unionize<C, T>(*result, other);
+			return result;
+		})
+		.def("__ior__",  [](C &s, const iterable &other) {
+			unionize<C, T>(s, other);
+			return s;
+		})
+		.def("intersection",  [](const C &s, const args &others) {
+			auto result = new C(s);
+			for (const auto &arg: others) {
+				auto arg_iterable = reinterpret_borrow<iterable>(arg);
+				intersect<C, T>(*result, arg_iterable);
+			}
+			return result;
+		})
+		.def("__and__",  [](const C &s, const iterable &other) {
+			auto result = new C(s);
+			intersect<C, T>(*result, other);
+			return result;
+		})
+		.def("__iand__",  [](C &s, const iterable &other) {
+			intersect<C, T>(s, other);
+			return s;
+		})
+		.def("difference",  [](const C &s, const args &others) {
+			auto result = new C(s);
+			for (const auto &arg: others) {
+				auto arg_iterable = reinterpret_borrow<iterable>(arg);
+				difference<C, T>(*result, arg_iterable);
+			}
+			return result;
+		})
+		.def("__sub__",  [](const C &s, const iterable &other) {
+			auto result = new C(s);
+			difference<C, T>(*result, other);
+			return result;
+		})
+		.def("__isub__",  [](C &s, const iterable &other) {
+			difference<C, T>(s, other);
+			return s;
+		})
+		.def("symmetric_difference",  [](const C &s, const iterable &other) {
+			auto result = new C(s);
+			symmetric_difference<C, T>(*result, other);
+			return result;
+		})
+		.def("__xor__",  [](const C &s, const iterable &other) {
+			auto result = new C(s);
+			symmetric_difference<C, T>(*result, other);
+			return result;
+		})
+		.def("__ixor__",  [](C &s, const iterable &other) {
+			symmetric_difference<C, T>(s, other);
+			return s;
+		})
 		.def("copy", [](const C &s) {
 			return new C(s);
 		})
@@ -107,20 +269,29 @@ void bind_pool(module &m, const char *name_cstr) {
 		.def("__iter__", [](const C &s){
 			return make_iterator(s.begin(), s.end());
 		}, keep_alive<0,1>())
-		.def("__repr__", [name_cstr](const C &s){
+		.def("__repr__", [name_cstr](const py::iterable &s){
-			return std::string("<") + name_cstr + " size=" + std::to_string(s.size()) + ">";
+			// repr(set(s)) where s is iterable would be more terse/robust
+			// but are there concerns with copying?
+			str representation = str(name_cstr) + str("({");
+			str comma(", ");
+			for (const auto &element: s) {
+				representation += repr(element);
+				representation += comma; // python supports trailing commas
+			}
+			representation += str("})");
+			return representation;
 		});
 }
 
 // shim
 template <typename C, typename T>
-void bind_set(module &m, const char *name_cstr) {
+void bind_pool(module &m, const char *name_cstr) {
-	bind_pool<C, T>(m, name_cstr);
+	bind_set<C, T>(m, name_cstr);
 }
 
 
 template <typename C, typename K, typename V>
-void update_dict(C *target, iterable &iterable_or_mapping) {
+void update_dict(C *target, const iterable &iterable_or_mapping) {
 	if (is_mapping(iterable_or_mapping)) {
 		for (const auto &key: iterable_or_mapping) {
 			(*target)[cast<K>(key)] = cast<V>(iterable_or_mapping[key]);
@@ -137,10 +308,14 @@ void update_dict(C *target, iterable &iterable_or_mapping) {
 
 template <typename C, typename K, typename V>
 void bind_dict(module &m, const char *name_cstr) {
-	std::string {name_cstr};
+	auto cls = class_<C>(m, name_cstr)
-
-	class_<C>(m, name_cstr)
 		.def(init<>())
+		.def(init<const C &>()) // copy constructor
+		.def(init([](const iterable &other){ // copy instructor from arbitrary iterables and mappings
+			auto s = new C();
+			update_dict<C, K, V>(s, other);
+			return s;
+		}))
 		.def("__len__", [](const C &s){ return (size_t)s.size(); })
 		.def("__getitem__", [](const C &s, const K &k) { return s.at(k); })
 		.def("__setitem__", [](C &s, const K &k, const V &v) { s[k] = v; })
@@ -210,9 +385,29 @@ void bind_dict(module &m, const char *name_cstr) {
 			return s;
 		})
 		.def("__bool__", [](const C &s) { return s.size() != 0; })
-		.def("__repr__", [name_cstr](const C &s){
+		.def("__repr__", [name_cstr](const C &s) {
-			return std::string("<") + name_cstr + " size=" + std::to_string(s.size()) + ">";
+			// repr(dict(s)) where s is iterable would be more terse/robust
+			// but are there concerns with copying?
+			str representation = str(name_cstr) + str("({");
+			str colon(": ");
+			str comma(", ");
+			for (const auto &item: s) {
+				representation += repr(cast(item.first));
+				representation += colon;
+				representation += repr(cast(item.second));
+				representation += comma; // python supports trailing commas
+			}
+			representation += str("})");
+			return representation;
 		});
+
+	// Inherit from collections.abc.Mapping so update operators (and a bunch
+	// of other things) work.
+	auto collections_abc = module_::import("collections.abc");
+	auto mapping = getattr(collections_abc, "Mapping");
+	auto current_bases = list(getattr(cls, "__bases__"));
+	current_bases.append(mapping);
+	setattr(cls, "__bases__", tuple(current_bases));
 }
 
 // idict is a special bijection and doesn't map cleanly to dict
@@ -221,10 +416,9 @@ void bind_dict(module &m, const char *name_cstr) {
 // the hashable as key and the integer as value
 template <typename C, typename K>
 void bind_idict(module &m, const char *name_cstr) {
-	std::string {name_cstr};
-
 	auto cls = class_<C>(m, name_cstr)
 		.def(init<>())
+		.def(init<const C &>()) // copy constructor
 		.def("__len__", [](const C &s){ return (size_t)s.size(); })
 		.def("__getitem__", [](const C &s, int v) { return s[v]; })
 		.def("__call__", [](C &s, const K &k) { return s(k); })
@@ -276,7 +470,16 @@ void bind_idict(module &m, const char *name_cstr) {
 		})
 		.def("__bool__", [](const C &s) { return s.size() != 0; })
 		.def("__repr__", [name_cstr](const C &s){
-			return std::string("<") + name_cstr + " size=" + std::to_string(s.size()) + ">";
+			// repr(dict(s)) where s is iterable would be more terse/robust
+			// but are there concerns with copying?
+			str representation = str(name_cstr) + str("() | {");
+			str comma(", ");
+			for (const auto &item: s) {
+				representation += repr(cast(item));
+				representation += comma; // python supports trailing commas
+			}
+			representation += str("}");
+			return representation;
 		});
 
 	for (const char *mutator: {"__setitem__", "__delitem__", "pop", "popitem", "setdefault"}) {

pyosys/wrappers_tpl.cc

@@ -34,12 +34,6 @@ using namespace RTLIL;
 #include "wrappers.inc.cc"
 
 namespace YOSYS_PYTHON {
-
-	[[noreturn]] static void log_python_exception_as_error() {
-		PyErr_Print();
-		log_error("Python interpreter encountered an exception.\\n");
-	}
-
 	struct YosysStatics{};
 
 	// Trampolines for Classes with Python-Overridable Virtual Methods

pyproject.toml

@@ -5,3 +5,7 @@ requires = [
 	"cxxheaderparser",
 ]
 build-backend = "setuptools.build_meta"
+
+[tool.ruff]
+target-version = "py38"
+lint.ignore = ["F541"]

tests/pyosys/test_dict.py

@@ -1,6 +1,12 @@
+from typing import Mapping
 from pyosys import libyosys as ys
 
-my_dict = ys.StringToStringDict()
+StringToStringDict = ys.StringToStringDict
+
+my_dict = StringToStringDict()
+
+assert isinstance(my_dict, Mapping)
+
 my_dict["foo"] = "bar"
 my_dict.update([("first", "second")])
 my_dict.update({"key": "value"})
@@ -11,3 +17,13 @@ new_dict = my_dict | {"tomato": "tomato"}
 del new_dict["foo"]
 assert set(my_dict.keys()) == {"first", "key", "foo"}
 assert set(new_dict.keys()) == {"first", "key", "tomato"}
+
+constructor_test_1 = ys.StringToStringDict(new_dict)
+constructor_test_2 = ys.StringToStringDict([("tomato", "tomato")])
+constructor_test_3 = ys.StringToStringDict({ "im running": "out of string ideas" })
+
+the_great_or = constructor_test_1 | constructor_test_2 | constructor_test_3
+
+assert set(the_great_or) ==  {"first", "key", "tomato", "im running"}
+repr_test = eval(repr(the_great_or))
+assert repr_test == the_great_or

tests/pyosys/test_import.py

@@ -1,3 +1,20 @@
+import os
+import sys
+
 from pyosys import libyosys as ys
 
-ys.log("Hello, world!")
+print(ys)
+
+ys.log("Hello, world!\n")
+
+from pyosys.libyosys import log
+
+print(log)
+
+log("Goodbye, world!\n")
+
+import pyosys
+
+if os.path.basename(sys.executable) == "yosys":
+	# make sure it's not importing the directory
+	assert "built-in" in repr(pyosys)

tests/pyosys/test_set.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+from pyosys.libyosys import StringSet, StringPool
+
+for cls in [StringSet, StringPool]:
+	print(f"Testing {cls.__name__}...")
+	A = cls()
+	A.add("a")
+
+	B = cls()
+	B = A | {"b"}
+
+	assert A < B
+	assert A <= B
+
+	A.add("b")
+
+	assert A == B
+	assert A <= B
+	assert not A < B
+
+	A.add("c")
+
+	assert A > B
+
+	A &= B
+	assert A == B
+
+	Ø = A - B
+	assert len(Ø) == 0
+
+	C = cls({"A", "B", "C"})
+	D = cls()
+	C |= {"A", "B", "C"}
+	D |= {"C", "D", "E"}
+	c_symdiff_d = (C ^ D)
+	assert (c_symdiff_d) == {"A", "B", "D", "E"}
+
+	repr_test = eval(repr(c_symdiff_d))
+	c_symdiff_d == repr_test
+
+
+print("Done.")