From 54799bb8be85f784b9682f5643e0f41f93684863 Mon Sep 17 00:00:00 2001 From: Mohamed Gaber Date: Tue, 23 Sep 2025 03:44:34 +0300 Subject: [PATCH] pyosys: globals, set operators for opaque types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is so much templating going on that compiling wrappers.cc now takes 1m1.668s on an Apple M4… --- Makefile | 2 +- kernel/yosys.cc | 10 +- pyosys/generator.py | 48 ++++--- pyosys/hashlib.h | 241 +++++++++++++++++++++++++++++++++--- pyosys/wrappers_tpl.cc | 6 - pyproject.toml | 4 + tests/pyosys/test_dict.py | 18 ++- tests/pyosys/test_import.py | 19 ++- tests/pyosys/test_set.py | 42 +++++++ 9 files changed, 343 insertions(+), 47 deletions(-) create mode 100644 tests/pyosys/test_set.py diff --git a/Makefile b/Makefile index c9cbd0810..027f18a7a 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ all: top-all YOSYS_SRC := $(dir $(firstword $(MAKEFILE_LIST))) VPATH := $(YOSYS_SRC) -CXXSTD ?= c++17 +export CXXSTD ?= c++17 CXXFLAGS := $(CXXFLAGS) -Wall -Wextra -ggdb -I. -I"$(YOSYS_SRC)" -MD -MP -D_YOSYS_ -fPIC -I$(PREFIX)/include LIBS := $(LIBS) -lstdc++ -lm PLUGIN_LINKFLAGS := diff --git a/kernel/yosys.cc b/kernel/yosys.cc index 9cab12bf6..47057c1ca 100644 --- a/kernel/yosys.cc +++ b/kernel/yosys.cc @@ -185,13 +185,12 @@ bool already_setup = false; bool already_shutdown = false; #ifdef WITH_PYTHON -// Include pyosys as a module so 'from pyosys import libyosys' also works -// in interpreter mode. +// Include pyosys as a package for some compatibility with wheels. // // This should not affect using wheels as the dylib has to actually be called -// pyosys.so for this module to be interacted with at all. +// pyosys.so for this function to be interacted with at all. PYBIND11_MODULE(pyosys, m) { - m.add_object("libyosys", m.import("libyosys")); + m.add_object("__path__", py::list()); } #endif @@ -209,7 +208,8 @@ void yosys_setup() // initialized platform fails (such as when libyosys is imported // from a Python interpreter) if (!Py_IsInitialized()) { - PyImport_AppendInittab((char*)"libyosys", PyInit_libyosys); + PyImport_AppendInittab((char*)"pyosys.libyosys", PyInit_libyosys); + // compatibility with wheels PyImport_AppendInittab((char*)"pyosys", PyInit_pyosys); Py_Initialize(); PyRun_SimpleString("import sys"); diff --git a/pyosys/generator.py b/pyosys/generator.py index b8a7aa8dc..b0cec99e3 100644 --- a/pyosys/generator.py +++ b/pyosys/generator.py @@ -18,7 +18,19 @@ # Written by Mohamed Gaber # # Inspired by py_wrap_generator.py by Benedikt Tutzer +""" +This generates: +- Wrapper calls for opaque container types +- Full wrappers for select classes and all enums, global functions and global + variables +Jump to "MARK: Inclusion and Exclusion" to control the above. + +Please run ruff on this file in particular to make sure it parses with Python +<= 3.12. There is so much f-string use here that the outer quote thing +is a common problem. ``ruff check pyosys/generator.py`` suffices. +""" +import os import io import shutil from pathlib import Path @@ -58,15 +70,24 @@ class PyosysClass: :param name: The base name of the class (without extra qualifiers) :param ref_only: Whether this class can be copied to Python, or only referenced. - :param string_expr: A C++ expression that will be used for the __str__ method in Python - :param hash_expr: A C++ expression that will be fed to ``run_hash`` to declare a __hash__ method for Python + :param string_expr: + A C++ expression that will be used for the ``__str__`` method in Python. + + The object will be available as a const reference with the identifier + `s`. + :param hash_expr: + A C++ expression that will be fed to ``run_hash`` to declare a + ``__hash__`` method for Python. + + The object will be vailable as a const reference with the identifier + `s`. :param denylist: If specified, one or more methods can be excluded from wrapping. """ name: str ref_only: bool = False - # in the format s.(method or property) + # in the format s.(method or property) (or just s) string_expr: Optional[str] = None hash_expr: Optional[str] = None @@ -89,9 +110,7 @@ class PyosysHeader: for cls in classes: self.classes_by_name[cls.name] = cls -""" -Add headers and classes here! -""" +# MARK: Inclusion and Exclusion global_denylist = frozenset( { # deprecated @@ -327,10 +346,11 @@ class PyosysWrapperGenerator(object): def make_preprocessor_options(self): py_include = get_paths()["include"] preprocessor_bin = shutil.which("clang++") or "g++" + cxx_std = os.getenv("CXX_STD", "c++17") return ParserOptions( preprocessor=make_gcc_preprocessor( defines=["_YOSYS_", "WITH_PYTHON"], - gcc_args=[preprocessor_bin, "-fsyntax-only"], + gcc_args=[preprocessor_bin, "-fsyntax-only", f"-std={cxx_std}"], include_paths=[str(__yosys_root__), py_include, pybind11.get_include()], ), ) @@ -476,7 +496,7 @@ class PyosysWrapperGenerator(object): if function.static: definition_fn = "def_static" - print(f"\t\t\t.{definition_fn}({", ".join(self.get_definition_args(function, class_basename, python_name_override))})", file=self.f) + print(f"\t\t\t.{definition_fn}({', '.join(self.get_definition_args(function, class_basename, python_name_override))})", file=self.f) def process_function(self, function: Function): if ( @@ -496,7 +516,7 @@ class PyosysWrapperGenerator(object): self.register_containers(function) - print(f"\t\t\tm.def({", ".join(self.get_definition_args(function, None))});", file=self.f) + print(f"\t\t\tm.def({', '.join(self.get_definition_args(function, None))});", file=self.f) def process_field(self, field: Field, class_basename: str): if field.access != "public": @@ -511,7 +531,8 @@ class PyosysWrapperGenerator(object): unique_ptrs = self.find_containers(("unique_ptr",), field.type) if len(unique_ptrs): - # TODO: figure out how to bridge unique pointers + # TODO: figure out how to bridge unique pointers maybe does anyone + # care return self.register_containers(field) @@ -559,10 +580,10 @@ class PyosysWrapperGenerator(object): self.process_method(method, basename) visited_anonymous_unions = set() - for field in cls.fields: - if field.name in metadata.denylist: + for field_ in cls.fields: + if field_.name in metadata.denylist: continue - self.process_field(field, basename) + self.process_field(field_, basename) # Handle anonymous unions for subclass in cls.classes: @@ -663,7 +684,6 @@ class PyosysWrapperGenerator(object): keyword_aliases = { - "in": "in_", "False": "False_", "None": "None_", "True": "True_", diff --git a/pyosys/hashlib.h b/pyosys/hashlib.h index 2ecadc3f1..b772a2bb3 100644 --- a/pyosys/hashlib.h +++ b/pyosys/hashlib.h @@ -37,7 +37,7 @@ // things like mutating containers that are class properties. // // All methods should be vaguely in the same order as the python reference -// https://docs.python.org/3/library/stdtypes.html +// https://docs.python.org/3.13/library/stdtypes.html // #include // optional maps cleanest to methods that accept None in Python @@ -60,26 +60,188 @@ bool is_mapping(object obj) { return isinstance(obj, mapping); } +// Set Operations +bool is_subset(const iterable &lhs, const iterable &rhs, bool strict = false) { + for (auto &element: lhs) { + if (!rhs.contains(element)) { + return false; + } + } + if (strict) { + return len(rhs) > len(lhs); + } + return true; +} + +template +void unionize(C &lhs, const iterable &rhs) { + for (auto &element: rhs) { + lhs.insert(cast(element)); + } +} + +template +void difference(C &lhs, const iterable &rhs) { + for (auto &element: rhs) { + auto element_cxx = cast(element); + if (lhs.count(element_cxx)) { + lhs.erase(element_cxx); + } + } +} + +template +void intersect(C &lhs, const iterable &rhs) { + // Doing it in-place is a lot slower + // TODO?: Leave modifying lhs to caller (saves a copy) but complicates + // chaining intersections. + C storage(lhs); + + for (auto &element_cxx: lhs) { + if (!rhs.contains(cast(element_cxx))) { + storage.erase(element_cxx); + } + } + lhs = std::move(storage); +} + +template +void symmetric_difference(C &lhs, const iterable &rhs) { + C storage(lhs); + + for (auto &element: rhs) { + auto element_cxx = cast(element); + if (lhs.count(element_cxx)) { + storage.erase(element_cxx); + } else { + storage.insert(element_cxx); + } + } + for (auto &element_cxx: lhs) { + if (rhs.contains(cast(element_cxx))) { + storage.erase(element_cxx); + } + } + lhs = std::move(storage); +} + // shim template void bind_vector(module &m, const char *name_cstr) { pybind11::bind_vector(m, name_cstr); } -// also used for std::set because the semantics are close enough +// also used for hashlib pool because the semantics are close enough template -void bind_pool(module &m, const char *name_cstr) { - std::string {name_cstr}; - +void bind_set(module &m, const char *name_cstr) { class_(m, name_cstr) .def(init<>()) + .def(init()) // copy constructor + .def(init([](const iterable &other){ // copy instructor from arbitrary iterables + auto s = new C(); + unionize(*s, other); + return s; + })) .def("__len__", [](const C &s){ return (size_t)s.size(); }) .def("__contains__", [](const C &s, const T &v){ return s.count(v); }) .def("__delitem__", [](C &s, const T &v) { auto n = s.erase(v); if (n == 0) throw key_error(str(cast(v))); }) - // TODO: disjoint, subset, union, intersection, difference, symdif + .def("disjoint", [](const C &s, const iterable &other) { + for (const auto &element: other) { + if (s.count(cast(element))) { + return false; + } + } + return true; + }) + .def("issubset", [](const iterable &s, const iterable &other) { + return is_subset(s, other); + }) + .def("__eq__", [](const iterable &s, const iterable &other) { + return is_subset(s, other) && len(s) == len(other); + }) + .def("__le__", [](const iterable &s, const iterable &other) { + return is_subset(s, other); + }) + .def("__lt__", [](const iterable &s, const iterable &other) { + return is_subset(s, other, true); + }) + .def("issuperset", [](const iterable &s, const iterable &other) { + return is_subset(other, s); + }) + .def("__ge__", [](const iterable &s, const iterable &other) { + return is_subset(other, s); + }) + .def("__gt__", [](const iterable &s, const iterable &other) { + return is_subset(other, s, true); + }) + .def("union", [](const C &s, const args &others) { + auto result = new C(s); + for (const auto &arg: others) { + auto arg_iterable = reinterpret_borrow(arg); + unionize(*result, arg_iterable); + } + return result; + }) + .def("__or__", [](const C &s, const iterable &other) { + auto result = new C(s); + unionize(*result, other); + return result; + }) + .def("__ior__", [](C &s, const iterable &other) { + unionize(s, other); + return s; + }) + .def("intersection", [](const C &s, const args &others) { + auto result = new C(s); + for (const auto &arg: others) { + auto arg_iterable = reinterpret_borrow(arg); + intersect(*result, arg_iterable); + } + return result; + }) + .def("__and__", [](const C &s, const iterable &other) { + auto result = new C(s); + intersect(*result, other); + return result; + }) + .def("__iand__", [](C &s, const iterable &other) { + intersect(s, other); + return s; + }) + .def("difference", [](const C &s, const args &others) { + auto result = new C(s); + for (const auto &arg: others) { + auto arg_iterable = reinterpret_borrow(arg); + difference(*result, arg_iterable); + } + return result; + }) + .def("__sub__", [](const C &s, const iterable &other) { + auto result = new C(s); + difference(*result, other); + return result; + }) + .def("__isub__", [](C &s, const iterable &other) { + difference(s, other); + return s; + }) + .def("symmetric_difference", [](const C &s, const iterable &other) { + auto result = new C(s); + symmetric_difference(*result, other); + return result; + }) + .def("__xor__", [](const C &s, const iterable &other) { + auto result = new C(s); + symmetric_difference(*result, other); + return result; + }) + .def("__ixor__", [](C &s, const iterable &other) { + symmetric_difference(s, other); + return s; + }) .def("copy", [](const C &s) { return new C(s); }) @@ -107,20 +269,29 @@ void bind_pool(module &m, const char *name_cstr) { .def("__iter__", [](const C &s){ return make_iterator(s.begin(), s.end()); }, keep_alive<0,1>()) - .def("__repr__", [name_cstr](const C &s){ - return std::string("<") + name_cstr + " size=" + std::to_string(s.size()) + ">"; + .def("__repr__", [name_cstr](const py::iterable &s){ + // repr(set(s)) where s is iterable would be more terse/robust + // but are there concerns with copying? + str representation = str(name_cstr) + str("({"); + str comma(", "); + for (const auto &element: s) { + representation += repr(element); + representation += comma; // python supports trailing commas + } + representation += str("})"); + return representation; }); } // shim template -void bind_set(module &m, const char *name_cstr) { - bind_pool(m, name_cstr); +void bind_pool(module &m, const char *name_cstr) { + bind_set(m, name_cstr); } template -void update_dict(C *target, iterable &iterable_or_mapping) { +void update_dict(C *target, const iterable &iterable_or_mapping) { if (is_mapping(iterable_or_mapping)) { for (const auto &key: iterable_or_mapping) { (*target)[cast(key)] = cast(iterable_or_mapping[key]); @@ -137,10 +308,14 @@ void update_dict(C *target, iterable &iterable_or_mapping) { template void bind_dict(module &m, const char *name_cstr) { - std::string {name_cstr}; - - class_(m, name_cstr) + auto cls = class_(m, name_cstr) .def(init<>()) + .def(init()) // copy constructor + .def(init([](const iterable &other){ // copy instructor from arbitrary iterables and mappings + auto s = new C(); + update_dict(s, other); + return s; + })) .def("__len__", [](const C &s){ return (size_t)s.size(); }) .def("__getitem__", [](const C &s, const K &k) { return s.at(k); }) .def("__setitem__", [](C &s, const K &k, const V &v) { s[k] = v; }) @@ -210,9 +385,29 @@ void bind_dict(module &m, const char *name_cstr) { return s; }) .def("__bool__", [](const C &s) { return s.size() != 0; }) - .def("__repr__", [name_cstr](const C &s){ - return std::string("<") + name_cstr + " size=" + std::to_string(s.size()) + ">"; + .def("__repr__", [name_cstr](const C &s) { + // repr(dict(s)) where s is iterable would be more terse/robust + // but are there concerns with copying? + str representation = str(name_cstr) + str("({"); + str colon(": "); + str comma(", "); + for (const auto &item: s) { + representation += repr(cast(item.first)); + representation += colon; + representation += repr(cast(item.second)); + representation += comma; // python supports trailing commas + } + representation += str("})"); + return representation; }); + + // Inherit from collections.abc.Mapping so update operators (and a bunch + // of other things) work. + auto collections_abc = module_::import("collections.abc"); + auto mapping = getattr(collections_abc, "Mapping"); + auto current_bases = list(getattr(cls, "__bases__")); + current_bases.append(mapping); + setattr(cls, "__bases__", tuple(current_bases)); } // idict is a special bijection and doesn't map cleanly to dict @@ -221,10 +416,9 @@ void bind_dict(module &m, const char *name_cstr) { // the hashable as key and the integer as value template void bind_idict(module &m, const char *name_cstr) { - std::string {name_cstr}; - auto cls = class_(m, name_cstr) .def(init<>()) + .def(init()) // copy constructor .def("__len__", [](const C &s){ return (size_t)s.size(); }) .def("__getitem__", [](const C &s, int v) { return s[v]; }) .def("__call__", [](C &s, const K &k) { return s(k); }) @@ -276,7 +470,16 @@ void bind_idict(module &m, const char *name_cstr) { }) .def("__bool__", [](const C &s) { return s.size() != 0; }) .def("__repr__", [name_cstr](const C &s){ - return std::string("<") + name_cstr + " size=" + std::to_string(s.size()) + ">"; + // repr(dict(s)) where s is iterable would be more terse/robust + // but are there concerns with copying? + str representation = str(name_cstr) + str("() | {"); + str comma(", "); + for (const auto &item: s) { + representation += repr(cast(item)); + representation += comma; // python supports trailing commas + } + representation += str("}"); + return representation; }); for (const char *mutator: {"__setitem__", "__delitem__", "pop", "popitem", "setdefault"}) { diff --git a/pyosys/wrappers_tpl.cc b/pyosys/wrappers_tpl.cc index d235b2c55..84e201657 100644 --- a/pyosys/wrappers_tpl.cc +++ b/pyosys/wrappers_tpl.cc @@ -34,12 +34,6 @@ using namespace RTLIL; #include "wrappers.inc.cc" namespace YOSYS_PYTHON { - - [[noreturn]] static void log_python_exception_as_error() { - PyErr_Print(); - log_error("Python interpreter encountered an exception.\\n"); - } - struct YosysStatics{}; // Trampolines for Classes with Python-Overridable Virtual Methods diff --git a/pyproject.toml b/pyproject.toml index 5a218c19d..d5882084c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,7 @@ requires = [ "cxxheaderparser", ] build-backend = "setuptools.build_meta" + +[tool.ruff] +target-version = "py38" +lint.ignore = ["F541"] diff --git a/tests/pyosys/test_dict.py b/tests/pyosys/test_dict.py index 81e7b5516..916d69b92 100644 --- a/tests/pyosys/test_dict.py +++ b/tests/pyosys/test_dict.py @@ -1,6 +1,12 @@ +from typing import Mapping from pyosys import libyosys as ys -my_dict = ys.StringToStringDict() +StringToStringDict = ys.StringToStringDict + +my_dict = StringToStringDict() + +assert isinstance(my_dict, Mapping) + my_dict["foo"] = "bar" my_dict.update([("first", "second")]) my_dict.update({"key": "value"}) @@ -11,3 +17,13 @@ new_dict = my_dict | {"tomato": "tomato"} del new_dict["foo"] assert set(my_dict.keys()) == {"first", "key", "foo"} assert set(new_dict.keys()) == {"first", "key", "tomato"} + +constructor_test_1 = ys.StringToStringDict(new_dict) +constructor_test_2 = ys.StringToStringDict([("tomato", "tomato")]) +constructor_test_3 = ys.StringToStringDict({ "im running": "out of string ideas" }) + +the_great_or = constructor_test_1 | constructor_test_2 | constructor_test_3 + +assert set(the_great_or) == {"first", "key", "tomato", "im running"} +repr_test = eval(repr(the_great_or)) +assert repr_test == the_great_or diff --git a/tests/pyosys/test_import.py b/tests/pyosys/test_import.py index 48e911033..b6a36b0c1 100644 --- a/tests/pyosys/test_import.py +++ b/tests/pyosys/test_import.py @@ -1,3 +1,20 @@ +import os +import sys + from pyosys import libyosys as ys -ys.log("Hello, world!") +print(ys) + +ys.log("Hello, world!\n") + +from pyosys.libyosys import log + +print(log) + +log("Goodbye, world!\n") + +import pyosys + +if os.path.basename(sys.executable) == "yosys": + # make sure it's not importing the directory + assert "built-in" in repr(pyosys) diff --git a/tests/pyosys/test_set.py b/tests/pyosys/test_set.py new file mode 100644 index 000000000..d89c5243e --- /dev/null +++ b/tests/pyosys/test_set.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +from pyosys.libyosys import StringSet, StringPool + +for cls in [StringSet, StringPool]: + print(f"Testing {cls.__name__}...") + A = cls() + A.add("a") + + B = cls() + B = A | {"b"} + + assert A < B + assert A <= B + + A.add("b") + + assert A == B + assert A <= B + assert not A < B + + A.add("c") + + assert A > B + + A &= B + assert A == B + + Ø = A - B + assert len(Ø) == 0 + + C = cls({"A", "B", "C"}) + D = cls() + C |= {"A", "B", "C"} + D |= {"C", "D", "E"} + c_symdiff_d = (C ^ D) + assert (c_symdiff_d) == {"A", "B", "D", "E"} + + repr_test = eval(repr(c_symdiff_d)) + c_symdiff_d == repr_test + + +print("Done.")