mirrors/yosys
3
0
Fork 0
mirror of https://github.com/YosysHQ/yosys synced 2025-10-31 11:42:30 +00:00
Code Activity
yosys/pyosys/generator.py
Mohamed Gaber 80fcce64da
pyosys: fix ref-only classes, implicit conversions 
+ cleanup
2025-10-03 11:54:44 +03:00

788 lines
27 KiB
Python
Raw Blame History

#!/usr/bin/env python3
# yosys -- Yosys Open SYnthesis Suite
#
# Copyright (C) 2012 Claire Xenia Wolf <claire@yosyshq.com>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# Written by Mohamed Gaber <me@donn.website>
#
# Inspired by py_wrap_generator.py by Benedikt Tutzer
"""
This generates:
- Wrapper calls for opaque container types
- Full wrappers for select classes and all enums, global functions and global
variables
Jump to "MARK: Inclusion and Exclusion" to control the above.
Please run ruff on this file in particular to make sure it parses with Python
<= 3.12. There is so much f-string use here that the outer quote thing
is a common problem. ``ruff check pyosys/generator.py`` suffices.
"""
import os
import io
import shutil
import argparse
from pathlib import Path
from sysconfig import get_paths
from dataclasses import dataclass, field
from typing import Any, Dict, FrozenSet, Iterable, Tuple, Union, Optional, List
import pybind11
from cxxheaderparser.simple import parse_file, ClassScope, NamespaceScope, EnumDecl
from cxxheaderparser.options import ParserOptions
from cxxheaderparser.preprocessor import make_gcc_preprocessor
from cxxheaderparser.types import (
PQName,
Type,
Pointer,
Reference,
MoveReference,
AnonymousName,
Method,
Function,
Field,
Variable,
Array,
FundamentalSpecifier,
)
__file_dir__ = Path(__file__).absolute().parent
__yosys_root__ = __file_dir__.parent
@dataclass
class PyosysClass:
"""
Metadata about classes or structs intended to be wrapped using Pyosys.
:param name: The base name of the class (without extra qualifiers)
:param ref_only: Whether this class can be copied to Python, or only
referenced.
:param string_expr:
A C++ expression that will be used for the ``__str__`` method in Python.
The object will be available as a const reference with the identifier
`s`.
:param hash_expr:
A C++ expression that will be fed to ``run_hash`` to declare a
``__hash__`` method for Python.
The object will be vailable as a const reference with the identifier
`s`.
:param denylist: If specified, one or more methods can be excluded from
wrapping.
"""
name: str
ref_only: bool = False
# in the format s.(method or property) (or just s)
string_expr: Optional[str] = None
hash_expr: Optional[str] = None
denylist: FrozenSet[str] = frozenset({})
@dataclass
class PyosysHeader:
"""
:param name: The name of the header, i.e., its relative path to the Yosys root
:param classes: A list of ``PyosysClass`` classes to be wrapped
:param enums: A list of enums to be wrapped
"""
name: str
classes: List[PyosysClass] = field(default_factory=lambda: [])
def __post_init__(self):
self.classes_by_name = {}
if classes := self.classes:
for cls in classes:
self.classes_by_name[cls.name] = cls
# MARK: Inclusion and Exclusion
global_denylist = frozenset(
{
# deprecated
"builtin_ff_cell_types",
"logv_file_error",
# no implementation
"set_verific_logging",
# can't bridge to python cleanly
## std::regex
"log_warn_regexes",
"log_nowarn_regexes",
"log_werror_regexes",
## function pointers
"log_error_atexit",
"log_verific_callback",
}
)
pyosys_headers = [
# Headers for incomplete types
PyosysHeader("kernel/binding.h"),
PyosysHeader("libs/sha1/sha1.h"),
# Headers for globals
PyosysHeader("kernel/log.h"),
PyosysHeader("kernel/yosys.h"),
PyosysHeader("kernel/cost.h"),
# Headers with classes
PyosysHeader(
"kernel/celltypes.h",
[PyosysClass("CellType", hash_expr="s.type"), PyosysClass("CellTypes")],
),
PyosysHeader("kernel/consteval.h", [PyosysClass("ConstEval")]),
PyosysHeader(
"kernel/register.h",
[
# PyosysClass("Pass") # Virtual methods, manually bridged
],
),
PyosysHeader(
"kernel/rtlil.h",
[
PyosysClass(
"IdString",
string_expr="s.str()",
hash_expr="s.str()",
denylist=frozenset(
# shouldn't be messed with from python in general
{
"global_id_storage_",
"global_id_index_",
"global_refcount_storage_",
"global_free_idx_list_",
"last_created_idx_ptr_",
"last_created_idx_",
"builtin_ff_cell_types",
}
),
),
PyosysClass(
"Const",
string_expr="s.as_string()",
denylist=frozenset({"bits", "bitvectorize"}),
),
PyosysClass("AttrObject", denylist=frozenset({"get_blackbox_attribute"})),
PyosysClass("NamedObject", denylist=frozenset({"get_blackbox_attribute"})),
PyosysClass("Selection"),
# PyosysClass("Monitor"), # Virtual methods, manually bridged
PyosysClass("CaseRule", denylist=frozenset({"get_blackbox_attribute"})),
PyosysClass("SwitchRule", denylist=frozenset({"get_blackbox_attribute"})),
PyosysClass("SyncRule"),
PyosysClass(
"Process",
ref_only=True,
string_expr="s.name.c_str()",
hash_expr="s.name",
),
PyosysClass("SigChunk"),
PyosysClass("SigBit", hash_expr="s"),
PyosysClass("SigSpec", hash_expr="s"),
PyosysClass(
"Cell",
ref_only=True,
string_expr="s.name.c_str()",
hash_expr="s",
),
PyosysClass(
"Wire",
ref_only=True,
string_expr="s.name.c_str()",
hash_expr="s",
),
PyosysClass(
"Memory",
ref_only=True,
string_expr="s.name.c_str()",
hash_expr="s",
),
PyosysClass(
"Module",
ref_only=True,
string_expr="s.name.c_str()",
hash_expr="s",
denylist=frozenset({"Pow"}), # has no implementation
),
PyosysClass(
"Design",
string_expr="s.hashidx_",
hash_expr="s",
denylist=frozenset({"selected_whole_modules"}), # deprecated
),
],
),
]
@dataclass(frozen=True) # hashable
class PyosysType:
"""
Bit of a hacky object all-around: this is more or less used to encapsulate
container types so they can be later made opaque using pybind.
"""
base: str
specialization: Tuple["PyosysType", ...]
const: bool = False
@classmethod
def from_type(Self, type_obj, drop_const=False) -> "PyosysType":
const = type_obj.const and not drop_const
if isinstance(type_obj, Pointer):
ptr_to = Self.from_type(type_obj.ptr_to)
return Self("ptr", (ptr_to,), const)
elif isinstance(type_obj, Reference):
ref_to = Self.from_type(type_obj.ref_to)
return Self("ref", (ref_to,), const)
assert isinstance(
type_obj, Type
), f"unexpected c++ type object of type {type(type_obj)}"
last_segment = type_obj.typename.segments[-1]
base = last_segment.name
specialization = tuple()
if (
hasattr(last_segment, "specialization")
and last_segment.specialization is not None
):
for template_arg in last_segment.specialization.args:
specialization = (*specialization, Self.from_type(template_arg.arg))
return Self(base, specialization, const)
def generate_identifier(self):
title = self.base.title()
if len(self.specialization) == 0:
return title
if title == "Dict":
key, value = self.specialization
return f"{key.generate_identifier()}To{value.generate_identifier()}{title}"
return (
"".join(spec.generate_identifier() for spec in self.specialization) + title
)
def generate_cpp_name(self):
const_prefix = "const " * self.const
if len(self.specialization) == 0:
return const_prefix + self.base
elif self.base == "ptr":
return const_prefix + f"{self.specialization[0].generate_cpp_name()} *"
elif self.base == "ref":
return const_prefix + f"{self.specialization[0].generate_cpp_name()} &"
else:
return (
const_prefix
+ f"{self.base}<{', '.join(spec.generate_cpp_name() for spec in self.specialization)}>"
)
class PyosysWrapperGenerator(object):
def __init__(
self,
from_headers: Iterable[PyosysHeader],
wrapper_stream: io.TextIOWrapper,
header_stream: io.TextIOWrapper,
):
self.headers = from_headers
self.f = wrapper_stream
self.f_inc = header_stream
self.found_containers: Dict[PyosysType, Any] = {}
self.class_registry: Dict[str, ClassScope] = {}
# entry point
def generate(self):
tpl = __file_dir__ / "wrappers_tpl.cc"
preprocessor_opts = self.make_preprocessor_options()
with open(tpl, encoding="utf8") as f:
do_line_directive = True
for i, line in enumerate(f):
if do_line_directive:
self.f.write(f'#line {i + 1} "{tpl}"\n')
do_line_directive = False
if "<!-- generated includes -->" in line:
for header in self.headers:
self.f.write(f'#include "{header.name}"\n')
do_line_directive = True
elif "<!-- generated pymod-level code -->" in line:
for header in self.headers:
header_path = __yosys_root__ / header.name
parsed = parse_file(header_path, options=preprocessor_opts)
global_namespace = parsed.namespace
self.process_namespace(header, global_namespace)
else:
self.f.write(line)
for container, _ in self.found_containers.items():
identifier = container.generate_identifier()
print(
f"using {identifier} = {container.generate_cpp_name()};",
file=self.f_inc,
)
print(f"PYBIND11_MAKE_OPAQUE({identifier})", file=self.f_inc)
print(
f"static void bind_autogenerated_opaque_containers(py::module &m) {{",
file=self.f_inc,
)
for container, _ in self.found_containers.items():
identifier = container.generate_identifier()
cxx = container.generate_cpp_name()
tpl_args = [cxx]
for spec in container.specialization:
tpl_args.append(spec.generate_cpp_name())
print(
f'\tpy::hashlib::bind_{container.base}<{", ".join(tpl_args)}>(m, "{container.generate_identifier()}");',
file=self.f_inc,
)
print(
f"\tpy::implicitly_convertible<py::iterable, {identifier}>();",
file=self.f_inc,
)
print(f"}}", file=self.f_inc)
# helpers
def make_preprocessor_options(self):
py_include = get_paths()["include"]
preprocessor_bin = shutil.which("clang++") or "g++"
cxx_std = os.getenv("CXX_STD", "c++17")
return ParserOptions(
preprocessor=make_gcc_preprocessor(
defines=["_YOSYS_", "YOSYS_ENABLE_PYTHON"],
gcc_args=[preprocessor_bin, "-fsyntax-only", f"-std={cxx_std}"],
include_paths=[str(__yosys_root__), py_include, pybind11.get_include()],
),
)
@staticmethod
def find_containers(
containers: Iterable[str], type_info: Any
) -> Dict[PyosysType, Any]:
if isinstance(type_info, Pointer):
return PyosysWrapperGenerator.find_containers(containers, type_info.ptr_to)
if isinstance(type_info, MoveReference):
return PyosysWrapperGenerator.find_containers(
containers, type_info.moveref_to
)
if isinstance(type_info, Reference):
return PyosysWrapperGenerator.find_containers(containers, type_info.ref_to)
if not isinstance(type_info, Type):
return ()
segments = type_info.typename.segments
containers_found = {}
for segment in segments:
if isinstance(segment, FundamentalSpecifier):
continue
if segment.name in containers:
containers_found.update(
{PyosysType.from_type(type_info, drop_const=True): type_info}
)
if segment.specialization is not None:
for arg in segment.specialization.args:
sub_containers = PyosysWrapperGenerator.find_containers(
containers, arg.arg
)
containers_found.update(sub_containers)
return containers_found
@staticmethod
def find_anonymous_union(cls: ClassScope):
if cls.class_decl.typename.classkey != "union":
return None
for s in cls.class_decl.typename.segments:
if isinstance(s, AnonymousName):
return s
return None # named union
@staticmethod
def get_parameter_types(function: Function) -> str:
return ", ".join(p.type.format() for p in function.parameters)
def register_containers(self, target: Union[Function, Field, Variable]):
supported = ("dict", "idict", "pool", "set", "vector")
if isinstance(target, Function):
self.found_containers.update(
self.find_containers(supported, target.return_type)
)
for parameter in target.parameters:
self.found_containers.update(
self.find_containers(supported, parameter.type)
)
else:
self.found_containers.update(self.find_containers(supported, target.type))
# processors
def get_overload_cast(
self, function: Function, class_basename: Optional[str]
) -> str:
is_method = isinstance(function, Method)
function_return_type = function.return_type.format()
if class_basename == "Const" and function_return_type in {
"iterator",
"const_iterator",
}:
# HACK: qualify Const's iterators
function_return_type = f"{class_basename}::{function_return_type}"
pointer_kind = (
f"{class_basename}::*" if (is_method and not function.static) else "*"
)
retval = f"static_cast <"
retval += function_return_type
retval += f"({pointer_kind})"
retval += f"({self.get_parameter_types(function)})"
if is_method and function.const:
retval += " const"
retval += ">"
retval += "(&"
if is_method:
retval += f"{class_basename}::"
retval += function.name.segments[-1].format()
retval += ")"
return retval
def get_definition_args(
self,
function: Function,
class_basename: Optional[str],
python_name_override: Optional[str] = None,
) -> List[str]:
function_basename = function.name.segments[-1].format()
python_function_basename = python_name_override or keyword_aliases.get(
function_basename, function_basename
)
def_args = [f'"{python_function_basename}"']
def_args.append(self.get_overload_cast(function, class_basename))
for parameter in function.parameters:
# ASSUMPTION: there are no unnamed parameters in the yosys codebase
parameter_arg = f'py::arg("{parameter.name}")'
if parameter.default is not None:
parameter_arg += f" = {parameter.default.format()}"
def_args.append(parameter_arg)
return def_args
def process_method(self, metadata: PyosysClass, function: Method):
if (
function.deleted
or function.template
or function.vararg
or function.access != "public"
or function.pure_virtual
or function.destructor
):
return
if any(isinstance(p.type, MoveReference) for p in function.parameters):
# skip move constructors
return
if len(function.name.segments) > 1:
# can't handle, skip
return
if function.constructor:
if (
not metadata.ref_only
): # ref-only classes should not be constructed from python
print(
f"\t\t\t.def(py::init<{self.get_parameter_types(function)}>())",
file=self.f,
)
return
python_name_override = None
if function.operator is not None:
if function.operator == "==":
python_name_override = "__eq__"
elif function.operator == "!=":
python_name_override = "__ne__"
elif function.operator == "<":
python_name_override = "__lt__"
else:
return
self.register_containers(function)
definition_fn = "def"
if function.static:
definition_fn = "def_static"
print(
f"\t\t\t.{definition_fn}({', '.join(self.get_definition_args(function, metadata.name, python_name_override))})",
file=self.f,
)
def process_function(self, function: Function):
if function.deleted or function.template or function.vararg or function.static:
return
if function.operator is not None:
# Python doesn't do global operators
return
if function.name.segments[-1].format() in global_denylist:
return
self.register_containers(function)
print(
f"\t\t\tm.def({', '.join(self.get_definition_args(function, None))});",
file=self.f,
)
def process_field(self, metadata: PyosysClass, field: Field):
if field.access != "public":
return
if field.name is None:
# anonymous structs and unions
# unions handled in calling function
# ASSUMPTION: No anonymous structs in the yosys codebase
# (they're not in the C++ standard anyway)
return
unique_ptrs = self.find_containers(("unique_ptr",), field.type)
if len(unique_ptrs):
# TODO: figure out how to bridge unique pointers maybe does anyone
# care
return
self.register_containers(field)
definition_fn = f"def_{'readonly' if field.type.const else 'readwrite'}"
if field.static:
definition_fn += "_static"
field_python_basename = keyword_aliases.get(field.name, field.name)
print(
f'\t\t\t.{definition_fn}("{field_python_basename}", &{metadata.name}::{field.name})',
file=self.f,
)
def process_variable(self, variable: Variable):
if isinstance(variable.type, Array):
return
variable_basename = variable.name.segments[-1].format()
if variable_basename in global_denylist:
return
self.register_containers(variable)
definition_fn = (
f"def_{'readonly' if variable.type.const else 'readwrite'}_static"
)
variable_python_basename = keyword_aliases.get(
variable_basename, variable_basename
)
variable_name = variable.name.format()
print(
f'\t\t\tglobal_variables.{definition_fn}("{variable_python_basename}", &{variable_name});',
file=self.f,
)
def process_class_members(
self, metadata: PyosysClass, cls: ClassScope, basename: str
):
for method in cls.methods:
if method.name.segments[-1].name in metadata.denylist:
continue
self.process_method(metadata, method)
visited_anonymous_unions = set()
for field_ in cls.fields:
if field_.name in metadata.denylist:
continue
self.process_field(metadata, field_)
# Handle anonymous unions
for subclass in cls.classes:
if subclass.class_decl.access != "public":
continue
if au := self.find_anonymous_union(subclass):
if au.id in visited_anonymous_unions:
continue
visited_anonymous_unions.add(au.id)
for subfield in subclass.fields:
self.process_field(metadata, subfield)
def process_class(
self,
metadata: PyosysClass,
cls: ClassScope,
namespace_components: Tuple[str, ...],
):
pqname: PQName = cls.class_decl.typename
full_path = list(namespace_components) + [
segment.format() for segment in pqname.segments
]
basename = full_path.pop()
self.class_registry[basename] = cls
declaration_namespace = "::".join(full_path)
tpl_args = [basename]
if metadata.ref_only:
tpl_args.append(f"std::unique_ptr<{basename}, py::nodelete>")
print(
f'\t\t{{using namespace {declaration_namespace}; py::class_<{", ".join(tpl_args)}>(m, "{basename}")',
file=self.f,
)
self.process_class_members(metadata, cls, basename)
for base in cls.class_decl.bases:
if base.access != "public":
continue
name = base.typename.segments[-1].format()
if base_scope := self.class_registry.get(name):
self.process_class_members(metadata, base_scope, basename)
if expr := metadata.string_expr:
print(
f'\t\t.def("__str__", [](const {basename} &s) {{ return {expr}; }})',
file=self.f,
)
if expr := metadata.hash_expr:
print(
f'\t\t.def("__hash__", [](const {basename} &s) {{ return run_hash({expr}); }})',
file=self.f,
)
print(f"\t\t;}}", file=self.f)
def process_enum(
self,
enum: EnumDecl,
namespace_components: Tuple[str, ...],
):
pqname: PQName = enum.typename
full_path = list(namespace_components) + [
segment.format() for segment in pqname.segments
]
basename = full_path.pop()
declaration_namespace = "::".join(full_path)
print(
f'\t\t{{using namespace {declaration_namespace}; py::native_enum<{basename}>(m, "{basename}", "enum.Enum")',
file=self.f,
)
enum_class = enum.typename.classkey == "enum class"
for value in enum.values:
enum_class_qualifier = f"{basename}::" * enum_class
print(
f'\t\t\t.value("{value.name}", {enum_class_qualifier}{value.name})',
file=self.f,
)
print(f"\t\t\t.finalize();}}", file=self.f)
def process_namespace(
self,
header: PyosysHeader,
ns: NamespaceScope,
namespace_components: Tuple[str, ...] = (),
):
for name, subns in ns.namespaces.items():
self.process_namespace(header, subns, (*namespace_components, name))
if len(namespace_components) and (len(ns.functions) + len(ns.variables)):
# TODO: Not essential but maybe move namespace usage into
# process_function for consistency?
print(
f"\t\t{{ using namespace {'::'.join(namespace_components)};",
file=self.f,
)
for function in ns.functions:
self.process_function(function)
for variable in ns.variables:
self.process_variable(variable)
print(f"\t\t}}", file=self.f)
for enum in ns.enums:
self.process_enum(enum, namespace_components)
for cls in ns.classes:
pqname = cls.class_decl.typename
declared_name_str = pqname.segments[-1].format()
if cls_metadata := header.classes_by_name.get(declared_name_str):
self.process_class(cls_metadata, cls, namespace_components)
keyword_aliases = {
"False": "False_",
"None": "None_",
"True": "True_",
"and": "and_",
"as": "as_",
"assert": "assert_",
"break": "break_",
"class": "class_",
"continue": "continue_",
"def": "def_",
"del": "del_",
"elif": "elif_",
"else": "else_",
"except": "except_",
"for": "for_",
"from": "from_",
"global": "global_",
"if": "if_",
"import": "import_",
"in": "in_",
"is": "is_",
"lambda": "lambda_",
"nonlocal": "nonlocal_",
"not": "not_",
"or": "or_",
"pass": "pass_",
"raise": "raise_",
"return": "return_",
"try": "try_",
"while": "while_",
"with": "with_",
"yield": "yield_",
}
def print_includes():
for header in pyosys_headers:
print(header.name)
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--debug", default=0, type=int)
group = ap.add_mutually_exclusive_group(required=True)
group.add_argument("--print-includes", action="store_true")
group.add_argument("output", nargs="?")
ns = ap.parse_args()
if ns.print_includes:
print_includes()
exit(0)
out_path = Path(ns.output)
out_inc = out_path.parent / (out_path.stem + ".inc.cc")
with open(out_path, "w", encoding="utf8") as f, open(
out_inc, "w", encoding="utf8"
) as inc_f:
generator = PyosysWrapperGenerator(
from_headers=pyosys_headers, wrapper_stream=f, header_stream=inc_f
)
generator.generate()
View git blame Copy permalink