3
0
Fork 0
mirror of https://github.com/YosysHQ/yosys synced 2025-10-08 17:01:57 +00:00

pyosys: rewrite using pybind11

- Rewrite all Python features to use the pybind11 library instead of boost::python.
  Unlike boost::python, pybind11 is a header-only library that is just included by Pyosys code, saving a lot of compile time on wheels.
- Factor out as much "translation" code from the generator into proper C++ files
- Fix running the embedded interpreter not supporting "from pyosys import libyosys as ys" like wheels
- Move Python-related elements to `pyosys` directory at the root of the repo
- Slight shift in bridging semantics:
  - Containers are declared as "opaque types" and are passed by reference to Python - many methods have been implemented to make them feel right at home without the overhead/ambiguity of copying to Python and then copying back after mutation
  - Monitor/Pass use "trampoline" pattern to support virual methods overridable in Python: virtual methods no longer require `py_` prefix
- Create really short test set for pyosys that just exercises basic functionality
This commit is contained in:
Mohamed Gaber 2025-09-21 22:36:27 +03:00
parent f7120e9c2a
commit 88be728353
No known key found for this signature in database
27 changed files with 2879 additions and 2674 deletions

View file

@ -55,11 +55,6 @@ jobs:
submodules: true
persist-credentials: false
- uses: actions/setup-python@v5
- name: Get Boost Source
shell: bash
run: |
mkdir -p boost
curl -L https://github.com/boostorg/boost/releases/download/boost-1.86.0/boost-1.86.0-b2-nodocs.tar.gz | tar --strip-components=1 -xzC boost
- name: Get FFI
shell: bash
run: |
@ -103,21 +98,16 @@ jobs:
CIBW_BEFORE_ALL: bash ./.github/workflows/wheels/cibw_before_all.sh
CIBW_ENVIRONMENT: >
OPTFLAGS=-O3
CXXFLAGS=-I./boost/pfx/include
LINKFLAGS=-L./boost/pfx/lib
PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a'
PATH="$PWD/bison/src:$PATH"
CIBW_ENVIRONMENT_MACOS: >
OPTFLAGS=-O3
CXXFLAGS=-I./boost/pfx/include
LINKFLAGS=-L./boost/pfx/lib
PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
MACOSX_DEPLOYMENT_TARGET=11
makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a CONFIG=clang'
makeFlags='CONFIG=clang'
PATH="$PWD/bison/src:$PATH"
CIBW_BEFORE_BUILD: bash ./.github/workflows/wheels/cibw_before_build.sh
CIBW_TEST_COMMAND: python3 {project}/tests/arch/ecp5/add_sub.py
CIBW_TEST_COMMAND: python3 {project}/tests/pyosys/run_tests.py python3
- uses: actions/upload-artifact@v4
with:
name: python-wheels-${{ matrix.os.runner }}

View file

@ -11,24 +11,3 @@ if [ "$(uname)" != "Linux" ]; then
fi
python3 --version
python3-config --includes
# Build boost
cd ./boost
## Delete the artefacts from previous builds (if any)
rm -rf ./pfx
## Bootstrap bjam
./bootstrap.sh --prefix=./pfx
## Build Boost against current version of Python, only for
## static linkage (Boost is statically linked because system boost packages
## wildly vary in versions, including the libboost_python3 version)
./b2\
-j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)\
--prefix=./pfx\
--with-filesystem\
--with-system\
--with-python\
cxxflags="$(python3-config --includes) -std=c++17 -fPIC"\
cflags="$(python3-config --includes) -fPIC"\
link=static\
variant=release\
install

View file

@ -38,7 +38,8 @@ techlibs/gowin/ @pepijndevos
techlibs/gatemate/ @pu-cc
# pyosys
misc/*.py @btut
pyosys/* @donn
setup.py @donn
backends/firrtl @ucbjrl @azidar

View file

@ -133,10 +133,6 @@ LINKFLAGS += -rdynamic
ifneq ($(shell :; command -v brew),)
BREW_PREFIX := $(shell brew --prefix)/opt
$(info $$BREW_PREFIX is [${BREW_PREFIX}])
ifeq ($(ENABLE_PYOSYS),1)
CXXFLAGS += -I$(BREW_PREFIX)/boost/include
LINKFLAGS += -L$(BREW_PREFIX)/boost/lib -L$(BREW_PREFIX)/boost-python3/lib
endif
CXXFLAGS += -I$(BREW_PREFIX)/readline/include -I$(BREW_PREFIX)/flex/include
LINKFLAGS += -L$(BREW_PREFIX)/readline/lib -L$(BREW_PREFIX)/flex/lib
PKG_CONFIG_PATH := $(BREW_PREFIX)/libffi/lib/pkgconfig:$(PKG_CONFIG_PATH)
@ -353,26 +349,14 @@ ifeq ($(ENABLE_PYOSYS),1)
LINKFLAGS += $(filter-out -l%,$(shell $(PYTHON_CONFIG) --ldflags))
LIBS += $(shell $(PYTHON_CONFIG) --libs)
EXE_LIBS += $(filter-out $(LIBS),$(shell $(PYTHON_CONFIG_FOR_EXE) --libs))
PYBIND11_INCLUDE ?= $(shell $(PYTHON_EXECUTABLE) -m pybind11 --includes)
CXXFLAGS += -I$(PYBIND11_INCLUDE) -DWITH_PYTHON
CXXFLAGS += $(shell $(PYTHON_CONFIG) --includes) -DWITH_PYTHON
# Detect name of boost_python library. Some distros use boost_python-py<version>, other boost_python<version>, some only use the major version number, some a concatenation of major and minor version numbers
CHECK_BOOST_PYTHON = (echo "int main(int argc, char ** argv) {return 0;}" | $(CXX) -xc -o /dev/null $(LINKFLAGS) $(EXE_LIBS) $(LIBS) -l$(1) - > /dev/null 2>&1 && echo "-l$(1)")
BOOST_PYTHON_LIB ?= $(shell \
$(call CHECK_BOOST_PYTHON,boost_python-py$(subst .,,$(PYTHON_VERSION))) || \
$(call CHECK_BOOST_PYTHON,boost_python-py$(PYTHON_MAJOR_VERSION)) || \
$(call CHECK_BOOST_PYTHON,boost_python$(subst .,,$(PYTHON_VERSION))) || \
$(call CHECK_BOOST_PYTHON,boost_python$(PYTHON_MAJOR_VERSION)) \
)
ifeq ($(BOOST_PYTHON_LIB),)
$(error BOOST_PYTHON_LIB could not be detected. Please define manually)
endif
LIBS += $(BOOST_PYTHON_LIB) -lboost_system -lboost_filesystem
PY_WRAPPER_FILE = kernel/python_wrappers
PY_WRAPPER_FILE = pyosys/wrappers
OBJS += $(PY_WRAPPER_FILE).o
PY_GEN_SCRIPT= py_wrap_generator
PY_WRAP_INCLUDES := $(shell $(PYTHON_EXECUTABLE) -c "from misc import $(PY_GEN_SCRIPT); $(PY_GEN_SCRIPT).print_includes()")
PY_GEN_SCRIPT = pyosys/generator.py
PY_WRAP_INCLUDES := $(shell $(PYTHON_EXECUTABLE) $(PY_GEN_SCRIPT) --print-includes)
endif # ENABLE_PYOSYS
ifeq ($(ENABLE_READLINE),1)
@ -779,9 +763,9 @@ endif
$(P) cat $< | grep -E -v "#[ ]*(include|error)" | $(CXX) $(CXXFLAGS) -x c++ -o $@ -E -P -
ifeq ($(ENABLE_PYOSYS),1)
$(PY_WRAPPER_FILE).cc: misc/$(PY_GEN_SCRIPT).py $(PY_WRAP_INCLUDES)
$(PY_WRAPPER_FILE).cc: $(PY_GEN_SCRIPT) pyosys/wrappers_tpl.cc $(PY_WRAP_INCLUDES) pyosys/hashlib.h
$(Q) mkdir -p $(dir $@)
$(P) $(PYTHON_EXECUTABLE) -c "from misc import $(PY_GEN_SCRIPT); $(PY_GEN_SCRIPT).gen_wrappers(\"$(PY_WRAPPER_FILE).cc\")"
$(P) $(PYTHON_EXECUTABLE) $(PY_GEN_SCRIPT) $(PY_WRAPPER_FILE).cc
endif
%.o: %.cpp

View file

@ -36,7 +36,7 @@ The main characteristics are:
all compilers, standard libraries and architectures.
In addition to ``dict<K, T>`` and ``pool<T>`` there is also an ``idict<K>`` that
creates a bijective map from ``K`` to the integers. For example:
creates a bijective map from ``K`` to incrementing integers. For example:
::

View file

@ -1,22 +1,25 @@
#!/usr/bin/python3
import libyosys as ys
from pyosys import libyosys as ys
from pathlib import Path
import matplotlib.pyplot as plt
import numpy as np
__file_dir__ = Path(__file__).absolute().parent
class CellStatsPass(ys.Pass):
def __init__(self):
super().__init__("cell_stats", "Shows cell stats as plot")
def py_help(self):
def help(self):
ys.log("This pass uses the matplotlib library to display cell stats\n")
def py_execute(self, args, design):
def execute(self, args, design):
ys.log_header(design, "Plotting cell stats\n")
cell_stats = {}
for module in design.selected_whole_modules_warn():
for module in design.all_selected_whole_modules():
for cell in module.selected_cells():
if cell.type.str() in cell_stats:
cell_stats[cell.type.str()] += 1
@ -29,4 +32,11 @@ class CellStatsPass(ys.Pass):
def py_clear_flags(self):
ys.log("Clear Flags - CellStatsPass\n")
p = CellStatsPass()
p = CellStatsPass() # register
if __name__ == "__main__":
design = ys.Design()
ys.run_pass(f"read_verilog {__file_dir__.parents[1] / 'tests' / 'simple' / 'fiedler-cooley.v'}", design)
ys.run_pass("prep", design)
ys.run_pass("opt -full", design)
ys.run_pass("cell_stats", design)

View file

@ -92,8 +92,9 @@ int main(int argc, char **argv)
yosys_banner();
yosys_setup();
#ifdef WITH_PYTHON
PyRun_SimpleString(("sys.path.append(\""+proc_self_dirname()+"\")").c_str());
PyRun_SimpleString(("sys.path.append(\""+proc_share_dirname()+"plugins\")").c_str());
py::object sys = py::module_::import("sys");
sys.attr("path").attr("append")(proc_self_dirname());
sys.attr("path").attr("append")(proc_share_dirname());
#endif
if (argc == 2)
@ -516,8 +517,9 @@ int main(int argc, char **argv)
yosys_setup();
#ifdef WITH_PYTHON
PyRun_SimpleString(("sys.path.append(\""+proc_self_dirname()+"\")").c_str());
PyRun_SimpleString(("sys.path.append(\""+proc_share_dirname()+"plugins\")").c_str());
py::object sys = py::module_::import("sys");
sys.attr("path").attr("append")(proc_self_dirname());
sys.attr("path").attr("append")(proc_share_dirname());
#endif
log_error_atexit = yosys_atexit;
@ -567,21 +569,18 @@ int main(int argc, char **argv)
#endif
} else if (scriptfile_python) {
#ifdef WITH_PYTHON
PyObject *sys = PyImport_ImportModule("sys");
py::list new_argv;
int py_argc = special_args.size() + 1;
PyObject *new_argv = PyList_New(py_argc);
PyList_SetItem(new_argv, 0, PyUnicode_FromString(scriptfile.c_str()));
new_argv.append(scriptfile);
for (int i = 1; i < py_argc; ++i)
PyList_SetItem(new_argv, i, PyUnicode_FromString(special_args[i - 1].c_str()));
new_argv.append(special_args[i - 1]);
PyObject *old_argv = PyObject_GetAttrString(sys, "argv");
PyObject_SetAttrString(sys, "argv", new_argv);
Py_DECREF(old_argv);
py::setattr(sys, "argv", new_argv);
PyObject *py_path = PyUnicode_FromString(scriptfile.c_str());
PyObject_SetAttrString(sys, "_yosys_script_path", py_path);
Py_DECREF(py_path);
PyRun_SimpleString("import os, sys; sys.path.insert(0, os.path.dirname(os.path.abspath(sys._yosys_script_path)))");
py::object Path = py::module_::import("pathlib").attr("Path");
py::object scriptfile_python_path = Path(scriptfile).attr("parent");
sys.attr("path").attr("insert")(0, py::str(scriptfile_python_path));
FILE *scriptfp = fopen(scriptfile.c_str(), "r");
if (scriptfp == nullptr) {

View file

@ -64,13 +64,8 @@
#endif
#ifdef WITH_PYTHON
#if PY_MAJOR_VERSION >= 3
# define INIT_MODULE PyInit_libyosys
extern "C" PyObject* INIT_MODULE();
#else
# define INIT_MODULE initlibyosys
extern "C" void INIT_MODULE();
#endif
extern "C" PyObject* PyInit_libyosys();
extern "C" PyObject* PyInit_pyosys();
#include <signal.h>
#endif
@ -189,6 +184,17 @@ int run_command(const std::string &command, std::function<void(const std::string
bool already_setup = false;
bool already_shutdown = false;
#ifdef WITH_PYTHON
// Include pyosys as a module so 'from pyosys import libyosys' also works
// in interpreter mode.
//
// This should not affect using wheels as the dylib has to actually be called
// pyosys.so for this module to be interacted with at all.
PYBIND11_MODULE(pyosys, m) {
m.add_object("libyosys", m.import("libyosys"));
}
#endif
void yosys_setup()
{
if(already_setup)
@ -199,11 +205,12 @@ void yosys_setup()
IdString::ensure_prepopulated();
#ifdef WITH_PYTHON
// With Python 3.12, calling PyImport_AppendInittab on an already
// Starting Python 3.12, calling PyImport_AppendInittab on an already
// initialized platform fails (such as when libyosys is imported
// from a Python interpreter)
if (!Py_IsInitialized()) {
PyImport_AppendInittab((char*)"libyosys", INIT_MODULE);
PyImport_AppendInittab((char*)"libyosys", PyInit_libyosys);
PyImport_AppendInittab((char*)"pyosys", PyInit_pyosys);
Py_Initialize();
PyRun_SimpleString("import sys");
signal(SIGINT, SIG_DFL);

View file

@ -55,6 +55,9 @@
#ifdef WITH_PYTHON
#include <Python.h>
#include <pybind11/pybind11.h>
namespace py = pybind11;
#endif
#ifndef _YOSYS_

File diff suppressed because it is too large Load diff

View file

@ -24,12 +24,6 @@
# include <dlfcn.h>
#endif
#ifdef WITH_PYTHON
# include <boost/algorithm/string/predicate.hpp>
# include <Python.h>
# include <boost/filesystem.hpp>
#endif
YOSYS_NAMESPACE_BEGIN
std::map<std::string, void*> loaded_plugins;
@ -57,23 +51,23 @@ void load_plugin(std::string filename, std::vector<std::string> aliases)
if (!is_loaded) {
// Check if we're loading a python script
if(filename.find(".py") != std::string::npos)
{
if (filename.rfind(".py") != std::string::npos) {
#ifdef WITH_PYTHON
boost::filesystem::path full_path(filename);
std::string path(full_path.parent_path().c_str());
filename = full_path.filename().c_str();
filename = filename.substr(0,filename.size()-3);
PyRun_SimpleString(("sys.path.insert(0,\""+path+"\")").c_str());
PyErr_Print();
PyObject *module_p = PyImport_ImportModule(filename.c_str());
if(module_p == NULL)
{
PyErr_Print();
log_cmd_error("Can't load python module `%s'\n", full_path.filename());
py::object Path = py::module_::import("pathlib").attr("Path");
py::object full_path = Path(py::cast(filename));
py::object plugin_python_path = full_path.attr("parent");
auto basename = py::cast<std::string>(full_path.attr("stem"));
py::object sys = py::module_::import("sys");
sys.attr("path").attr("insert")(0, py::str(plugin_python_path));
try {
auto module_container = py::module_::import(basename.c_str());
loaded_python_plugins[orig_filename] = module_container.ptr();
} catch (py::error_already_set &e) {
log_cmd_error("Can't load python module `%s': %s\n", basename, e.what());
return;
}
loaded_python_plugins[orig_filename] = module_p;
Pass::init_register();
#else
log_error(

1
pyosys/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
wrappers.cc

View file

@ -1,3 +1,4 @@
import os
import sys

2039
pyosys/generator.py Normal file

File diff suppressed because it is too large Load diff

275
pyosys/hashlib.h Normal file
View file

@ -0,0 +1,275 @@
// -------------------------------------------------------
// Written by Mohamed Gaber in 2025 <me@donn.website>
// Based on kernel/hashlib.h by Claire Xenia Wolf <claire@yosyshq.com>
// -------------------------------------------------------
// This header is free and unencumbered software released into the public domain.
//
// Anyone is free to copy, modify, publish, use, compile, sell, or
// distribute this software, either in source code form or as a compiled
// binary, for any purpose, commercial or non-commercial, and by any
// means.
//
// In jurisdictions that recognize copyright laws, the author or authors
// of this software dedicate any and all copyright interest in the
// software to the public domain. We make this dedication for the benefit
// of the public at large and to the detriment of our heirs and
// successors. We intend this dedication to be an overt act of
// relinquishment in perpetuity of all present and future rights to this
// software under copyright law.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
//
// For more information, please refer to <https://unlicense.org/>
// -------------------------------------------------------
//
// pybind11 bridging headers for hashlib template
//
// These are various binding functions that expose hashlib templates as opaque
// types (https://pybind11.readthedocs.io/en/latest/advanced/cast/stl.html#making-opaque-types).
//
// Opaque types cross language barries by reference, not value. This allows
// things like mutating containers that are class properties.
//
// All methods should be vaguely in the same order as the python reference
// https://docs.python.org/3/library/stdtypes.html
//
#include <optional> // optional maps cleanest to methods that accept None in Python
#include <pybind11/pybind11.h> // base
#include <pybind11/stl.h> // std::optional
#include <pybind11/operators.h> // easier operator binding
#include "kernel/hashlib.h"
namespace pybind11 {
template<typename T>
struct is_pointer { static const bool value = false; };
template<typename T>
struct is_pointer<T*> { static const bool value = true; };
bool is_mapping(object obj) {
object mapping = module_::import("collections.abc").attr("Mapping");
return isinstance(obj, mapping);
}
// also used for std::set because the semantics are close enough
template <typename C, typename T>
void bind_pool(module &m, const char *name_cstr) {
std::string {name_cstr};
class_<C>(m, name_cstr)
.def(init<>())
.def("__len__", [](const C &s){ return (size_t)s.size(); })
.def("__contains__", [](const C &s, const T &v){ return s.count(v); })
.def("__delitem__", [](C &s, const T &v) {
auto n = s.erase(v);
if (n == 0) throw key_error(str(cast(v)));
})
// TODO: disjoint, subset, union, intersection, difference, symdif
.def("copy", [](const C &s) {
return new C(s);
})
.def("update", [](C &s, iterable iterable) {
for (auto item: iterable) {
s.insert(item.cast<T>());
}
})
.def("add", [](C &s, const T &v){ s.insert(v); })
.def("remove", [](C &s, const T &v){
auto n = s.erase(v);
if (n == 0) throw key_error(str(cast(v)));
})
.def("discard", [](C &s, const T &v){ s.erase(v); })
.def("clear", [](C &s){ s.clear(); })
.def("pop", [](C &s){
if (s.size() == 0) {
throw key_error("empty pool");
}
auto result = *s.begin();
s.erase(result);
return result;
})
.def("__bool__", [](const C &s) { return s.size() != 0; })
.def("__iter__", [](const C &s){
return make_iterator(s.begin(), s.end());
}, keep_alive<0,1>())
.def("__repr__", [name_cstr](const C &s){
return std::string("<") + name_cstr + " size=" + std::to_string(s.size()) + ">";
});
}
template <typename C, typename K, typename V>
void update_dict(C *target, iterable &iterable_or_mapping) {
if (is_mapping(iterable_or_mapping)) {
for (const auto &key: iterable_or_mapping) {
(*target)[cast<K>(key)] = cast<V>(iterable_or_mapping[key]);
}
} else {
for (const auto &pair: iterable_or_mapping) {
if (len(pair) != 2) {
throw value_error(str("iterable element %s has more than two elements").format(str(pair)));
}
(*target)[cast<K>(pair[cast(0)])] = cast<V>(pair[cast(1)]);
}
}
}
template <typename C, typename K, typename V>
void bind_dict(module &m, const char *name_cstr) {
std::string {name_cstr};
class_<C>(m, name_cstr)
.def(init<>())
.def("__len__", [](const C &s){ return (size_t)s.size(); })
.def("__getitem__", [](const C &s, const K &k) { return s.at(k); })
.def("__setitem__", [](C &s, const K &k, const V &v) { s[k] = v; })
.def("__delitem__", [](C &s, const K &k) {
auto n = s.erase(k);
if (n == 0) throw key_error("remove: key not found");
})
.def("__contains__", [](const C &s, const K &k) { return s.count(k) != 0; })
.def("__iter__", [](const C &s){
return make_key_iterator(s.begin(), s.end());
}, keep_alive<0,1>())
.def("clear", [](C &s){ s.clear(); })
.def("copy", [](const C &s) {
return new C(s);
})
.def("get", [](const C &s, const K& k, std::optional<const V> &default_) {
if (default_.has_value()) {
return s.at(k, *default_);
} else {
return s.at(k);
}
}, arg("key"), arg("default") = std::nullopt)
.def("items", [](const C &s){
return make_iterator(s.begin(), s.end());
}, keep_alive<0,1>())
.def("keys", [](const C &s){
return make_key_iterator(s.begin(), s.end());
}, keep_alive<0,1>())
.def("pop", [](const C &s, const K& k, std::optional<const V> &default_) {
if (default_.has_value()) {
return s.at(k, *default_);
} else {
return s.at(k);
}
}, arg("key"), arg("default") = std::nullopt)
.def("popitem", [name_cstr](args _) { throw std::runtime_error(std::string(name_cstr) + " is not an ordered dictionary"); })
.def("setdefault", [name_cstr](C &s, const K& k, std::optional<const V> &default_) {
auto it = s.find(k);
if (it != s.end()) {
return it->second;
}
if (default_.has_value()) {
s[k] = *default_;
return *default_;
}
// if pointer, nullptr can be our default
if constexpr (is_pointer<V>::value) {
s[k] = nullptr;
return (V)nullptr;
}
// TODO: std::optional? do we care?
throw type_error(std::string("the value type of ") + name_cstr + " is not nullable");
}, arg("key"), arg("default") = std::nullopt)
.def("update", [](C &s, iterable iterable_or_mapping) {
update_dict<C, K, V>(&s, iterable_or_mapping);
}, arg("iterable_or_mapping"))
.def("values", [](const C &s){
return make_value_iterator(s.begin(), s.end());
}, keep_alive<0,1>())
.def("__or__", [](const C &s, iterable iterable_or_mapping) {
auto result = new C(s);
update_dict<C, K, V>(result, iterable_or_mapping);
return result;
})
.def("__ior__", [](C &s, iterable iterable_or_mapping) {
update_dict<C, K, V>(&s, iterable_or_mapping);
return s;
})
.def("__bool__", [](const C &s) { return s.size() != 0; })
.def("__repr__", [name_cstr](const C &s){
return std::string("<") + name_cstr + " size=" + std::to_string(s.size()) + ">";
});
}
// idict is a special bijection and doesn't map cleanly to dict
//
// it's cleanest, despite the inconsistency with __getitem__, to just think of
// the hashable as key and the integer as value
template <typename C, typename K>
void bind_idict(module &m, const char *name_cstr) {
std::string {name_cstr};
auto cls = class_<C>(m, name_cstr)
.def(init<>())
.def("__len__", [](const C &s){ return (size_t)s.size(); })
.def("__getitem__", [](const C &s, int v) { return s[v]; })
.def("__call__", [](C &s, const K &k) { return s(k); })
.def("__contains__", [](const C &s, const K &k) {
return s.count(k) != 0;
})
.def("__iter__", [](const C &s){
return make_iterator(s.begin(), s.end());
}, keep_alive<0,1>())
.def("clear", [](C &s) {
s.clear();
})
.def("copy", [](const C &s) {
return new C(s);
})
.def("get", [](const C &s, const K& k, std::optional<int> &default_) {
if (default_.has_value()) {
return s.at(k, *default_);
} else {
return s.at(k);
}
}, arg("key"), arg("default") = std::nullopt)
.def("keys", [](const C &s){
return make_iterator(s.begin(), s.end());
})
.def("values", [](args _){
throw type_error("idicts do not support iteration on the integers");
})
.def("items", [](args _){
throw type_error("idicts do not support pairwise iteration");
})
.def("update", [](C &s, iterable iterable) {
for (auto &e: iterable) {
s(cast<K>(e));
}
})
.def("__or__", [](const C &s, iterable iterable) {
auto result = new C(s);
for (auto &e: iterable) {
(*result)(cast<K>(e));
}
return result;
})
.def("__ior__", [](C &s, iterable iterable) {
for (auto &e: iterable) {
s(cast<K>(e));
}
return s;
})
.def("__bool__", [](const C &s) { return s.size() != 0; })
.def("__repr__", [name_cstr](const C &s){
return std::string("<") + name_cstr + " size=" + std::to_string(s.size()) + ">";
});
for (const char *mutator: {"__setitem__", "__delitem__", "pop", "popitem", "setdefault"}) {
cls.def(mutator, [](args _) {
throw type_error("idicts do not support arbitrary element mutation");
});
}
}
}; // namespace pybind11

248
pyosys/wrappers_tpl.cc Normal file
View file

@ -0,0 +1,248 @@
/*
* yosys -- Yosys Open SYnthesis Suite
*
* Copyright (C) 2012 Claire Xenia Wolf <claire@yosyshq.com>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#ifdef WITH_PYTHON
// <!-- generated includes -->
#include <pybind11/stl_bind.h>
#include <pybind11/native_enum.h>
#include "pyosys/hashlib.h"
USING_YOSYS_NAMESPACE
// <!-- generated top-level code -->
namespace YOSYS_PYTHON {
[[noreturn]] static void log_python_exception_as_error() {
PyErr_Print();
log_error("Python interpreter encountered an exception.\\n");
}
struct YosysStatics{};
// <!-- generated YOSYS_PYTHON namespace-level code -->
// Trampolines for Classes with Python-Overridable Virtual Methods
// https://pybind11.readthedocs.io/en/stable/advanced/classes.html#overriding-virtual-functions-in-python
class PassTrampoline : public Pass {
public:
using Pass::Pass;
void help() override {
PYBIND11_OVERRIDE(void, Pass, help);
}
bool formatted_help() override {
PYBIND11_OVERRIDE(bool, Pass, formatted_help);
}
void clear_flags() override {
PYBIND11_OVERRIDE(void, Pass, clear_flags);
}
void execute(std::vector<std::string> args, RTLIL::Design *design) override {
PYBIND11_OVERRIDE_PURE(
void,
Pass,
execute,
args,
design
);
}
void on_register() override {
PYBIND11_OVERRIDE(void, Pass, on_register);
}
void on_shutdown() override {
PYBIND11_OVERRIDE(void, Pass, on_shutdown);
}
bool replace_existing_pass() const override {
PYBIND11_OVERRIDE(
bool,
Pass,
replace_existing_pass
);
}
};
class MonitorTrampoline : public RTLIL::Monitor {
public:
using RTLIL::Monitor::Monitor;
void notify_module_add(RTLIL::Module *module) override {
PYBIND11_OVERRIDE(
void,
RTLIL::Monitor,
notify_module_add,
module
);
}
void notify_module_del(RTLIL::Module *module) override {
PYBIND11_OVERRIDE(
void,
RTLIL::Monitor,
notify_module_del,
module
);
}
void notify_connect(
RTLIL::Cell *cell,
const RTLIL::IdString &port,
const RTLIL::SigSpec &old_sig,
const RTLIL::SigSpec &sig
) override {
PYBIND11_OVERRIDE(
void,
RTLIL::Monitor,
notify_connect,
cell,
port,
old_sig,
sig
);
}
void notify_connect(
RTLIL::Module *module,
const RTLIL::SigSig &sigsig
) override {
PYBIND11_OVERRIDE(
void,
RTLIL::Monitor,
notify_connect,
module,
sigsig
);
}
void notify_connect(
RTLIL::Module *module,
const std::vector<RTLIL::SigSig> &sigsig_vec
) override {
PYBIND11_OVERRIDE(
void,
RTLIL::Monitor,
notify_connect,
module,
sigsig_vec
);
}
void notify_blackout(
RTLIL::Module *module
) override {
PYBIND11_OVERRIDE(
void,
RTLIL::Monitor,
notify_blackout,
module
);
}
};
/// @brief Use an auxiliary function to adapt the legacy function.
void log_to_stream(py::object object)
{
// TODO
};
PYBIND11_MODULE(libyosys, m) {
// this code is run on import
m.doc() = "python access to libyosys";
if (!yosys_already_setup()) {
log_streams.push_back(&std::cout);
log_error_stderr = true;
yosys_setup();
// Cleanup
m.add_object("_cleanup_handle", py::capsule([](){
yosys_shutdown();
}));
}
m.def("log_to_stream", &log_to_stream, "pipes yosys logs to a Python stream");
// Trampoline Classes
py::class_<Pass, YOSYS_PYTHON::PassTrampoline, std::unique_ptr<Pass, py::nodelete>>(m, "Pass")
.def(py::init([](std::string name, std::string short_help) {
auto created = new YOSYS_PYTHON::PassTrampoline(name, short_help);
Pass::init_register();
return created;
}), py::arg("name"), py::arg("short_help"))
.def("help", &Pass::help)
.def("formatted_help", &Pass::formatted_help)
.def("execute", &Pass::execute)
.def("clear_flags", &Pass::clear_flags)
.def("on_register", &Pass::on_register)
.def("on_shutdown", &Pass::on_shutdown)
.def("replace_existing_pass", &Pass::replace_existing_pass)
.def("experimental", &Pass::experimental)
.def("internal", &Pass::internal)
.def("pre_execute", &Pass::pre_execute)
.def("post_execute", &Pass::post_execute)
.def("cmd_log_args", &Pass::cmd_log_args)
.def("cmd_error", &Pass::cmd_error)
.def("extra_args", &Pass::extra_args)
.def("call", py::overload_cast<RTLIL::Design *,std::string>(&Pass::call))
.def("call", py::overload_cast<RTLIL::Design *,std::vector<std::string>>(&Pass::call))
;
py::class_<RTLIL::Monitor, YOSYS_PYTHON::MonitorTrampoline>(m, "Monitor")
.def(py::init([]() {
return new YOSYS_PYTHON::MonitorTrampoline();
}))
.def("notify_module_add", &RTLIL::Monitor::notify_module_add)
.def("notify_module_del", &RTLIL::Monitor::notify_module_del)
.def(
"notify_connect",
py::overload_cast<
RTLIL::Cell *,
const RTLIL::IdString &,
const RTLIL::SigSpec &,
const RTLIL::SigSpec &
>(&RTLIL::Monitor::notify_connect)
)
.def(
"notify_connect",
py::overload_cast<
RTLIL::Module *,
const RTLIL::SigSig &
>(&RTLIL::Monitor::notify_connect)
)
.def(
"notify_connect",
py::overload_cast<
RTLIL::Module *,
const std::vector<RTLIL::SigSig> &
>(&RTLIL::Monitor::notify_connect)
)
.def("notify_blackout", &RTLIL::Monitor::notify_blackout)
;
// <!-- generated pymod-level code -->
};
};
#endif // WITH_PYTHON

6
pyproject.toml Normal file
View file

@ -0,0 +1,6 @@
[build-system]
requires = [
"setuptools>=42",
"pybind11>=3,<4",
]
build-backend = "setuptools.build_meta"

View file

@ -16,18 +16,19 @@ import os
import re
import shlex
import shutil
from pathlib import Path
from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext
__dir__ = os.path.dirname(os.path.abspath(__file__))
import pybind11
from pybind11.setup_helpers import build_ext
__yosys_root__ = Path(__file__).parent
yosys_version_rx = re.compile(r"YOSYS_VER\s*:=\s*([\w\-\+\.]+)")
version = yosys_version_rx.search(
open(os.path.join(__dir__, "Makefile"), encoding="utf8").read()
)[1].replace(
"+", "."
) # Convert to patch version
with open(__yosys_root__ / "Makefile", encoding="utf8") as f:
# Extract and convert + to patch version
version = yosys_version_rx.search(f.read())[1].replace("+", ".")
class libyosys_so_ext(Extension):
@ -38,7 +39,13 @@ class libyosys_so_ext(Extension):
"libyosys.so",
[],
)
# when iterating locally, you probably want to set this variable
# to avoid mass rebuilds bec of pybind11's include path changing
pybind_include = os.getenv("_FORCE_PYBIND_INCLUDE", pybind11.get_include())
self.args = [
f"PYBIND11_INCLUDE={pybind_include}",
"ENABLE_PYOSYS=1",
# Would need to be installed separately by the user
"ENABLE_TCL=0",
@ -73,22 +80,23 @@ class libyosys_so_ext(Extension):
*self.args,
]
)
build_path = os.path.dirname(os.path.dirname(bext.get_ext_fullpath(self.name)))
pyosys_path = os.path.join(build_path, "pyosys")
ext_fullpath = Path(bext.get_ext_fullpath(self.name))
build_path = ext_fullpath.parents[1]
pyosys_path = build_path / "pyosys"
os.makedirs(pyosys_path, exist_ok=True)
# libyosys.so
target = os.path.join(pyosys_path, os.path.basename(self.name))
target = pyosys_path / self.name
shutil.copy(self.name, target)
bext.spawn(["strip", "-S", target])
bext.spawn(["strip", "-S", str(target)])
# yosys-abc
yosys_abc_target = os.path.join(pyosys_path, "yosys-abc")
yosys_abc_target = pyosys_path / "yosys-abc"
shutil.copy("yosys-abc", yosys_abc_target)
bext.spawn(["strip", "-S", yosys_abc_target])
bext.spawn(["strip", "-S", str(yosys_abc_target)])
# share directory
share_target = os.path.join(pyosys_path, "share")
share_target = pyosys_path / "share"
try:
shutil.rmtree(share_target)
except FileNotFoundError:
@ -104,14 +112,16 @@ class custom_build_ext(build_ext):
return ext.custom_build(self)
with open(__yosys_root__ / "README.md", encoding="utf8") as f:
long_description = f.read()
setup(
name="pyosys",
packages=["pyosys"],
version=version,
description="Python access to libyosys",
long_description=open(os.path.join(__dir__, "README.md")).read(),
long_description=long_description,
long_description_content_type="text/markdown",
install_requires=["wheel", "setuptools"],
license="MIT",
classifiers=[
"Programming Language :: Python :: 3",
@ -119,7 +129,6 @@ setup(
"Operating System :: POSIX :: Linux",
"Operating System :: MacOS :: MacOS X",
],
package_dir={"pyosys": "misc"},
python_requires=">=3.8",
ext_modules=[libyosys_so_ext()],
cmdclass={

39
tests/pyosys/run_tests.py Normal file
View file

@ -0,0 +1,39 @@
from pathlib import Path
import shutil
import subprocess
import sys
__file_dir__ = Path(__file__).absolute().parent
if len(sys.argv) != 2:
print(f"Usage: {sys.argv[0]} {sys.argv[1]}")
exit(64)
binary = []
if sys.argv[1] in ["yosys"]:
binary = [__file_dir__.parents[1] / "yosys", "-Qy"]
else:
binary = [sys.argv[1]]
tests = __file_dir__.glob("test_*.py")
errors = False
log_dir = __file_dir__ / "logs"
try:
shutil.rmtree(log_dir)
except FileNotFoundError:
pass
for test in tests:
print(f"* {test.name} ", end="")
log_dir.mkdir(parents=True, exist_ok=True)
log = log_dir / (test.stem + ".log")
result = subprocess.run([
*binary,
test
], stdout=open(log, "w"), stderr=subprocess.STDOUT)
if result.returncode == 0:
print("OK!")
else:
print(f"FAILED: {log}")
errors = True
if errors:
exit(1)

BIN
tests/pyosys/spm.cut.v.gz Normal file

Binary file not shown.

View file

@ -0,0 +1,45 @@
from pyosys import libyosys as ys
from pathlib import Path
__file_dir__ = Path(__file__).absolute().parent
d = ys.Design()
ys.run_pass(f"read_verilog {__file_dir__ / 'spm.cut.v.gz'}", d)
ys.run_pass("hierarchy -top spm", d)
name_by_tv_location = []
name_by_au_location = []
# test both dictionary mapping and equiv operators working fine
module = None
print(d.modules_)
for idstr, module_obj in d.modules_.items():
if idstr != ys.IdString("\\spm"):
continue
if idstr.str() != "\\spm":
continue
module = module_obj
break
assert module == d.top_module(), "top module search failed"
for name in module.ports:
wire = module.wires_[name]
name_str = name.str()
if name_str.endswith(".d"): # single reg output, in au
name_by_au_location.append(name_str[1:-2])
elif name_str.endswith(".q"): # single reg input, in tv
name_by_tv_location.append(name_str[1:-2])
else: # port/boundary scan
frm = wire.start_offset + wire.width
to = wire.start_offset
for i in range(frm - 1, to - 1, -1):
bit_name = name_str[1:] + f"\\[{i}\\]"
if wire.port_input:
name_by_tv_location.append(bit_name)
elif wire.port_output:
name_by_au_location.append(bit_name)
assert name_by_tv_location == ['x\\[0\\]', 'a\\[31\\]', 'a\\[30\\]', 'a\\[29\\]', 'a\\[28\\]', 'a\\[27\\]', 'a\\[26\\]', 'a\\[25\\]', 'a\\[24\\]', 'a\\[23\\]', 'a\\[22\\]', 'a\\[21\\]', 'a\\[20\\]', 'a\\[19\\]', 'a\\[18\\]', 'a\\[17\\]', 'a\\[16\\]', 'a\\[15\\]', 'a\\[14\\]', 'a\\[13\\]', 'a\\[12\\]', 'a\\[11\\]', 'a\\[10\\]', 'a\\[9\\]', 'a\\[8\\]', 'a\\[7\\]', 'a\\[6\\]', 'a\\[5\\]', 'a\\[4\\]', 'a\\[3\\]', 'a\\[2\\]', 'a\\[1\\]', 'a\\[0\\]', '_315_', '_314_', '_313_', '_312_', '_311_', '_310_', '_309_', '_308_', '_307_', '_306_', '_305_', '_304_', '_303_', '_302_', '_301_', '_300_', '_299_', '_298_', '_297_', '_296_', '_295_', '_294_', '_293_', '_292_', '_291_', '_290_', '_289_', '_288_', '_287_', '_286_', '_285_', '_284_', '_283_', '_282_', '_281_', '_280_', '_279_', '_278_', '_277_', '_276_', '_275_', '_274_', '_273_', '_272_', '_271_', '_270_', '_269_', '_268_', '_267_', '_266_', '_265_', '_264_', '_263_', '_262_', '_261_', '_260_', '_259_', '_258_', '_257_', '_256_', '_255_', '_254_', '_253_', '_252_'], "failed to extract test vector register locations"
assert name_by_au_location == ['y\\[0\\]', '_315_', '_314_', '_313_', '_312_', '_311_', '_310_', '_309_', '_308_', '_307_', '_306_', '_305_', '_304_', '_303_', '_302_', '_301_', '_300_', '_299_', '_298_', '_297_', '_296_', '_295_', '_294_', '_293_', '_292_', '_291_', '_290_', '_289_', '_288_', '_287_', '_286_', '_285_', '_284_', '_283_', '_282_', '_281_', '_280_', '_279_', '_278_', '_277_', '_276_', '_275_', '_274_', '_273_', '_272_', '_271_', '_270_', '_269_', '_268_', '_267_', '_266_', '_265_', '_264_', '_263_', '_262_', '_261_', '_260_', '_259_', '_258_', '_257_', '_256_', '_255_', '_254_', '_253_', '_252_'], "failed to extract golden output register locations"
print("ok!")

13
tests/pyosys/test_dict.py Normal file
View file

@ -0,0 +1,13 @@
from pyosys import libyosys as ys
my_dict = ys.StringToStringDict()
my_dict["foo"] = "bar"
my_dict.update([("first", "second")])
my_dict.update({"key": "value"})
for key, value in my_dict.items():
print(key, value)
new_dict = my_dict | {"tomato": "tomato"}
del new_dict["foo"]
assert set(my_dict.keys()) == {"first", "key", "foo"}
assert set(new_dict.keys()) == {"first", "key", "tomato"}

View file

@ -0,0 +1,31 @@
from pyosys import libyosys as ys
my_idict = ys.IdstringIdict()
print(my_idict(ys.IdString("\\hello")))
print(my_idict(ys.IdString("\\world")))
print(my_idict.get(ys.IdString("\\world")))
try:
print(my_idict.get(ys.IdString("\\dummy")))
except IndexError as e:
print(f"{repr(e)}")
print(my_idict[0])
print(my_idict[1])
try:
print(my_idict[2])
except IndexError as e:
print(f"{repr(e)}")
for i in my_idict:
print(i)
current_len = len(my_idict)
assert current_len == 2, "copy"
my_copy = my_idict.copy()
my_copy(ys.IdString("\\copy"))
assert len(my_idict) == current_len, "copy seemed to have mutate original idict"
assert len(my_copy) == current_len + 1, "copy not behaving as expected"
current_copy_len = len(my_copy)
my_copy |= (ys.IdString(e) for e in ("\\the", "\\world")) # 1 new element
assert len(my_copy) == current_copy_len + 1, "or operator returned unexpected result"

View file

@ -0,0 +1,3 @@
from pyosys import libyosys as ys
ys.log("Hello, world!")

View file

@ -0,0 +1,22 @@
from pyosys import libyosys as ys
from pathlib import Path
__file_dir__ = Path(__file__).absolute().parent
d = ys.Design()
class Monitor(ys.Monitor):
def __init__(self):
super().__init__()
self.mods = []
def notify_module_add(self, mod):
self.mods.append(mod.name.str())
m = Monitor()
d.monitors.add(m)
ys.run_pass(f"read_verilog {__file_dir__ / 'spm.cut.v.gz'}", d)
ys.run_pass("hierarchy -top spm", d)
assert m.mods == ["\\spm"]

34
tests/pyosys/test_pass.py Normal file
View file

@ -0,0 +1,34 @@
from pyosys import libyosys as ys
import json
from pathlib import Path
__file_dir__ = Path(__file__).absolute().parent
class CellStatsPass(ys.Pass):
def __init__(self):
super().__init__(
"cell_stats",
"dumps cell statistics in JSON format"
)
def execute(self, args, design):
ys.log_header(design, "Dumping cell stats\n")
ys.log_push()
cell_stats = {}
for module in design.all_selected_whole_modules():
for cell in module.selected_cells():
if cell.type.str() in cell_stats:
cell_stats[cell.type.str()] += 1
else:
cell_stats[cell.type.str()] = 1
ys.log(json.dumps(cell_stats))
ys.log_pop()
p = CellStatsPass() # registration
design = ys.Design()
ys.run_pass(f"read_verilog {__file_dir__.parent / 'simple' / 'fiedler-cooley.v'}", design)
ys.run_pass("prep", design)
ys.run_pass("opt -full", design)
ys.run_pass("cell_stats", design)

View file

@ -0,0 +1,21 @@
from pathlib import Path
from pyosys import libyosys as ys
__file_dir__ = Path(__file__).absolute().parent
add_sub = __file_dir__.parent / "arch" / "common" / "add_sub.v"
base = ys.Design()
ys.run_pass(f"read_verilog {add_sub}", base)
ys.run_pass("hierarchy -top top", base)
ys.run_pass("proc", base)
ys.run_pass("equiv_opt -assert -map +/ecp5/cells_sim.v synth_ecp5", base)
postopt = ys.Design()
ys.run_pass("design -load postopt", postopt)
ys.run_pass("cd top", postopt)
ys.run_pass("select -assert-min 25 t:LUT4", postopt)
ys.run_pass("select -assert-max 26 t:LUT4", postopt)
ys.run_pass("select -assert-count 10 t:PFUMX", postopt)
ys.run_pass("select -assert-count 6 t:L6MUX21", postopt)
ys.run_pass("select -assert-none t:LUT4 t:PFUMX t:L6MUX21 %% t:* %D", postopt)