Pyosys Wheels

* Created `setup.py`: Python package manifest to build `pyosys` wheels with a custom extension to build and include `libyosys.so` using Make
* `.gitignore`: Added byproducts of the Python wheel build process
* `Makefile`: Added `-undefined dynamic_lookup` to `libyosys.so` so missing symbols can be resolved by importing into a Python interpreter
* `kernel/yosys.cc`: Gated `PyImport_AppendInittab` with `!Py_IsInitialized`; as of Python 3.12, the interpreter is already initialized and `PyImport_AppendInittab` would cause an exception to be raised
* Created `wheels.yml`: CI workflow for building wheels for CPython on:
  * Linux (glibc, musl) and Darwin
  * x86-64 and arm64

.github/workflows/wheels.yml

@@ -0,0 +1,111 @@
+name: Build Wheels for PyPI
+on:
+  workflow_dispatch:
+  push:
+    # tags: ["yosys-*"]
+jobs:
+  build_wheels:
+    strategy:
+      matrix:
+        os: [
+            # You can't cross-compile on Linux, so might as well do the emulated
+            # workload on its own
+            {
+              name: "Ubuntu 22.04",
+              family: "linux",
+              runner: "ubuntu-22.04",
+              archs: "x86_64",
+            },
+            {
+              name: "Ubuntu 22.04",
+              family: "linux",
+              runner: "ubuntu-22.04",
+              archs: "aarch64",
+            },
+            # While you can cross-compile on macOS, this is less of a headache
+            {
+              name: "macOS 13",
+              family: "macos",
+              runner: "macos-13",
+              archs: "x86_64",
+            },
+            {
+              name: "macOS 14",
+              family: "macos",
+              runner: "macos-14",
+              archs: "arm64",
+            },
+            # TODO: Make Windows Work
+            # {
+            #   name: "Windows Server 2019",
+            #   family: "windows",
+            #   runner: "windows-2019",
+            #   archs: "AMD64 ARM64",
+            # },
+          ]
+    name: Build Wheels | ${{ matrix.os.name }} | ${{ matrix.os.archs }}
+    runs-on: ${{ matrix.os.runner }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
+      - if: ${{ matrix.os.family == 'linux' }}
+        name: "[Linux] Set up QEMU"
+        uses: docker/setup-qemu-action@v2
+      - uses: actions/setup-python@v3
+      - name: Get Boost Source
+        run: |
+          mkdir -p boost
+          curl -L https://sourceforge.net/projects/boost/files/boost/1.83.0/boost_1_83_0.tar.bz2 | tar --strip-components=1 -xjC boost
+      - name: Seed Makefile.bak
+        # For every sed, a .bak is created so it can be copied over Makefile to
+        # rever the change for the next Python version to be built.
+        #
+        # This creates a .bak for the first Python version's cp to consume.
+        run: |
+          cp Makefile Makefile.bak
+      - if: ${{ matrix.os.family == 'macos' }}
+        name: "[macOS] Flex/Bison"
+        run: |
+          brew install flex bison
+          echo "PATH=$(brew --prefix flex)/bin:$PATH" >> $GITHUB_ENV
+          echo "PATH=$(brew --prefix bison)/bin:$PATH" >> $GITHUB_ENV
+      - if: ${{ matrix.os.family == 'macos' && matrix.os.archs == 'arm64' }}
+        name: "[macOS/arm64] Install Python 3.8 (see: https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64)"
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.8
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.20.0
+        env:
+          # Explaining the build steps:
+          #   1. Revert previous seds (if any)
+          #   2. Delete the libboost static archives from previous builds (if any)
+          #   3. Navigate to boost source extracted in previous GHA step
+          #   4-5. Build Boost against current version of Python, only for
+          #        static linkage
+          #        (Boost is statically linked because system boost packages
+          #         wildly vary in versions, including the libboost_python3
+          #         version)
+          #   6-7. Return to package directory and sed the Makefile to point at
+          #        the newly compiled libboost_python
+          CIBW_SKIP: pp* # The Makefile requires python3-config which is not in pypy
+          CIBW_ARCHS: ${{ matrix.os.archs }}
+          CIBW_BUILD_VERBOSITY: "1"
+          CIBW_BEFORE_ALL_LINUX: yum install -y libffi-devel flex bison || apk add libffi-dev flex bison
+          CIBW_BEFORE_ALL_MAC: brew install libffi
+          CIBW_BEFORE_BUILD: |
+            cp Makefile.bak Makefile &&\
+              cd ./boost &&\
+              rm -rf ./pfx &&\
+              ./bootstrap.sh --prefix=./pfx &&\
+              ./b2 --prefix=./pfx --with-filesystem --with-system --with-python cxxflags="$(python3-config --includes) -std=c++17 -fPIC" cflags="$(python3-config --includes) -fPIC" link=static variant=release install &&\
+              ls ./pfx/lib/libboost_python*.a &&\
+              cd {package} &&\
+              sed -i'.bak' -e "1i\\
+                LINKFLAGS = -L./boost/pfx/lib" -e "1i\\
+                CXXFLAGS = -I./boost/pfx/include" -e "s@BOOST_PYTHON_LIB ?=@BOOST_PYTHON_LIB = $(ls ./boost/pfx/lib/libboost_python*.a)\nTrash =@" Makefile
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./dist/*.whl

.gitignore

@@ -46,3 +46,8 @@ __pycache__
 /tests/unit/objtest/
 /tests/ystests
 /result
+/dist
+/*.egg-info
+/build
+/venv
+/boost

Makefile

@@ -742,7 +742,7 @@ $(PROGRAM_PREFIX)yosys$(EXE): $(OBJS)
 
 libyosys.so: $(filter-out kernel/driver.o,$(OBJS))
 ifeq ($(OS), Darwin)
-	$(P) $(CXX) -o libyosys.so -shared -Wl,-install_name,$(LIBDIR)/libyosys.so $(LINKFLAGS) $^ $(LIBS) $(LIBS_VERIFIC)
+	$(P) $(CXX) -o libyosys.so -shared -undefined dynamic_lookup -Wl,-install_name,$(LIBDIR)/libyosys.so $(LINKFLAGS) $^ $(LIBS) $(LIBS_VERIFIC)
 else
 	$(P) $(CXX) -o libyosys.so -shared -Wl,-soname,$(LIBDIR)/libyosys.so $(LINKFLAGS) $^ $(LIBS) $(LIBS_VERIFIC)
 endif

kernel/yosys.cc

@@ -555,10 +555,15 @@ void yosys_setup()
 #undef X
 
 	#ifdef WITH_PYTHON
-		PyImport_AppendInittab((char*)"libyosys", INIT_MODULE);
+		// With Python 3.12, calling PyImport_AppendInittab on an already
-		Py_Initialize();
+		// initialized platform fails (such as when libyosys is imported
-		PyRun_SimpleString("import sys");
+		// from a Python interpreter)
-		signal(SIGINT, SIG_DFL);
+		if (!Py_IsInitialized()) {
+			PyImport_AppendInittab((char*)"libyosys", INIT_MODULE);
+			Py_Initialize();
+			PyRun_SimpleString("import sys");
+			signal(SIGINT, SIG_DFL);
+		}
 	#endif
 
 	Pass::init_register();

setup.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+# Copyright (C) 2024 Efabless Corporation
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+import os
+import re
+import shlex
+import shutil
+from setuptools import setup, Extension
+from setuptools.command.build_ext import build_ext
+
+__dir__ = os.path.dirname(os.path.abspath(__file__))
+
+yosys_version_rx = re.compile(r"YOSYS_VER\s*:=\s*([\w\-\+\.]+)")
+
+version = yosys_version_rx.search(
+    open(os.path.join(__dir__, "Makefile"), encoding="utf8").read()
+)[1]
+
+
+class libyosys_so_ext(Extension):
+    def __init__(
+        self,
+    ) -> None:
+        super().__init__(
+            "libyosys.so",
+            [],
+        )
+        self.args = [
+            "ENABLE_PYOSYS=1",
+            # Wheel meant to be imported from interpreter
+            "ENABLE_PYTHON_CONFIG_EMBED=0",
+            # Would need to be installed separately by the user
+            "ENABLE_TCL=0",
+            # Would need to be installed separately by the user
+            "ENABLE_READLINE=0",
+            # Show compile commands
+            "PRETTY=0",
+        ]
+
+    def custom_build(self, bext: build_ext):
+        bext.spawn(
+            ["make", f"-j{os.cpu_count() or 1}", self.name]
+            + shlex.split(os.getenv("makeFlags", ""))
+            + self.args
+        )
+        build_path = os.path.dirname(os.path.dirname(bext.get_ext_fullpath(self.name)))
+        pyosys_path = os.path.join(build_path, "pyosys")
+        target = os.path.join(pyosys_path, os.path.basename(self.name))
+        os.makedirs(pyosys_path, exist_ok=True)
+        shutil.copyfile(self.name, target)
+
+        # I don't know how debug info is getting here.
+        bext.spawn(["strip", "-S", target])
+
+
+class custom_build_ext(build_ext):
+    def build_extension(self, ext) -> None:
+        if not hasattr(ext, "custom_build"):
+            return super().build_extension(ext)
+        return ext.custom_build(self)
+
+
+setup(
+    name="pyosys",
+    packages=["pyosys"],
+    version=version,
+    description="Python access to libyosys",
+    long_description=open(os.path.join(__dir__, "README.md")).read(),
+    long_description_content_type="text/markdown",
+    author="Claire Xenia Wolf",
+    author_email="claire@yosyshq.com",
+    install_requires=["wheel", "setuptools"],
+    classifiers=[
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3",
+        "Intended Audience :: Developers",
+        "Operating System :: POSIX :: Linux",
+        "Operating System :: MacOS :: MacOS X",
+    ],
+    package_dir={"pyosys": "misc"},
+    python_requires=">=3.8",
+    ext_modules=[libyosys_so_ext()],
+    cmdclass={
+        "build_ext": custom_build_ext,
+    },
+)