Merge pull request #5300 from donn/wheel_fixes

wheels: bison 3.8 on almalinux + memory pressure easing

.github/workflows/wheels.yml

@@ -4,7 +4,7 @@ name: Build Wheels for PyPI
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 10 * * 0'
+    - cron: "0 10 * * 0"
 
 jobs:
   build_wheels:
@@ -54,9 +54,6 @@ jobs:
           fetch-depth: 0
           submodules: true
           persist-credentials: false
-      - if: ${{ matrix.os.family == 'linux' }}
-        name: "[Linux] Set up QEMU"
-        uses: docker/setup-qemu-action@v3
       - uses: actions/setup-python@v5
       - name: Get Boost Source
         shell: bash
@@ -68,14 +65,19 @@ jobs:
         run: |
           mkdir -p ffi
           curl -L https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz | tar --strip-components=1 -xzC ffi
+      - if: ${{ matrix.os.family == 'linux' }}
+        name: "[Linux] Bison 3.8.2"
+        shell: bash
+        run: |
+          mkdir -p bison
+          curl -L https://ftpmirror.gnu.org/gnu/bison/bison-3.8.2.tar.gz | tar --strip-components=1 -xzC bison
       ## Software installed by default in GitHub Action Runner VMs:
       ##  https://github.com/actions/runner-images
       - if: ${{ matrix.os.family == 'macos' }}
         name: "[macOS] Flex/Bison"
         run: |
           brew install flex bison
-          echo "PATH=$(brew --prefix flex)/bin:$PATH" >> $GITHUB_ENV
-          echo "PATH=$(brew --prefix bison)/bin:$PATH" >> $GITHUB_ENV
+          echo "PATH=$(brew --prefix flex)/bin:$(brew --prefix bison)/bin:$PATH" >> $GITHUB_ENV
       - if: ${{ matrix.os.family == 'windows' }}
         name: "[Windows] Flex/Bison"
         run: |
@@ -100,16 +102,20 @@ jobs:
           CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
           CIBW_BEFORE_ALL: bash ./.github/workflows/wheels/cibw_before_all.sh
           CIBW_ENVIRONMENT: >
+            OPTFLAGS=-O3
             CXXFLAGS=-I./boost/pfx/include
             LINKFLAGS=-L./boost/pfx/lib
             PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
             makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a'
+            PATH="$PWD/bison/src:$PATH"
           CIBW_ENVIRONMENT_MACOS: >
+            OPTFLAGS=-O3
             CXXFLAGS=-I./boost/pfx/include
             LINKFLAGS=-L./boost/pfx/lib
             PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
             MACOSX_DEPLOYMENT_TARGET=11
             makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a CONFIG=clang'
+            PATH="$PWD/bison/src:$PATH"
           CIBW_BEFORE_BUILD: bash ./.github/workflows/wheels/cibw_before_build.sh
           CIBW_TEST_COMMAND: python3 {project}/tests/arch/ecp5/add_sub.py
       - uses: actions/upload-artifact@v4

.github/workflows/wheels/_run_cibw_linux.py

@@ -20,11 +20,19 @@ import os
 import yaml
 import platform
 import subprocess
+from pathlib import Path
 
-__dir__ = os.path.dirname(os.path.abspath(__file__))
+__yosys_root__ = Path(__file__).absolute().parents[3]
 
+for source in ["boost", "ffi", "bison"]:
+    if not (__yosys_root__ / source).is_dir():
+        print(
+            "You need to download boost, ffi and bison in a similar manner to wheels.yml first."
+        )
+        exit(-1)
 
-workflow = yaml.safe_load(open(os.path.join(os.path.dirname(__dir__), "wheels.yml")))
+with open(__yosys_root__ / ".github" / "workflows" / "wheels.yml") as f:
+    workflow = yaml.safe_load(f)
 
 env = os.environ.copy()
 
@@ -40,5 +48,5 @@ for key, value in cibw_step["env"].items():
         continue
     env[key] = value
 
-env["CIBW_ARCHS"] = os.getenv("CIBW_ARCHS") or platform.machine()
+env["CIBW_ARCHS"] = os.getenv("CIBW_ARCHS", platform.machine())
 subprocess.check_call(["cibuildwheel"], env=env)

.github/workflows/wheels/cibw_before_all.sh

@@ -1,23 +1,37 @@
-set -e
-set -x
+#!/bin/bash
+set -e -x
 
 # Build-time dependencies
 ## Linux Docker Images
 if command -v yum &> /dev/null; then
-    yum install -y flex bison
+    yum install -y flex # manylinux's bison versions are hopelessly out of date
 fi
 
 if command -v apk &> /dev/null; then
     apk add flex bison
 fi
 
+if ! printf '%s\n' '%require "3.8"' '%%' 'start: ;' | bison -o /dev/null /dev/stdin ; then
+	(
+		set -e -x
+		cd bison
+		./configure
+		make clean
+		make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
+	)
+fi
+
 ## macOS/Windows -- installed in GitHub Action itself, not container
 
-# Build Static FFI (platform-dependent but not Python version dependent)
-cd ffi
-## Ultimate libyosys.so will be shared, so we need fPIC for the static libraries
-CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=$PWD/pfx
-## Without this, SHELL has a space in its path which breaks the makefile
-make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
-## Forces static library to be used in all situations
-sed -i.bak 's@-L${toolexeclibdir} -lffi@${toolexeclibdir}/libffi.a@' ./pfx/lib/pkgconfig/libffi.pc
+# Runtime Dependencies
+## Build Static FFI (platform-dependent but not Python version dependent)
+(
+	set -e -x
+	cd ffi
+	## Ultimate libyosys.so will be shared, so we need fPIC for the static libraries
+	CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=$PWD/pfx
+	make clean
+	make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
+	## Forces static library to be used in all situations
+	sed -i.bak 's@-L${toolexeclibdir} -lffi@${toolexeclibdir}/libffi.a@' ./pfx/lib/pkgconfig/libffi.pc
+)

.gitignore

@@ -47,6 +47,7 @@
 /kernel/python_wrappers.cc
 /boost
 /ffi
+/bison
 /venv
 /*.whl
 /*.egg-info

setup.py

@@ -51,16 +51,27 @@ class libyosys_so_ext(Extension):
         ]
 
     def custom_build(self, bext: build_ext):
+        make_flags_split = shlex.split(os.getenv("makeFlags", ""))
+        # abc linking takes a lot of memory, best get it out of the way first
+        bext.spawn(
+            [
+                "make",
+                f"-j{os.cpu_count() or 1}",
+                "yosys-abc",
+                *make_flags_split,
+                *self.args,
+            ]
+        )
+        # build libyosys and share with abc out of the way
         bext.spawn(
             [
                 "make",
                 f"-j{os.cpu_count() or 1}",
                 self.name,
-                "yosys-abc",
                 "share",
+                *make_flags_split,
+                *self.args,
             ]
-            + shlex.split(os.getenv("makeFlags", ""))
-            + self.args
         )
         build_path = os.path.dirname(os.path.dirname(bext.get_ext_fullpath(self.name)))
         pyosys_path = os.path.join(build_path, "pyosys")
@@ -85,6 +96,7 @@ class libyosys_so_ext(Extension):
 
         shutil.copytree("share", share_target)
 
+
 class custom_build_ext(build_ext):
     def build_extension(self, ext) -> None:
         if not hasattr(ext, "custom_build"):
@@ -100,8 +112,8 @@ setup(
     long_description=open(os.path.join(__dir__, "README.md")).read(),
     long_description_content_type="text/markdown",
     install_requires=["wheel", "setuptools"],
+    license="MIT",
     classifiers=[
-        "License :: OSI Approved :: MIT License",
         "Programming Language :: Python :: 3",
         "Intended Audience :: Developers",
         "Operating System :: POSIX :: Linux",