Merge pull request #5300 from donn/wheel_fixes

wheels: bison 3.8 on almalinux + memory pressure easing

.github/workflows/wheels.yml

@@ -4,7 +4,7 @@ name: Build Wheels for PyPI
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 10 * * 0'
+    - cron: "0 10 * * 0"
 
 jobs:
   build_wheels:
@@ -54,9 +54,6 @@ jobs:
           fetch-depth: 0
           submodules: true
           persist-credentials: false
-      - if: ${{ matrix.os.family == 'linux' }}
-        name: "[Linux] Set up QEMU"
-        uses: docker/setup-qemu-action@v3
       - uses: actions/setup-python@v5
       - name: Get Boost Source
         shell: bash
@@ -68,14 +65,19 @@ jobs:
         run: |
           mkdir -p ffi
           curl -L https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz | tar --strip-components=1 -xzC ffi
+      - if: ${{ matrix.os.family == 'linux' }}
+        name: "[Linux] Bison 3.8.2"
+        shell: bash
+        run: |
+          mkdir -p bison
+          curl -L https://ftpmirror.gnu.org/gnu/bison/bison-3.8.2.tar.gz | tar --strip-components=1 -xzC bison
       ## Software installed by default in GitHub Action Runner VMs:
       ##  https://github.com/actions/runner-images
       - if: ${{ matrix.os.family == 'macos' }}
         name: "[macOS] Flex/Bison"
         run: |
           brew install flex bison
-          echo "PATH=$(brew --prefix flex)/bin:$PATH" >> $GITHUB_ENV
-          echo "PATH=$(brew --prefix bison)/bin:$PATH" >> $GITHUB_ENV
+          echo "PATH=$(brew --prefix flex)/bin:$(brew --prefix bison)/bin:$PATH" >> $GITHUB_ENV
       - if: ${{ matrix.os.family == 'windows' }}
         name: "[Windows] Flex/Bison"
         run: |
@@ -100,16 +102,20 @@ jobs:
           CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
           CIBW_BEFORE_ALL: bash ./.github/workflows/wheels/cibw_before_all.sh
           CIBW_ENVIRONMENT: >
+            OPTFLAGS=-O3
             CXXFLAGS=-I./boost/pfx/include
             LINKFLAGS=-L./boost/pfx/lib
             PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
             makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a'
+            PATH="$PWD/bison/src:$PATH"
           CIBW_ENVIRONMENT_MACOS: >
+            OPTFLAGS=-O3
             CXXFLAGS=-I./boost/pfx/include
             LINKFLAGS=-L./boost/pfx/lib
             PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
             MACOSX_DEPLOYMENT_TARGET=11
             makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a CONFIG=clang'
+            PATH="$PWD/bison/src:$PATH"
           CIBW_BEFORE_BUILD: bash ./.github/workflows/wheels/cibw_before_build.sh
           CIBW_TEST_COMMAND: python3 {project}/tests/arch/ecp5/add_sub.py
       - uses: actions/upload-artifact@v4

.github/workflows/wheels/_run_cibw_linux.py

@@ -20,11 +20,19 @@ import os
 import yaml
 import platform
 import subprocess
+from pathlib import Path
 
-__dir__ = os.path.dirname(os.path.abspath(__file__))
+__yosys_root__ = Path(__file__).absolute().parents[3]
 
+for source in ["boost", "ffi", "bison"]:
+    if not (__yosys_root__ / source).is_dir():
+        print(
+            "You need to download boost, ffi and bison in a similar manner to wheels.yml first."
+        )
+        exit(-1)
 
-workflow = yaml.safe_load(open(os.path.join(os.path.dirname(__dir__), "wheels.yml")))
+with open(__yosys_root__ / ".github" / "workflows" / "wheels.yml") as f:
+    workflow = yaml.safe_load(f)
 
 env = os.environ.copy()
 
@@ -40,5 +48,5 @@ for key, value in cibw_step["env"].items():
         continue
     env[key] = value
 
-env["CIBW_ARCHS"] = os.getenv("CIBW_ARCHS") or platform.machine()
+env["CIBW_ARCHS"] = os.getenv("CIBW_ARCHS", platform.machine())
 subprocess.check_call(["cibuildwheel"], env=env)

.github/workflows/wheels/cibw_before_all.sh

@@ -1,23 +1,37 @@
-set -e
-set -x
+#!/bin/bash
+set -e -x
 
 # Build-time dependencies
 ## Linux Docker Images
 if command -v yum &> /dev/null; then
-    yum install -y flex bison
+    yum install -y flex # manylinux's bison versions are hopelessly out of date
 fi
 
 if command -v apk &> /dev/null; then
     apk add flex bison
 fi
 
+if ! printf '%s\n' '%require "3.8"' '%%' 'start: ;' | bison -o /dev/null /dev/stdin ; then
+	(
+		set -e -x
+		cd bison
+		./configure
+		make clean
+		make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
+	)
+fi
+
 ## macOS/Windows -- installed in GitHub Action itself, not container
 
-# Build Static FFI (platform-dependent but not Python version dependent)
-cd ffi
-## Ultimate libyosys.so will be shared, so we need fPIC for the static libraries
-CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=$PWD/pfx
-## Without this, SHELL has a space in its path which breaks the makefile
-make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
-## Forces static library to be used in all situations
-sed -i.bak 's@-L${toolexeclibdir} -lffi@${toolexeclibdir}/libffi.a@' ./pfx/lib/pkgconfig/libffi.pc
+# Runtime Dependencies
+## Build Static FFI (platform-dependent but not Python version dependent)
+(
+	set -e -x
+	cd ffi
+	## Ultimate libyosys.so will be shared, so we need fPIC for the static libraries
+	CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=$PWD/pfx
+	make clean
+	make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
+	## Forces static library to be used in all situations
+	sed -i.bak 's@-L${toolexeclibdir} -lffi@${toolexeclibdir}/libffi.a@' ./pfx/lib/pkgconfig/libffi.pc
+)