From 2bc2105d8287d43c629404914d3d7ac9d3c8b78a Mon Sep 17 00:00:00 2001 From: Donn Date: Wed, 9 Apr 2025 10:42:40 +0200 Subject: [PATCH 1/3] wheels: fix builds, add linux aarch64 to ci Essentially, something is attempting to build the Yosys EXE when you build libyosys.so now. With `ENABLE_PYTHON_CONFIG_EMBED=0`, the Yosys EXE will always fail to build. Furthermore, because `ENABLE_PYOSYS` now attempts to build a wheel, building a wheel has become recursive. This commit uses a supplementary set of libs for the EXE (EXE_LIBS) so it and libyosys.so can be built simultaneously, as well as a new Makefile flag, `ENABLE_WHEEL`, to prevent the aforementioned recursion. I also enabled aarch64 Linux in the CI because it's publicly available now. --- .github/workflows/wheels.yml | 16 ++++++---------- Makefile | 15 ++++++++++----- setup.py | 4 ++-- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index d66239a16..3a3d240c4 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -14,16 +14,12 @@ jobs: runner: "ubuntu-22.04", archs: "x86_64", }, - ## Aarch64 is disabled for now: GitHub is committing to EOY - ## for free aarch64 runners for open-source projects and - ## emulation times out: - ## https://github.com/orgs/community/discussions/19197#discussioncomment-10550689 - # { - # name: "Ubuntu 22.04", - # family: "linux", - # runner: "ubuntu-22.04", - # archs: "aarch64", - # }, + { + name: "Ubuntu 22.04", + family: "linux", + runner: "ubuntu-22.04-arm", + archs: "aarch64", + }, { name: "macOS 13", family: "macos", diff --git a/Makefile b/Makefile index 89723e99d..74951b8cb 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ ENABLE_ZLIB := 1 # python wrappers ENABLE_PYOSYS := 0 +ENABLE_WHEEL := $(ENABLE_PYOSYS) # other configuration flags ENABLE_GCOV := 0 @@ -102,6 +103,7 @@ LIBS := $(LIBS) -lstdc++ -lm PLUGIN_LINKFLAGS := PLUGIN_LIBS := EXE_LINKFLAGS := +EXE_LIBS := ifeq ($(OS), MINGW) EXE_LINKFLAGS := -Wl,--export-all-symbols -Wl,--out-implib,libyosys_exe.a PLUGIN_LINKFLAGS += -L"$(LIBDIR)" @@ -209,11 +211,11 @@ PYTHON_VERSION_TESTCODE := "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.versi PYTHON_VERSION := $(shell $(PYTHON_EXECUTABLE) -c ""$(PYTHON_VERSION_TESTCODE)"") PYTHON_MAJOR_VERSION := $(shell echo $(PYTHON_VERSION) | cut -f1 -d.) -ENABLE_PYTHON_CONFIG_EMBED ?= $(shell $(PYTHON_EXECUTABLE)-config --embed --libs > /dev/null && echo 1) -ifeq ($(ENABLE_PYTHON_CONFIG_EMBED),1) -PYTHON_CONFIG := $(PYTHON_EXECUTABLE)-config --embed -else PYTHON_CONFIG := $(PYTHON_EXECUTABLE)-config +PYTHON_CONFIG_FOR_EXE := $(PYTHON_CONFIG) +PYTHON_CONFIG_EMBED_AVAILABLE ?= $(shell $(PYTHON_EXECUTABLE)-config --embed --libs > /dev/null && echo 1) +ifeq ($(PYTHON_CONFIG_EMBED_AVAILABLE),1) +PYTHON_CONFIG_FOR_EXE := $(PYTHON_CONFIG) --embed endif PYTHON_DESTDIR := $(shell $(PYTHON_EXECUTABLE) -c "import site; print(site.getsitepackages()[-1]);") @@ -346,8 +348,11 @@ ifeq ($(ENABLE_PYOSYS),1) # python-config --ldflags includes -l and -L, but LINKFLAGS is only -L LINKFLAGS += $(filter-out -l%,$(shell $(PYTHON_CONFIG) --ldflags)) LIBS += $(shell $(PYTHON_CONFIG) --libs) +EXE_LIBS += $(filter-out $(LIBS),$(shell $(PYTHON_CONFIG_FOR_EXE) --libs)) CXXFLAGS += $(shell $(PYTHON_CONFIG) --includes) -DWITH_PYTHON +ifeq ($(ENABLE_WHEEL),1) EXTRA_TARGETS += wheel +endif # Detect name of boost_python library. Some distros use boost_python-py, other boost_python, some only use the major version number, some a concatenation of major and minor version numbers CHECK_BOOST_PYTHON = (echo "int main(int argc, char ** argv) {return 0;}" | $(CXX) -xc -o /dev/null $(LINKFLAGS) $(LIBS) -l$(1) - > /dev/null 2>&1 && echo "-l$(1)") @@ -735,7 +740,7 @@ share: $(EXTRA_TARGETS) @echo "" $(PROGRAM_PREFIX)yosys$(EXE): $(OBJS) - $(P) $(CXX) -o $(PROGRAM_PREFIX)yosys$(EXE) $(EXE_LINKFLAGS) $(LINKFLAGS) $(OBJS) $(LIBS) $(LIBS_VERIFIC) + $(P) $(CXX) -o $(PROGRAM_PREFIX)yosys$(EXE) $(EXE_LINKFLAGS) $(LINKFLAGS) $(OBJS) $(EXE_LIBS) $(LIBS) $(LIBS_VERIFIC) libyosys.so: $(filter-out kernel/driver.o,$(OBJS)) ifeq ($(OS), Darwin) diff --git a/setup.py b/setup.py index b39b579e0..8a37a91b3 100644 --- a/setup.py +++ b/setup.py @@ -40,8 +40,8 @@ class libyosys_so_ext(Extension): ) self.args = [ "ENABLE_PYOSYS=1", - # Wheel meant to be imported from interpreter - "ENABLE_PYTHON_CONFIG_EMBED=0", + # Prevent recursive wheel build + "ENABLE_WHEEL=0", # Would need to be installed separately by the user "ENABLE_TCL=0", "ENABLE_READLINE=0", From c18f1310a534e84db70b19ebce238c19cb9ac73e Mon Sep 17 00:00:00 2001 From: Donn Date: Mon, 14 Apr 2025 14:45:41 +0200 Subject: [PATCH 2/3] Revert #4901 Partially reverts commit 9c5bffcf93183df0756e01f3c4774d2f83688b8e. The reasoning behind this is that setup.py is intended to strictly consume the Makefile and not be consumed by it. The attempt at using them recursively has caused a number of issues and has rendered Pyosys unusable to some users: See https://github.com/YosysHQ/yosys/issues/5012 Additionally, unlike the previous pyosys installation target, the wheel installation does not respect PREFIX=, only venvs. For installation inside a venv, the intended method should remain a user manually executing `pip3 install .` instead of relying on the Makefile. --- Makefile | 30 ++++++++++-------------------- passes/techmap/Makefile.inc | 2 +- setup.py | 5 ----- 3 files changed, 11 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 74951b8cb..977e1cf40 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,6 @@ ENABLE_ZLIB := 1 # python wrappers ENABLE_PYOSYS := 0 -ENABLE_WHEEL := $(ENABLE_PYOSYS) # other configuration flags ENABLE_GCOV := 0 @@ -350,9 +349,6 @@ LINKFLAGS += $(filter-out -l%,$(shell $(PYTHON_CONFIG) --ldflags)) LIBS += $(shell $(PYTHON_CONFIG) --libs) EXE_LIBS += $(filter-out $(LIBS),$(shell $(PYTHON_CONFIG_FOR_EXE) --libs)) CXXFLAGS += $(shell $(PYTHON_CONFIG) --includes) -DWITH_PYTHON -ifeq ($(ENABLE_WHEEL),1) -EXTRA_TARGETS += wheel -endif # Detect name of boost_python library. Some distros use boost_python-py, other boost_python, some only use the major version number, some a concatenation of major and minor version numbers CHECK_BOOST_PYTHON = (echo "int main(int argc, char ** argv) {return 0;}" | $(CXX) -xc -o /dev/null $(LINKFLAGS) $(LIBS) -l$(1) - > /dev/null 2>&1 && echo "-l$(1)") @@ -971,20 +967,6 @@ unit-test: libyosys.so clean-unit-test: @$(MAKE) -C $(UNITESTPATH) clean -ifeq ($(ENABLE_PYOSYS),1) -wheel: $(TARGETS) - $(PYTHON_EXECUTABLE) -m pip wheel . - -install-wheel: wheel - $(PYTHON_EXECUTABLE) -m pip install pyosys-$(YOSYS_MAJOR).$(YOSYS_MINOR).$(YOSYS_COMMIT)-*.whl --force-reinstall -else -wheel: - $(error Pyosys is not enabled. Set ENABLE_PYOSYS=1 to enable it.) - -install-wheel: - $(error Pyosys is not enabled. Set ENABLE_PYOSYS=1 to enable it.) -endif - install: $(TARGETS) $(EXTRA_TARGETS) $(INSTALL_SUDO) mkdir -p $(DESTDIR)$(BINDIR) $(INSTALL_SUDO) cp $(filter-out libyosys.so,$(TARGETS)) $(DESTDIR)$(BINDIR) @@ -1004,7 +986,13 @@ ifeq ($(ENABLE_LIBYOSYS),1) $(INSTALL_SUDO) cp libyosys.so $(DESTDIR)$(LIBDIR)/ $(INSTALL_SUDO) $(STRIP) -S $(DESTDIR)$(LIBDIR)/libyosys.so ifeq ($(ENABLE_PYOSYS),1) - $(INSTALL_SUDO) @$(MAKE) install-wheel + $(INSTALL_SUDO) mkdir -p $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys + $(INSTALL_SUDO) cp libyosys.so $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys/libyosys.so + $(INSTALL_SUDO) cp -r share $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys +ifeq ($(ENABLE_ABC),1) + $(INSTALL_SUDO) cp yosys-abc $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys/yosys-abc +endif + $(INSTALL_SUDO) cp misc/__init__.py $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys/ endif endif ifeq ($(ENABLE_PLUGINS),1) @@ -1020,7 +1008,9 @@ uninstall: ifeq ($(ENABLE_LIBYOSYS),1) $(INSTALL_SUDO) rm -vf $(DESTDIR)$(LIBDIR)/libyosys.so ifeq ($(ENABLE_PYOSYS),1) - $(INSTALL_SUDO) $(PYTHON_EXECUTABLE) -m pip uninstall -y pyosys + $(INSTALL_SUDO) rm -vf $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys/libyosys.so + $(INSTALL_SUDO) rm -vf $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys/__init__.py + $(INSTALL_SUDO) rmdir $(DESTDIR)$(PYTHON_DESTDIR)/$(subst -,_,$(PROGRAM_PREFIX))pyosys endif endif diff --git a/passes/techmap/Makefile.inc b/passes/techmap/Makefile.inc index e6a910a0d..f34faf37c 100644 --- a/passes/techmap/Makefile.inc +++ b/passes/techmap/Makefile.inc @@ -62,5 +62,5 @@ EXTRA_OBJS += passes/techmap/filterlib.o $(PROGRAM_PREFIX)yosys-filterlib$(EXE): passes/techmap/filterlib.o $(Q) mkdir -p $(dir $@) - $(P) $(CXX) -o $(PROGRAM_PREFIX)yosys-filterlib$(EXE) $(LINKFLAGS) $^ $(LIBS) + $(P) $(CXX) -o $(PROGRAM_PREFIX)yosys-filterlib$(EXE) $(LINKFLAGS) $^ $(EXE_LIBS) $(LIBS) endif diff --git a/setup.py b/setup.py index 8a37a91b3..a199530d0 100644 --- a/setup.py +++ b/setup.py @@ -40,8 +40,6 @@ class libyosys_so_ext(Extension): ) self.args = [ "ENABLE_PYOSYS=1", - # Prevent recursive wheel build - "ENABLE_WHEEL=0", # Would need to be installed separately by the user "ENABLE_TCL=0", "ENABLE_READLINE=0", @@ -87,9 +85,6 @@ class libyosys_so_ext(Extension): shutil.copytree("share", share_target) - # I don't know how debug info is getting here. - - class custom_build_ext(build_ext): def build_extension(self, ext) -> None: if not hasattr(ext, "custom_build"): From b7062143e1fa76d8adbbe6f6cea013e4f2fc807a Mon Sep 17 00:00:00 2001 From: Donn Date: Mon, 14 Apr 2025 14:54:38 +0200 Subject: [PATCH 3/3] hotfix: fix CHECK_BOOST_PYTHON missing libpython --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 977e1cf40..2c145edfc 100644 --- a/Makefile +++ b/Makefile @@ -351,7 +351,7 @@ EXE_LIBS += $(filter-out $(LIBS),$(shell $(PYTHON_CONFIG_FOR_EXE) --libs)) CXXFLAGS += $(shell $(PYTHON_CONFIG) --includes) -DWITH_PYTHON # Detect name of boost_python library. Some distros use boost_python-py, other boost_python, some only use the major version number, some a concatenation of major and minor version numbers -CHECK_BOOST_PYTHON = (echo "int main(int argc, char ** argv) {return 0;}" | $(CXX) -xc -o /dev/null $(LINKFLAGS) $(LIBS) -l$(1) - > /dev/null 2>&1 && echo "-l$(1)") +CHECK_BOOST_PYTHON = (echo "int main(int argc, char ** argv) {return 0;}" | $(CXX) -xc -o /dev/null $(LINKFLAGS) $(EXE_LIBS) $(LIBS) -l$(1) - > /dev/null 2>&1 && echo "-l$(1)") BOOST_PYTHON_LIB ?= $(shell \ $(call CHECK_BOOST_PYTHON,boost_python-py$(subst .,,$(PYTHON_VERSION))) || \ $(call CHECK_BOOST_PYTHON,boost_python-py$(PYTHON_MAJOR_VERSION)) || \