New python packaging and tests (#7356)

* Simplify/modernize python packaging

* Modify azure CI to utilize new python packaging

azure-pipelines.yml

@@ -43,8 +43,20 @@ jobs:
     - ${{if eq(variables['runRegressions'], 'True')}}:
       - template: scripts/test-regressions.yml
 
-- job: LinuxBuildsArm64
-  displayName: "ManyLinux ARM64 build"
+- job: "ManylinuxPythonBuildAmd64"
+  displayName: "Python bindings (manylinux Centos AMD64) build"
+  pool:
+    vmImage: "ubuntu-latest"
+  container: "quay.io/pypa/manylinux2014_x86_64:latest"
+  steps:
+  - script: "/opt/python/cp38-cp38/bin/python -m venv $PWD/env"
+  - script: 'echo "##vso[task.prependpath]$PWD/env/bin"'
+  - script: "pip install build git+https://github.com/rhelmot/auditwheel"  # @TODO remove when patches make it upstream
+  - script: "cd src/api/python && python -m build && AUDITWHEEL_PLAT= auditwheel repair --best-plat dist/*.whl && cd ../../.."
+  - script: "pip install ./src/api/python/wheelhouse/*.whl && python - <src/api/python/z3test.py z3 && python - <src/api/python/z3test.py z3num"
+
+- job: ManyLinuxPythonBuildArm64
+  displayName: "Python bindings (manylinux Centos ARM64 cross) build"
   variables:
     name: ManyLinux
     python: "/opt/python/cp37-cp37m/bin/python"
@@ -55,17 +67,14 @@ jobs:
   - script: curl -L -o /tmp/arm-toolchain.tar.xz 'https://developer.arm.com/-/media/Files/downloads/gnu/11.2-2022.02/binrel/gcc-arm-11.2-2022.02-x86_64-aarch64-none-linux-gnu.tar.xz?rev=33c6e30e5ac64e6dba8f0431f2c35f1b&hash=9918A05BF47621B632C7A5C8D2BB438FB80A4480'
   - script: mkdir -p /tmp/arm-toolchain/
   - script: tar xf /tmp/arm-toolchain.tar.xz -C /tmp/arm-toolchain/ --strip-components=1
+  - script: "/opt/python/cp38-cp38/bin/python -m venv $PWD/env"
+  - script: 'echo "##vso[task.prependpath]$PWD/env/bin"'
   - script: echo '##vso[task.prependpath]/tmp/arm-toolchain/bin'
   - script: echo '##vso[task.prependpath]/tmp/arm-toolchain/aarch64-none-linux-gnu/libc/usr/bin'
   - script: echo $PATH
-  - script: stat /tmp/arm-toolchain/bin/aarch64-none-linux-gnu-gcc
-  - task: PythonScript@0
-    displayName: Build
-    inputs:
-      scriptSource: 'filepath'
-      scriptPath: scripts/mk_unix_dist.py
-      arguments: --nodotnet --nojava --arch=arm64
-      pythonInterpreter: $(python)
+  - script: "stat `which aarch64-none-linux-gnu-gcc`"
+  - script: "pip install build git+https://github.com/rhelmot/auditwheel"  # @TODO remove when patches make it upstream
+  - script: "cd src/api/python && CC=aarch64-none-linux-gnu-gcc CXX=aarch64-none-linux-gnu-g++ AR=aarch64-none-linux-gnu-ar LD=aarch64-none-linux-gnu-ld python -m build && AUDITWHEEL_PLAT= auditwheel repair --best-plat dist/*.whl && cd ../../.."
 
 - job: "Ubuntu20OCaml"
   displayName: "Ubuntu 20 with OCaml"

src/api/python/pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
-requires = ["setuptools>=46.4.0", "wheel", "cmake"]
+requires = ["setuptools>=59", "wheel", "cmake"]
 build-backend = "setuptools.build_meta"

src/api/python/setup.py

@@ -7,14 +7,12 @@ import multiprocessing
 import re
 import glob
 from setuptools import setup
-from distutils.util import get_platform
-from distutils.errors import LibError
-from distutils.command.build import build as _build
-from distutils.command.sdist import sdist as _sdist
-from distutils.command.clean import clean as _clean
+from setuptools.command.build import build as _build
+from setuptools.command.sdist import sdist as _sdist
 from setuptools.command.develop import develop as _develop
-from setuptools.command.bdist_egg import bdist_egg as _bdist_egg
 
+class LibError(Exception):
+    pass
 
 build_env = dict(os.environ)
 build_env['PYTHON'] = sys.executable
@@ -238,27 +236,12 @@ class develop(_develop):
         self.execute(_copy_bins, (), msg="Copying binaries")
         _develop.run(self)
 
-class bdist_egg(_bdist_egg):
-    def run(self):
-        self.run_command('build')
-        _bdist_egg.run(self)
-
 class sdist(_sdist):
     def run(self):
         self.execute(_clean_bins, (), msg="Cleaning binary files and headers")
         self.execute(_copy_sources, (), msg="Copying source files")
         _sdist.run(self)
 
-class clean(_clean):
-    def run(self):
-        self.execute(_clean_bins, (), msg="Cleaning binary files and headers")
-        self.execute(_clean_native_build, (), msg="Cleaning native build")
-        _clean.run(self)
-
-# the build directory needs to exist
-#try: os.makedirs(os.path.join(ROOT_DIR, 'build'))
-#except OSError: pass
-
 # platform.freedesktop_os_release was added in 3.10
 os_id = ''
 if hasattr(platform, 'freedesktop_os_release'):
@@ -269,75 +252,6 @@ if hasattr(platform, 'freedesktop_os_release'):
     except OSError:
         pass
 
-if 'bdist_wheel' in sys.argv and '--plat-name' not in sys.argv:
-    if RELEASE_DIR is None:
-        name = get_platform()
-        if 'linux' in name:
-            # linux_* platform tags are disallowed because the python ecosystem is fubar
-            # linux builds should be built in the centos 5 vm for maximum compatibility
-            # see https://github.com/pypa/manylinux
-            # see also https://github.com/angr/angr-dev/blob/master/admin/bdist.py
-            plat_name = 'manylinux_2_28_' + platform.machine()
-        elif 'mingw' in name:
-            if platform.architecture()[0] == '64bit':
-                plat_name = 'win_amd64'
-            else:
-                plat_name ='win32'
-        else:
-            # https://www.python.org/dev/peps/pep-0425/
-            plat_name = name.replace('.', '_').replace('-', '_')
-    else:
-        # extract the architecture of the release from the directory name
-        arch = RELEASE_METADATA[1]
-        distos = RELEASE_METADATA[2]
-        if distos in ('debian', 'ubuntu'):
-            raise Exception(
-                "Linux binary distributions must be built on centos to conform to PEP 513 or alpine if targeting musl"
-            )
-        elif distos == 'glibc':
-            if arch == 'x64':
-                plat_name = 'manylinux_2_28_x86_64'
-            elif arch == 'arm64' or arch == 'aarch64':
-                # context on why are we match on arm64
-                # but use aarch64 on the plat_name is
-                # due to a workaround current python
-                # legacy build doesn't support aarch64
-                # so using the currently supported arm64
-                # build and simply rename it to aarch64
-                # see full context on #7148
-                plat_name = 'manylinux_2_28_aarch64'                
-            else:
-                plat_name = 'manylinux_2_28_i686'
-        elif distos == 'linux' and os_id == 'alpine':
-            if arch == 'x64':
-                plat_name = 'musllinux_1_1_x86_64'
-            else:
-                plat_name = 'musllinux_1_1_i686'
-        elif distos == 'win':
-            if arch == 'x64':
-                plat_name = 'win_amd64'
-            else:
-                plat_name = 'win32'
-        elif distos == 'osx':
-            osver = RELEASE_METADATA[3]
-            if osver.count('.') > 1:
-                osver = '.'.join(osver.split('.')[:2])
-            if osver.startswith("11"):
-                osver = "11_0"
-            if arch == 'x64':
-                plat_name ='macosx_%s_x86_64' % osver.replace('.', '_')
-            elif arch == 'arm64':
-                plat_name ='macosx_%s_arm64' % osver.replace('.', '_')
-            else:
-                raise Exception(f"idk how os {distos} {osver} works. what goes here?")
-        else:
-            raise Exception(f"idk how to translate between this z3 release os {distos} and the python naming scheme")
-
-    idx = sys.argv.index('bdist_wheel') + 1
-    sys.argv.insert(idx, '--plat-name')
-    sys.argv.insert(idx + 1, plat_name)
-    sys.argv.insert(idx + 2, '--universal')   # supports py2+py3. if --plat-name is not specified this will also mean that the package can be installed on any machine regardless of architecture, so watch out!
-
 setup(
     name='z3-solver',
     version=_z3_version(),
@@ -356,5 +270,5 @@ setup(
         'z3': [os.path.join('lib', '*'), os.path.join('include', '*.h'), os.path.join('include', 'c++', '*.h')]
     },
     data_files=[('bin',[os.path.join('bin',EXECUTABLE_FILE)])],
-    cmdclass={'build': build, 'develop': develop, 'sdist': sdist, 'bdist_egg': bdist_egg, 'clean': clean},
+    cmdclass={'build': build, 'develop': develop, 'sdist': sdist},
 )