From 6fa85ad65404c0dbc8adf3e120fa076bd5c6d003 Mon Sep 17 00:00:00 2001 From: Audrey Dutcher Date: Mon, 10 Jun 2019 16:23:26 -0700 Subject: [PATCH] Allow building python wheels with binaries from a prebuilt release --- src/api/python/setup.py | 128 +++++++++++++++++++++++++++++----------- 1 file changed, 92 insertions(+), 36 deletions(-) diff --git a/src/api/python/setup.py b/src/api/python/setup.py index 2d5e5c75d..30dd56549 100644 --- a/src/api/python/setup.py +++ b/src/api/python/setup.py @@ -19,19 +19,38 @@ build_env = dict(os.environ) build_env['PYTHON'] = sys.executable build_env['CXXFLAGS'] = build_env.get('CXXFLAGS', '') + " -std=c++11" +# determine where we're building and where sources are ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) SRC_DIR_LOCAL = os.path.join(ROOT_DIR, 'core') SRC_DIR_REPO = os.path.join(ROOT_DIR, '..', '..', '..') SRC_DIR = SRC_DIR_LOCAL if os.path.exists(SRC_DIR_LOCAL) else SRC_DIR_REPO -BUILD_DIR = os.path.join(SRC_DIR, 'build') # implicit in configure script + +# determine where binaries are +RELEASE_DIR = os.environ.get('PACKAGE_FROM_RELEASE', None) +if RELEASE_DIR is None: + BUILD_DIR = os.path.join(SRC_DIR, 'build') # implicit in configure script + HEADER_DIRS = [os.path.join(SRC_DIR, 'src', 'api'), os.path.join(SRC_DIR, 'src', 'api', 'c++')] + RELEASE_METADATA = None + BUILD_PLATFORM = sys.platform +else: + BUILD_DIR = os.path.join(RELEASE_DIR, 'bin') + HEADER_DIRS = [os.path.join(RELEASE_DIR, 'include')] + RELEASE_METADATA = os.path.basename(RELEASE_DIR).split('-') + if RELEASE_METADATA[0] != 'z3' or len(RELEASE_METADATA) not in (4, 5): + raise Exception("RELEASE_DIR must be in the format z3-version-arch-os[-osversion] so we can extract metadata from it. Sorry!") + RELEASE_METADATA.pop(0) + BUILD_PLATFORM = RELEASE_METADATA[2] + +# determine where destinations are LIBS_DIR = os.path.join(ROOT_DIR, 'z3', 'lib') HEADERS_DIR = os.path.join(ROOT_DIR, 'z3', 'include') BINS_DIR = os.path.join(ROOT_DIR, 'bin') -if sys.platform == 'darwin': +# determine platform-specific filenames +if BUILD_PLATFORM in ('darwin', 'osx'): LIBRARY_FILE = "libz3.dylib" EXECUTABLE_FILE = "z3" -elif sys.platform in ('win32', 'cygwin'): +elif BUILD_PLATFORM in ('win32', 'cygwin', 'win'): LIBRARY_FILE = "libz3.dll" EXECUTABLE_FILE = "z3.exe" else: @@ -58,14 +77,20 @@ def _clean_native_build(): def _z3_version(): post = os.getenv('Z3_VERSION_SUFFIX', '') - fn = os.path.join(SRC_DIR, 'scripts', 'mk_project.py') - if os.path.exists(fn): - with open(fn) as f: - for line in f: - n = re.match(".*set_version\((.*), (.*), (.*), (.*)\).*", line) - if not n is None: - return n.group(1) + '.' + n.group(2) + '.' + n.group(3) + '.' + n.group(4) + post - return "?.?.?.?" + if RELEASE_DIR is None: + fn = os.path.join(SRC_DIR, 'scripts', 'mk_project.py') + if os.path.exists(fn): + with open(fn) as f: + for line in f: + n = re.match(r".*set_version\((.*), (.*), (.*), (.*)\).*", line) + if not n is None: + return n.group(1) + '.' + n.group(2) + '.' + n.group(3) + '.' + n.group(4) + post + return "?.?.?.?" + else: + version = RELEASE_METADATA[0] + if version.count('.') == 2: + version += '.0' + return version + post def _configure_z3(): # bail out early if we don't need to do this - it forces a rebuild every time otherwise @@ -98,23 +123,28 @@ def _copy_bins(): _clean_bins() - if SRC_DIR == SRC_DIR_LOCAL: - shutil.copy(os.path.join(SRC_DIR, 'src', 'api', 'python', 'z3', 'z3core.py'), os.path.join(ROOT_DIR, 'z3')) - shutil.copy(os.path.join(SRC_DIR, 'src', 'api', 'python', 'z3', 'z3consts.py'), os.path.join(ROOT_DIR, 'z3')) + python_dir = None + if RELEASE_DIR is not None: + python_dir = os.path.join(RELEASE_DIR, 'bin', 'python') + elif SRC_DIR == SRC_DIR_LOCAL: + python_dir = os.path.join(SRC_DIR, 'src', 'api', 'python') + if python_dir is not None: + shutil.copy(os.path.join(python_dir, 'z3', 'z3core.py'), os.path.join(ROOT_DIR, 'z3')) + shutil.copy(os.path.join(python_dir, 'z3', 'z3consts.py'), os.path.join(ROOT_DIR, 'z3')) # STEP 2: Copy the shared library, the executable and the headers os.mkdir(LIBS_DIR) os.mkdir(BINS_DIR) os.mkdir(HEADERS_DIR) - os.mkdir(os.path.join(HEADERS_DIR, 'c++')) shutil.copy(os.path.join(BUILD_DIR, LIBRARY_FILE), LIBS_DIR) shutil.copy(os.path.join(BUILD_DIR, EXECUTABLE_FILE), BINS_DIR) - header_files = [x for x in os.listdir(os.path.join(SRC_DIR, 'src', 'api')) if x.endswith('.h')] - header_files += [os.path.join('c++', x) for x in os.listdir(os.path.join(SRC_DIR, 'src', 'api', 'c++')) if x.endswith('.h')] - for fname in header_files: - shutil.copy(os.path.join(SRC_DIR, 'src', 'api', fname), os.path.join(HEADERS_DIR, fname)) + for header_dir in HEADER_DIRS: + for fname in os.listdir(header_dir): + if not fname.endswith('.h'): + continue + shutil.copy(os.path.join(header_dir, fname), os.path.join(HEADERS_DIR, fname)) def _copy_sources(): """ @@ -138,8 +168,9 @@ def _copy_sources(): class build(_build): def run(self): - self.execute(_configure_z3, (), msg="Configuring Z3") - self.execute(_build_z3, (), msg="Building Z3") + if RELEASE_DIR is None: + self.execute(_configure_z3, (), msg="Configuring Z3") + self.execute(_build_z3, (), msg="Building Z3") self.execute(_copy_bins, (), msg="Copying binaries") _build.run(self) @@ -172,23 +203,48 @@ class clean(_clean): #except OSError: pass if 'bdist_wheel' in sys.argv and '--plat-name' not in sys.argv: + if RELEASE_DIR is None: + name = get_platform() + if 'linux' in name: + # linux_* platform tags are disallowed because the python ecosystem is fubar + # linux builds should be built in the centos 5 vm for maximum compatibility + # see https://github.com/pypa/manylinux + # see also https://github.com/angr/angr-dev/blob/master/admin/bdist.py + plat_name = 'manylinux1_' + platform.machine() + elif 'mingw' in name: + if platform.architecture()[0] == '64bit': + plat_name = 'win_amd64' + else: + plat_name ='win32' + else: + # https://www.python.org/dev/peps/pep-0425/ + plat_name = name.replace('.', '_').replace('-', '_') + else: + # extract the architecture of the release from the directory name + arch = RELEASE_METADATA[1] + distos = RELEASE_METADATA[2] + if distos in ('debian', 'ubuntu') or 'linux' in distos: + raise Exception("Linux binary distributions must be built on centos to conform to PEP 513") + if distos == 'win': + if arch == 'x64': + plat_name = 'win_amd64' + else: + plat_name = 'win32' + elif distos == 'osx': + osver = RELEASE_METADATA[3] + if osver.count('.') > 1: + osver = '.'.join(osver.split('.')[:2]) + if arch == 'x64': + plat_name ='macosx_%s_x86_64' % osver.replace('.', '_') + else: + raise Exception('idk how os x works. what goes here?') + else: + raise Exception("idk how to translate between this z3 release os and the python naming scheme") + idx = sys.argv.index('bdist_wheel') + 1 sys.argv.insert(idx, '--plat-name') - name = get_platform() - if 'linux' in name: - # linux_* platform tags are disallowed because the python ecosystem is fubar - # linux builds should be built in the centos 5 vm for maximum compatibility - # see https://github.com/pypa/manylinux - # see also https://github.com/angr/angr-dev/blob/master/admin/bdist.py - sys.argv.insert(idx + 1, 'manylinux1_' + platform.machine()) - elif 'mingw' in name: - if platform.architecture()[0] == '64bit': - sys.argv.insert(idx + 1, 'win_amd64') - else: - sys.argv.insert(idx + 1, 'win32') - else: - # https://www.python.org/dev/peps/pep-0425/ - sys.argv.insert(idx + 1, name.replace('.', '_').replace('-', '_')) + sys.argv.insert(idx + 1, plat_name) + sys.argv.insert(idx + 2, '--universal') # supports py2+py3. if --plat-name is not specified this will also mean that the package can be installed on any machine regardless of architecture, so watch out! setup(