Allow building python wheels with binaries from a prebuilt release

src/api/python/setup.py

@@ -19,19 +19,38 @@ build_env = dict(os.environ)
 build_env['PYTHON'] = sys.executable
 build_env['CXXFLAGS'] = build_env.get('CXXFLAGS', '') + " -std=c++11"
 
+# determine where we're building and where sources are
 ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
 SRC_DIR_LOCAL = os.path.join(ROOT_DIR, 'core')
 SRC_DIR_REPO = os.path.join(ROOT_DIR, '..', '..', '..')
 SRC_DIR = SRC_DIR_LOCAL if os.path.exists(SRC_DIR_LOCAL) else SRC_DIR_REPO
-BUILD_DIR = os.path.join(SRC_DIR, 'build') # implicit in configure script
+
+# determine where binaries are
+RELEASE_DIR = os.environ.get('PACKAGE_FROM_RELEASE', None)
+if RELEASE_DIR is None:
+    BUILD_DIR = os.path.join(SRC_DIR, 'build') # implicit in configure script
+    HEADER_DIRS = [os.path.join(SRC_DIR, 'src', 'api'), os.path.join(SRC_DIR, 'src', 'api', 'c++')]
+    RELEASE_METADATA = None
+    BUILD_PLATFORM = sys.platform
+else:
+    BUILD_DIR = os.path.join(RELEASE_DIR, 'bin')
+    HEADER_DIRS = [os.path.join(RELEASE_DIR, 'include')]
+    RELEASE_METADATA = os.path.basename(RELEASE_DIR).split('-')
+    if RELEASE_METADATA[0] != 'z3' or len(RELEASE_METADATA) not in (4, 5):
+        raise Exception("RELEASE_DIR must be in the format z3-version-arch-os[-osversion] so we can extract metadata from it. Sorry!")
+    RELEASE_METADATA.pop(0)
+    BUILD_PLATFORM = RELEASE_METADATA[2]
+
+# determine where destinations are
 LIBS_DIR = os.path.join(ROOT_DIR, 'z3', 'lib')
 HEADERS_DIR = os.path.join(ROOT_DIR, 'z3', 'include')
 BINS_DIR = os.path.join(ROOT_DIR, 'bin')
 
-if sys.platform == 'darwin':
+# determine platform-specific filenames
+if BUILD_PLATFORM in ('darwin', 'osx'):
     LIBRARY_FILE = "libz3.dylib"
     EXECUTABLE_FILE = "z3"
-elif sys.platform in ('win32', 'cygwin'):
+elif BUILD_PLATFORM in ('win32', 'cygwin', 'win'):
     LIBRARY_FILE = "libz3.dll"
     EXECUTABLE_FILE = "z3.exe"
 else:
@@ -58,14 +77,20 @@ def _clean_native_build():
 
 def _z3_version():
     post = os.getenv('Z3_VERSION_SUFFIX', '')
-    fn = os.path.join(SRC_DIR, 'scripts', 'mk_project.py')
+    if RELEASE_DIR is None:
-    if os.path.exists(fn):
+        fn = os.path.join(SRC_DIR, 'scripts', 'mk_project.py')
-        with open(fn) as f:
+        if os.path.exists(fn):
-            for line in f:
+            with open(fn) as f:
-                n = re.match(".*set_version\((.*), (.*), (.*), (.*)\).*", line)
+                for line in f:
-                if not n is None:
+                    n = re.match(r".*set_version\((.*), (.*), (.*), (.*)\).*", line)
-                    return n.group(1) + '.' + n.group(2) + '.' + n.group(3) + '.' + n.group(4) + post
+                    if not n is None:
-    return "?.?.?.?"
+                        return n.group(1) + '.' + n.group(2) + '.' + n.group(3) + '.' + n.group(4) + post
+        return "?.?.?.?"
+    else:
+        version = RELEASE_METADATA[0]
+        if version.count('.') == 2:
+            version += '.0'
+        return version + post
 
 def _configure_z3():
     # bail out early if we don't need to do this - it forces a rebuild every time otherwise
@@ -98,23 +123,28 @@ def _copy_bins():
 
     _clean_bins()
 
-    if SRC_DIR == SRC_DIR_LOCAL:
+    python_dir = None
-        shutil.copy(os.path.join(SRC_DIR, 'src', 'api', 'python', 'z3', 'z3core.py'), os.path.join(ROOT_DIR, 'z3'))
+    if RELEASE_DIR is not None:
-        shutil.copy(os.path.join(SRC_DIR, 'src', 'api', 'python', 'z3', 'z3consts.py'), os.path.join(ROOT_DIR, 'z3'))
+        python_dir = os.path.join(RELEASE_DIR, 'bin', 'python')
+    elif SRC_DIR == SRC_DIR_LOCAL:
+        python_dir = os.path.join(SRC_DIR, 'src', 'api', 'python')
+    if python_dir is not None:
+        shutil.copy(os.path.join(python_dir, 'z3', 'z3core.py'), os.path.join(ROOT_DIR, 'z3'))
+        shutil.copy(os.path.join(python_dir, 'z3', 'z3consts.py'), os.path.join(ROOT_DIR, 'z3'))
 
     # STEP 2: Copy the shared library, the executable and the headers
 
     os.mkdir(LIBS_DIR)
     os.mkdir(BINS_DIR)
     os.mkdir(HEADERS_DIR)
-    os.mkdir(os.path.join(HEADERS_DIR, 'c++'))
     shutil.copy(os.path.join(BUILD_DIR, LIBRARY_FILE), LIBS_DIR)
     shutil.copy(os.path.join(BUILD_DIR, EXECUTABLE_FILE), BINS_DIR)
 
-    header_files = [x for x in os.listdir(os.path.join(SRC_DIR, 'src', 'api')) if x.endswith('.h')]
+    for header_dir in HEADER_DIRS:
-    header_files += [os.path.join('c++', x) for x in os.listdir(os.path.join(SRC_DIR, 'src', 'api', 'c++')) if x.endswith('.h')]
+        for fname in os.listdir(header_dir):
-    for fname in header_files:
+            if not fname.endswith('.h'):
-        shutil.copy(os.path.join(SRC_DIR, 'src', 'api', fname), os.path.join(HEADERS_DIR, fname))
+                continue
+            shutil.copy(os.path.join(header_dir, fname), os.path.join(HEADERS_DIR, fname))
 
 def _copy_sources():
     """
@@ -138,8 +168,9 @@ def _copy_sources():
 
 class build(_build):
     def run(self):
-        self.execute(_configure_z3, (), msg="Configuring Z3")
+        if RELEASE_DIR is None:
-        self.execute(_build_z3, (), msg="Building Z3")
+            self.execute(_configure_z3, (), msg="Configuring Z3")
+            self.execute(_build_z3, (), msg="Building Z3")
         self.execute(_copy_bins, (), msg="Copying binaries")
         _build.run(self)
 
@@ -172,23 +203,48 @@ class clean(_clean):
 #except OSError: pass
 
 if 'bdist_wheel' in sys.argv and '--plat-name' not in sys.argv:
+    if RELEASE_DIR is None:
+        name = get_platform()
+        if 'linux' in name:
+            # linux_* platform tags are disallowed because the python ecosystem is fubar
+            # linux builds should be built in the centos 5 vm for maximum compatibility
+            # see https://github.com/pypa/manylinux
+            # see also https://github.com/angr/angr-dev/blob/master/admin/bdist.py
+            plat_name = 'manylinux1_' + platform.machine()
+        elif 'mingw' in name:
+            if platform.architecture()[0] == '64bit':
+                plat_name = 'win_amd64'
+            else:
+                plat_name ='win32'
+        else:
+            # https://www.python.org/dev/peps/pep-0425/
+            plat_name = name.replace('.', '_').replace('-', '_')
+    else:
+        # extract the architecture of the release from the directory name
+        arch = RELEASE_METADATA[1]
+        distos = RELEASE_METADATA[2]
+        if distos in ('debian', 'ubuntu') or 'linux' in distos:
+            raise Exception("Linux binary distributions must be built on centos to conform to PEP 513")
+        if distos == 'win':
+            if arch == 'x64':
+                plat_name = 'win_amd64'
+            else:
+                plat_name = 'win32'
+        elif distos == 'osx':
+            osver = RELEASE_METADATA[3]
+            if osver.count('.') > 1:
+                osver = '.'.join(osver.split('.')[:2])
+            if arch == 'x64':
+                plat_name ='macosx_%s_x86_64' % osver.replace('.', '_')
+            else:
+                raise Exception('idk how os x works. what goes here?')
+        else:
+            raise Exception("idk how to translate between this z3 release os and the python naming scheme")
+
     idx = sys.argv.index('bdist_wheel') + 1
     sys.argv.insert(idx, '--plat-name')
-    name = get_platform()
+    sys.argv.insert(idx + 1, plat_name)
-    if 'linux' in name:
+    sys.argv.insert(idx + 2, '--universal')   # supports py2+py3. if --plat-name is not specified this will also mean that the package can be installed on any machine regardless of architecture, so watch out!
-        # linux_* platform tags are disallowed because the python ecosystem is fubar
-        # linux builds should be built in the centos 5 vm for maximum compatibility
-        # see https://github.com/pypa/manylinux
-        # see also https://github.com/angr/angr-dev/blob/master/admin/bdist.py
-        sys.argv.insert(idx + 1, 'manylinux1_' + platform.machine())
-    elif 'mingw' in name:
-        if platform.architecture()[0] == '64bit':
-            sys.argv.insert(idx + 1, 'win_amd64')
-        else:
-            sys.argv.insert(idx + 1, 'win32')
-    else:
-        # https://www.python.org/dev/peps/pep-0425/
-        sys.argv.insert(idx + 1, name.replace('.', '_').replace('-', '_'))
 
 
 setup(