From ec5b148eccfb5359d974d441f5c206e502427c28 Mon Sep 17 00:00:00 2001 From: Audrey Dutcher Date: Tue, 13 Aug 2019 19:17:13 -0700 Subject: [PATCH] Add python packaging build and deployment with Azure --- scripts/mk_unix_dist.py | 2 +- scripts/nightly.yaml | 48 ++++++++++++++++++++++++++- scripts/release.yml | 73 +++++++++++++++++++++++++++++++++++------ src/api/python/setup.py | 11 +++++-- 4 files changed, 120 insertions(+), 14 deletions(-) diff --git a/scripts/mk_unix_dist.py b/scripts/mk_unix_dist.py index 05dcf2d1e..9935e3f88 100644 --- a/scripts/mk_unix_dist.py +++ b/scripts/mk_unix_dist.py @@ -107,7 +107,7 @@ def check_build_dir(path): # Create a build directory using mk_make.py def mk_build_dir(path): if not check_build_dir(path) or FORCE_MK: - opts = ["python", os.path.join('scripts', 'mk_make.py'), "-b", path, "--staticlib"] + opts = [sys.executable, os.path.join('scripts', 'mk_make.py'), "-b", path, "--staticlib"] if DOTNET_CORE_ENABLED: opts.append('--dotnet') if not DOTNET_KEY_FILE is None: diff --git a/scripts/nightly.yaml b/scripts/nightly.yaml index 441b5ba98..922fb133a 100644 --- a/scripts/nightly.yaml +++ b/scripts/nightly.yaml @@ -17,7 +17,6 @@ jobs: artifactName: 'Mac' targetPath: $(Build.ArtifactStagingDirectory) - - job: Ubuntu displayName: "Ubuntu build" pool: @@ -35,6 +34,23 @@ jobs: artifactName: 'Ubuntu' targetPath: $(Build.ArtifactStagingDirectory) +- job: Manylinux + displayName: "Manylinux build" + pool: + vmImage: "ubuntu-16.04" + container: "rhelmot/manylinux1_x86_64:latest" + variables: + python: "/opt/python/cp35-cp35m/bin/python" + steps: + - script: $(python) scripts/mk_unix_dist.py --nodotnet --nojava + - script: git clone https://github.com/z3prover/z3test z3test + - script: $(python) z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2 + - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/ + - task: PublishPipelineArtifact@0 + inputs: + artifactName: 'Manylinux' + targetPath: $(Build.ArtifactStagingDirectory) + - job: Windows displayName: "Windows build" pool: @@ -52,6 +68,36 @@ jobs: artifactName: 'Windows' targetPath: $(Build.ArtifactStagingDirectory) +- job: Python + displayName: "Python packaging" + dependsOn: + - Manylinux + - Windows + pool: + vmImage: "ubuntu-16.04" + steps: + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: 'Windows' + targetPath: $(Agent.TempDirectory) + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: 'Manylinux' + targetPath: $(Agent.TempDirectory) + - script: cd $(Agent.TempDirectory); mkdir linux-bin; cd linux-bin; unzip ../*centos*.zip + - script: cd $(Agent.TempDirectory); mkdir win32-bin; cd win32-bin; unzip ../*x86-win*.zip + - script: cd $(Agent.TempDirectory); mkdir win64-bin; cd win64-bin; unzip ../*x64-win*.zip + - script: python -m pip install --user -U setuptools wheel + - script: cd src/api/python; python setup.py sdist + # take a look at this PREMIUM HACK I came up with to get around the fact that the azure variable syntax overloads the bash syntax for subshells + - script: cd src/api/python; echo $(Agent.TempDirectory)/linux-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel + - script: cd src/api/python; echo $(Agent.TempDirectory)/win32-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel + - script: cd src/api/python; echo $(Agent.TempDirectory)/win64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel + - task: PublishPipelineArtifact@0 + inputs: + artifactName: 'Python packages' + targetPath: src/api/python/dist + - job: Deploy displayName: "Deploy into GitHub" dependsOn: diff --git a/scripts/release.yml b/scripts/release.yml index 3e96bef86..d9101ce18 100644 --- a/scripts/release.yml +++ b/scripts/release.yml @@ -10,14 +10,13 @@ jobs: secureFile: 'z3.snk' - script: python scripts/mk_unix_dist.py --dotnet-key=$(Agent.TempDirectory)/z3.snk - script: git clone https://github.com/z3prover/z3test z3test - - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2 + - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2 - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/. - task: PublishPipelineArtifact@0 inputs: artifactName: 'Mac' targetPath: $(Build.ArtifactStagingDirectory) - - job: Ubuntu displayName: "Ubuntu build" pool: @@ -28,13 +27,30 @@ jobs: secureFile: 'z3.snk' - script: python scripts/mk_unix_dist.py --dotnet-key=$(Agent.TempDirectory)/z3.snk - script: git clone https://github.com/z3prover/z3test z3test - - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2 + - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2 - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/. - task: PublishPipelineArtifact@0 inputs: artifactName: 'Ubuntu' targetPath: $(Build.ArtifactStagingDirectory) +- job: Manylinux + displayName: "Manylinux build" + pool: + vmImage: "ubuntu-16.04" + container: "rhelmot/manylinux1_x86_64:latest" + variables: + python: "/opt/python/cp35-cp35m/bin/python" + steps: + - script: $(python) scripts/mk_unix_dist.py --nodotnet --nojava + - script: git clone https://github.com/z3prover/z3test z3test + - script: $(python) z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2 + - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/ + - task: PublishPipelineArtifact@0 + inputs: + artifactName: 'Manylinux' + targetPath: $(Build.ArtifactStagingDirectory) + - job: Windows displayName: "Windows build" pool: @@ -43,19 +59,50 @@ jobs: - task: DownloadSecureFile@1 inputs: secureFile: 'z3.snk' - - script: scripts\mk_win_dist.cmd + - script: scripts\mk_win_dist.cmd - script: xcopy dist\*.zip $(Build.ArtifactStagingDirectory)\* /y - task: PublishPipelineArtifact@0 inputs: artifactName: 'Windows' targetPath: $(Build.ArtifactStagingDirectory) +- job: Python + displayName: "Python packaging" + dependsOn: + - Manylinux + - Windows + pool: + vmImage: "ubuntu-16.04" + steps: + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: 'Windows' + targetPath: $(Agent.TempDirectory) + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: 'Manylinux' + targetPath: $(Agent.TempDirectory) + - script: cd $(Agent.TempDirectory); mkdir linux-bin; cd linux-bin; unzip ../*centos*.zip + - script: cd $(Agent.TempDirectory); mkdir win32-bin; cd win32-bin; unzip ../*x86-win*.zip + - script: cd $(Agent.TempDirectory); mkdir win64-bin; cd win64-bin; unzip ../*x64-win*.zip + - script: python -m pip install --user -U setuptools wheel + - script: cd src/api/python; python setup.py sdist + # take a look at this PREMIUM HACK I came up with to get around the fact that the azure variable syntax overloads the bash syntax for subshells + - script: cd src/api/python; echo $(Agent.TempDirectory)/linux-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel + - script: cd src/api/python; echo $(Agent.TempDirectory)/win32-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel + - script: cd src/api/python; echo $(Agent.TempDirectory)/win64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel + - task: PublishPipelineArtifact@0 + inputs: + artifactName: 'Python packages' + targetPath: src/api/python/dist + - job: Deploy - displayName: "Deploy into GitHub" - dependsOn: + displayName: "Deploy into GitHub and PyPI" + dependsOn: - Mac - Ubuntu - Windows + - Python steps: - task: DownloadPipelineArtifact@0 inputs: @@ -69,10 +116,6 @@ jobs: inputs: artifactName: 'Ubuntu' targetPath: tmp - - task: CopyFiles@2 - inputs: - sourceFolder: $(Build.ArtifactStagingDirectory) - targetFolder: tmp # TBD: build NuGet package # TBD: this script should build a specific pre-specified tag - task: GitHubRelease@0 @@ -89,6 +132,16 @@ jobs: assets: 'tmp/*' isDraft: true isPreRelease: true + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: 'Python packages' + targetPath: dist + - task: DownloadSecureFile@1 + name: pypirc + inputs: + secureFile: 'pypirc' + - script: python -m pip install --user -U setuptools wheel twine + - script: python -m twine upload --config-file $(pypirc.secureFilePath) -r $(pypiReleaseServer) dist/* # TBD: run regression tests on generated binaries. diff --git a/src/api/python/setup.py b/src/api/python/setup.py index 30dd56549..1bd982cbc 100644 --- a/src/api/python/setup.py +++ b/src/api/python/setup.py @@ -33,11 +33,13 @@ if RELEASE_DIR is None: RELEASE_METADATA = None BUILD_PLATFORM = sys.platform else: + if not os.path.isdir(RELEASE_DIR): + raise Exception("RELEASE_DIR (%s) is not a directory!" % RELEASE_DIR) BUILD_DIR = os.path.join(RELEASE_DIR, 'bin') HEADER_DIRS = [os.path.join(RELEASE_DIR, 'include')] RELEASE_METADATA = os.path.basename(RELEASE_DIR).split('-') if RELEASE_METADATA[0] != 'z3' or len(RELEASE_METADATA) not in (4, 5): - raise Exception("RELEASE_DIR must be in the format z3-version-arch-os[-osversion] so we can extract metadata from it. Sorry!") + raise Exception("RELEASE_DIR (%s) must be in the format z3-version-arch-os[-osversion] so we can extract metadata from it. Sorry!" % RELEASE_DIR) RELEASE_METADATA.pop(0) BUILD_PLATFORM = RELEASE_METADATA[2] @@ -225,7 +227,12 @@ if 'bdist_wheel' in sys.argv and '--plat-name' not in sys.argv: distos = RELEASE_METADATA[2] if distos in ('debian', 'ubuntu') or 'linux' in distos: raise Exception("Linux binary distributions must be built on centos to conform to PEP 513") - if distos == 'win': + elif distos == 'centos': + if arch == 'x64': + plat_name = 'manylinux1_x86_64' + else: + plat_name = 'manylinux1_i686' + elif distos == 'win': if arch == 'x64': plat_name = 'win_amd64' else: